diff options
Diffstat (limited to 'sql')
187 files changed, 41352 insertions, 12090 deletions
diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 709306e8073..cacc87356d4 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -16,7 +16,8 @@ INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql -${CMAKE_SOURCE_DIR}/regex +${CMAKE_BINARY_DIR}/pcre +${CMAKE_SOURCE_DIR}/pcre ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} ${CMAKE_BINARY_DIR}/sql @@ -38,6 +39,7 @@ ENDIF() SET (SQL_SOURCE ../sql-common/client.c derror.cc des_key_file.cc discover.cc ../libmysql/errmsg.c field.cc field_conv.cc + filesort_utils.cc filesort.cc gstream.cc sha2.cc signal_handler.cc handler.cc hash_filo.h sql_plugin_services.h @@ -61,7 +63,8 @@ SET (SQL_SOURCE sql_list.cc sql_load.cc sql_manager.cc sql_parse.cc sql_partition.cc sql_plugin.cc sql_prepare.cc sql_rename.cc debug_sync.cc debug_sync.h - sql_repl.cc sql_select.cc sql_show.cc sql_state.c sql_string.cc + sql_repl.cc sql_select.cc sql_show.cc sql_state.c + sql_statistics.cc sql_string.cc sql_table.cc sql_test.cc sql_trigger.cc sql_udf.cc sql_union.cc sql_update.cc sql_view.cc strfunc.cc table.cc thr_malloc.cc sql_time.cc tztime.cc uniques.cc unireg.cc item_xmlfunc.cc @@ -77,6 +80,7 @@ SET (SQL_SOURCE sql_reload.cc # added in MariaDB: + sql_explain.h sql_explain.cc sql_lifo_buffer.h sql_join_cache.h sql_join_cache.cc create_options.cc multi_range_read.cc opt_index_cond_pushdown.cc opt_subselect.cc @@ -84,6 +88,8 @@ SET (SQL_SOURCE gcalc_slicescan.cc gcalc_tools.cc threadpool_common.cc ../sql-common/mysql_async.c + my_apc.cc my_apc.h + rpl_gtid.cc rpl_parallel.cc ${CMAKE_CURRENT_BINARY_DIR}/sql_builtin.cc ${GEN_SOURCES} ${MYSYS_LIBWRAP_SOURCE} @@ -102,7 +108,7 @@ ADD_LIBRARY(sql STATIC ${SQL_SOURCE}) ADD_DEPENDENCIES(sql GenServerSource) DTRACE_INSTRUMENT(sql) TARGET_LINK_LIBRARIES(sql ${MYSQLD_STATIC_PLUGIN_LIBS} - mysys dbug strings vio regex ${LIBJEMALLOC} + mysys dbug strings vio pcre ${LIBJEMALLOC} ${LIBWRAP} ${LIBCRYPT} ${LIBDL} ${CMAKE_THREAD_LIBS_INIT} ${SSL_LIBRARIES}) diff --git a/sql/bounded_queue.h b/sql/bounded_queue.h new file mode 100644 index 00000000000..2d4e6cff96d --- /dev/null +++ b/sql/bounded_queue.h @@ -0,0 +1,195 @@ +/* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#ifndef BOUNDED_QUEUE_INCLUDED +#define BOUNDED_QUEUE_INCLUDED + +#include <string.h> +#include "my_global.h" +#include "my_base.h" +#include "my_sys.h" +#include "queues.h" + +class Sort_param; + +/** + A priority queue with a fixed, limited size. + + This is a wrapper on top of QUEUE and the queue_xxx() functions. + It keeps the top-N elements which are inserted. + + Elements of type Element_type are pushed into the queue. + For each element, we call a user-supplied keymaker_function, + to generate a key of type Key_type for the element. + Instances of Key_type are compared with the user-supplied compare_function. + + The underlying QUEUE implementation needs one extra element for replacing + the lowest/highest element when pushing into a full queue. + */ +template<typename Element_type, typename Key_type> +class Bounded_queue +{ +public: + Bounded_queue() + { + memset(&m_queue, 0, sizeof(m_queue)); + } + + ~Bounded_queue() + { + delete_queue(&m_queue); + } + + /** + Function for making sort-key from input data. + @param param Sort parameters. + @param to Where to put the key. + @param from The input data. + */ + typedef void (*keymaker_function)(Sort_param *param, + Key_type *to, + Element_type *from); + + /** + Function for comparing two keys. + @param n Pointer to number of bytes to compare. + @param a First key. + @param b Second key. + @retval -1, 0, or 1 depending on whether the left argument is + less than, equal to, or greater than the right argument. + */ + typedef int (*compare_function)(size_t *n, Key_type **a, Key_type **b); + + /** + Initialize the queue. + + @param max_elements The size of the queue. + @param max_at_top Set to true if you want biggest element on top. + false: We keep the n largest elements. + pop() will return the smallest key in the result set. + true: We keep the n smallest elements. + pop() will return the largest key in the result set. + @param compare Compare function for elements, takes 3 arguments. + If NULL, we use get_ptr_compare(compare_length). + @param compare_length Length of the data (i.e. the keys) used for sorting. + @param keymaker Function which generates keys for elements. + @param sort_param Sort parameters. + @param sort_keys Array of pointers to keys to sort. + + @retval 0 OK, 1 Could not allocate memory. + + We do *not* take ownership of any of the input pointer arguments. + */ + int init(ha_rows max_elements, bool max_at_top, + compare_function compare, size_t compare_length, + keymaker_function keymaker, Sort_param *sort_param, + Key_type **sort_keys); + + /** + Pushes an element on the queue. + If the queue is already full, we discard one element. + Calls keymaker_function to generate a key for the element. + + @param element The element to be pushed. + */ + void push(Element_type *element); + + /** + Removes the top element from the queue. + + @retval Pointer to the (key of the) removed element. + + @note This function is for unit testing, where we push elements into to the + queue, and test that the appropriate keys are retained. + Interleaving of push() and pop() operations has not been tested. + */ + Key_type **pop() + { + // Don't return the extra element to the client code. + if (queue_is_full((&m_queue))) + queue_remove(&m_queue, 0); + DBUG_ASSERT(m_queue.elements > 0); + if (m_queue.elements == 0) + return NULL; + return reinterpret_cast<Key_type**>(queue_remove(&m_queue, 0)); + } + + /** + The number of elements in the queue. + */ + uint num_elements() const { return m_queue.elements; } + + /** + Is the queue initialized? + */ + bool is_initialized() const { return m_queue.max_elements > 0; } + +private: + Key_type **m_sort_keys; + size_t m_compare_length; + keymaker_function m_keymaker; + Sort_param *m_sort_param; + st_queue m_queue; +}; + + +template<typename Element_type, typename Key_type> +int Bounded_queue<Element_type, Key_type>::init(ha_rows max_elements, + bool max_at_top, + compare_function compare, + size_t compare_length, + keymaker_function keymaker, + Sort_param *sort_param, + Key_type **sort_keys) +{ + DBUG_ASSERT(sort_keys != NULL); + + m_sort_keys= sort_keys; + m_compare_length= compare_length; + m_keymaker= keymaker; + m_sort_param= sort_param; + // init_queue() takes an uint, and also does (max_elements + 1) + if (max_elements >= (UINT_MAX - 1)) + return 1; + if (compare == NULL) + compare= + reinterpret_cast<compare_function>(get_ptr_compare(compare_length)); + // We allocate space for one extra element, for replace when queue is full. + return init_queue(&m_queue, (uint) max_elements + 1, + 0, max_at_top, + reinterpret_cast<queue_compare>(compare), + &m_compare_length, 0, 0); +} + + +template<typename Element_type, typename Key_type> +void Bounded_queue<Element_type, Key_type>::push(Element_type *element) +{ + DBUG_ASSERT(is_initialized()); + if (queue_is_full((&m_queue))) + { + // Replace top element with new key, and re-order the queue. + Key_type **pq_top= reinterpret_cast<Key_type **>(queue_top(&m_queue)); + (*m_keymaker)(m_sort_param, *pq_top, element); + queue_replace_top(&m_queue); + } else { + // Insert new key into the queue. + (*m_keymaker)(m_sort_param, m_sort_keys[m_queue.elements], element); + queue_insert(&m_queue, + reinterpret_cast<uchar*>(&m_sort_keys[m_queue.elements])); + } +} + +#endif // BOUNDED_QUEUE_INCLUDED diff --git a/sql/create_options.cc b/sql/create_options.cc index 5cedfa03a63..f12120bd0a1 100644 --- a/sql/create_options.cc +++ b/sql/create_options.cc @@ -21,6 +21,7 @@ #include "create_options.h" #include <my_getopt.h> +#include "set_var.h" #define FRM_QUOTED_VALUE 0x8000 @@ -74,7 +75,7 @@ void engine_option_value::link(engine_option_value **start, } static bool report_wrong_value(THD *thd, const char *name, const char *val, - my_bool suppress_warning) + bool suppress_warning) { if (suppress_warning) return 0; @@ -92,7 +93,7 @@ static bool report_wrong_value(THD *thd, const char *name, const char *val, } static bool report_unknown_option(THD *thd, engine_option_value *val, - my_bool suppress_warning) + bool suppress_warning) { DBUG_ENTER("report_unknown_option"); @@ -115,8 +116,8 @@ static bool report_unknown_option(THD *thd, engine_option_value *val, } static bool set_one_value(ha_create_table_option *opt, - THD *thd, LEX_STRING *value, void *base, - my_bool suppress_warning, + THD *thd, const LEX_STRING *value, void *base, + bool suppress_warning, MEM_ROOT *root) { DBUG_ENTER("set_one_value"); @@ -126,6 +127,8 @@ static bool set_one_value(ha_create_table_option *opt, (value->str ? value->str : "<DEFAULT>"))); switch (opt->type) { + case HA_OPTION_TYPE_SYSVAR: + DBUG_ASSERT(0); // HA_OPTION_TYPE_SYSVAR's are replaced in resolve_sysvars() case HA_OPTION_TYPE_ULL: { ulonglong *val= (ulonglong*)((char*)base + opt->offset); @@ -257,52 +260,92 @@ static const size_t ha_option_type_sizeof[]= @retval FALSE OK */ -my_bool parse_option_list(THD* thd, void *option_struct_arg, - engine_option_value *option_list, - ha_create_table_option *rules, - my_bool suppress_warning, - MEM_ROOT *root) +bool parse_option_list(THD* thd, handlerton *hton, void *option_struct_arg, + engine_option_value **option_list, + ha_create_table_option *rules, + bool suppress_warning, MEM_ROOT *root) { ha_create_table_option *opt; size_t option_struct_size= 0; - engine_option_value *val= option_list; + engine_option_value *val, *last; void **option_struct= (void**)option_struct_arg; DBUG_ENTER("parse_option_list"); DBUG_PRINT("enter", - ("struct: 0x%lx list: 0x%lx rules: 0x%lx suppres %u root 0x%lx", - (ulong) *option_struct, (ulong)option_list, (ulong)rules, - (uint) suppress_warning, (ulong) root)); + ("struct: %p list: %p rules: %p suppress_warning: %u root: %p", + *option_struct, *option_list, rules, + (uint) suppress_warning, root)); if (rules) { - LEX_STRING default_val= {NULL, 0}; for (opt= rules; opt->name; opt++) set_if_bigger(option_struct_size, opt->offset + ha_option_type_sizeof[opt->type]); *option_struct= alloc_root(root, option_struct_size); - - /* set all values to default */ - for (opt= rules; opt->name; opt++) - set_one_value(opt, thd, &default_val, *option_struct, - suppress_warning, root); } - for (; val; val= val->next) + for (opt= rules; opt && opt->name; opt++) { - for (opt= rules; opt && opt->name; opt++) + bool seen=false; + for (val= *option_list; val; val= val->next) { + last= val; if (my_strnncoll(system_charset_info, (uchar*)opt->name, opt->name_length, (uchar*)val->name.str, val->name.length)) continue; + seen=true; + + if (val->parsed && !val->value.str) + continue; + if (set_one_value(opt, thd, &val->value, *option_struct, suppress_warning || val->parsed, root)) DBUG_RETURN(TRUE); val->parsed= true; break; } + if (!seen) + { + LEX_STRING default_val= null_lex_str; + + /* + If it's CREATE/ALTER TABLE parsing mode (options are created in the + transient thd->mem_root, not in the long living TABLE_SHARE::mem_root), + and variable-backed option was not explicitly set. + + If it's not create, but opening of the existing frm (that was, + probably, created with the older version of the storage engine and + does not have this option stored), we take the *default* value of the + sysvar, not the *current* value. Because we don't want to have + different option values for the same table if it's opened many times. + */ + if (root == thd->mem_root && opt->var) + { + // take a value from the variable and add it to the list + sys_var *sysvar= find_hton_sysvar(hton, opt->var); + DBUG_ASSERT(sysvar); + + char buf[256]; + String sbuf(buf, sizeof(buf), system_charset_info), *str; + if ((str= sysvar->val_str(&sbuf, thd, OPT_SESSION, 0))) + { + LEX_STRING name= { const_cast<char*>(opt->name), opt->name_length }; + default_val.str= strmake_root(root, str->ptr(), str->length()); + default_val.length= str->length(); + val= new (root) engine_option_value(name, default_val, true, + option_list, &last); + val->parsed= true; + } + } + set_one_value(opt, thd, &default_val, *option_struct, + suppress_warning, root); + } + } + + for (val= *option_list; val; val= val->next) + { if (report_unknown_option(thd, val, suppress_warning)) DBUG_RETURN(TRUE); val->parsed= true; @@ -313,6 +356,102 @@ my_bool parse_option_list(THD* thd, void *option_struct_arg, /** + Resolves all HA_OPTION_TYPE_SYSVAR elements. + + This is done when an engine is loaded. +*/ +static bool resolve_sysvars(handlerton *hton, ha_create_table_option *rules) +{ + for (ha_create_table_option *opt= rules; opt && opt->name; opt++) + { + if (opt->type == HA_OPTION_TYPE_SYSVAR) + { + struct my_option optp; + plugin_opt_set_limits(&optp, opt->var); + switch(optp.var_type) { + case GET_ULL: + case GET_ULONG: + case GET_UINT: + opt->type= HA_OPTION_TYPE_ULL; + opt->def_value= (ulonglong)optp.def_value; + opt->min_value= (ulonglong)optp.min_value; + opt->max_value= (ulonglong)optp.max_value; + opt->block_size= (ulonglong)optp.block_size; + break; + case GET_STR: + case GET_STR_ALLOC: + opt->type= HA_OPTION_TYPE_STRING; + break; + case GET_BOOL: + opt->type= HA_OPTION_TYPE_BOOL; + opt->def_value= optp.def_value; + break; + case GET_ENUM: + { + opt->type= HA_OPTION_TYPE_ENUM; + opt->def_value= optp.def_value; + + char buf[256]; + String str(buf, sizeof(buf), system_charset_info); + for (const char **s= optp.typelib->type_names; *s; s++) + { + if (str.append(*s) || str.append(',')) + return 1; + } + DBUG_ASSERT(str.length()); + opt->values= my_strndup(str.ptr(), str.length()-1, MYF(MY_WME)); + if (!opt->values) + return 1; + break; + } + default: + DBUG_ASSERT(0); + } + } + } + return 0; +} + +bool resolve_sysvar_table_options(handlerton *hton) +{ + return resolve_sysvars(hton, hton->table_options) || + resolve_sysvars(hton, hton->field_options) || + resolve_sysvars(hton, hton->index_options); +} + +/* + Restore HA_OPTION_TYPE_SYSVAR options back as they were + before resolve_sysvars(). + + This is done when the engine is unloaded, so that we could + call resolve_sysvars() if the engine is installed again. +*/ +static void free_sysvars(handlerton *hton, ha_create_table_option *rules) +{ + for (ha_create_table_option *opt= rules; opt && opt->name; opt++) + { + if (opt->var) + { + my_free(const_cast<char*>(opt->values)); + opt->type= HA_OPTION_TYPE_SYSVAR; + opt->def_value= 0; + opt->min_value= 0; + opt->max_value= 0; + opt->block_size= 0; + opt->values= 0; + } + } +} + +void free_sysvar_table_options(handlerton *hton) +{ + free_sysvars(hton, hton->table_options); + free_sysvars(hton, hton->field_options); + free_sysvars(hton, hton->index_options); +} + + +/** Parses all table/fields/keys options @param thd thread handler @@ -323,27 +462,27 @@ my_bool parse_option_list(THD* thd, void *option_struct_arg, @retval FALSE OK */ -my_bool parse_engine_table_options(THD *thd, handlerton *ht, - TABLE_SHARE *share) +bool parse_engine_table_options(THD *thd, handlerton *ht, TABLE_SHARE *share) { MEM_ROOT *root= &share->mem_root; DBUG_ENTER("parse_engine_table_options"); - if (parse_option_list(thd, &share->option_struct, share->option_list, + if (parse_option_list(thd, ht, &share->option_struct, & share->option_list, ht->table_options, TRUE, root)) DBUG_RETURN(TRUE); for (Field **field= share->field; *field; field++) { - if (parse_option_list(thd, &(*field)->option_struct, (*field)->option_list, + if (parse_option_list(thd, ht, &(*field)->option_struct, + & (*field)->option_list, ht->field_options, TRUE, root)) DBUG_RETURN(TRUE); } for (uint index= 0; index < share->keys; index ++) { - if (parse_option_list(thd, &share->key_info[index].option_struct, - share->key_info[index].option_list, + if (parse_option_list(thd, ht, &share->key_info[index].option_struct, + & share->key_info[index].option_list, ht->index_options, TRUE, root)) DBUG_RETURN(TRUE); } @@ -543,8 +682,8 @@ uchar *engine_option_value::frm_read(const uchar *buff, engine_option_value **st @retval FALSE OK */ -my_bool engine_table_options_frm_read(const uchar *buff, uint length, - TABLE_SHARE *share) +bool engine_table_options_frm_read(const uchar *buff, uint length, + TABLE_SHARE *share) { const uchar *buff_end= buff + length; engine_option_value *UNINIT_VAR(end); diff --git a/sql/create_options.h b/sql/create_options.h index ae918f6cea1..ea05bf75fac 100644 --- a/sql/create_options.h +++ b/sql/create_options.h @@ -69,16 +69,15 @@ class engine_option_value: public Sql_alloc typedef struct st_key KEY; class Create_field; -my_bool parse_engine_table_options(THD *thd, handlerton *ht, +bool resolve_sysvar_table_options(handlerton *hton); +void free_sysvar_table_options(handlerton *hton); +bool parse_engine_table_options(THD *thd, handlerton *ht, TABLE_SHARE *share); +bool parse_option_list(THD* thd, handlerton *hton, void *option_struct, + engine_option_value **option_list, + ha_create_table_option *rules, + bool suppress_warning, MEM_ROOT *root); +bool engine_table_options_frm_read(const uchar *buff, uint length, TABLE_SHARE *share); -my_bool parse_option_list(THD* thd, void *option_struct, - engine_option_value *option_list, - ha_create_table_option *rules, - my_bool suppress_warning, - MEM_ROOT *root); -my_bool engine_table_options_frm_read(const uchar *buff, - uint length, - TABLE_SHARE *share); engine_option_value *merge_engine_table_options(engine_option_value *source, engine_option_value *changes, MEM_ROOT *root); diff --git a/sql/datadict.cc b/sql/datadict.cc index 4e4fcafa31b..deeedcd512d 100644 --- a/sql/datadict.cc +++ b/sql/datadict.cc @@ -18,6 +18,22 @@ #include "sql_class.h" #include "sql_table.h" +static int read_string(File file, uchar**to, size_t length) +{ + DBUG_ENTER("read_string"); + + my_free(*to); + if (!(*to= (uchar*) my_malloc(length+1,MYF(MY_WME))) || + mysql_file_read(file, *to, length, MYF(MY_NABP))) + { + my_free(*to); + *to= 0; + DBUG_RETURN(1); + } + *((char*) *to+length)= '\0'; // C-style safety + DBUG_RETURN (0); +} + /** Check type of .frm if we are not going to parse it. @@ -60,9 +76,7 @@ frm_type_enum dd_frm_type(THD *thd, char *path, enum legacy_db_type *dbt) if the following test is true (arg #3). This should not have effect on return value from this function (default FRMTYPE_TABLE) */ - if (header[0] != (uchar) 254 || header[1] != 1 || - (header[2] != FRM_VER && header[2] != FRM_VER+1 && - (header[2] < FRM_VER+3 || header[2] > FRM_VER+4))) + if (!is_binary_frm_header(header)) goto err; *dbt= (enum legacy_db_type) (uint) *(header + 3); @@ -84,9 +98,9 @@ frm_type_enum dd_frm_type(THD *thd, char *path, enum legacy_db_type *dbt) if ((n_length= uint4korr(frm_image+55))) { - uint record_offset= (uint2korr(frm_image+6)+ + uint record_offset= uint2korr(frm_image+6)+ ((uint2korr(frm_image+14) == 0xffff ? - uint4korr(frm_image+47) : uint2korr(frm_image+14)))); + uint4korr(frm_image+47) : uint2korr(frm_image+14))); uint reclength= uint2korr(frm_image+16); uchar *next_chunk= frm_image + record_offset + reclength; @@ -117,117 +131,44 @@ err: } -/** - Given a table name, check type of .frm and legacy table type. - - @param[in] thd The current session. - @param[in] db Table schema. - @param[in] table_name Table database. - @param[out] table_type handlerton of the table if FRMTYPE_TABLE, - otherwise undefined. - - @return FALSE if FRMTYPE_TABLE and storage engine found. TRUE otherwise. -*/ - -bool dd_frm_storage_engine(THD *thd, const char *db, const char *table_name, - handlerton **table_type) -{ - char path[FN_REFLEN + 1]; - enum legacy_db_type db_type; - LEX_STRING db_name = {(char *) db, strlen(db)}; - - /* There should be at least some lock on the table. */ - DBUG_ASSERT(thd->mdl_context.is_lock_owner(MDL_key::TABLE, db, - table_name, MDL_SHARED)); - - if (check_db_name(&db_name)) - { - my_error(ER_WRONG_DB_NAME, MYF(0), db_name.str); - return TRUE; - } - - if (check_table_name(table_name, strlen(table_name), FALSE)) - { - my_error(ER_WRONG_TABLE_NAME, MYF(0), table_name); - return TRUE; - } - - (void) build_table_filename(path, sizeof(path) - 1, db, - table_name, reg_ext, 0); - - dd_frm_type(thd, path, &db_type); - - /* Type is unknown if the object is not found or is not a table. */ - if (db_type == DB_TYPE_UNKNOWN || - !(*table_type= ha_resolve_by_legacy_type(thd, db_type))) - { - my_error(ER_NO_SUCH_TABLE, MYF(0), db, table_name); - return TRUE; - } - - return FALSE; -} - - -/** - Given a table name, check if the storage engine for the - table referred by this name supports an option 'flag'. - Return an error if the table does not exist or is not a - base table. - - @pre Any metadata lock on the table. - - @param[in] thd The current session. - @param[in] db Table schema. - @param[in] table_name Table database. - @param[in] flag The option to check. - @param[out] yes_no The result. Undefined if error. -*/ - -bool dd_check_storage_engine_flag(THD *thd, - const char *db, const char *table_name, - uint32 flag, bool *yes_no) -{ - handlerton *table_type; - - if (dd_frm_storage_engine(thd, db, table_name, &table_type)) - return TRUE; - - *yes_no= ha_check_storage_engine_flag(table_type, flag); - - return FALSE; -} - - /* Regenerate a metadata locked table. @param thd Thread context. @param db Name of the database to which the table belongs to. @param name Table name. + @param path For temporary tables only - path to table files. + Otherwise NULL (the path is calculated from db and table names). @retval FALSE Success. @retval TRUE Error. */ -bool dd_recreate_table(THD *thd, const char *db, const char *table_name) +bool dd_recreate_table(THD *thd, const char *db, const char *table_name, + const char *path) { bool error= TRUE; HA_CREATE_INFO create_info; - char path[FN_REFLEN + 1]; + char path_buf[FN_REFLEN + 1]; DBUG_ENTER("dd_recreate_table"); - /* There should be a exclusive metadata lock on the table. */ - DBUG_ASSERT(thd->mdl_context.is_lock_owner(MDL_key::TABLE, db, table_name, - MDL_EXCLUSIVE)); - memset(&create_info, 0, sizeof(create_info)); - /* Create a path to the table, but without a extension. */ - build_table_filename(path, sizeof(path) - 1, db, table_name, "", 0); + if (path) + create_info.options|= HA_LEX_CREATE_TMP_TABLE; + else + { + build_table_filename(path_buf, sizeof(path_buf) - 1, + db, table_name, "", 0); + path= path_buf; + + /* There should be a exclusive metadata lock on the table. */ + DBUG_ASSERT(thd->mdl_context.is_lock_owner(MDL_key::TABLE, db, table_name, + MDL_EXCLUSIVE)); + } /* Attempt to reconstruct the table. */ - error= ha_create_table(thd, path, db, table_name, &create_info, TRUE); + error= ha_create_table(thd, path, db, table_name, &create_info, NULL); DBUG_RETURN(error); } diff --git a/sql/datadict.h b/sql/datadict.h index f852b02f52c..dd80942daca 100644 --- a/sql/datadict.h +++ b/sql/datadict.h @@ -28,14 +28,22 @@ enum frm_type_enum FRMTYPE_VIEW }; +/* + Take extra care when using dd_frm_type() - it only checks the .frm file, + and it won't work for any engine that supports discovery. + + Prefer to use ha_table_exists() instead. + To check whether it's an frm of a view, use dd_frm_is_view(). +*/ frm_type_enum dd_frm_type(THD *thd, char *path, enum legacy_db_type *dbt); -bool dd_frm_storage_engine(THD *thd, const char *db, const char *table_name, - handlerton **table_type); -bool dd_check_storage_engine_flag(THD *thd, - const char *db, const char *table_name, - uint32 flag, - bool *yes_no); -bool dd_recreate_table(THD *thd, const char *db, const char *table_name); +static inline bool dd_frm_is_view(THD *thd, char *path) +{ + enum legacy_db_type not_used; + return dd_frm_type(thd, path, ¬_used) == FRMTYPE_VIEW; +} + +bool dd_recreate_table(THD *thd, const char *db, const char *table_name, + const char *path = NULL); #endif // DATADICT_INCLUDED diff --git a/sql/debug_sync.cc b/sql/debug_sync.cc index 4cff1c09ba7..156296972fc 100644 --- a/sql/debug_sync.cc +++ b/sql/debug_sync.cc @@ -239,7 +239,8 @@ void debug_sync_init_thread(THD *thd) if (opt_debug_sync_timeout) { thd->debug_sync_control= (st_debug_sync_control*) - my_malloc(sizeof(st_debug_sync_control), MYF(MY_WME | MY_ZEROFILL)); + my_malloc(sizeof(st_debug_sync_control), + MYF(MY_WME | MY_ZEROFILL | MY_THREAD_SPECIFIC)); if (!thd->debug_sync_control) { /* @@ -984,6 +985,7 @@ static bool debug_sync_eval_action(THD *thd, char *action_str) DBUG_ENTER("debug_sync_eval_action"); DBUG_ASSERT(thd); DBUG_ASSERT(action_str); + DBUG_PRINT("debug_sync", ("action_str='%s'", action_str)); /* Get debug sync point name. Or a special command. diff --git a/sql/discover.cc b/sql/discover.cc index b9dba92a780..cc0dece031a 100644 --- a/sql/discover.cc +++ b/sql/discover.cc @@ -45,7 +45,7 @@ 3 Could not allocate data for read. Could not read file */ -int readfrm(const char *name, uchar **frmdata, size_t *len) +int readfrm(const char *name, const uchar **frmdata, size_t *len) { int error; char index_file[FN_REFLEN]; @@ -70,13 +70,17 @@ int readfrm(const char *name, uchar **frmdata, size_t *len) error= 2; if (mysql_file_fstat(file, &state, MYF(0))) goto err; - read_len= (size_t)state.st_size; + read_len= (size_t)min(FRM_MAX_SIZE, state.st_size); // safety // Read whole frm file error= 3; - read_data= 0; // Nothing to free - if (read_string(file, &read_data, read_len)) + if (!(read_data= (uchar*)my_malloc(read_len, MYF(MY_WME)))) goto err; + if (mysql_file_read(file, read_data, read_len, MYF(MY_NABP))) + { + my_free(read_data); + goto err; + } // Setup return data *frmdata= (uchar*) read_data; @@ -96,7 +100,7 @@ int readfrm(const char *name, uchar **frmdata, size_t *len) Write the content of a frm data pointer to a frm file. - @param name path to table-file "db/name" + @param path path to table-file "db/name" @param frmdata frm data @param len length of the frmdata @@ -106,29 +110,160 @@ int readfrm(const char *name, uchar **frmdata, size_t *len) 2 Could not write file */ -int writefrm(const char *name, const uchar *frmdata, size_t len) +int writefrm(const char *path, const char *db, const char *table, + bool tmp_table, const uchar *frmdata, size_t len) { - File file; - char index_file[FN_REFLEN]; + char file_name[FN_REFLEN+1]; int error; + int create_flags= O_RDWR | O_TRUNC; DBUG_ENTER("writefrm"); - DBUG_PRINT("enter",("name: '%s' len: %lu ",name, (ulong) len)); + DBUG_PRINT("enter",("name: '%s' len: %lu ",path, (ulong) len)); - error= 0; - if ((file= mysql_file_create(key_file_frm, - fn_format(index_file, name, "", reg_ext, - MY_UNPACK_FILENAME | MY_APPEND_EXT), - CREATE_MODE, O_RDWR | O_TRUNC, - MYF(MY_WME))) >= 0) + if (tmp_table) + create_flags|= O_EXCL | O_NOFOLLOW; + + strxnmov(file_name, sizeof(file_name)-1, path, reg_ext, NullS); + + File file= mysql_file_create(key_file_frm, file_name, + CREATE_MODE, create_flags, MYF(0)); + + if ((error= file < 0)) { - if (mysql_file_write(file, frmdata, len, MYF(MY_WME | MY_NABP))) - error= 2; - (void) mysql_file_close(file, MYF(0)); + if (my_errno == ENOENT) + my_error(ER_BAD_DB_ERROR, MYF(0), db); + else + my_error(ER_CANT_CREATE_TABLE, MYF(0), db, table, my_errno); + } + else + { + error= mysql_file_write(file, frmdata, len, MYF(MY_WME | MY_NABP)); + + if (!error && !tmp_table && opt_sync_frm) + error= mysql_file_sync(file, MYF(MY_WME)) || + my_sync_dir_by_file(file_name, MYF(MY_WME)); + + error|= mysql_file_close(file, MYF(MY_WME)); } DBUG_RETURN(error); } /* writefrm */ +static inline void advance(FILEINFO* &from, FILEINFO* &to, + FILEINFO* cur, bool &skip) +{ + if (skip) // if not copying + from= cur; // just advance the start pointer + else // if copying + if (to == from) // but to the same place (not shifting the data) + from= to= cur; // advance both pointers + else // otherwise + while (from < cur) // have to copy [from...cur) to [to...) + *to++ = *from++; + skip= false; +} + +/** + Go through the directory listing looking for files with a specified + extension and add them to the result list + + @details + This function may be called many times on the same directory listing + but with different extensions. To avoid discovering the same table twice, + whenever a table file is discovered, all files with the same name + (independently from the extensions) are removed from the list. + Example: the list contained + { "db.opt", "t1.MYD", "t1.MYI", "t1.frm", "t2.ARZ", "t3.ARZ", "t3.frm" } + on discovering all ".frm" files, tables "t1" and "t3" will be found, + and list will become + { "db.opt", "t2.ARZ" } + and now ".ARZ" discovery can discover the table "t2" + @note + This function assumes that the directory listing is sorted alphabetically. + @note Partitioning makes this more complicated. A partitioned table t1 might + have files, like t1.frm, t1#P#part1.ibd, t1#P#foo.ibd, etc. + That means we need to compare file names only up to the first '#' or '.' + whichever comes first. +*/ +int extension_based_table_discovery(MY_DIR *dirp, const char *ext_meta, + handlerton::discovered_list *result) +{ + CHARSET_INFO *cs= character_set_filesystem; + size_t ext_meta_len= strlen(ext_meta); + FILEINFO *from, *to, *cur, *end; + bool skip= false; + + from= to= cur= dirp->dir_entry; + end= cur + dirp->number_of_files; + while (cur < end) + { + char *octothorp= strrchr(cur->name + 1, '#'); + char *ext= strchr(octothorp ? octothorp : cur->name, FN_EXTCHAR); + + if (ext) + { + size_t len= (octothorp ? octothorp : ext) - cur->name; + if (from != cur && + (my_strnncoll(cs, (uchar*)from->name, len, (uchar*)cur->name, len) || + (from->name[len] != FN_EXTCHAR && from->name[len] != '#'))) + advance(from, to, cur, skip); + + if (my_strnncoll(cs, (uchar*)ext, strlen(ext), + (uchar*)ext_meta, ext_meta_len) == 0) + { + *ext = 0; + if (result->add_file(cur->name)) + return 1; + *ext = FN_EXTCHAR; + skip= true; // table discovered, skip all files with the same name + } + } + else + { + advance(from, to, cur, skip); + from++; + } + + cur++; + } + advance(from, to, cur, skip); + dirp->number_of_files= to - dirp->dir_entry; + return 0; +} + +/** + Simple, not reusable file-based table discovery + + @details + simplified version of extension_based_table_discovery(), that does not + modify the list of files. It cannot be called many times for the same + directory listing, otherwise it'll produce duplicate results. +*/ +int ext_table_discovery_simple(MY_DIR *dirp, + handlerton::discovered_list *result) +{ + CHARSET_INFO *cs= character_set_filesystem; + FILEINFO *cur, *end; + + cur= dirp->dir_entry; + end= cur + dirp->number_of_files; + while (cur < end) + { + char *ext= strrchr(cur->name, FN_EXTCHAR); + + if (ext) + { + if (my_strnncoll(cs, (uchar*)ext, strlen(ext), + (uchar*)reg_ext, reg_ext_length) == 0) + { + *ext = 0; + if (result->add_file(cur->name)) + return 1; + } + } + cur++; + } + return 0; +} diff --git a/sql/discover.h b/sql/discover.h index a663e44128d..fbf94891c74 100644 --- a/sql/discover.h +++ b/sql/discover.h @@ -18,7 +18,16 @@ #include "my_global.h" /* uchar */ -int readfrm(const char *name, uchar **data, size_t *length); -int writefrm(const char* name, const uchar* data, size_t len); +int extension_based_table_discovery(MY_DIR *dirp, const char *ext, + handlerton::discovered_list *tl); + +#ifdef MYSQL_SERVER +int readfrm(const char *name, const uchar **data, size_t *length); +int writefrm(const char *path, const char *db, const char *table, + bool tmp_table, const uchar *frmdata, size_t len); + +int ext_table_discovery_simple(MY_DIR *dirp, + handlerton::discovered_list *result); +#endif #endif /* DISCOVER_INCLUDED */ diff --git a/sql/event_data_objects.cc b/sql/event_data_objects.cc index dfd35583581..bc683a76ecb 100644 --- a/sql/event_data_objects.cc +++ b/sql/event_data_objects.cc @@ -209,7 +209,7 @@ Event_basic::Event_basic() { DBUG_ENTER("Event_basic::Event_basic"); /* init memory root */ - init_sql_alloc(&mem_root, 256, 512); + init_sql_alloc(&mem_root, 256, 512, MYF(0)); dbname.str= name.str= NULL; dbname.length= name.length= 0; time_zone= NULL; diff --git a/sql/event_db_repository.cc b/sql/event_db_repository.cc index 37dff0da714..233fd6a3bd6 100644 --- a/sql/event_db_repository.cc +++ b/sql/event_db_repository.cc @@ -163,7 +163,7 @@ const TABLE_FIELD_TYPE event_table_fields[ET_FIELD_COUNT] = }; static const TABLE_FIELD_DEF - event_table_def= {ET_FIELD_COUNT, event_table_fields}; +event_table_def= {ET_FIELD_COUNT, event_table_fields, 0, (uint*) 0}; class Event_db_intact : public Table_check_intact { @@ -835,9 +835,6 @@ Event_db_repository::update_event(THD *thd, Event_parse_data *parse_data, (int) table->field[ET_FIELD_ON_COMPLETION]->val_int())) goto end; - /* Don't update create on row update. */ - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - /* mysql_event_fill_row() calls my_error() in case of error so no need to handle it here @@ -1118,17 +1115,15 @@ update_timing_fields_for_event(THD *thd, TABLE *table= NULL; Field **fields; int ret= 1; - bool save_binlog_row_based; + enum_binlog_format save_binlog_format; MYSQL_TIME time; - DBUG_ENTER("Event_db_repository::update_timing_fields_for_event"); /* Turn off row binlogging of event timing updates. These are not used for RBR of events replicated to the slave. */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); DBUG_ASSERT(thd->security_ctx->master_access & SUPER_ACL); @@ -1141,8 +1136,6 @@ update_timing_fields_for_event(THD *thd, goto end; store_record(table, record[1]); - /* Don't update create on row update. */ - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; my_tz_OFFSET0->gmt_sec_to_TIME(&time, last_executed); fields[ET_FIELD_LAST_EXECUTED]->set_notnull(); @@ -1163,10 +1156,7 @@ end: if (table) close_mysql_tables(thd); - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); + thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(test(ret)); } diff --git a/sql/event_parse_data.cc b/sql/event_parse_data.cc index ad812a6aa5d..4316a9f1fb8 100644 --- a/sql/event_parse_data.cc +++ b/sql/event_parse_data.cc @@ -574,8 +574,8 @@ void Event_parse_data::check_originator_id(THD *thd) status= Event_parse_data::SLAVESIDE_DISABLED; status_changed= true; } - originator = thd->server_id; + originator = thd->variables.server_id; } else - originator = server_id; + originator = global_system_variables.server_id; } diff --git a/sql/event_scheduler.cc b/sql/event_scheduler.cc index f8d177ac0c1..f005f8787ed 100644 --- a/sql/event_scheduler.cc +++ b/sql/event_scheduler.cc @@ -132,11 +132,11 @@ post_init_event_thread(THD *thd) return TRUE; } + thread_safe_increment32(&thread_count, &thread_count_lock); mysql_mutex_lock(&LOCK_thread_count); threads.append(thd); - thread_count++; - inc_thread_running(); mysql_mutex_unlock(&LOCK_thread_count); + inc_thread_running(); return FALSE; } @@ -153,15 +153,9 @@ void deinit_event_thread(THD *thd) { thd->proc_info= "Clearing"; - DBUG_ASSERT(thd->net.buff != 0); - net_end(&thd->net); DBUG_PRINT("exit", ("Event thread finishing")); - mysql_mutex_lock(&LOCK_thread_count); - thread_count--; - dec_thread_running(); - delete thd; - mysql_cond_broadcast(&COND_thread_count); - mysql_mutex_unlock(&LOCK_thread_count); + + delete_running_thd(thd); } @@ -182,15 +176,17 @@ deinit_event_thread(THD *thd) void pre_init_event_thread(THD* thd) { + THD *orig_thd= current_thd; DBUG_ENTER("pre_init_event_thread"); + + set_current_thd(thd); thd->client_capabilities= 0; thd->security_ctx->master_access= 0; thd->security_ctx->db_access= 0; thd->security_ctx->host_or_ip= (char*)my_localhost; - my_net_init(&thd->net, NULL); + my_net_init(&thd->net, NULL, MYF(MY_THREAD_SPECIFIC)); thd->security_ctx->set_user((char*)"event_scheduler"); thd->net.read_timeout= slave_net_timeout; - thd->slave_thread= 0; thd->variables.option_bits|= OPTION_AUTO_IS_NULL; thd->client_capabilities|= CLIENT_MULTI_RESULTS; mysql_mutex_lock(&LOCK_thread_count); @@ -208,6 +204,7 @@ pre_init_event_thread(THD* thd) /* Do not use user-supplied timeout value for system threads. */ thd->variables.lock_wait_timeout= LONG_TIMEOUT; + set_current_thd(orig_thd); DBUG_VOID_RETURN; } @@ -409,6 +406,7 @@ Event_scheduler::start(int *err_no) ret= true; goto end; } + pre_init_event_thread(new_thd); new_thd->system_thread= SYSTEM_THREAD_EVENT_SCHEDULER; new_thd->command= COM_DAEMON; @@ -420,6 +418,7 @@ Event_scheduler::start(int *err_no) */ new_thd->security_ctx->master_access |= SUPER_ACL; + /* This should not be marked with MY_THREAD_SPECIFIC */ scheduler_param_value= (struct scheduler_param *)my_malloc(sizeof(struct scheduler_param), MYF(0)); scheduler_param_value->thd= new_thd; @@ -542,6 +541,7 @@ Event_scheduler::execute_top(Event_queue_element_for_exec *event_name) pthread_t th; int res= 0; DBUG_ENTER("Event_scheduler::execute_top"); + if (!(new_thd= new THD())) goto error; diff --git a/sql/events.cc b/sql/events.cc index 763c75e77b0..b0d7c00478c 100644 --- a/sql/events.cc +++ b/sql/events.cc @@ -306,7 +306,8 @@ Events::create_event(THD *thd, Event_parse_data *parse_data, bool if_not_exists) { bool ret; - bool save_binlog_row_based, event_already_exists; + bool event_already_exists; + enum_binlog_format save_binlog_format; DBUG_ENTER("Events::create_event"); if (check_if_system_tables_error()) @@ -338,8 +339,7 @@ Events::create_event(THD *thd, Event_parse_data *parse_data, Turn off row binlogging of this statement and use statement-based so that all supporting tables are updated for CREATE EVENT command. */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); if (lock_object_name(thd, MDL_key::EVENT, parse_data->dbname.str, parse_data->name.str)) @@ -398,10 +398,8 @@ Events::create_event(THD *thd, Event_parse_data *parse_data, } } } - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); + + thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); } @@ -431,7 +429,7 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, LEX_STRING *new_dbname, LEX_STRING *new_name) { int ret; - bool save_binlog_row_based; + enum_binlog_format save_binlog_format; Event_queue_element *new_element; DBUG_ENTER("Events::update_event"); @@ -478,8 +476,7 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, Turn off row binlogging of this statement and use statement-based so that all supporting tables are updated for UPDATE EVENT command. */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); if (lock_object_name(thd, MDL_key::EVENT, parse_data->dbname.str, parse_data->name.str)) @@ -513,11 +510,8 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, ret= write_bin_log(thd, TRUE, thd->query(), thd->query_length()); } } - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); + thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); } @@ -550,7 +544,7 @@ bool Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) { int ret; - bool save_binlog_row_based; + enum_binlog_format save_binlog_format; DBUG_ENTER("Events::drop_event"); if (check_if_system_tables_error()) @@ -563,8 +557,7 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) Turn off row binlogging of this statement and use statement-based so that all supporting tables are updated for DROP EVENT command. */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); if (lock_object_name(thd, MDL_key::EVENT, dbname.str, name.str)) @@ -578,10 +571,8 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) DBUG_ASSERT(thd->query() && thd->query_length()); ret= write_bin_log(thd, TRUE, thd->query(), thd->query_length()); } - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); + + thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); } @@ -889,7 +880,7 @@ end: } delete thd; /* Remember that we don't have a THD */ - my_pthread_setspecific_ptr(THR_THD, NULL); + set_current_thd(0); DBUG_RETURN(res); } diff --git a/sql/field.cc b/sql/field.cc index e9cd153ad75..1e025784617 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2008, 2013, Monty Program Ab + Copyright (c) 2008, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -50,11 +50,6 @@ Instansiate templates and static variables *****************************************************************************/ -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class List<Create_field>; -template class List_iterator<Create_field>; -#endif - static const char *zero_timestamp="0000-00-00 00:00:00.000000"; /* number of bytes to store second_part part of the TIMESTAMP(N) */ @@ -73,10 +68,10 @@ const char field_separator=','; #define LONGLONG_TO_STRING_CONVERSION_BUFFER_SIZE 128 #define DECIMAL_TO_STRING_CONVERSION_BUFFER_SIZE 128 #define BLOB_PACK_LENGTH_TO_MAX_LENGH(arg) \ -((ulong) ((LL(1) << min(arg, 4) * 8) - LL(1))) +((ulong) ((1LL << min(arg, 4) * 8) - 1)) #define ASSERT_COLUMN_MARKED_FOR_READ DBUG_ASSERT(!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))) -#define ASSERT_COLUMN_MARKED_FOR_WRITE_OR_COMPUTED DBUG_ASSERT(!table || (!table->write_set || bitmap_is_set(table->write_set, field_index) || bitmap_is_set(table->vcol_set, field_index))) +#define ASSERT_COLUMN_MARKED_FOR_WRITE_OR_COMPUTED DBUG_ASSERT(is_stat_field || !table || (!table->write_set || bitmap_is_set(table->write_set, field_index) || bitmap_is_set(table->vcol_set, field_index))) #define FLAGSTR(S,F) ((S) & (F) ? #F " " : "") @@ -1195,11 +1190,11 @@ int Field_num::check_int(CHARSET_INFO *cs, const char *str, int length, if (str == int_end || error == MY_ERRNO_EDOM) { ErrConvString err(str, length, cs); - push_warning_printf(table->in_use, MYSQL_ERROR::WARN_LEVEL_WARN, + push_warning_printf(get_thd(), MYSQL_ERROR::WARN_LEVEL_WARN, ER_TRUNCATED_WRONG_VALUE_FOR_FIELD, ER(ER_TRUNCATED_WRONG_VALUE_FOR_FIELD), "integer", err.ptr(), field_name, - (ulong) table->in_use->warning_info->current_row_for_warning()); + (ulong) get_thd()->warning_info->current_row_for_warning()); return 1; } /* Test if we have garbage at the end of the given string. */ @@ -1268,7 +1263,7 @@ bool Field_num::get_int(CHARSET_INFO *cs, const char *from, uint len, goto out_of_range; } } - if (table->in_use->count_cuted_fields && + if (get_thd()->count_cuted_fields && check_int(cs, from, len, end, error)) return 1; return 0; @@ -1278,6 +1273,37 @@ out_of_range: return 1; } + +/** + @brief + Determine the relative position of the field value in a numeric interval + + @details + The function returns a double number between 0.0 and 1.0 as the relative + position of the value of the this field in the numeric interval of [min,max]. + If the value is not in the interval the the function returns 0.0 when + the value is less than min, and, 1.0 when the value is greater than max. + + @param min value of the left end of the interval + @param max value of the right end of the interval + + @return + relative position of the field value in the numeric interval [min,max] +*/ + +double Field_num::pos_in_interval(Field *min, Field *max) +{ + double n, d; + n= val_real() - min->val_real(); + if (n < 0) + return 0.0; + d= max->val_real() - min->val_real(); + if (d <= 0) + return 1.0; + return min(n/d, 1.0); +} + + /** Process decimal library return codes and issue warnings for overflow and truncation. @@ -1339,13 +1365,18 @@ Field::Field(uchar *ptr_arg,uint32 length_arg,uchar *null_ptr_arg, option_struct(0), key_start(0), part_of_key(0), part_of_key_not_clustered(0), part_of_sortkey(0), unireg_check(unireg_check_arg), field_length(length_arg), - null_bit(null_bit_arg), is_created_from_null_item(FALSE), vcol_info(0), + null_bit(null_bit_arg), is_created_from_null_item(FALSE), + read_stats(NULL), collected_stats(0), + vcol_info(0), stored_in_db(TRUE) { flags=null_ptr ? 0: NOT_NULL_FLAG; comment.str= (char*) ""; comment.length=0; - field_index= 0; + field_index= 0; + is_stat_field= FALSE; + cond_selectivity= 1.0; + next_equal_field= NULL; } @@ -1445,10 +1476,11 @@ int Field::store(const char *to, uint length, CHARSET_INFO *cs, enum_check_fields check_level) { int res; - enum_check_fields old_check_level= table->in_use->count_cuted_fields; - table->in_use->count_cuted_fields= check_level; + THD *thd= get_thd(); + enum_check_fields old_check_level= thd->count_cuted_fields; + thd->count_cuted_fields= check_level; res= store(to, length, cs); - table->in_use->count_cuted_fields= old_check_level; + thd->count_cuted_fields= old_check_level; return res; } @@ -1832,6 +1864,10 @@ Field *Field::new_field(MEM_ROOT *root, TABLE *new_table, tmp->key_start.init(0); tmp->part_of_key.init(0); tmp->part_of_sortkey.init(0); + /* + TODO: it is not clear why this method needs to reset unireg_check. + Try not to reset it, or explain why it needs to be reset. + */ tmp->unireg_check= Field::NONE; tmp->flags&= (NOT_NULL_FLAG | BLOB_FLAG | UNSIGNED_FLAG | ZEROFILL_FLAG | BINARY_FLAG | ENUM_FLAG | SET_FLAG); @@ -1870,6 +1906,32 @@ Field *Field::clone(MEM_ROOT *root, TABLE *new_table) } + +Field *Field::clone(MEM_ROOT *root, TABLE *new_table, my_ptrdiff_t diff, + bool stat_flag) +{ + Field *tmp; + if ((tmp= (Field*) memdup_root(root,(char*) this,size_of()))) + { + tmp->init(new_table); + tmp->move_field_offset(diff); + } + tmp->is_stat_field= stat_flag; + return tmp; +} + + +Field *Field::clone(MEM_ROOT *root, my_ptrdiff_t diff) +{ + Field *tmp; + if ((tmp= (Field*) memdup_root(root,(char*) this,size_of()))) + { + tmp->move_field_offset(diff); + } + return tmp; +} + + /**************************************************************************** Field_null, a field that always return NULL ****************************************************************************/ @@ -1984,7 +2046,7 @@ int Field_decimal::store(const char *from_arg, uint len, CHARSET_INFO *cs) uchar *left_wall,*right_wall; uchar tmp_char; /* - To remember if table->in_use->cuted_fields has already been incremented, + To remember if get_thd()->cuted_fields has already been incremented, to do that only once */ bool is_cuted_fields_incr=0; @@ -2075,7 +2137,7 @@ int Field_decimal::store(const char *from_arg, uint len, CHARSET_INFO *cs) it makes the code easer to read. */ - if (table->in_use->count_cuted_fields) + if (get_thd()->count_cuted_fields) { // Skip end spaces for (;from != end && my_isspace(&my_charset_bin, *from); from++) ; @@ -2227,7 +2289,7 @@ int Field_decimal::store(const char *from_arg, uint len, CHARSET_INFO *cs) /* Write digits of the frac_% parts ; - Depending on table->in_use->count_cutted_fields, we may also want + Depending on get_thd()->count_cutted_fields, we may also want to know if some non-zero tail of these parts will be truncated (for example, 0.002->0.00 will generate a warning, while 0.000->0.00 will not) @@ -2245,7 +2307,7 @@ int Field_decimal::store(const char *from_arg, uint len, CHARSET_INFO *cs) { if (pos == right_wall) { - if (table->in_use->count_cuted_fields && !is_cuted_fields_incr) + if (get_thd()->count_cuted_fields && !is_cuted_fields_incr) break; // Go on below to see if we lose non zero digits return 0; } @@ -2666,20 +2728,21 @@ int Field_new_decimal::store(const char *from, uint length, ASSERT_COLUMN_MARKED_FOR_WRITE_OR_COMPUTED; int err; my_decimal decimal_value; + THD *thd= get_thd(); DBUG_ENTER("Field_new_decimal::store(char*)"); if ((err= str2my_decimal(E_DEC_FATAL_ERROR & ~(E_DEC_OVERFLOW | E_DEC_BAD_NUM), from, length, charset_arg, &decimal_value)) && - table->in_use->abort_on_warning) + thd->abort_on_warning) { ErrConvString errmsg(from, length, &my_charset_bin); - push_warning_printf(table->in_use, MYSQL_ERROR::WARN_LEVEL_WARN, + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, ER_TRUNCATED_WRONG_VALUE_FOR_FIELD, ER(ER_TRUNCATED_WRONG_VALUE_FOR_FIELD), "decimal", errmsg.ptr(), field_name, - (ulong) table->in_use->warning_info->current_row_for_warning()); + (ulong) thd->warning_info->current_row_for_warning()); DBUG_RETURN(err); } @@ -2695,11 +2758,11 @@ int Field_new_decimal::store(const char *from, uint length, case E_DEC_BAD_NUM: { ErrConvString errmsg(from, length, &my_charset_bin); - push_warning_printf(table->in_use, MYSQL_ERROR::WARN_LEVEL_WARN, + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, ER_TRUNCATED_WRONG_VALUE_FOR_FIELD, ER(ER_TRUNCATED_WRONG_VALUE_FOR_FIELD), "decimal", errmsg.ptr(), field_name, - (ulong) table->in_use->warning_info-> + (ulong) thd->warning_info-> current_row_for_warning()); my_decimal_set_zero(&decimal_value); break; @@ -2727,6 +2790,7 @@ int Field_new_decimal::store(double nr) ASSERT_COLUMN_MARKED_FOR_WRITE_OR_COMPUTED; my_decimal decimal_value; int err; + THD *thd= get_thd(); DBUG_ENTER("Field_new_decimal::store(double)"); err= double2my_decimal(E_DEC_FATAL_ERROR & ~E_DEC_OVERFLOW, nr, @@ -2736,11 +2800,11 @@ int Field_new_decimal::store(double nr) if (check_overflow(err)) set_value_on_overflow(&decimal_value, decimal_value.sign()); /* Only issue a warning if store_value doesn't issue an warning */ - table->in_use->got_warning= 0; + thd->got_warning= 0; } if (store_value(&decimal_value)) err= 1; - else if (err && !table->in_use->got_warning) + else if (err && !thd->got_warning) err= warn_if_overflow(err); DBUG_RETURN(err); } @@ -2758,11 +2822,11 @@ int Field_new_decimal::store(longlong nr, bool unsigned_val) if (check_overflow(err)) set_value_on_overflow(&decimal_value, decimal_value.sign()); /* Only issue a warning if store_value doesn't issue an warning */ - table->in_use->got_warning= 0; + get_thd()->got_warning= 0; } if (store_value(&decimal_value)) err= 1; - else if (err && !table->in_use->got_warning) + else if (err && !get_thd()->got_warning) err= warn_if_overflow(err); return err; } @@ -3612,7 +3676,7 @@ int Field_long::store(longlong nr, bool unsigned_val) res=0; error= 1; } - else if ((ulonglong) nr >= (LL(1) << 32)) + else if ((ulonglong) nr >= (1LL << 32)) { res=(int32) (uint32) ~0L; error= 1; @@ -3658,7 +3722,7 @@ longlong Field_long::val_int(void) ASSERT_COLUMN_MARKED_FOR_READ; int32 j; /* See the comment in Field_long::store(long long) */ - DBUG_ASSERT(table->in_use == current_thd); + DBUG_ASSERT(!table || table->in_use == current_thd); j=sint4korr(ptr); return unsigned_flag ? (longlong) (uint32) j : (longlong) j; } @@ -3740,7 +3804,7 @@ int Field_longlong::store(const char *from,uint len,CHARSET_INFO *cs) set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, 1); error= 1; } - else if (table->in_use->count_cuted_fields && + else if (get_thd()->count_cuted_fields && check_int(cs, from, len, end, error)) error= 1; else @@ -3892,7 +3956,7 @@ int Field_float::store(const char *from,uint len,CHARSET_INFO *cs) char *end; double nr= my_strntod(cs,(char*) from,len,&end,&error); if (error || (!len || ((uint) (end-from) != len && - table->in_use->count_cuted_fields))) + get_thd()->count_cuted_fields))) { set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, (error ? ER_WARN_DATA_OUT_OF_RANGE : WARN_DATA_TRUNCATED), 1); @@ -4080,7 +4144,7 @@ int Field_double::store(const char *from,uint len,CHARSET_INFO *cs) char *end; double nr= my_strntod(cs,(char*) from, len, &end, &error); if (error || (!len || ((uint) (end-from) != len && - table->in_use->count_cuted_fields))) + get_thd()->count_cuted_fields))) { set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, (error ? ER_WARN_DATA_OUT_OF_RANGE : WARN_DATA_TRUNCATED), 1); @@ -4384,16 +4448,10 @@ void Field_double::sql_type(String &res) const 2038-01-01 00:00:00 UTC stored as number of seconds since Unix Epoch in UTC. - Up to one of timestamps columns in the table can be automatically - set on row update and/or have NOW() as default value. - TABLE::timestamp_field points to Field object for such timestamp with - auto-set-on-update. TABLE::time_stamp holds offset in record + 1 for this - field, and is used by handler code which performs updates required. - Actually SQL-99 says that we should allow niladic functions (like NOW()) - as defaults for any field. Current limitations (only NOW() and only - for one TIMESTAMP field) are because of restricted binary .frm format - and should go away in the future. + as defaults for any field. The current limitation (only NOW() and only + for TIMESTAMP and DATETIME fields) are because of restricted binary .frm + format and should go away in the future. Also because of this limitation of binary .frm format we use 5 different unireg_check values with TIMESTAMP field to distinguish various cases of @@ -4434,10 +4492,12 @@ Field_timestamp::Field_timestamp(uchar *ptr_arg, uint32 len_arg, { /* For 4.0 MYD and 4.0 InnoDB compatibility */ flags|= UNSIGNED_FLAG | BINARY_FLAG; - if (unireg_check != NONE && !share->timestamp_field) + if (unireg_check != NONE) { - /* This timestamp has auto-update */ - share->timestamp_field= this; + /* + We mark the flag with TIMESTAMP_FLAG to indicate to the client that + this field will be automaticly updated on insert. + */ flags|= TIMESTAMP_FLAG; if (unireg_check != TIMESTAMP_DN_FIELD) flags|= ON_UPDATE_NOW_FLAG; @@ -4445,40 +4505,6 @@ Field_timestamp::Field_timestamp(uchar *ptr_arg, uint32 len_arg, } -/** - Get auto-set type for TIMESTAMP field. - - Returns value indicating during which operations this TIMESTAMP field - should be auto-set to current timestamp. -*/ -timestamp_auto_set_type Field_timestamp::get_auto_set_type() const -{ - switch (unireg_check) - { - case TIMESTAMP_DN_FIELD: - return TIMESTAMP_AUTO_SET_ON_INSERT; - case TIMESTAMP_UN_FIELD: - return TIMESTAMP_AUTO_SET_ON_UPDATE; - case TIMESTAMP_OLD_FIELD: - /* - Although we can have several such columns in legacy tables this - function should be called only for first of them (i.e. the one - having auto-set property). - */ - DBUG_ASSERT(table->timestamp_field == this); - /* Fall-through */ - case TIMESTAMP_DNUN_FIELD: - return TIMESTAMP_AUTO_SET_ON_BOTH; - default: - /* - Normally this function should not be called for TIMESTAMPs without - auto-set property. - */ - DBUG_ASSERT(0); - return TIMESTAMP_NO_AUTO_SET; - } -} - my_time_t Field_timestamp::get_timestamp(ulong *sec_part) const { ASSERT_COLUMN_MARKED_FOR_READ; @@ -4528,10 +4554,11 @@ int Field_timestamp::store_TIME_with_warning(THD *thd, MYSQL_TIME *l_time, int Field_timestamp::store_time_dec(MYSQL_TIME *ltime, uint dec) { - THD *thd= table->in_use; int unused; MYSQL_TIME l_time= *ltime; ErrConvTime str(ltime); + THD *thd= get_thd(); + bool valid= !check_date(&l_time, pack_time(&l_time) != 0, (thd->variables.sql_mode & MODE_NO_ZERO_DATE) | MODE_NO_ZERO_IN_DATE, &unused); @@ -4546,7 +4573,7 @@ int Field_timestamp::store(const char *from,uint len,CHARSET_INFO *cs) int error; int have_smth_to_conv; ErrConvString str(from, len, cs); - THD *thd= table->in_use; + THD *thd= get_thd(); /* We don't want to store invalid or fuzzy datetime values in TIMESTAMP */ have_smth_to_conv= (str_to_datetime(cs, from, len, &l_time, @@ -4563,7 +4590,7 @@ int Field_timestamp::store(double nr) MYSQL_TIME l_time; int error; ErrConvDouble str(nr); - THD *thd= table->in_use; + THD *thd= get_thd(); longlong tmp= double_to_datetime(nr, &l_time, (thd->variables.sql_mode & MODE_NO_ZERO_DATE) | @@ -4577,13 +4604,13 @@ int Field_timestamp::store(longlong nr, bool unsigned_val) MYSQL_TIME l_time; int error; ErrConvInteger str(nr); - THD *thd= table->in_use; + THD *thd= get_thd(); /* We don't want to store invalid or fuzzy datetime values in TIMESTAMP */ longlong tmp= number_to_datetime(nr, 0, &l_time, (thd->variables.sql_mode & MODE_NO_ZERO_DATE) | MODE_NO_ZERO_IN_DATE, &error); - return store_TIME_with_warning(thd, &l_time, &str, error, tmp != LL(-1)); + return store_TIME_with_warning(thd, &l_time, &str, error, tmp != -1); } @@ -4669,7 +4696,7 @@ String *Field_timestamp::val_str(String *val_buffer, String *val_ptr) bool Field_timestamp::get_date(MYSQL_TIME *ltime, ulonglong fuzzydate) { - THD *thd= table->in_use; + THD *thd= get_thd(); thd->time_zone_used= 1; ulong sec_part; my_time_t temp= get_timestamp(&sec_part); @@ -4722,12 +4749,40 @@ void Field_timestamp::sql_type(String &res) const int Field_timestamp::set_time() { - THD *thd= table->in_use; + THD *thd= get_thd(); set_notnull(); store_TIME(thd->query_start(), 0); return 0; } +/** + Mark the field as having an explicit default value. + + @param value if available, the value that the field is being set to + + @note + Fields that have an explicit default value should not be updated + automatically via the DEFAULT or ON UPDATE functions. The functions + that deal with data change functionality (INSERT/UPDATE/LOAD), + determine if there is an explicit value for each field before performing + the data change, and call this method to mark the field. + + For timestamp columns, the only case where a column is not marked + as been given a value are: + - It's explicitly assigned with DEFAULT + - We assign NULL to a timestamp field that is defined as NOT NULL. + This is how MySQL has worked since it's start. +*/ + +void Field_timestamp::set_explicit_default(Item *value) +{ + if (((value->type() == Item::DEFAULT_VALUE_ITEM && + !((Item_default_value*)value)->arg) || + (!maybe_null() && value->is_null()))) + return; + set_has_explicit_value(); +} + void Field_timestamp_hires::sql_type(String &res) const { CHARSET_INFO *cs=res.charset(); @@ -4871,7 +4926,7 @@ int Field_timestamp_hires::store_decimal(const my_decimal *d) int error; MYSQL_TIME ltime; longlong tmp; - THD *thd= table->in_use; + THD *thd= get_thd(); ErrConvDecimal str(d); if (my_decimal2seconds(d, &nr, &sec_part)) @@ -4889,7 +4944,7 @@ int Field_timestamp_hires::store_decimal(const my_decimal *d) int Field_timestamp_hires::set_time() { - THD *thd= table->in_use; + THD *thd= get_thd(); set_notnull(); store_TIME(thd->query_start(), thd->query_start_sec_part()); return 0; @@ -5008,7 +5063,7 @@ int Field_temporal::store(const char *from,uint len,CHARSET_INFO *cs) MYSQL_TIME ltime; int error; enum enum_mysql_timestamp_type func_res; - THD *thd= table->in_use; + THD *thd= get_thd(); ErrConvString str(from, len, cs); func_res= str_to_datetime(cs, from, len, <ime, @@ -5024,7 +5079,7 @@ int Field_temporal::store(double nr) { int error= 0; MYSQL_TIME ltime; - THD *thd= table->in_use; + THD *thd= get_thd(); ErrConvDouble str(nr); longlong tmp= double_to_datetime(nr, <ime, @@ -5041,7 +5096,7 @@ int Field_temporal::store(longlong nr, bool unsigned_val) int error; MYSQL_TIME ltime; longlong tmp; - THD *thd= table->in_use; + THD *thd= get_thd(); ErrConvInteger str(nr); tmp= number_to_datetime(nr, 0, <ime, (thd->variables.sql_mode & @@ -5104,7 +5159,7 @@ int Field_time::store(const char *from,uint len,CHARSET_INFO *cs) int was_cut; int have_smth_to_conv= str_to_time(cs, from, len, <ime, - table->in_use->variables.sql_mode & + get_thd()->variables.sql_mode & (MODE_NO_ZERO_DATE | MODE_NO_ZERO_IN_DATE | MODE_INVALID_DATES), &was_cut) > MYSQL_TIMESTAMP_ERROR; @@ -5210,10 +5265,10 @@ String *Field_time::val_str(String *val_buffer, bool Field_time::get_date(MYSQL_TIME *ltime, ulonglong fuzzydate) { - THD *thd= table->in_use; if (!(fuzzydate & TIME_TIME_ONLY) && (fuzzydate & TIME_NO_ZERO_IN_DATE)) { + THD *thd= get_thd(); push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, ER(ER_WARN_DATA_OUT_OF_RANGE), field_name, @@ -5401,7 +5456,7 @@ int Field_year::store(const char *from, uint len,CHARSET_INFO *cs) set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, 1); return 1; } - if (table->in_use->count_cuted_fields && + if (get_thd()->count_cuted_fields && (error= check_int(cs, from, len, end, error))) { if (error == 1) /* empty or incorrect string */ @@ -5770,8 +5825,8 @@ String *Field_datetime::val_str(String *val_buffer, Avoid problem with slow longlong arithmetic and sprintf */ - part1=(long) (tmp/LL(1000000)); - part2=(long) (tmp - (ulonglong) part1*LL(1000000)); + part1=(long) (tmp/1000000LL); + part2=(long) (tmp - (ulonglong) part1*1000000LL); pos=(char*) val_buffer->ptr() + MAX_DATETIME_WIDTH; *pos--=0; @@ -5802,8 +5857,8 @@ bool Field_datetime::get_date(MYSQL_TIME *ltime, ulonglong fuzzydate) { longlong tmp=Field_datetime::val_int(); uint32 part1,part2; - part1=(uint32) (tmp/LL(1000000)); - part2=(uint32) (tmp - (ulonglong) part1*LL(1000000)); + part1=(uint32) (tmp/1000000LL); + part2=(uint32) (tmp - (ulonglong) part1*1000000LL); ltime->time_type= MYSQL_TIMESTAMP_DATETIME; ltime->neg= 0; @@ -5848,6 +5903,20 @@ void Field_datetime::sql_type(String &res) const res.set_ascii(STRING_WITH_LEN("datetime")); } + +int Field_datetime::set_time() +{ + THD *thd= table->in_use; + MYSQL_TIME now_time; + thd->variables.time_zone->gmt_sec_to_TIME(&now_time, thd->query_start()); + now_time.second_part= thd->query_start_sec_part(); + set_notnull(); + store_TIME(&now_time); + thd->time_zone_used= 1; + return 0; +} + + void Field_datetime_hires::store_TIME(MYSQL_TIME *ltime) { ulonglong packed= sec_part_shift(pack_time(ltime), dec); @@ -5861,7 +5930,7 @@ int Field_datetime_hires::store_decimal(const my_decimal *d) int error; MYSQL_TIME ltime; longlong tmp; - THD *thd= table->in_use; + THD *thd= get_thd(); ErrConvDecimal str(d); if (my_decimal2seconds(d, &nr, &sec_part)) @@ -5997,7 +6066,9 @@ check_string_copy_error(Field_str *field, { const char *pos; char tmp[32]; - THD *thd= field->table->in_use; + THD *thd; + + thd= field->get_thd(); if (!(pos= well_formed_error_pos) && !(pos= cannot_convert_error_pos)) @@ -6039,11 +6110,12 @@ int Field_longstr::report_if_important_data(const char *pstr, const char *end, bool count_spaces) { - if ((pstr < end) && table->in_use->count_cuted_fields) + THD *thd= get_thd(); + if ((pstr < end) && thd->count_cuted_fields) { if (test_if_important_data(field_charset, pstr, end)) { - if (table->in_use->abort_on_warning) + if (thd->abort_on_warning) set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_DATA_TOO_LONG, 1); else set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, WARN_DATA_TRUNCATED, 1); @@ -6070,7 +6142,7 @@ int Field_string::store(const char *from,uint length,CHARSET_INFO *cs) const char *from_end_pos; /* See the comment for Field_long::store(long long) */ - DBUG_ASSERT(table->in_use == current_thd); + DBUG_ASSERT(!table || table->in_use == current_thd); copy_length= well_formed_copy_nchars(field_charset, (char*) ptr, field_length, @@ -6116,7 +6188,7 @@ int Field_str::store(double nr) if (error) { - if (table->in_use->abort_on_warning) + if (get_thd()->abort_on_warning) set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_DATA_TOO_LONG, 1); else set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, WARN_DATA_TRUNCATED, 1); @@ -6124,6 +6196,79 @@ int Field_str::store(double nr) return store(buff, length, &my_charset_numeric); } +static +inline ulonglong char_prefix_to_ulonglong(uchar *src) +{ + uint sz= sizeof(ulonglong); + for (uint i= 0; i < sz/2; i++) + { + uchar tmp= src[i]; + src[i]= src[sz-1-i]; + src[sz-1-i]= tmp; + } + return uint8korr(src); +} + +/** + @brief + Determine the relative position of the field value in a string interval + + @details + The function returns a double number between 0.0 and 1.0 as the relative + position of the value of the this field in the string interval of [min,max]. + If the value is not in the interval the the function returns 0.0 when + the value is less than min, and, 1.0 when the value is greater than max. + + @note + To calculate the relative position of the string value v in the interval + [min, max] the function first converts the beginning of these three + strings v, min, max into the strings that are used for byte comparison. + For each string not more sizeof(ulonglong) first bytes are taken + from the result of conversion. Then these bytes are interpreted as the + big-endian representation of an ulonglong integer. The values of these + integer numbers obtained for the strings v, min, max are used to calculate + the position of v in [min,max] in the same way is it's done for numeric + fields (see Field_num::pos_in_interval). + + @todo + Improve the procedure for the case when min and max have the same + beginning + + @param min value of the left end of the interval + @param max value of the right end of the interval + + @return + relative position of the field value in the string interval [min,max] +*/ + +double Field_str::pos_in_interval(Field *min, Field *max) +{ + uchar mp_prefix[sizeof(ulonglong)]; + uchar minp_prefix[sizeof(ulonglong)]; + uchar maxp_prefix[sizeof(ulonglong)]; + ulonglong mp, minp, maxp; + my_strnxfrm(charset(), mp_prefix, sizeof(mp), + ptr + length_size(), + data_length()); + my_strnxfrm(charset(), minp_prefix, sizeof(minp), + min->ptr + length_size(), + min->data_length()); + my_strnxfrm(charset(), maxp_prefix, sizeof(maxp), + max->ptr + length_size(), + max->data_length()); + mp= char_prefix_to_ulonglong(mp_prefix); + minp= char_prefix_to_ulonglong(minp_prefix); + maxp= char_prefix_to_ulonglong(maxp_prefix); + double n, d; + n= mp - minp; + if (n < 0) + return 0.0; + d= maxp - minp; + if (d <= 0) + return 1.0; + return min(n/d, 1.0); +} + uint Field::is_equal(Create_field *new_field) { @@ -6176,7 +6321,7 @@ double Field_string::val_real(void) double result; result= my_strntod(cs,(char*) ptr,field_length,&end,&error); - if (!table->in_use->no_errors && + if (!get_thd()->no_errors && (error || (field_length != (uint32)(end - (char*) ptr) && !check_if_only_end_space(cs, end, (char*) ptr + field_length)))) @@ -6200,7 +6345,7 @@ longlong Field_string::val_int(void) longlong result; result= my_strntoll(cs, (char*) ptr,field_length,10,&end,&error); - if (!table->in_use->no_errors && + if (!get_thd()->no_errors && (error || (field_length != (uint32)(end - (char*) ptr) && !check_if_only_end_space(cs, end, (char*) ptr + field_length)))) @@ -6220,9 +6365,9 @@ String *Field_string::val_str(String *val_buffer __attribute__((unused)), { ASSERT_COLUMN_MARKED_FOR_READ; /* See the comment for Field_long::store(long long) */ - DBUG_ASSERT(table->in_use == current_thd); + DBUG_ASSERT(!table || table->in_use == current_thd); uint length; - if (table->in_use->variables.sql_mode & + if (get_thd()->variables.sql_mode & MODE_PAD_CHAR_TO_FULL_LENGTH) length= my_charpos(field_charset, ptr, ptr + field_length, field_length / field_charset->mbmaxlen); @@ -6239,7 +6384,7 @@ my_decimal *Field_string::val_decimal(my_decimal *decimal_value) ASSERT_COLUMN_MARKED_FOR_READ; int err= str2my_decimal(E_DEC_FATAL_ERROR, (char*) ptr, field_length, charset(), decimal_value); - if (!table->in_use->no_errors && err) + if (!get_thd()->no_errors && err) { ErrConvString errmsg((char*) ptr, field_length, charset()); push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN, @@ -6623,7 +6768,7 @@ double Field_varstring::val_real(void) uint length= length_bytes == 1 ? (uint) *ptr : uint2korr(ptr); result= my_strntod(cs, (char*)ptr+length_bytes, length, &end, &error); - if (!table->in_use->no_errors && + if (!get_thd()->no_errors && (error || (length != (uint)(end - (char*)ptr+length_bytes) && !check_if_only_end_space(cs, end, (char*)ptr+length_bytes+length)))) { @@ -6646,7 +6791,7 @@ longlong Field_varstring::val_int(void) longlong result= my_strntoll(cs, (char*) ptr+length_bytes, length, 10, &end, &error); - if (!table->in_use->no_errors && + if (!get_thd()->no_errors && (error || (length != (uint)(end - (char*)ptr+length_bytes) && !check_if_only_end_space(cs, end, (char*)ptr+length_bytes+length)))) { @@ -6675,7 +6820,7 @@ my_decimal *Field_varstring::val_decimal(my_decimal *decimal_value) int error= str2my_decimal(E_DEC_FATAL_ERROR, (char*) ptr+length_bytes, length, cs, decimal_value); - if (!table->in_use->no_errors && error) + if (!get_thd()->no_errors && error) { push_numerical_conversion_warning(current_thd, (char*)ptr+length_bytes, length, cs, "DECIMAL", @@ -7670,7 +7815,7 @@ int Field_enum::store(const char *from,uint length,CHARSET_INFO *cs) tmp=0; set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, WARN_DATA_TRUNCATED, 1); } - if (!table->in_use->count_cuted_fields) + if (!get_thd()->count_cuted_fields) err= 0; } else @@ -7694,7 +7839,7 @@ int Field_enum::store(longlong nr, bool unsigned_val) if ((ulonglong) nr > typelib->count || nr == 0) { set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, WARN_DATA_TRUNCATED, 1); - if (nr != 0 || table->in_use->count_cuted_fields) + if (nr != 0 || get_thd()->count_cuted_fields) { nr= 0; error= 1; @@ -7866,7 +8011,7 @@ int Field_set::store(longlong nr, bool unsigned_val) if (sizeof(ulonglong)*8 <= typelib->count) max_nr= ULONGLONG_MAX; else - max_nr= (ULL(1) << typelib->count) - 1; + max_nr= (1ULL << typelib->count) - 1; if ((ulonglong) nr > max_nr) { @@ -8224,7 +8369,7 @@ int Field_bit::store(const char *from, uint length, CHARSET_INFO *cs) { set_rec_bits((1 << bit_len) - 1, bit_ptr, bit_ofs, bit_len); memset(ptr, 0xff, bytes_in_rec); - if (table->in_use->really_abort_on_warning()) + if (get_thd()->really_abort_on_warning()) set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_DATA_TOO_LONG, 1); else set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, 1); @@ -8335,6 +8480,36 @@ my_decimal *Field_bit::val_decimal(my_decimal *deciaml_value) } +/** + @brief + Determine the relative position of the field value in a bit interval + + @details + The function returns a double number between 0.0 and 1.0 as the relative + position of the value of the this field in the bit interval of [min,max]. + If the value is not in the interval the the function returns 0.0 when + the value is less than min, and, 1.0 when the value is greater than max. + + @param min value of the left end of the interval + @param max value of the right end of the interval + + @return + relative position of the field value in the bit interval [min,max] +*/ + +double Field_bit::pos_in_interval(Field *min, Field *max) +{ + double n, d; + n= val_real() - min->val_real(); + if (n < 0) + return 0.0; + d= max->val_real() - min->val_real(); + if (d <= 0) + return 1.0; + return min(n/d, 1.0); +} + + /* Compare two bit fields using pointers within the record. SYNOPSIS @@ -8360,7 +8535,9 @@ int Field_bit::cmp_max(const uchar *a, const uchar *b, uint max_len) if ((flag= (int) (bits_a - bits_b))) return flag; } - return memcmp(a, b, field_length); + if (!bytes_in_rec) + return 0; + return memcmp(a, b, bytes_in_rec); } @@ -8659,7 +8836,7 @@ int Field_bit_as_char::store(const char *from, uint length, CHARSET_INFO *cs) memset(ptr, 0xff, bytes_in_rec); if (bits) *ptr&= ((1 << bits) - 1); /* set first uchar */ - if (table->in_use->really_abort_on_warning()) + if (get_thd()->really_abort_on_warning()) set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_DATA_TOO_LONG, 1); else set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, 1); @@ -8841,6 +9018,7 @@ void Create_field::init_for_tmp_table(enum_field_types sql_type_arg, FLAGSTR(pack_flag, FIELDFLAG_DECIMAL), f_packtype(pack_flag))); vcol_info= 0; + create_if_not_exists= FALSE; stored_in_db= TRUE; DBUG_VOID_RETURN; @@ -8878,20 +9056,41 @@ bool Create_field::init(THD *thd, char *fld_name, enum_field_types fld_type, char *fld_change, List<String> *fld_interval_list, CHARSET_INFO *fld_charset, uint fld_geom_type, Virtual_column_info *fld_vcol_info, - engine_option_value *create_opt) + engine_option_value *create_opt, bool check_exists) { uint sign_len, allowed_type_modifier= 0; ulong max_field_charlength= MAX_FIELD_CHARLENGTH; + const bool on_update_is_function= + (fld_on_update_value != NULL && + fld_on_update_value->type() == Item::FUNC_ITEM); DBUG_ENTER("Create_field::init()"); field= 0; field_name= fld_name; - def= fld_default_value; flags= fld_type_modifier; option_list= create_opt; - unireg_check= (fld_type_modifier & AUTO_INCREMENT_FLAG ? - Field::NEXT_NUMBER : Field::NONE); + + if (fld_default_value != NULL && fld_default_value->type() == Item::FUNC_ITEM) + { + /* There is a function default for insertions. */ + def= NULL; + unireg_check= (on_update_is_function ? + Field::TIMESTAMP_DNUN_FIELD : // for insertions and for updates. + Field::TIMESTAMP_DN_FIELD); // only for insertions. + } + else + { + /* No function default for insertions. Either NULL or a constant. */ + def= fld_default_value; + if (on_update_is_function) + unireg_check= Field::TIMESTAMP_UN_FIELD; // function default for updates + else + unireg_check= ((fld_type_modifier & AUTO_INCREMENT_FLAG) != 0 ? + Field::NEXT_NUMBER : // Automatic increment. + Field::NONE); + } + decimals= fld_decimals ? (uint)atoi(fld_decimals) : 0; if (decimals >= NOT_FIXED_DEC) { @@ -8911,6 +9110,7 @@ bool Create_field::init(THD *thd, char *fld_name, enum_field_types fld_type, comment= *fld_comment; vcol_info= fld_vcol_info; + create_if_not_exists= check_exists; stored_in_db= TRUE; /* Initialize data for a computed field */ @@ -9114,44 +9314,6 @@ bool Create_field::init(THD *thd, char *fld_name, enum_field_types fld_type, } length+= MAX_DATETIME_WIDTH + (length ? 1 : 0); flags|= UNSIGNED_FLAG; - - if (fld_default_value) - { - /* Grammar allows only NOW() value for ON UPDATE clause */ - if (fld_default_value->type() == Item::FUNC_ITEM && - ((Item_func*)fld_default_value)->functype() == Item_func::NOW_FUNC) - { - unireg_check= (fld_on_update_value ? Field::TIMESTAMP_DNUN_FIELD: - Field::TIMESTAMP_DN_FIELD); - /* - We don't need default value any longer moreover it is dangerous. - Everything handled by unireg_check further. - */ - def= 0; - } - else - unireg_check= (fld_on_update_value ? Field::TIMESTAMP_UN_FIELD: - Field::NONE); - } - else - { - /* - If we have default TIMESTAMP NOT NULL column without explicit DEFAULT - or ON UPDATE values then for the sake of compatiblity we should treat - this column as having DEFAULT NOW() ON UPDATE NOW() (when we don't - have another TIMESTAMP column with auto-set option before this one) - or DEFAULT 0 (in other cases). - So here we are setting TIMESTAMP_OLD_FIELD only temporary, and will - replace this value by TIMESTAMP_DNUN_FIELD or NONE later when - information about all TIMESTAMP fields in table will be availiable. - - If we have TIMESTAMP NULL column without explicit DEFAULT value - we treat it as having DEFAULT NULL attribute. - */ - unireg_check= (fld_on_update_value ? Field::TIMESTAMP_UN_FIELD : - (flags & NOT_NULL_FLAG ? Field::TIMESTAMP_OLD_FIELD : - Field::NONE)); - } break; case MYSQL_TYPE_DATE: /* We don't support creation of MYSQL_TYPE_DATE anymore */ @@ -9557,6 +9719,7 @@ Create_field::Create_field(Field *old_field,Field *orig_field) comment= old_field->comment; decimals= old_field->decimals(); vcol_info= old_field->vcol_info; + create_if_not_exists= FALSE; stored_in_db= old_field->stored_in_db; option_list= old_field->option_list; option_struct= old_field->option_struct; @@ -9617,11 +9780,18 @@ Create_field::Create_field(Field *old_field,Field *orig_field) def=0; char_length= length; - if (!(flags & (NO_DEFAULT_VALUE_FLAG | BLOB_FLAG)) && - old_field->ptr && orig_field && - (sql_type != MYSQL_TYPE_TIMESTAMP || /* set def only if */ - old_field->table->timestamp_field != old_field || /* timestamp field */ - unireg_check == Field::TIMESTAMP_UN_FIELD)) /* has default val */ + /* + Copy the default value from the column object orig_field, if: + 1) The column has a constant default value. + 2) The column type is not a BLOB type. + 3) The original column (old_field) was properly initialized with a record + buffer pointer. + 4) The original column doesn't have a default function to auto-initialize + the column on INSERT + */ + if (!(flags & (NO_DEFAULT_VALUE_FLAG | BLOB_FLAG)) && // 1) 2) + old_field->ptr && orig_field && // 3) + !old_field->has_insert_default_function()) // 4) { char buff[MAX_FIELD_WIDTH]; String tmp(buff,sizeof(buff), charset); @@ -9775,7 +9945,7 @@ void Field::set_datetime_warning(MYSQL_ERROR::enum_warning_level level, uint code, const ErrConv *str, timestamp_type ts_type, int cuted_increment) { - THD *thd= table->in_use; + THD *thd= get_thd(); if (thd->really_abort_on_warning() && level >= MYSQL_ERROR::WARN_LEVEL_WARN) make_truncated_value_warning(thd, level, str, ts_type, field_name); else @@ -9805,3 +9975,29 @@ key_map Field::get_possible_keys() return (table->pos_in_table_list->is_materialized_derived() ? part_of_key : key_start); } + + +/** + Mark the field as having an explicit default value. + + @param value if available, the value that the field is being set to + + @note + Fields that have an explicit default value should not be updated + automatically via the DEFAULT or ON UPDATE functions. The functions + that deal with data change functionality (INSERT/UPDATE/LOAD), + determine if there is an explicit value for each field before performing + the data change, and call this method to mark the field. + + If the 'value' parameter is NULL, then the field is marked unconditionally + as having an explicit value. If 'value' is not NULL, then it can be further + analyzed to check if it really should count as a value. +*/ + +void Field::set_explicit_default(Item *value) +{ + if (value->type() == Item::DEFAULT_VALUE_ITEM && + !((Item_default_value*)value)->arg) + return; + set_has_explicit_value(); +} diff --git a/sql/field.h b/sql/field.h index 10b854001cd..8aa02440e22 100644 --- a/sql/field.h +++ b/sql/field.h @@ -36,6 +36,8 @@ class Protocol; class Create_field; class Relay_log_info; class Field; +class Column_statistics; +class Column_statistics_collected; enum enum_check_fields { @@ -66,6 +68,8 @@ enum Derivation /* The length of the header part for each virtual column in the .frm file */ #define FRM_VCOL_HEADER_SIZE(b) (3 + test(b)) +class Count_distinct_field; + struct ha_field_option_struct; struct st_cache_field; @@ -244,6 +248,35 @@ public: */ bool is_created_from_null_item; + /* TRUE in Field objects created for column min/max values */ + bool is_stat_field; + + /* + Selectivity of the range condition over this field. + When calculating this selectivity a range predicate + is taken into account only if: + - it is extracted from the WHERE clause + - it depends only on the table the field belongs to + */ + double cond_selectivity; + + /* + The next field in the class of equal fields at the top AND level + of the WHERE clause + */ + Field *next_equal_field; + + /* + This structure is used for statistical data on the column + that has been read from the statistical table column_stat + */ + Column_statistics *read_stats; + /* + This structure is used for statistical data on the column that + is collected by the function collect_statistics_for_table + */ + Column_statistics_collected *collected_stats; + /* This is additional data provided for any computed(virtual) field. In particular it includes a pointer to the item by which this field @@ -336,6 +369,26 @@ public: virtual uint32 data_length() { return pack_length(); } virtual uint32 sort_length() const { return pack_length(); } + /* + Get the number bytes occupied by the value in the field. + CHAR values are stripped of trailing spaces. + Flexible values are stripped of their length. + */ + virtual uint32 value_length() + { + uint len; + if (!zero_pack() && + (type() == MYSQL_TYPE_STRING && + (len= pack_length()) >= 4 && len < 256)) + { + uchar *str, *end; + for (str= ptr, end= str+len; end > str && end[-1] == ' '; end--) {} + len=(uint) (end-str); + return len; + } + return data_length(); + } + /** Get the maximum size of the data in packed format. @@ -357,6 +410,46 @@ public: *null_ptr= ((*null_ptr & (uchar) ~null_bit) | (null_ptr[l_offset] & null_bit)); } + + bool has_insert_default_function() const + { + return unireg_check == TIMESTAMP_DN_FIELD || + unireg_check == TIMESTAMP_DNUN_FIELD; + } + + bool has_update_default_function() const + { + return unireg_check == TIMESTAMP_UN_FIELD || + unireg_check == TIMESTAMP_DNUN_FIELD; + } + + /* + Mark the field as having a value supplied by the client, thus it should + not be auto-updated. + */ + void set_has_explicit_value() + { + flags|= HAS_EXPLICIT_VALUE; + } + + virtual void set_explicit_default(Item *value); + + /** + Evaluates the @c INSERT default function and stores the result in the + field. If no such function exists for the column, or the function is not + valid for the column's data type, invoking this function has no effect. + */ + virtual int evaluate_insert_default_function() { return 0; } + + + /** + Evaluates the @c UPDATE default function, if one exists, and stores the + result in the record buffer. If no such function exists for the column, + or the function is not valid for the column's data type, invoking this + function has no effect. + */ + virtual int evaluate_update_default_function() { return 0; } + virtual bool binary() const { return 1; } virtual bool zero_pack() const { return 1; } virtual enum ha_base_keytype key_type() const { return HA_KEYTYPE_BINARY; } @@ -377,6 +470,40 @@ public: { return cmp(a, b); } virtual int key_cmp(const uchar *str, uint length) { return cmp(ptr,str); } + /* + Update the value m of the 'min_val' field with the current value v + of this field if force_update is set to TRUE or if v < m. + Return TRUE if the value has been updated. + */ + virtual bool update_min(Field *min_val, bool force_update) + { + bool update_fl= force_update || cmp(ptr, min_val->ptr) < 0; + if (update_fl) + { + min_val->set_notnull(); + memcpy(min_val->ptr, ptr, pack_length()); + } + return update_fl; + } + /* + Update the value m of the 'max_val' field with the current value v + of this field if force_update is set to TRUE or if v > m. + Return TRUE if the value has been updated. + */ + virtual bool update_max(Field *max_val, bool force_update) + { + bool update_fl= force_update || cmp(ptr, max_val->ptr) > 0; + if (update_fl) + { + max_val->set_notnull(); + memcpy(max_val->ptr, ptr, pack_length()); + } + return update_fl; + } + virtual void store_field_value(uchar *val, uint len) + { + memcpy(ptr, val, len); + } virtual uint decimals() const { return 0; } /* Caller beware: sql_type can change str.Ptr, so check @@ -412,6 +539,8 @@ public: */ inline bool real_maybe_null(void) { return null_ptr != 0; } + inline THD *get_thd() { return table ? table->in_use : current_thd; } + enum { LAST_NULL_BYTE_UNDEF= 0 }; @@ -449,6 +578,9 @@ public: uchar *new_ptr, uchar *new_null_ptr, uint new_null_bit); Field *clone(MEM_ROOT *mem_root, TABLE *new_table); + Field *clone(MEM_ROOT *mem_root, TABLE *new_table, my_ptrdiff_t diff, + bool stat_flag= FALSE); + Field *clone(MEM_ROOT *mem_root, my_ptrdiff_t diff); inline void move_field(uchar *ptr_arg,uchar *null_ptr_arg,uchar null_bit_arg) { ptr=ptr_arg; null_ptr=null_ptr_arg; null_bit=null_bit_arg; @@ -619,6 +751,12 @@ public: virtual bool hash_join_is_possible() { return TRUE; } virtual bool eq_cmp_as_binary() { return TRUE; } + /* Position of the field value within the interval of [min, max] */ + virtual double pos_in_interval(Field *min, Field *max) + { + return (double) 0.5; + } + friend int cre_myisam(char * name, register TABLE *form, uint options, ulonglong auto_increment_value); friend class Copy_field; @@ -737,6 +875,7 @@ public: bool get_int(CHARSET_INFO *cs, const char *from, uint len, longlong *rnd, ulonglong unsigned_max, longlong signed_min, longlong signed_max); + double pos_in_interval(Field *min, Field *max); }; @@ -782,6 +921,8 @@ public: virtual bool str_needs_quotes() { return TRUE; } uint is_equal(Create_field *new_field); bool eq_cmp_as_binary() { return test(flags & BINARY_FLAG); } + virtual uint length_size() { return 0; } + double pos_in_interval(Field *min, Field *max); }; /* base class for Field_string, Field_varstring and Field_blob */ @@ -1268,12 +1409,26 @@ public: virtual int set_time(); virtual void set_default() { - if (table->timestamp_field == this && - unireg_check != TIMESTAMP_UN_FIELD) + if (has_insert_default_function()) set_time(); else Field::set_default(); } + virtual void set_explicit_default(Item *value); + virtual int evaluate_insert_default_function() + { + int res= 0; + if (has_insert_default_function()) + res= set_time(); + return res; + } + virtual int evaluate_update_default_function() + { + int res= 0; + if (has_update_default_function()) + res= set_time(); + return res; + } /* Get TIMESTAMP field value as seconds since begging of Unix Epoch */ virtual my_time_t get_timestamp(ulong *sec_part) const; virtual void store_TIME(my_time_t timestamp, ulong sec_part) @@ -1281,7 +1436,6 @@ public: int4store(ptr,timestamp); } bool get_date(MYSQL_TIME *ltime, ulonglong fuzzydate); - timestamp_auto_set_type get_auto_set_type() const; uchar *pack(uchar *to, const uchar *from, uint max_length __attribute__((unused))) { @@ -1532,6 +1686,28 @@ public: void sql_type(String &str) const; bool zero_pack() const { return 1; } bool get_date(MYSQL_TIME *ltime, ulonglong fuzzydate); + virtual int set_time(); + virtual void set_default() + { + if (has_insert_default_function()) + set_time(); + else + Field::set_default(); + } + virtual int evaluate_insert_default_function() + { + int res= 0; + if (has_insert_default_function()) + res= set_time(); + return res; + } + virtual int evaluate_update_default_function() + { + int res= 0; + if (has_update_default_function()) + res= set_time(); + return res; + } uchar *pack(uchar* to, const uchar *from, uint max_length __attribute__((unused))) { @@ -1775,6 +1951,7 @@ public: uint new_null_bit); uint is_equal(Create_field *new_field); void hash(ulong *nr, ulong *nr2); + uint length_size() { return length_bytes; } private: int do_save_field_metadata(uchar *first_byte); }; @@ -1840,6 +2017,10 @@ public: int cmp_binary(const uchar *a,const uchar *b, uint32 max_length=~0L); int key_cmp(const uchar *,const uchar*); int key_cmp(const uchar *str, uint length); + /* Never update the value of min_val for a blob field */ + bool update_min(Field *min_val, bool force_update) { return FALSE; } + /* Never update the value of max_val for a blob field */ + bool update_max(Field *max_val, bool force_update) { return FALSE; } uint32 key_length() const { return 0; } void sort_string(uchar *buff,uint length); uint32 pack_length() const @@ -1857,6 +2038,7 @@ public: { return (uint32) (packlength); } uint row_pack_length() { return pack_length_no_ptr(); } uint32 sort_length() const; + uint32 value_length() { return get_length(); } virtual uint32 max_data_length() const { return (uint32) (((ulonglong) 1 << (packlength*8)) -1); @@ -2117,6 +2299,33 @@ public: { return cmp_binary((uchar *) a, (uchar *) b); } int key_cmp(const uchar *str, uint length); int cmp_offset(uint row_offset); + bool update_min(Field *min_val, bool force_update) + { + longlong val= val_int(); + bool update_fl= force_update || val < min_val->val_int(); + if (update_fl) + { + min_val->set_notnull(); + min_val->store(val, FALSE); + } + return update_fl; + } + bool update_max(Field *max_val, bool force_update) + { + longlong val= val_int(); + bool update_fl= force_update || val > max_val->val_int(); + if (update_fl) + { + max_val->set_notnull(); + max_val->store(val, FALSE); + } + return update_fl; + } + void store_field_value(uchar *val, uint len) + { + store(*((longlong *)val), TRUE); + } + double pos_in_interval(Field *min, Field *max); void get_image(uchar *buff, uint length, CHARSET_INFO *cs) { get_key_image(buff, length, itRAW); } void set_image(const uchar *buff,uint length, CHARSET_INFO *cs) @@ -2225,10 +2434,11 @@ public: /** structure with parsed options (for comparing fields in ALTER TABLE) */ ha_field_option_struct *option_struct; - uint8 row,col,sc_length,interval_id; // For rea_create_table + uint8 interval_id; // For rea_create_table uint offset,pack_flag; + bool create_if_not_exists; // Used in ALTER TABLE IF NOT EXISTS - /* + /* This is additinal data provided for any computed(virtual) field. In particular it includes a pointer to the item by which this field can be computed from other fields. @@ -2241,7 +2451,8 @@ public: */ bool stored_in_db; - Create_field() :after(0), option_list(NULL), option_struct(NULL) + Create_field() :after(0), option_list(NULL), option_struct(NULL), + create_if_not_exists(FALSE) {} Create_field(Field *field, Field *orig_field); /* Used to make a clone of this object for ALTER/CREATE TABLE */ @@ -2259,7 +2470,7 @@ public: Item *on_update_value, LEX_STRING *comment, char *change, List<String> *interval_list, CHARSET_INFO *cs, uint uint_geom_type, Virtual_column_info *vcol_info, - engine_option_value *option_list); + engine_option_value *option_list, bool check_exists); bool field_flags_are_binary() { diff --git a/sql/filesort.cc b/sql/filesort.cc index 4183c28f05b..be84dd0736c 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -34,6 +34,9 @@ #include "sql_base.h" // update_virtual_fields #include "sql_test.h" // TEST_filesort #include "opt_range.h" // SQL_SELECT +#include "bounded_queue.h" +#include "filesort_utils.h" +#include "sql_select.h" #include "log_slow.h" #include "debug_sync.h" #include "sql_base.h" @@ -47,22 +50,63 @@ if (my_b_write((file),(uchar*) (from),param->ref_length)) \ static uchar *read_buffpek_from_file(IO_CACHE *buffer_file, uint count, uchar *buf); -static ha_rows find_all_keys(SORTPARAM *param,SQL_SELECT *select, - uchar * *sort_keys, uchar *sort_keys_buf, - IO_CACHE *buffer_file, IO_CACHE *tempfile); -static bool write_keys(SORTPARAM *param,uchar * *sort_keys, - uint count, IO_CACHE *buffer_file, IO_CACHE *tempfile); -static void make_sortkey(SORTPARAM *param,uchar *to, uchar *ref_pos); -static void register_used_fields(SORTPARAM *param); -static bool save_index(SORTPARAM *param,uchar **sort_keys, uint count, - FILESORT_INFO *table_sort); +static ha_rows find_all_keys(Sort_param *param,SQL_SELECT *select, + Filesort_info *fs_info, + IO_CACHE *buffer_file, + IO_CACHE *tempfile, + Bounded_queue<uchar, uchar> *pq, + ha_rows *found_rows); +static bool write_keys(Sort_param *param, Filesort_info *fs_info, + uint count, IO_CACHE *buffer_file, IO_CACHE *tempfile); +static void make_sortkey(Sort_param *param, uchar *to, uchar *ref_pos); +static void register_used_fields(Sort_param *param); +static bool save_index(Sort_param *param, uint count, + Filesort_info *table_sort); static uint suffix_length(ulong string_length); static uint sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length, bool *multi_byte_charset); -static SORT_ADDON_FIELD *get_addon_fields(THD *thd, Field **ptabfield, +static SORT_ADDON_FIELD *get_addon_fields(ulong max_length_for_sort_data, + Field **ptabfield, uint sortlength, uint *plength); static void unpack_addon_fields(struct st_sort_addon_field *addon_field, uchar *buff, uchar *buff_end); +static bool check_if_pq_applicable(Sort_param *param, Filesort_info *info, + TABLE *table, + ha_rows records, ulong memory_available); + + +void Sort_param::init_for_filesort(uint sortlen, TABLE *table, + ulong max_length_for_sort_data, + ha_rows maxrows, bool sort_positions) +{ + sort_length= sortlen; + ref_length= table->file->ref_length; + if (!(table->file->ha_table_flags() & HA_FAST_KEY_READ) && + !table->fulltext_searched && !sort_positions) + { + /* + Get the descriptors of all fields whose values are appended + to sorted fields and get its total length in addon_length. + */ + addon_field= get_addon_fields(max_length_for_sort_data, + table->field, sort_length, &addon_length); + } + if (addon_field) + res_length= addon_length; + else + { + res_length= ref_length; + /* + The reference to the record is considered + as an additional sorted field + */ + sort_length+= ref_length; + } + rec_length= sort_length + addon_length; + max_rows= maxrows; +} + + /** Sort a table. Creates a set of pointers that can be used to read the rows @@ -75,15 +119,17 @@ static void unpack_addon_fields(struct st_sort_addon_field *addon_field, The result set is stored in table->io_cache or table->record_pointers. - @param thd Current thread - @param table Table to sort - @param sortorder How to sort the table - @param s_length Number of elements in sortorder - @param select condition to apply to the rows - @param max_rows Return only this many rows - @param sort_positions Set to 1 if we want to force sorting by position - (Needed by UPDATE/INSERT or ALTER TABLE) - @param examined_rows Store number of examined rows here + @param thd Current thread + @param table Table to sort + @param sortorder How to sort the table + @param s_length Number of elements in sortorder + @param select Condition to apply to the rows + @param max_rows Return only this many rows + @param sort_positions Set to TRUE if we want to force sorting by position + (Needed by UPDATE/INSERT or ALTER TABLE or + when rowids are required by executor) + @param[out] examined_rows Store number of examined rows here + @param[out] found_rows Store the number of found rows here @note If we sort by position (like if sort_positions is 1) filesort() will @@ -93,31 +139,30 @@ static void unpack_addon_fields(struct st_sort_addon_field *addon_field, HA_POS_ERROR Error @retval \# Number of rows - @retval - examined_rows will be set to number of examined rows */ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, SQL_SELECT *select, ha_rows max_rows, - bool sort_positions, ha_rows *examined_rows) + bool sort_positions, + ha_rows *examined_rows, + ha_rows *found_rows) { int error; size_t memory_available= thd->variables.sortbuff_size; - size_t min_sort_memory; - size_t sort_buff_sz; uint maxbuffer; BUFFPEK *buffpek; ha_rows num_rows= HA_POS_ERROR; - uchar **sort_keys= 0; IO_CACHE tempfile, buffpek_pointers, *outfile; - SORTPARAM param; + Sort_param param; bool multi_byte_charset; + Bounded_queue<uchar, uchar> pq; + DBUG_ENTER("filesort"); DBUG_EXECUTE("info",TEST_filesort(sortorder,s_length);); #ifdef SKIP_DBUG_IN_FILESORT DBUG_PUSH(""); /* No DBUG here */ #endif - FILESORT_INFO table_sort; + Filesort_info table_sort= table->sort; TABLE_LIST *tab= table->pos_in_table_list; Item_subselect *subselect= tab ? tab->containing_subselect() : 0; @@ -135,7 +180,6 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, QUICK_INDEX_MERGE_SELECT. Work with a copy and put it back at the end when index_merge select has finished with it. */ - memcpy(&table_sort, &table->sort, sizeof(FILESORT_INFO)); table->sort.io_cache= NULL; DBUG_ASSERT(table_sort.record_pointers == NULL); @@ -144,43 +188,22 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, my_b_clear(&buffpek_pointers); buffpek=0; error= 1; - bzero((char*) ¶m,sizeof(param)); - param.sort_length= sortlength(thd, sortorder, s_length, &multi_byte_charset); - param.ref_length= table->file->ref_length; - if (!(table->file->ha_table_flags() & HA_FAST_KEY_READ) && - !table->fulltext_searched && !sort_positions) - { - /* - Get the descriptors of all fields whose values are appended - to sorted fields and get its total length in param.spack_length. - */ - param.addon_field= get_addon_fields(thd, table->field, - param.sort_length, - ¶m.addon_length); - } + + param.init_for_filesort(sortlength(thd, sortorder, s_length, + &multi_byte_charset), + table, + thd->variables.max_length_for_sort_data, + max_rows, sort_positions); table_sort.addon_buf= 0; table_sort.addon_length= param.addon_length; table_sort.addon_field= param.addon_field; table_sort.unpack= unpack_addon_fields; - if (param.addon_field) - { - param.res_length= param.addon_length; - if (!(table_sort.addon_buf= (uchar *) my_malloc(param.addon_length, - MYF(MY_WME)))) - goto err; - } - else - { - param.res_length= param.ref_length; - /* - The reference to the record is considered - as an additional sorted field - */ - param.sort_length+= param.ref_length; - } - param.rec_length= param.sort_length+param.addon_length; - param.max_rows= max_rows; + if (param.addon_field && + !(table_sort.addon_buf= + (uchar *) my_malloc(param.addon_length, MYF(MY_WME | + MY_THREAD_SPECIFIC)))) + goto err; if (select && select->quick) status_var_increment(thd->status_var.filesort_range_count); @@ -192,25 +215,58 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, num_rows= table->file->estimate_rows_upper_bound(); if (multi_byte_charset && - !(param.tmp_buffer= (char*) my_malloc(param.sort_length,MYF(MY_WME)))) + !(param.tmp_buffer= (char*) my_malloc(param.sort_length, + MYF(MY_WME | MY_THREAD_SPECIFIC)))) goto err; - min_sort_memory= max(MIN_SORT_MEMORY, param.sort_length * MERGEBUFF2); - set_if_bigger(min_sort_memory, sizeof(BUFFPEK*)*MERGEBUFF2); - if (!table_sort.sort_keys) + if (check_if_pq_applicable(¶m, &table_sort, + table, num_rows, memory_available)) + { + DBUG_PRINT("info", ("filesort PQ is applicable")); + const size_t compare_length= param.sort_length; + if (pq.init(param.max_rows, + true, // max_at_top + NULL, // compare_function + compare_length, + &make_sortkey, ¶m, table_sort.get_sort_keys())) + { + /* + If we fail to init pq, we have to give up: + out of memory means my_malloc() will call my_error(). + */ + DBUG_PRINT("info", ("failed to allocate PQ")); + table_sort.free_sort_buffer(); + DBUG_ASSERT(thd->is_error()); + goto err; + } + // For PQ queries (with limit) we initialize all pointers. + table_sort.init_record_pointers(); + } + else { + DBUG_PRINT("info", ("filesort PQ is not applicable")); + + size_t min_sort_memory= max(MIN_SORT_MEMORY, param.sort_length*MERGEBUFF2); + set_if_bigger(min_sort_memory, sizeof(BUFFPEK*)*MERGEBUFF2); while (memory_available >= min_sort_memory) { ulonglong keys= memory_available / (param.rec_length + sizeof(char*)); - table_sort.keys= (uint) min(num_rows, keys); - sort_buff_sz= table_sort.keys*(param.rec_length+sizeof(char*)); - set_if_bigger(sort_buff_sz, param.rec_length * MERGEBUFF2); - - DBUG_EXECUTE_IF("make_sort_keys_alloc_fail", - DBUG_SET("+d,simulate_out_of_memory");); - - if ((table_sort.sort_keys= - (uchar**) my_malloc(sort_buff_sz, MYF(0)))) + param.max_keys_per_buffer= (uint) min(num_rows, keys); + if (table_sort.get_sort_keys()) + { + // If we have already allocated a buffer, it better have same size! + if (!table_sort.check_sort_buffer_properties(param.max_keys_per_buffer, + param.rec_length)) + { + /* + table->sort will still have a pointer to the same buffer, + but that will be overwritten by the assignment below. + */ + table_sort.free_sort_buffer(); + } + } + table_sort.alloc_sort_buffer(param.max_keys_per_buffer, param.rec_length); + if (table_sort.get_sort_keys()) break; size_t old_memory_available= memory_available; memory_available= memory_available/4*3; @@ -218,40 +274,37 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, old_memory_available > min_sort_memory) memory_available= min_sort_memory; } + if (memory_available < min_sort_memory) + { + my_error(ER_OUT_OF_SORTMEMORY,MYF(ME_ERROR + ME_FATALERROR)); + goto err; + } } - sort_keys= table_sort.sort_keys; - param.keys= table_sort.keys; - if (memory_available < min_sort_memory) - { - my_error(ER_OUT_OF_SORTMEMORY,MYF(ME_ERROR + ME_FATALERROR)); - goto err; - } if (open_cached_file(&buffpek_pointers,mysql_tmpdir,TEMP_PREFIX, - DISK_BUFFER_SIZE, MYF(ME_ERROR | MY_WME))) + DISK_BUFFER_SIZE, MYF(MY_WME))) goto err; param.sort_form= table; param.end=(param.local_sortorder=sortorder)+s_length; - num_rows= find_all_keys(¶m, - select, - sort_keys, - (uchar *)(sort_keys+param.keys), + num_rows= find_all_keys(¶m, select, + &table_sort, &buffpek_pointers, - &tempfile); + &tempfile, + pq.is_initialized() ? &pq : NULL, + found_rows); if (num_rows == HA_POS_ERROR) goto err; + maxbuffer= (uint) (my_b_tell(&buffpek_pointers)/sizeof(*buffpek)); if (maxbuffer == 0) // The whole set is in memory { - if (save_index(¶m,sort_keys,(uint) num_rows, &table_sort)) + if (save_index(¶m, (uint) num_rows, &table_sort)) goto err; } else { - thd->query_plan_flags|= QPLAN_FILESORT_DISK; - /* filesort cannot handle zero-length records during merge. */ DBUG_ASSERT(param.sort_length != 0); @@ -270,7 +323,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, /* Open cached file if it isn't open */ if (! my_b_inited(outfile) && open_cached_file(outfile,mysql_tmpdir,TEMP_PREFIX,READ_RECORD_BUFFER, - MYF(ME_ERROR | MY_WME))) + MYF(MY_WME))) goto err; if (reinit_io_cache(outfile,WRITE_CACHE,0L,0,0)) goto err; @@ -279,18 +332,20 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, Use also the space previously used by string pointers in sort_buffer for temporary key storage. */ - param.keys=((param.keys * - (param.rec_length+sizeof(char*))) / - param.rec_length - 1); + param.max_keys_per_buffer=((param.max_keys_per_buffer * + (param.rec_length + sizeof(char*))) / + param.rec_length - 1); maxbuffer--; // Offset from 0 - if (merge_many_buff(¶m,(uchar*) sort_keys,buffpek,&maxbuffer, + if (merge_many_buff(¶m, + (uchar*) table_sort.get_sort_keys(), + buffpek,&maxbuffer, &tempfile)) goto err; if (flush_io_cache(&tempfile) || reinit_io_cache(&tempfile,READ_CACHE,0L,0,0)) goto err; if (merge_index(¶m, - (uchar*) sort_keys, + (uchar*) table_sort.get_sort_keys(), buffpek, maxbuffer, &tempfile, @@ -305,12 +360,11 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, } error= 0; - err: + err: my_free(param.tmp_buffer); if (!subselect || !subselect->is_uncacheable()) { - my_free(sort_keys); - table_sort.sort_keys= 0; + table_sort.free_sort_buffer(); my_free(buffpek); table_sort.buffpek= 0; table_sort.buffpek_len= 0; @@ -349,7 +403,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, thd->killed == ABORT_QUERY ? "" : thd->stmt_da->message()); if (global_system_variables.log_warnings > 1) - { + { sql_print_warning("%s, host: %s, user: %s, thread: %lu, query: %-.4096s", ER_THD(thd, ER_FILSORT_ABORT), thd->security_ctx->host_or_ip, @@ -369,8 +423,12 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, /* table->sort.io_cache should be free by this time */ DBUG_ASSERT(NULL == table->sort.io_cache); - memcpy(&table->sort, &table_sort, sizeof(FILESORT_INFO)); - DBUG_PRINT("exit",("num_rows: %ld", (long) num_rows)); + // Assign the copy back! + table->sort= table_sort; + + DBUG_PRINT("exit", + ("num_rows: %ld examined_rows: %ld found_rows: %ld", + (long) num_rows, (long) *examined_rows, (long) *found_rows)); MYSQL_FILESORT_DONE(error, num_rows); DBUG_RETURN(error ? HA_POS_ERROR : num_rows); } /* filesort */ @@ -384,8 +442,7 @@ void filesort_free_buffers(TABLE *table, bool full) if (full) { - my_free(table->sort.sort_keys); - table->sort.sort_keys= NULL; + table->sort.free_sort_buffer(); my_free(table->sort.buffpek); table->sort.buffpek= NULL; table->sort.buffpek_len= 0; @@ -410,7 +467,7 @@ static uchar *read_buffpek_from_file(IO_CACHE *buffpek_pointers, uint count, if (count > UINT_MAX/sizeof(BUFFPEK)) return 0; /* sizeof(BUFFPEK)*count will overflow */ if (!tmp) - tmp= (uchar *)my_malloc(length, MYF(MY_WME)); + tmp= (uchar *)my_malloc(length, MYF(MY_WME | MY_THREAD_SPECIFIC)); if (tmp) { if (reinit_io_cache(buffpek_pointers,READ_CACHE,0L,0,0) || @@ -553,8 +610,10 @@ static void dbug_print_record(TABLE *table, bool print_rowid) #endif + /** - Search after sort_keys and write them into tempfile. + Search after sort_keys, and write them into tempfile + (if we run out of space in the sort_keys buffer). All produced sequences are guaranteed to be non-empty. @param param Sorting parameter @@ -563,19 +622,28 @@ static void dbug_print_record(TABLE *table, bool print_rowid) @param buffpek_pointers File to write BUFFPEKs describing sorted segments in tempfile. @param tempfile File to write sorted sequences of sortkeys to. + @param pq If !NULL, use it for keeping top N elements + @param [out] found_rows The number of FOUND_ROWS(). + For a query with LIMIT, this value will typically + be larger than the function return value. @note Basic idea: @verbatim while (get_next_sortkey()) { - if (no free space in sort_keys buffers) + if (using priority queue) + push sort key into queue + else { - sort sort_keys buffer; - dump sorted sequence to 'tempfile'; - dump BUFFPEK describing sequence location into 'buffpek_pointers'; + if (no free space in sort_keys buffers) + { + sort sort_keys buffer; + dump sorted sequence to 'tempfile'; + dump BUFFPEK describing sequence location into 'buffpek_pointers'; + } + put sort key into 'sort_keys'; } - put sort key into 'sort_keys'; } if (sort_keys has some elements && dumped at least once) sort-dump-dump as above; @@ -589,10 +657,12 @@ static void dbug_print_record(TABLE *table, bool print_rowid) HA_POS_ERROR on error. */ -static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select, - uchar **sort_keys, uchar *sort_keys_buf, +static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select, + Filesort_info *fs_info, IO_CACHE *buffpek_pointers, - IO_CACHE *tempfile) + IO_CACHE *tempfile, + Bounded_queue<uchar, uchar> *pq, + ha_rows *found_rows) { int error,flag,quick_select; uint idx,indexpos,ref_length; @@ -600,10 +670,9 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select, my_off_t record; TABLE *sort_form; THD *thd= current_thd; - volatile killed_state *killed= &thd->killed; handler *file; MY_BITMAP *save_read_set, *save_write_set, *save_vcol_set; - uchar *next_sort_key= sort_keys_buf; + DBUG_ENTER("find_all_keys"); DBUG_PRINT("info",("using: %s", (select ? select->quick ? "ranges" : "where": @@ -617,10 +686,16 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select, ref_pos= ref_buff; quick_select=select && select->quick; record=0; + *found_rows= 0; flag= ((file->ha_table_flags() & HA_REC_NOT_IN_SEQ) || quick_select); if (flag) ref_pos= &file->ref[0]; next_pos=ref_pos; + + DBUG_EXECUTE_IF("show_explain_in_find_all_keys", + dbug_serve_apcs(thd, 1); + ); + if (!quick_select) { next_pos=(uchar*) 0; /* Find records in sequence */ @@ -632,7 +707,6 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select, current_thd->variables.read_buff_size); } - /* Remember original bitmaps */ save_read_set= sort_form->read_set; save_write_set= sort_form->write_set; @@ -687,7 +761,7 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select, break; } - if (*killed) + if (thd->check_killed()) { DBUG_PRINT("info",("Sort killed by user")); if (!quick_select) @@ -732,18 +806,23 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select, if (write_record) { - if (idx == param->keys) + ++(*found_rows); + if (pq) { - if (write_keys(param, sort_keys, - idx, buffpek_pointers, tempfile)) - DBUG_RETURN(HA_POS_ERROR); - idx= 0; - next_sort_key= sort_keys_buf; - indexpos++; + pq->push(ref_pos); + idx= pq->num_elements(); + } + else + { + if (idx == param->max_keys_per_buffer) + { + if (write_keys(param, fs_info, idx, buffpek_pointers, tempfile)) + DBUG_RETURN(HA_POS_ERROR); + idx= 0; + indexpos++; + } + make_sortkey(param, fs_info->get_record_buffer(idx++), ref_pos); } - sort_keys[idx++]= next_sort_key; - make_sortkey(param, next_sort_key, ref_pos); - next_sort_key+= param->rec_length; } /* It does not make sense to read more keys in case of a fatal error */ @@ -777,12 +856,12 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select, DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ } if (indexpos && idx && - write_keys(param, sort_keys, - idx, buffpek_pointers, tempfile)) + write_keys(param, fs_info, idx, buffpek_pointers, tempfile)) DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ const ha_rows retval= my_b_inited(tempfile) ? (ha_rows) (my_b_tell(tempfile)/param->rec_length) : idx; + DBUG_PRINT("info", ("find_all_keys return %u", (uint) retval)); DBUG_RETURN(retval); } /* find_all_keys */ @@ -810,21 +889,19 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select, */ static bool -write_keys(SORTPARAM *param, register uchar **sort_keys, uint count, +write_keys(Sort_param *param, Filesort_info *fs_info, uint count, IO_CACHE *buffpek_pointers, IO_CACHE *tempfile) { - size_t sort_length, rec_length; + size_t rec_length; uchar **end; BUFFPEK buffpek; DBUG_ENTER("write_keys"); - sort_length= param->sort_length; rec_length= param->rec_length; -#ifdef MC68000 - quicksort(sort_keys,count,sort_length); -#else - my_string_ptr_sort((uchar*) sort_keys, (uint) count, sort_length); -#endif + uchar **sort_keys= fs_info->get_sort_keys(); + + fs_info->sort_buffer(param, count); + if (!my_b_inited(tempfile) && open_cached_file(tempfile, mysql_tmpdir, TEMP_PREFIX, DISK_BUFFER_SIZE, MYF(MY_WME))) @@ -873,8 +950,8 @@ static inline void store_length(uchar *to, uint length, uint pack_length) /** Make a sort-key from record. */ -static void make_sortkey(register SORTPARAM *param, - register uchar *to, uchar *ref_pos) +static void make_sortkey(register Sort_param *param, + register uchar *to, uchar *ref_pos) { reg3 Field *field; reg1 SORT_FIELD *sort_field; @@ -898,7 +975,7 @@ static void make_sortkey(register SORTPARAM *param, switch (sort_field->result_type) { case STRING_RESULT: { - CHARSET_INFO *cs=item->collation.collation; + const CHARSET_INFO *cs=item->collation.collation; char fill_char= ((cs->state & MY_CS_BINSORT) ? (char) 0 : ' '); if (maybe_null) @@ -909,7 +986,7 @@ static void make_sortkey(register SORTPARAM *param, if (!res) { if (maybe_null) - bzero((char*) to-1,sort_field->length+1); + memset(to-1, 0, sort_field->length+1); else { /* purecov: begin deadcode */ @@ -921,7 +998,7 @@ static void make_sortkey(register SORTPARAM *param, DBUG_ASSERT(0); DBUG_PRINT("warning", ("Got null on something that shouldn't be null")); - bzero((char*) to,sort_field->length); // Avoid crash + memset(to, 0, sort_field->length); // Avoid crash /* purecov: end */ } break; @@ -984,12 +1061,19 @@ static void make_sortkey(register SORTPARAM *param, } if (maybe_null) { + *to++=1; /* purecov: inspected */ if (item->null_value) { - bzero((char*) to++, sort_field->length+1); + if (maybe_null) + memset(to-1, 0, sort_field->length+1); + else + { + DBUG_PRINT("warning", + ("Got null on something that shouldn't be null")); + memset(to, 0, sort_field->length); + } break; } - *to++=1; /* purecov: inspected */ } to[7]= (uchar) value; to[6]= (uchar) (value >> 8); @@ -1011,7 +1095,8 @@ static void make_sortkey(register SORTPARAM *param, { if (item->null_value) { - bzero((char*) to++, sort_field->length+1); + memset(to, 0, sort_field->length+1); + to++; break; } *to++=1; @@ -1028,7 +1113,7 @@ static void make_sortkey(register SORTPARAM *param, { if (item->null_value) { - bzero((char*) to,sort_field->length+1); + memset(to, 0, sort_field->length+1); to++; break; } @@ -1073,7 +1158,7 @@ static void make_sortkey(register SORTPARAM *param, SORT_ADDON_FIELD *addonf= param->addon_field; uchar *nulls= to; DBUG_ASSERT(addonf != 0); - bzero((char *) nulls, addonf->offset); + memset(nulls, 0, addonf->offset); to+= addonf->offset; for ( ; (field= addonf->field) ; addonf++) { @@ -1112,7 +1197,7 @@ static void make_sortkey(register SORTPARAM *param, Register fields used by sorting in the sorted table's read set */ -static void register_used_fields(SORTPARAM *param) +static void register_used_fields(Sort_param *param) { reg1 SORT_FIELD *sort_field; TABLE *table=param->sort_form; @@ -1157,21 +1242,20 @@ static void register_used_fields(SORTPARAM *param) } -static bool save_index(SORTPARAM *param, uchar **sort_keys, uint count, - FILESORT_INFO *table_sort) +static bool save_index(Sort_param *param, uint count, Filesort_info *table_sort) { uint offset,res_length; uchar *to; DBUG_ENTER("save_index"); - my_string_ptr_sort((uchar*) sort_keys, (uint) count, param->sort_length); + table_sort->sort_buffer(param, count); res_length= param->res_length; offset= param->rec_length-res_length; - if ((ha_rows) count > param->max_rows) - count=(uint) param->max_rows; if (!(to= table_sort->record_pointers= - (uchar*) my_malloc(res_length*count, MYF(MY_WME)))) + (uchar*) my_malloc(res_length*count, + MYF(MY_WME | MY_THREAD_SPECIFIC)))) DBUG_RETURN(1); /* purecov: inspected */ + uchar **sort_keys= table_sort->get_sort_keys(); for (uchar **end= sort_keys+count ; sort_keys != end ; sort_keys++) { memcpy(to, *sort_keys+offset, res_length); @@ -1181,10 +1265,150 @@ static bool save_index(SORTPARAM *param, uchar **sort_keys, uint count, } +/** + Test whether priority queue is worth using to get top elements of an + ordered result set. If it is, then allocates buffer for required amount of + records + + @param param Sort parameters. + @param filesort_info Filesort information. + @param table Table to sort. + @param num_rows Estimate of number of rows in source record set. + @param memory_available Memory available for sorting. + + DESCRIPTION + Given a query like this: + SELECT ... FROM t ORDER BY a1,...,an LIMIT max_rows; + This function tests whether a priority queue should be used to keep + the result. Necessary conditions are: + - estimate that it is actually cheaper than merge-sort + - enough memory to store the <max_rows> records. + + If we don't have space for <max_rows> records, but we *do* have + space for <max_rows> keys, we may rewrite 'table' to sort with + references to records instead of additional data. + (again, based on estimates that it will actually be cheaper). + + @retval + true - if it's ok to use PQ + false - PQ will be slower than merge-sort, or there is not enough memory. +*/ + +bool check_if_pq_applicable(Sort_param *param, + Filesort_info *filesort_info, + TABLE *table, ha_rows num_rows, + ulong memory_available) +{ + DBUG_ENTER("check_if_pq_applicable"); + + /* + How much Priority Queue sort is slower than qsort. + Measurements (see unit test) indicate that PQ is roughly 3 times slower. + */ + const double PQ_slowness= 3.0; + + if (param->max_rows == HA_POS_ERROR) + { + DBUG_PRINT("info", ("No LIMIT")); + DBUG_RETURN(NULL); + } + + if (param->max_rows + 2 >= UINT_MAX) + { + DBUG_PRINT("info", ("Too large LIMIT")); + DBUG_RETURN(NULL); + } + + ulong num_available_keys= + memory_available / (param->rec_length + sizeof(char*)); + // We need 1 extra record in the buffer, when using PQ. + param->max_keys_per_buffer= (uint) param->max_rows + 1; + + if (num_rows < num_available_keys) + { + // The whole source set fits into memory. + if (param->max_rows < num_rows/PQ_slowness ) + { + filesort_info->alloc_sort_buffer(param->max_keys_per_buffer, + param->rec_length); + DBUG_RETURN(filesort_info->get_sort_keys() != NULL); + } + else + { + // PQ will be slower. + DBUG_RETURN(false); + } + } + + // Do we have space for LIMIT rows in memory? + if (param->max_keys_per_buffer < num_available_keys) + { + filesort_info->alloc_sort_buffer(param->max_keys_per_buffer, + param->rec_length); + DBUG_RETURN(filesort_info->get_sort_keys() != NULL); + } + + // Try to strip off addon fields. + if (param->addon_field) + { + const ulong row_length= + param->sort_length + param->ref_length + sizeof(char*); + num_available_keys= memory_available / row_length; + + // Can we fit all the keys in memory? + if (param->max_keys_per_buffer < num_available_keys) + { + const double sort_merge_cost= + get_merge_many_buffs_cost_fast(num_rows, + num_available_keys, + row_length); + /* + PQ has cost: + (insert + qsort) * log(queue size) / TIME_FOR_COMPARE_ROWID + + cost of file lookup afterwards. + The lookup cost is a bit pessimistic: we take scan_time and assume + that on average we find the row after scanning half of the file. + A better estimate would be lookup cost, but note that we are doing + random lookups here, rather than sequential scan. + */ + const double pq_cpu_cost= + (PQ_slowness * num_rows + param->max_keys_per_buffer) * + log((double) param->max_keys_per_buffer) / TIME_FOR_COMPARE_ROWID; + const double pq_io_cost= + param->max_rows * table->file->scan_time() / 2.0; + const double pq_cost= pq_cpu_cost + pq_io_cost; + + if (sort_merge_cost < pq_cost) + DBUG_RETURN(false); + + filesort_info->alloc_sort_buffer(param->max_keys_per_buffer, + param->sort_length + param->ref_length); + if (filesort_info->get_sort_keys()) + { + // Make attached data to be references instead of fields. + my_free(filesort_info->addon_buf); + my_free(filesort_info->addon_field); + filesort_info->addon_buf= NULL; + filesort_info->addon_field= NULL; + param->addon_field= NULL; + param->addon_length= 0; + + param->res_length= param->ref_length; + param->sort_length+= param->ref_length; + param->rec_length= param->sort_length; + + DBUG_RETURN(true); + } + } + } + DBUG_RETURN(false); +} + + /** Merge buffers to make < MERGEBUFF2 buffers. */ -int merge_many_buff(SORTPARAM *param, uchar *sort_buffer, - BUFFPEK *buffpek, uint *maxbuffer, IO_CACHE *t_file) +int merge_many_buff(Sort_param *param, uchar *sort_buffer, + BUFFPEK *buffpek, uint *maxbuffer, IO_CACHE *t_file) { register uint i; IO_CACHE t_file2,*from_file,*to_file,*temp; @@ -1315,7 +1539,7 @@ void reuse_freed_buff(QUEUE *queue, BUFFPEK *reuse, uint key_length) other error */ -int merge_buffers(SORTPARAM *param, IO_CACHE *from_file, +int merge_buffers(Sort_param *param, IO_CACHE *from_file, IO_CACHE *to_file, uchar *sort_buffer, BUFFPEK *lastbuff, BUFFPEK *Fb, BUFFPEK *Tb, int flag) @@ -1333,18 +1557,13 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file, void *first_cmp_arg; element_count dupl_count= 0; uchar *src; - killed_state not_killable; uchar *unique_buff= param->unique_buff; - volatile killed_state *killed= ¤t_thd->killed; + const bool killable= !param->not_killable; + THD* const thd=current_thd; DBUG_ENTER("merge_buffers"); - status_var_increment(current_thd->status_var.filesort_merge_passes); - current_thd->query_plan_fsort_passes++; - if (param->not_killable) - { - killed= ¬_killable; - not_killable= NOT_KILLED; - } + status_var_increment(thd->status_var.filesort_merge_passes); + thd->query_plan_fsort_passes++; error=0; rec_length= param->rec_length; @@ -1357,7 +1576,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file, (flag && min_dupl_count ? sizeof(dupl_count) : 0)-res_length); uint wr_len= flag ? res_length : rec_length; uint wr_offset= flag ? offset : 0; - maxcount= (ulong) (param->keys/((uint) (Tb-Fb) +1)); + maxcount= (ulong) (param->max_keys_per_buffer/((uint) (Tb-Fb) +1)); to_start_filepos= my_b_tell(to_file); strpos= sort_buffer; org_max_rows=max_rows= param->max_rows; @@ -1421,7 +1640,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file, while (queue.elements > 1) { - if (*killed) + if (killable && thd->check_killed()) { error= 1; goto err; /* purecov: inspected */ } @@ -1497,7 +1716,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file, } buffpek= (BUFFPEK*) queue_top(&queue); buffpek->base= (uchar*) sort_buffer; - buffpek->max_keys= param->keys; + buffpek->max_keys= param->max_keys_per_buffer; /* As we know all entries in the buffer are unique, we only have to @@ -1587,9 +1806,9 @@ err: /* Do a merge to output-file (save only positions) */ -int merge_index(SORTPARAM *param, uchar *sort_buffer, - BUFFPEK *buffpek, uint maxbuffer, - IO_CACHE *tempfile, IO_CACHE *outfile) +int merge_index(Sort_param *param, uchar *sort_buffer, + BUFFPEK *buffpek, uint maxbuffer, + IO_CACHE *tempfile, IO_CACHE *outfile) { DBUG_ENTER("merge_index"); if (merge_buffers(param,tempfile,outfile,sort_buffer,buffpek,buffpek, @@ -1635,7 +1854,7 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length, bool *multi_byte_charset) { reg2 uint length; - CHARSET_INFO *cs; + const CHARSET_INFO *cs; *multi_byte_charset= 0; length=0; @@ -1736,7 +1955,8 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length, */ static SORT_ADDON_FIELD * -get_addon_fields(THD *thd, Field **ptabfield, uint sortlength, uint *plength) +get_addon_fields(ulong max_length_for_sort_data, + Field **ptabfield, uint sortlength, uint *plength) { Field **pfield; Field *field; @@ -1773,9 +1993,11 @@ get_addon_fields(THD *thd, Field **ptabfield, uint sortlength, uint *plength) return 0; length+= (null_fields+7)/8; - if (length+sortlength > thd->variables.max_length_for_sort_data || + if (length+sortlength > max_length_for_sort_data || !(addonf= (SORT_ADDON_FIELD *) my_malloc(sizeof(SORT_ADDON_FIELD)* - (fields+1), MYF(MY_WME)))) + (fields+1), + MYF(MY_WME | + MY_THREAD_SPECIFIC)))) return 0; *plength= length; @@ -1856,7 +2078,7 @@ void change_double_for_sort(double nr,uchar *to) if (nr == 0.0) { /* Change to zero string */ tmp[0]=(uchar) 128; - bzero((char*) tmp+1,sizeof(nr)-1); + memset(tmp+1, 0, sizeof(nr)-1); } else { diff --git a/sql/filesort.h b/sql/filesort.h index 8ee8999d055..8960fa6cb66 100644 --- a/sql/filesort.h +++ b/sql/filesort.h @@ -29,10 +29,8 @@ typedef struct st_sort_field SORT_FIELD; ha_rows filesort(THD *thd, TABLE *table, st_sort_field *sortorder, uint s_length, SQL_SELECT *select, ha_rows max_rows, bool sort_positions, - ha_rows *examined_rows); + ha_rows *examined_rows, ha_rows *found_rows); void filesort_free_buffers(TABLE *table, bool full); -double get_merge_many_buffs_cost(uint *buffer, uint last_n_elems, - int elem_size); void change_double_for_sort(double nr,uchar *to); #endif /* FILESORT_INCLUDED */ diff --git a/sql/filesort_utils.cc b/sql/filesort_utils.cc new file mode 100644 index 00000000000..f8f6d5c9420 --- /dev/null +++ b/sql/filesort_utils.cc @@ -0,0 +1,143 @@ +/* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include "filesort_utils.h" +#include "sql_const.h" +#include "sql_sort.h" +#include "table.h" +#include "my_sys.h" + + +namespace { +/** + A local helper function. See comments for get_merge_buffers_cost(). + */ +double get_merge_cost(ha_rows num_elements, ha_rows num_buffers, uint elem_size) +{ + return + 2.0 * ((double) num_elements * elem_size) / IO_SIZE + + (double) num_elements * log((double) num_buffers) / + (TIME_FOR_COMPARE_ROWID * M_LN2); +} +} + +/** + This is a simplified, and faster version of @see get_merge_many_buffs_cost(). + We calculate the cost of merging buffers, by simulating the actions + of @see merge_many_buff. For explanations of formulas below, + see comments for get_merge_buffers_cost(). + TODO: Use this function for Unique::get_use_cost(). +*/ +double get_merge_many_buffs_cost_fast(ha_rows num_rows, + ha_rows num_keys_per_buffer, + uint elem_size) +{ + ha_rows num_buffers= num_rows / num_keys_per_buffer; + ha_rows last_n_elems= num_rows % num_keys_per_buffer; + double total_cost; + + // Calculate CPU cost of sorting buffers. + total_cost= + ( num_buffers * num_keys_per_buffer * log(1.0 + num_keys_per_buffer) + + last_n_elems * log(1.0 + last_n_elems) ) + / TIME_FOR_COMPARE_ROWID; + + // Simulate behavior of merge_many_buff(). + while (num_buffers >= MERGEBUFF2) + { + // Calculate # of calls to merge_buffers(). + const ha_rows loop_limit= num_buffers - MERGEBUFF*3/2; + const ha_rows num_merge_calls= 1 + loop_limit/MERGEBUFF; + const ha_rows num_remaining_buffs= + num_buffers - num_merge_calls * MERGEBUFF; + + // Cost of merge sort 'num_merge_calls'. + total_cost+= + num_merge_calls * + get_merge_cost(num_keys_per_buffer * MERGEBUFF, MERGEBUFF, elem_size); + + // # of records in remaining buffers. + last_n_elems+= num_remaining_buffs * num_keys_per_buffer; + + // Cost of merge sort of remaining buffers. + total_cost+= + get_merge_cost(last_n_elems, 1 + num_remaining_buffs, elem_size); + + num_buffers= num_merge_calls; + num_keys_per_buffer*= MERGEBUFF; + } + + // Simulate final merge_buff call. + last_n_elems+= num_keys_per_buffer * num_buffers; + total_cost+= get_merge_cost(last_n_elems, 1 + num_buffers, elem_size); + return total_cost; +} + +uchar **Filesort_buffer::alloc_sort_buffer(uint num_records, uint record_length) +{ + ulong sort_buff_sz; + + DBUG_ENTER("alloc_sort_buffer"); + + DBUG_EXECUTE_IF("alloc_sort_buffer_fail", + DBUG_SET("+d,simulate_out_of_memory");); + + if (m_idx_array.is_null()) + { + sort_buff_sz= num_records * (record_length + sizeof(uchar*)); + set_if_bigger(sort_buff_sz, record_length * MERGEBUFF2); + uchar **sort_keys= + (uchar**) my_malloc(sort_buff_sz, MYF(MY_THREAD_SPECIFIC)); + m_idx_array= Idx_array(sort_keys, num_records); + m_record_length= record_length; + uchar **start_of_data= m_idx_array.array() + m_idx_array.size(); + m_start_of_data= reinterpret_cast<uchar*>(start_of_data); + } + else + { + DBUG_ASSERT(num_records == m_idx_array.size()); + DBUG_ASSERT(record_length == m_record_length); + } + DBUG_RETURN(m_idx_array.array()); +} + + +void Filesort_buffer::free_sort_buffer() +{ + my_free(m_idx_array.array()); + m_idx_array= Idx_array(); + m_record_length= 0; + m_start_of_data= NULL; +} + + +void Filesort_buffer::sort_buffer(const Sort_param *param, uint count) +{ + if (count <= 1) + return; + uchar **keys= get_sort_keys(); + uchar **buffer= NULL; + if (radixsort_is_appliccable(count, param->sort_length) && + (buffer= (uchar**) my_malloc(count*sizeof(char*), + MYF(MY_THREAD_SPECIFIC)))) + { + radixsort_for_str_ptr(keys, count, param->sort_length, buffer); + my_free(buffer); + return; + } + + size_t size= param->sort_length; + my_qsort2(keys, count, sizeof(uchar*), get_ptr_compare(size), &size); +} diff --git a/sql/filesort_utils.h b/sql/filesort_utils.h new file mode 100644 index 00000000000..4cccf8ffa02 --- /dev/null +++ b/sql/filesort_utils.h @@ -0,0 +1,129 @@ +/* Copyright (c) 2010, 2012 Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef FILESORT_UTILS_INCLUDED +#define FILESORT_UTILS_INCLUDED + +#include "my_global.h" +#include "my_base.h" +#include "sql_array.h" + +class Sort_param; +/* + Calculate cost of merge sort + + @param num_rows Total number of rows. + @param num_keys_per_buffer Number of keys per buffer. + @param elem_size Size of each element. + + Calculates cost of merge sort by simulating call to merge_many_buff(). + + @retval + Computed cost of merge sort in disk seeks. + + @note + Declared here in order to be able to unit test it, + since library dependencies have not been sorted out yet. + + See also comments get_merge_many_buffs_cost(). +*/ + +double get_merge_many_buffs_cost_fast(ha_rows num_rows, + ha_rows num_keys_per_buffer, + uint elem_size); + + +/** + A wrapper class around the buffer used by filesort(). + The buffer is a contiguous chunk of memory, + where the first part is <num_records> pointers to the actual data. + + We wrap the buffer in order to be able to do lazy initialization of the + pointers: the buffer is often much larger than what we actually need. + + The buffer must be kept available for multiple executions of the + same sort operation, so we have explicit allocate and free functions, + rather than doing alloc/free in CTOR/DTOR. +*/ +class Filesort_buffer +{ +public: + Filesort_buffer() : + m_idx_array(), m_record_length(0), m_start_of_data(NULL) + {} + + /** Sort me... */ + void sort_buffer(const Sort_param *param, uint count); + + /// Initializes a record pointer. + uchar *get_record_buffer(uint idx) + { + m_idx_array[idx]= m_start_of_data + (idx * m_record_length); + return m_idx_array[idx]; + } + + /// Initializes all the record pointers. + void init_record_pointers() + { + for (uint ix= 0; ix < m_idx_array.size(); ++ix) + (void) get_record_buffer(ix); + } + + /// Returns total size: pointer array + record buffers. + size_t sort_buffer_size() const + { + return m_idx_array.size() * (m_record_length + sizeof(uchar*)); + } + + /// Allocates the buffer, but does *not* initialize pointers. + uchar **alloc_sort_buffer(uint num_records, uint record_length); + + + /// Check <num_records, record_length> for the buffer + bool check_sort_buffer_properties(uint num_records, uint record_length) + { + return (static_cast<uint>(m_idx_array.size()) == num_records && + m_record_length == m_record_length); + } + + /// Frees the buffer. + void free_sort_buffer(); + + /// Getter, for calling routines which still use the uchar** interface. + uchar **get_sort_keys() { return m_idx_array.array(); } + + /** + We need an assignment operator, see filesort(). + This happens to have the same semantics as the one that would be + generated by the compiler. We still implement it here, to show shallow + assignment explicitly: we have two objects sharing the same array. + */ + Filesort_buffer &operator=(const Filesort_buffer &rhs) + { + m_idx_array= rhs.m_idx_array; + m_record_length= rhs.m_record_length; + m_start_of_data= rhs.m_start_of_data; + return *this; + } + +private: + typedef Bounds_checked_array<uchar*> Idx_array; + + Idx_array m_idx_array; + uint m_record_length; + uchar *m_start_of_data; +}; + +#endif // FILESORT_UTILS_INCLUDED diff --git a/sql/frm_crypt.cc b/sql/frm_crypt.cc deleted file mode 100644 index 5612908aea5..00000000000 --- a/sql/frm_crypt.cc +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - - -/* -** change the following to the output of password('our password') -** split into 2 parts of 8 characters each. -** This is done to make it impossible to search after a text string in the -** mysql binary. -*/ - -#include "sql_priv.h" -#include "frm_crypt.h" - -#ifdef HAVE_CRYPTED_FRM - -/* password('test') */ -ulong password_seed[2]={0x378b243e, 0x220ca493}; - -SQL_CRYPT *get_crypt_for_frm(void) -{ - return new SQL_CRYPT(password_seed); -} - -#endif diff --git a/sql/frm_crypt.h b/sql/frm_crypt.h deleted file mode 100644 index 0605644b3e0..00000000000 --- a/sql/frm_crypt.h +++ /dev/null @@ -1,23 +0,0 @@ -/* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -#ifndef FRM_CRYPT_INCLUDED -#define FRM_CRYPT_INCLUDED - -class SQL_CRYPT; - -SQL_CRYPT *get_crypt_for_frm(void); - -#endif /* FRM_CRYPT_INCLUDED */ diff --git a/sql/ha_ndbcluster.cc b/sql/ha_ndbcluster.cc index 2878f25ed14..6fc30fa4fa0 100644 --- a/sql/ha_ndbcluster.cc +++ b/sql/ha_ndbcluster.cc @@ -2906,8 +2906,6 @@ int ha_ndbcluster::write_row(uchar *record) } ha_statistic_increment(&SSV::ha_write_count); - if (table->timestamp_field_type & TIMESTAMP_AUTO_SET_ON_INSERT) - table->timestamp_field->set_time(); if (!(op= trans->getNdbOperation(m_table))) ERR_RETURN(trans->getNdbError()); @@ -3146,11 +3144,6 @@ int ha_ndbcluster::update_row(const uchar *old_data, uchar *new_data) } ha_statistic_increment(&SSV::ha_update_count); - if (table->timestamp_field_type & TIMESTAMP_AUTO_SET_ON_UPDATE) - { - table->timestamp_field->set_time(); - bitmap_set_bit(table->write_set, table->timestamp_field->field_index); - } if (m_use_partition_function && (error= get_parts_for_update(old_data, new_data, table->record[0], @@ -8656,7 +8649,7 @@ NDB_SHARE *ndbcluster_get_share(const char *key, TABLE *table, MEM_ROOT **root_ptr= my_pthread_getspecific_ptr(MEM_ROOT**, THR_MALLOC); MEM_ROOT *old_root= *root_ptr; - init_sql_alloc(&share->mem_root, 1024, 0); + init_sql_alloc(&share->mem_root, 1024, 0, MYF(0)); *root_ptr= &share->mem_root; // remember to reset before return share->state= NSS_INITIAL; /* enough space for key, db, and table_name */ @@ -9500,7 +9493,7 @@ pthread_handler_t ndb_util_thread_func(void *arg __attribute__((unused))) thd->init_for_queries(); thd->main_security_ctx.host_or_ip= ""; thd->client_capabilities = 0; - my_net_init(&thd->net, 0); + my_net_init(&thd->net, 0, MYF(MY_THREAD_SPECIFIC)); thd->main_security_ctx.master_access= ~0; thd->main_security_ctx.priv_user[0] = 0; /* Do not use user-supplied timeout value for system threads. */ @@ -9737,11 +9730,9 @@ next: mysql_mutex_lock(&LOCK_ndb_util_thread); ndb_util_thread_end: - net_end(&thd->net); ndb_util_thread_fail: if (share_list) delete [] share_list; - thd->cleanup(); delete thd; /* signal termination */ diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index 013929c24e0..075de1e1549 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -3672,7 +3672,7 @@ pthread_handler_t ndb_binlog_thread_func(void *arg) thd->system_thread= SYSTEM_THREAD_NDBCLUSTER_BINLOG; thd->main_security_ctx.host_or_ip= ""; thd->client_capabilities= 0; - my_net_init(&thd->net, 0); + my_net_init(&thd->net, 0, MYF(MY_THREAD_SPECIFIC)); thd->main_security_ctx.master_access= ~0; thd->main_security_ctx.priv_user[0]= 0; /* Do not use user-supplied timeout value for system threads. */ @@ -3971,7 +3971,7 @@ restart: my_pthread_getspecific_ptr(MEM_ROOT**, THR_MALLOC); MEM_ROOT *old_root= *root_ptr; MEM_ROOT mem_root; - init_sql_alloc(&mem_root, 4096, 0); + init_sql_alloc(&mem_root, 4096, 0, MYF(0)); List<Cluster_schema> post_epoch_log_list; List<Cluster_schema> post_epoch_unlock_list; *root_ptr= &mem_root; @@ -4371,8 +4371,6 @@ err: my_hash_free(&ndb_schema_objects); - net_end(&thd->net); - thd->cleanup(); delete thd; ndb_binlog_thread_running= -1; diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 0d30265ce9a..248b95751af 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -82,6 +82,16 @@ static uint alter_table_flags(uint flags); extern "C" int cmp_key_then_part_id(void *key_p, uchar *ref1, uchar *ref2); +/* + If frm_error() is called then we will use this to to find out what file + extensions exist for the storage engine. This is also used by the default + rename_table and delete_table method in handler.cc. +*/ +static const char *ha_partition_ext[]= +{ + ha_par_ext, NullS +}; + static int partition_initialize(void *p) { @@ -96,6 +106,7 @@ static int partition_initialize(void *p) partition_hton->flags= HTON_NOT_USER_SELECTABLE | HTON_HIDDEN | HTON_TEMPORARY_NOT_SUPPORTED; + partition_hton->tablefile_extensions= ha_partition_ext; return 0; } @@ -171,7 +182,7 @@ ha_partition::ha_partition(handlerton *hton, TABLE_SHARE *share) :handler(hton, share) { DBUG_ENTER("ha_partition::ha_partition(table)"); - init_alloc_root(&m_mem_root, 512, 512); + init_alloc_root(&m_mem_root, 512, 512, MYF(0)); init_handler_variables(); DBUG_VOID_RETURN; } @@ -193,7 +204,7 @@ ha_partition::ha_partition(handlerton *hton, partition_info *part_info) { DBUG_ENTER("ha_partition::ha_partition(part_info)"); DBUG_ASSERT(part_info); - init_alloc_root(&m_mem_root, 512, 512); + init_alloc_root(&m_mem_root, 512, 512, MYF(0)); init_handler_variables(); m_part_info= part_info; m_create_handler= TRUE; @@ -220,7 +231,7 @@ ha_partition::ha_partition(handlerton *hton, TABLE_SHARE *share, :handler(hton, share) { DBUG_ENTER("ha_partition::ha_partition(clone)"); - init_alloc_root(&m_mem_root, 512, 512); + init_alloc_root(&m_mem_root, 512, 512, MYF(0)); init_handler_variables(); m_part_info= part_info_arg; m_create_handler= TRUE; @@ -303,13 +314,6 @@ void ha_partition::init_handler_variables() } -const char *ha_partition::table_type() const -{ - // we can do this since we only support a single engine type - return m_file && m_file[0] ? m_file[0]->table_type() : "Unknown"; -} - - /* Destructor method @@ -511,7 +515,7 @@ int ha_partition::rename_table(const char *from, const char *to) Create the handler file (.par-file) SYNOPSIS - create_handler_files() + create_partitioning_metadata() name Full path of table name create_info Create info generated for CREATE TABLE @@ -520,19 +524,18 @@ int ha_partition::rename_table(const char *from, const char *to) 0 Success DESCRIPTION - create_handler_files is called to create any handler specific files + create_partitioning_metadata is called to create any handler specific files before opening the file with openfrm to later call ::create on the file object. In the partition handler this is used to store the names of partitions and types of engines in the partitions. */ -int ha_partition::create_handler_files(const char *path, +int ha_partition::create_partitioning_metadata(const char *path, const char *old_path, - int action_flag, - HA_CREATE_INFO *create_info) + int action_flag) { - DBUG_ENTER("ha_partition::create_handler_files()"); + DBUG_ENTER("ha_partition::create_partitioning_metadata()"); /* We need to update total number of parts since we might write the handler @@ -3498,8 +3501,8 @@ void ha_partition::try_semi_consistent_read(bool yes) ADDITIONAL INFO: - We have to set timestamp fields and auto_increment fields, because those - may be used in determining which partition the row should be written to. + We have to set auto_increment fields, because those may be used in + determining which partition the row should be written to. */ int ha_partition::write_row(uchar * buf) @@ -3510,7 +3513,6 @@ int ha_partition::write_row(uchar * buf) bool have_auto_increment= table->next_number_field && buf == table->record[0]; my_bitmap_map *old_map; THD *thd= ha_thd(); - timestamp_auto_set_type saved_timestamp_type= table->timestamp_field_type; ulonglong saved_sql_mode= thd->variables.sql_mode; bool saved_auto_inc_field_not_null= table->auto_increment_field_not_null; #ifdef NOT_NEEDED @@ -3519,11 +3521,6 @@ int ha_partition::write_row(uchar * buf) DBUG_ENTER("ha_partition::write_row"); DBUG_ASSERT(buf == m_rec0); - /* If we have a timestamp column, update it to the current time */ - if (table->timestamp_field_type & TIMESTAMP_AUTO_SET_ON_INSERT) - table->timestamp_field->set_time(); - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - /* If we have an auto_increment column and we are writing a changed row or a new row, then update the auto_increment value in the record. @@ -3599,7 +3596,6 @@ int ha_partition::write_row(uchar * buf) exit: thd->variables.sql_mode= saved_sql_mode; table->auto_increment_field_not_null= saved_auto_inc_field_not_null; - table->timestamp_field_type= saved_timestamp_type; DBUG_RETURN(error); } @@ -3634,19 +3630,9 @@ int ha_partition::update_row(const uchar *old_data, uchar *new_data) uint32 new_part_id, old_part_id; int error= 0; longlong func_value; - timestamp_auto_set_type orig_timestamp_type= table->timestamp_field_type; DBUG_ENTER("ha_partition::update_row"); m_err_rec= NULL; - /* - We need to set timestamp field once before we calculate - the partition. Then we disable timestamp calculations - inside m_file[*]->update_row() methods - */ - if (orig_timestamp_type & TIMESTAMP_AUTO_SET_ON_UPDATE) - table->timestamp_field->set_time(); - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - if ((error= get_parts_for_update(old_data, new_data, table->record[0], m_part_info, &old_part_id, &new_part_id, &func_value))) @@ -3739,7 +3725,6 @@ exit: info(HA_STATUS_AUTO); set_auto_increment_if_higher(table->found_next_number_field); } - table->timestamp_field_type= orig_timestamp_type; DBUG_RETURN(error); } @@ -3958,6 +3943,7 @@ int ha_partition::truncate_partition(Alter_info *alter_info, bool *binlog_stmt) SYNOPSIS start_bulk_insert() rows Number of rows to insert + flags Flags to control index creation RETURN VALUE NONE @@ -3965,7 +3951,7 @@ int ha_partition::truncate_partition(Alter_info *alter_info, bool *binlog_stmt) DESCRIPTION rows == 0 means we will probably insert many rows */ -void ha_partition::start_bulk_insert(ha_rows rows) +void ha_partition::start_bulk_insert(ha_rows rows, uint flags) { DBUG_ENTER("ha_partition::start_bulk_insert"); @@ -7592,21 +7578,6 @@ int ha_partition::final_drop_index(TABLE *table_arg) } -/* - If frm_error() is called then we will use this to to find out what file - extensions exist for the storage engine. This is also used by the default - rename_table and delete_table method in handler.cc. -*/ - -static const char *ha_partition_ext[]= -{ - ha_par_ext, NullS -}; - -const char **ha_partition::bas_ext() const -{ return ha_partition_ext; } - - uint ha_partition::min_of_the_max_uint( uint (handler::*operator_func)(void) const) const { diff --git a/sql/ha_partition.h b/sql/ha_partition.h index cec377b9e29..0108fb7994f 100644 --- a/sql/ha_partition.h +++ b/sql/ha_partition.h @@ -232,7 +232,7 @@ public: chance for the handler to add any interesting comments to the table comments not provided by the users comment. - create_handler_files is called before opening a new handler object + create_partitioning_metadata is called before opening a new handler object with openfrm to call create. It is used to create any local handler object needed in opening the object in openfrm ------------------------------------------------------------------------- @@ -241,9 +241,8 @@ public: virtual int rename_table(const char *from, const char *to); virtual int create(const char *name, TABLE *form, HA_CREATE_INFO *create_info); - virtual int create_handler_files(const char *name, - const char *old_name, int action_flag, - HA_CREATE_INFO *create_info); + virtual int create_partitioning_metadata(const char *name, + const char *old_name, int action_flag); virtual void update_create_info(HA_CREATE_INFO *create_info); virtual char *update_table_comment(const char *comment); virtual int change_partitions(HA_CREATE_INFO *create_info, @@ -379,7 +378,7 @@ public: virtual int delete_row(const uchar * buf); virtual int delete_all_rows(void); virtual int truncate(); - virtual void start_bulk_insert(ha_rows rows); + virtual void start_bulk_insert(ha_rows rows, uint flags); virtual int end_bulk_insert(); private: ha_rows guess_bulk_insert_rows(); @@ -643,9 +642,6 @@ public: */ virtual const char *index_type(uint inx); - /* The name of the table type that will be used for display purposes */ - virtual const char *table_type() const; - /* The name of the row type used for the underlying tables. */ virtual enum row_type get_row_type() const; @@ -890,10 +886,6 @@ public: */ virtual uint alter_table_flags(uint flags); /* - extensions of table handler files - */ - virtual const char **bas_ext() const; - /* unireg.cc will call the following to make sure that the storage engine can handle the data it is about to send. diff --git a/sql/handler.cc b/sql/handler.cc index 2e038be0092..7a90ac310ea 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -34,7 +34,7 @@ #include "sql_parse.h" // check_stack_overrun #include "sql_acl.h" // SUPER_ACL #include "sql_base.h" // free_io_cache -#include "discover.h" // writefrm +#include "discover.h" // extension_based_table_discovery, etc #include "log_event.h" // *_rows_log_event #include "create_options.h" #include "rpl_filter.h" @@ -44,7 +44,7 @@ #include "probes_mysql.h" #include "debug_sync.h" // DEBUG_SYNC #include "sql_audit.h" -#include "../mysys/my_handler_errors.h" +#include <my_handler_errors.h> #ifdef WITH_PARTITION_STORAGE_ENGINE #include "ha_partition.h" @@ -125,7 +125,7 @@ handlerton *ha_default_handlerton(THD *thd) { plugin_ref plugin= ha_default_plugin(thd); DBUG_ASSERT(plugin); - handlerton *hton= plugin_data(plugin, handlerton*); + handlerton *hton= plugin_hton(plugin); DBUG_ASSERT(hton); return hton; } @@ -156,7 +156,7 @@ redo: if ((plugin= my_plugin_lock_by_name(thd, name, MYSQL_STORAGE_ENGINE_PLUGIN))) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton && !(hton->flags & HTON_NOT_USER_SELECTABLE)) return plugin; @@ -204,7 +204,7 @@ handlerton *ha_resolve_by_legacy_type(THD *thd, enum legacy_db_type db_type) default: if (db_type > DB_TYPE_UNKNOWN && db_type < DB_TYPE_DEFAULT && (plugin= ha_lock_engine(thd, installed_htons[db_type]))) - return plugin_data(plugin, handlerton*); + return plugin_hton(plugin); /* fall through */ case DB_TYPE_UNKNOWN: return NULL; @@ -234,13 +234,6 @@ handlerton *ha_checktype(THD *thd, enum legacy_db_type database_type, RUN_HOOK(transaction, after_rollback, (thd, FALSE)); - switch (database_type) { - case DB_TYPE_MRG_ISAM: - return ha_resolve_by_legacy_type(thd, DB_TYPE_MRG_MYISAM); - default: - break; - } - return ha_default_handlerton(thd); } /* ha_checktype */ @@ -393,6 +386,34 @@ static int ha_finish_errors(void) return 0; } +static volatile int32 need_full_discover_for_existence= 0; +static volatile int32 engines_with_discover_table_names= 0; + +static int full_discover_for_existence(handlerton *, const char *, const char *) +{ return 1; } + +static int ext_based_existence(handlerton *, const char *, const char *) +{ return 1; } + +static int hton_ext_based_table_discovery(handlerton *hton, LEX_STRING *db, + MY_DIR *dir, handlerton::discovered_list *result) +{ + /* + tablefile_extensions[0] is the metadata file, see + the comment above tablefile_extensions declaration + */ + return extension_based_table_discovery(dir, hton->tablefile_extensions[0], + result); +} + +static void update_discovery_counters(handlerton *hton, int val) +{ + if (hton->discover_table_existence == full_discover_for_existence) + my_atomic_add32(&need_full_discover_for_existence, val); + + if (hton->discover_table_names) + my_atomic_add32(&engines_with_discover_table_names, val); +} int ha_finalize_handlerton(st_plugin_int *plugin) { @@ -430,6 +451,9 @@ int ha_finalize_handlerton(st_plugin_int *plugin) } } + free_sysvar_table_options(hton); + update_discovery_counters(hton, -1); + /* In case a plugin is uninstalled and re-installed later, it should reuse an array slot. Otherwise the number of uninstall/install @@ -453,12 +477,12 @@ int ha_finalize_handlerton(st_plugin_int *plugin) int ha_initialize_handlerton(st_plugin_int *plugin) { handlerton *hton; + static const char *no_exts[]= { 0 }; DBUG_ENTER("ha_initialize_handlerton"); DBUG_PRINT("plugin", ("initialize plugin: '%s'", plugin->name.str)); hton= (handlerton *)my_malloc(sizeof(handlerton), MYF(MY_WME | MY_ZEROFILL)); - if (hton == NULL) { sql_print_error("Unable to allocate memory for plugin '%s' handlerton.", @@ -466,6 +490,9 @@ int ha_initialize_handlerton(st_plugin_int *plugin) goto err_no_hton_memory; } + hton->tablefile_extensions= no_exts; + hton->discover_table_names= hton_ext_based_table_discovery; + hton->slot= HA_SLOT_UNDEF; /* Historical Requirement */ plugin->data= hton; // shortcut for the future @@ -476,6 +503,21 @@ int ha_initialize_handlerton(st_plugin_int *plugin) goto err; } + // hton_ext_based_table_discovery() works only when discovery + // is supported and the engine if file-based. + if (hton->discover_table_names == hton_ext_based_table_discovery && + (!hton->discover_table || !hton->tablefile_extensions[0])) + hton->discover_table_names= NULL; + + // default discover_table_existence implementation + if (!hton->discover_table_existence && hton->discover_table) + { + if (hton->tablefile_extensions[0]) + hton->discover_table_existence= ext_based_existence; + else + hton->discover_table_existence= full_discover_for_existence; + } + /* the switch below and hton->state should be removed when command-line options for plugins will be implemented @@ -571,6 +613,9 @@ int ha_initialize_handlerton(st_plugin_int *plugin) break; }; + resolve_sysvar_table_options(hton); + update_discovery_counters(hton, 1); + DBUG_RETURN(0); err_deinit: @@ -624,7 +669,7 @@ int ha_end() static my_bool dropdb_handlerton(THD *unused1, plugin_ref plugin, void *path) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->state == SHOW_OPTION_YES && hton->drop_database) hton->drop_database(hton, (char *)path); return FALSE; @@ -640,7 +685,7 @@ void ha_drop_database(char* path) static my_bool checkpoint_state_handlerton(THD *unused1, plugin_ref plugin, void *disable) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->state == SHOW_OPTION_YES && hton->checkpoint_state) hton->checkpoint_state(hton, (int) *(bool*) disable); return FALSE; @@ -653,11 +698,48 @@ void ha_checkpoint_state(bool disable) } +struct st_commit_checkpoint_request { + void *cookie; + void (*pre_hook)(void *); +}; + +static my_bool commit_checkpoint_request_handlerton(THD *unused1, plugin_ref plugin, + void *data) +{ + st_commit_checkpoint_request *st= (st_commit_checkpoint_request *)data; + handlerton *hton= plugin_hton(plugin); + if (hton->state == SHOW_OPTION_YES && hton->commit_checkpoint_request) + { + void *cookie= st->cookie; + if (st->pre_hook) + (*st->pre_hook)(cookie); + (*hton->commit_checkpoint_request)(hton, cookie); + } + return FALSE; +} + + +/* + Invoke commit_checkpoint_request() in all storage engines that implement it. + + If pre_hook is non-NULL, the hook will be called prior to each invocation. +*/ +void +ha_commit_checkpoint_request(void *cookie, void (*pre_hook)(void *)) +{ + st_commit_checkpoint_request st; + st.cookie= cookie; + st.pre_hook= pre_hook; + plugin_foreach(NULL, commit_checkpoint_request_handlerton, + MYSQL_STORAGE_ENGINE_PLUGIN, &st); +} + + static my_bool closecon_handlerton(THD *thd, plugin_ref plugin, void *unused) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); /* there's no need to rollback here as all transactions must be rolled back already @@ -684,7 +766,7 @@ void ha_close_connection(THD* thd) static my_bool kill_handlerton(THD *thd, plugin_ref plugin, void *level) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->state == SHOW_OPTION_YES && hton->kill_query && thd_get_ha_data(thd, hton)) @@ -1067,7 +1149,8 @@ int ha_prepare(THD *thd) else { push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, - ER_ILLEGAL_HA, ER(ER_ILLEGAL_HA), + ER_GET_ERRNO, ER(ER_GET_ERRNO), + HA_ERR_WRONG_COMMAND, ha_resolve_storage_engine_name(ht)); } } @@ -1171,6 +1254,8 @@ int ha_commit_trans(THD *thd, bool all) bool need_prepare_ordered, need_commit_ordered; my_xid xid; DBUG_ENTER("ha_commit_trans"); + DBUG_PRINT("info",("thd: %p option_bits: %lu all: %d", + thd, (ulong) thd->variables.option_bits, all)); /* Just a random warning to test warnings pushed during autocommit. */ DBUG_EXECUTE_IF("warn_during_ha_commit_trans", @@ -1216,7 +1301,10 @@ int ha_commit_trans(THD *thd, bool all) { /* Free resources and perform other cleanup even for 'empty' transactions. */ if (is_real_trans) + { thd->transaction.cleanup(); + thd->wakeup_subsequent_commits(error); + } DBUG_RETURN(0); } @@ -1230,6 +1318,8 @@ int ha_commit_trans(THD *thd, bool all) /* rw_trans is TRUE when we in a transaction changing data */ bool rw_trans= is_real_trans && (rw_ha_count > 0); MDL_request mdl_request; + DBUG_PRINT("info", ("is_real_trans: %d rw_trans: %d rw_ha_count: %d", + is_real_trans, rw_trans, rw_ha_count)); if (rw_trans) { @@ -1248,6 +1338,7 @@ int ha_commit_trans(THD *thd, bool all) thd->variables.lock_wait_timeout)) { ha_rollback_trans(thd, all); + thd->wakeup_subsequent_commits(1); DBUG_RETURN(1); } @@ -1308,11 +1399,13 @@ int ha_commit_trans(THD *thd, bool all) goto done; } + DEBUG_SYNC(thd, "ha_commit_trans_before_log_and_order"); cookie= tc_log->log_and_order(thd, xid, all, need_prepare_ordered, need_commit_ordered); if (!cookie) goto err; + DEBUG_SYNC(thd, "ha_commit_trans_after_log_and_order"); DBUG_EXECUTE_IF("crash_commit_after_log", DBUG_SUICIDE();); error= commit_one_phase_2(thd, all, trans, is_real_trans) ? 2 : 0; @@ -1333,6 +1426,7 @@ done: err: error= 1; /* Transaction was rolled back */ ha_rollback_trans(thd, all); + thd->wakeup_subsequent_commits(error); end: if (rw_trans && mdl_request.ticket) @@ -1376,8 +1470,15 @@ int ha_commit_one_phase(THD *thd, bool all) transaction.all.ha_list, see why in trans_register_ha()). */ bool is_real_trans=all || thd->transaction.all.ha_list == 0; + int res; DBUG_ENTER("ha_commit_one_phase"); - int res= commit_one_phase_2(thd, all, trans, is_real_trans); + if (is_real_trans) + { + DEBUG_SYNC(thd, "ha_commit_one_phase"); + if ((res= thd->wait_for_prior_commit())) + DBUG_RETURN(res); + } + res= commit_one_phase_2(thd, all, trans, is_real_trans); DBUG_RETURN(res); } @@ -1388,6 +1489,8 @@ commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) int error= 0; Ha_trx_info *ha_info= trans->ha_list, *ha_info_next; DBUG_ENTER("commit_one_phase_2"); + if (is_real_trans) + DEBUG_SYNC(thd, "commit_one_phase_2"); if (ha_info) { for (; ha_info; ha_info= ha_info_next) @@ -1416,7 +1519,10 @@ commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) } /* Free resources and perform other cleanup even for 'empty' transactions. */ if (is_real_trans) + { + thd->wakeup_subsequent_commits(error); thd->transaction.cleanup(); + } DBUG_RETURN(error); } @@ -1532,7 +1638,7 @@ struct xahton_st { static my_bool xacommit_handlerton(THD *unused1, plugin_ref plugin, void *arg) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->state == SHOW_OPTION_YES && hton->recover) { hton->commit_by_xid(hton, ((struct xahton_st *)arg)->xid); @@ -1544,7 +1650,7 @@ static my_bool xacommit_handlerton(THD *unused1, plugin_ref plugin, static my_bool xarollback_handlerton(THD *unused1, plugin_ref plugin, void *arg) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->state == SHOW_OPTION_YES && hton->recover) { hton->rollback_by_xid(hton, ((struct xahton_st *)arg)->xid); @@ -1650,7 +1756,7 @@ struct xarecover_st static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin, void *arg) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); struct xarecover_st *info= (struct xarecover_st *) arg; int got; @@ -1810,6 +1916,17 @@ bool mysql_xa_recover(THD *thd) DBUG_RETURN(0); } +/* + Called by engine to notify TC that a new commit checkpoint has been reached. + See comments on handlerton method commit_checkpoint_request() for details. +*/ +void +commit_checkpoint_notify_ha(handlerton *hton, void *cookie) +{ + tc_log->commit_checkpoint_notify(cookie); +} + + /** @details This function should be called when MySQL sends rows of a SELECT result set @@ -1926,7 +2043,7 @@ int ha_savepoint(THD *thd, SAVEPOINT *sv) } if ((err= ht->savepoint_set(ht, thd, (uchar *)(sv+1)+ht->savepoint_offset))) { // cannot happen - my_error(ER_GET_ERRNO, MYF(0), err); + my_error(ER_GET_ERRNO, MYF(0), err, hton_name(ht)->str); error=1; } status_var_increment(thd->status_var.ha_savepoint_count); @@ -1957,7 +2074,7 @@ int ha_release_savepoint(THD *thd, SAVEPOINT *sv) if ((err= ht->savepoint_release(ht, thd, (uchar *)(sv+1) + ht->savepoint_offset))) { // cannot happen - my_error(ER_GET_ERRNO, MYF(0), err); + my_error(ER_GET_ERRNO, MYF(0), err, hton_name(ht)->str); error=1; } } @@ -1968,7 +2085,7 @@ int ha_release_savepoint(THD *thd, SAVEPOINT *sv) static my_bool snapshot_handlerton(THD *thd, plugin_ref plugin, void *arg) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->state == SHOW_OPTION_YES && hton->start_consistent_snapshot) { @@ -2008,7 +2125,7 @@ int ha_start_consistent_snapshot(THD *thd) static my_bool flush_handlerton(THD *thd, plugin_ref plugin, void *arg) { - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->state == SHOW_OPTION_YES && hton->flush_logs && hton->flush_logs(hton)) return TRUE; @@ -2131,15 +2248,15 @@ int ha_delete_table(THD *thd, handlerton *table_type, const char *path, TABLE_SHARE dummy_share; DBUG_ENTER("ha_delete_table"); + /* table_type is NULL in ALTER TABLE when renaming only .frm files */ + if (table_type == NULL || table_type == view_pseudo_hton || + ! (file=get_new_handler((TABLE_SHARE*)0, thd->mem_root, table_type))) + DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); + bzero((char*) &dummy_table, sizeof(dummy_table)); bzero((char*) &dummy_share, sizeof(dummy_share)); dummy_table.s= &dummy_share; - /* DB_TYPE_UNKNOWN is used in ALTER TABLE when renaming only .frm files */ - if (table_type == NULL || - ! (file=get_new_handler((TABLE_SHARE*)0, thd->mem_root, table_type))) - DBUG_RETURN(ENOENT); - path= get_canonical_filename(file, path, tmp_path); if ((error= file->ha_delete_table(path)) && generate_warning) { @@ -2154,6 +2271,7 @@ int ha_delete_table(THD *thd, handlerton *table_type, const char *path, /* Fill up strucutures that print_error may need */ dummy_share.path.str= (char*) path; dummy_share.path.length= strlen(path); + dummy_share.normalized_path= dummy_share.path; dummy_share.db.str= (char*) db; dummy_share.db.length= strlen(db); dummy_share.table_name.str= (char*) alias; @@ -2893,7 +3011,7 @@ void handler::print_error(int error, myf errflag) DBUG_ENTER("handler::print_error"); DBUG_PRINT("enter",("error: %d",error)); - int textno=ER_GET_ERRNO; + int textno= -1; // impossible value switch (error) { case EACCES: textno=ER_OPEN_AS_READONLY; @@ -3022,7 +3140,9 @@ void handler::print_error(int error, myf errflag) textno=ER_OUT_OF_RESOURCES; break; case HA_ERR_WRONG_COMMAND: - textno=ER_ILLEGAL_HA; + my_error(ER_ILLEGAL_HA, MYF(0), table_type(), table_share->db.str, + table_share->table_name.str); + DBUG_VOID_RETURN; break; case HA_ERR_OLD_FILE: textno=ER_OLD_KEYFILE; @@ -3140,21 +3260,12 @@ void handler::print_error(int error, myf errflag) my_error(ER_GET_ERRMSG, errflag, error, str.c_ptr(), engine); } } - else if (error >= HA_ERR_FIRST && error <= HA_ERR_LAST) - { - const char* engine= table_type(); - const char *errmsg= handler_error_messages[error - HA_ERR_FIRST]; - my_error(ER_GET_ERRMSG, errflag, error, errmsg, engine); - SET_FATAL_ERROR; - } else - { - my_error(ER_GET_ERRNO, errflag,error); - /* SET_FATAL_ERROR; */ - } + my_error(ER_GET_ERRNO, errflag, error, table_type()); DBUG_VOID_RETURN; } } + DBUG_ASSERT(textno > 0); if (fatal_error) { /* Ensure this becomes a true error */ @@ -3168,7 +3279,17 @@ void handler::print_error(int error, myf errflag) errflag|= ME_NOREFRESH; } } - my_error(textno, errflag, table_share->table_name.str, error); + + /* if we got an OS error from a file-based engine, specify a path of error */ + if (error < HA_ERR_FIRST && bas_ext()[0]) + { + char buff[FN_REFLEN]; + strxnmov(buff, sizeof(buff), + table_share->normalized_path.str, bas_ext()[0], NULL); + my_error(textno, errflag, buff, error); + } + else + my_error(textno, errflag, table_share->table_name.str, error); DBUG_VOID_RETURN; } @@ -3370,9 +3491,14 @@ int handler::delete_table(const char *name) { int saved_error= 0; int error= 0; - int enoent_or_zero= ENOENT; // Error if no file was deleted + int enoent_or_zero; char buff[FN_REFLEN]; + if (ht->discover_table) + enoent_or_zero= 0; // the table may not exist in the engine, it's ok + else + enoent_or_zero= ENOENT; // the first file of bas_ext() *must* exist + for (const char **ext=bas_ext(); *ext ; ext++) { fn_format(buff, name, "", *ext, MY_UNPACK_FILENAME|MY_APPEND_EXT); @@ -3761,16 +3887,16 @@ handler::ha_create(const char *name, TABLE *form, HA_CREATE_INFO *info) /** Create handler files for CREATE TABLE: public interface. - @sa handler::create_handler_files() + @sa handler::create_partitioning_metadata() */ int -handler::ha_create_handler_files(const char *name, const char *old_name, - int action_flag, HA_CREATE_INFO *info) +handler::ha_create_partitioning_metadata(const char *name, const char *old_name, + int action_flag) { mark_trx_read_write(); - return create_handler_files(name, old_name, action_flag, info); + return create_partitioning_metadata(name, old_name, action_flag); } @@ -4058,8 +4184,7 @@ end: */ int ha_create_table(THD *thd, const char *path, const char *db, const char *table_name, - HA_CREATE_INFO *create_info, - bool update_create_info) + HA_CREATE_INFO *create_info, LEX_CUSTRING *frm) { int error= 1; TABLE table; @@ -4069,121 +4194,48 @@ int ha_create_table(THD *thd, const char *path, DBUG_ENTER("ha_create_table"); init_tmp_table_share(thd, &share, db, 0, table_name, path); - if (open_table_def(thd, &share, 0) || - open_table_from_share(thd, &share, "", 0, (uint) READ_ALL, 0, &table, - TRUE)) - goto err; - if (update_create_info) - update_create_info_from_table(create_info, &table); - - name= get_canonical_filename(table.file, share.path.str, name_buff); - - error= table.file->ha_create(name, &table, create_info); - (void) closefrm(&table, 0); - if (error) - { - strxmov(name_buff, db, ".", table_name, NullS); - my_error(ER_CANT_CREATE_TABLE, MYF(ME_BELL+ME_WAITTANG), name_buff, error); - } -err: - free_table_share(&share); - DBUG_RETURN(error != 0); -} - -/** - Try to discover table from engine. - - @note - If found, write the frm file to disk. - - @retval - -1 Table did not exists - @retval - 0 Table created ok - @retval - > 0 Error, table existed but could not be created -*/ -int ha_create_table_from_engine(THD* thd, const char *db, const char *name) -{ - int error; - uchar *frmblob; - size_t frmlen; - char path[FN_REFLEN + 1]; - HA_CREATE_INFO create_info; - TABLE table; - TABLE_SHARE share; - DBUG_ENTER("ha_create_table_from_engine"); - DBUG_PRINT("enter", ("name '%s'.'%s'", db, name)); - - bzero((uchar*) &create_info,sizeof(create_info)); - if ((error= ha_discover(thd, db, name, &frmblob, &frmlen))) + if (frm) { - /* Table could not be discovered and thus not created */ - DBUG_RETURN(error); - } + bool write_frm_now= !create_info->db_type->discover_table && + !create_info->tmp_table(); - /* - Table exists in handler and could be discovered - frmblob and frmlen are set, write the frm to disk - */ + share.frm_image= frm; - build_table_filename(path, sizeof(path) - 1, db, name, "", 0); - // Save the frm file - error= writefrm(path, frmblob, frmlen); - my_free(frmblob); - if (error) - DBUG_RETURN(2); - - init_tmp_table_share(thd, &share, db, 0, name, path); - if (open_table_def(thd, &share, 0)) - { - DBUG_RETURN(3); + // open an frm image + if (share.init_from_binary_frm_image(thd, write_frm_now, + frm->str, frm->length)) + goto err; } - if (open_table_from_share(thd, &share, "" ,0, 0, 0, &table, FALSE)) + else { - free_table_share(&share); - DBUG_RETURN(3); - } - - update_create_info_from_table(&create_info, &table); - create_info.table_options|= HA_OPTION_CREATE_FROM_ENGINE; - - get_canonical_filename(table.file, path, path); - error=table.file->ha_create(path, &table, &create_info); - (void) closefrm(&table, 1); - - DBUG_RETURN(error != 0); -} + // open an frm file + share.db_plugin= ha_lock_engine(thd, create_info->db_type); + if (open_table_def(thd, &share)) + goto err; + } + -/** - Try to find a table in a storage engine. + if (open_table_from_share(thd, &share, "", 0, READ_ALL, 0, &table, true)) + goto err; - @param db Normalized table schema name - @param name Normalized table name. - @param[out] exists Only valid if the function succeeded. + update_create_info_from_table(create_info, &table); - @retval TRUE An error is found - @retval FALSE Success, check *exists -*/ + name= get_canonical_filename(table.file, share.path.str, name_buff); -bool -ha_check_if_table_exists(THD* thd, const char *db, const char *name, - bool *exists) -{ - uchar *frmblob= NULL; - size_t frmlen; - DBUG_ENTER("ha_check_if_table_exists"); + error= table.file->ha_create(name, &table, create_info); - *exists= ! ha_discover(thd, db, name, &frmblob, &frmlen); - if (*exists) - my_free(frmblob); + (void) closefrm(&table, 0); - DBUG_RETURN(FALSE); + if (error) + my_error(ER_CANT_CREATE_TABLE, MYF(0), db, table_name, error); + +err: + free_table_share(&share); + DBUG_RETURN(error != 0); } - void st_ha_check_opt::init() { flags= sql_flags= 0; @@ -4306,149 +4358,379 @@ int ha_change_key_cache(KEY_CACHE *old_key_cache, } -/** - Try to discover one table from handler(s). - - @retval - -1 Table did not exists - @retval - 0 OK. In this case *frmblob and *frmlen are set - @retval - >0 error. frmblob and frmlen may not be set -*/ -struct st_discover_args -{ - const char *db; - const char *name; - uchar **frmblob; - size_t *frmlen; -}; - static my_bool discover_handlerton(THD *thd, plugin_ref plugin, void *arg) { - st_discover_args *vargs= (st_discover_args *)arg; - handlerton *hton= plugin_data(plugin, handlerton *); - if (hton->state == SHOW_OPTION_YES && hton->discover && - (!(hton->discover(hton, thd, vargs->db, vargs->name, - vargs->frmblob, - vargs->frmlen)))) - return TRUE; + TABLE_SHARE *share= (TABLE_SHARE *)arg; + handlerton *hton= plugin_hton(plugin); + if (hton->state == SHOW_OPTION_YES && hton->discover_table) + { + share->db_plugin= plugin; + int error= hton->discover_table(hton, thd, share); + if (error != HA_ERR_NO_SUCH_TABLE) + { + if (error) + { + DBUG_ASSERT(share->error); // get_cached_table_share needs that + /* + report an error, unless it is "generic" and a more + specific one was already reported + */ + if (error != HA_ERR_GENERIC || !thd->is_error()) + my_error(ER_GET_ERRNO, MYF(0), error, plugin_name(plugin)->str); + share->db_plugin= 0; + } + else + share->error= OPEN_FRM_OK; - return FALSE; + status_var_increment(thd->status_var.ha_discover_count); + return TRUE; // abort the search + } + share->db_plugin= 0; + } + + DBUG_ASSERT(share->error == OPEN_FRM_OPEN_ERROR); + return FALSE; // continue with the next engine } -int ha_discover(THD *thd, const char *db, const char *name, - uchar **frmblob, size_t *frmlen) +int ha_discover_table(THD *thd, TABLE_SHARE *share) { - int error= -1; // Table does not exist in any handler - DBUG_ENTER("ha_discover"); - DBUG_PRINT("enter", ("db: %s, name: %s", db, name)); - st_discover_args args= {db, name, frmblob, frmlen}; + DBUG_ENTER("ha_discover_table"); + int found; - if (is_prefix(name,tmp_file_prefix)) /* skip temporary tables */ - DBUG_RETURN(error); + DBUG_ASSERT(share->error == OPEN_FRM_OPEN_ERROR); // share is not OK yet - if (plugin_foreach(thd, discover_handlerton, - MYSQL_STORAGE_ENGINE_PLUGIN, &args)) - error= 0; + if (share->db_plugin) + found= discover_handlerton(thd, share->db_plugin, share); + else + found= plugin_foreach(thd, discover_handlerton, + MYSQL_STORAGE_ENGINE_PLUGIN, share); + + if (!found) + open_table_error(share, OPEN_FRM_OPEN_ERROR, ENOENT); // not found - if (!error) - status_var_increment(thd->status_var.ha_discover_count); - DBUG_RETURN(error); + DBUG_RETURN(share->error != OPEN_FRM_OK); } +static my_bool file_ext_exists(char *path, size_t path_len, const char *ext) +{ + strmake(path + path_len, ext, FN_REFLEN - path_len); + return !access(path, F_OK); +} -/** - Call this function in order to give the handler the possiblity - to ask engine if there are any new tables that should be written to disk - or any dropped tables that need to be removed from disk -*/ -struct st_find_files_args +struct st_discover_existence_args { - const char *db; - const char *path; - const char *wild; - bool dir; - List<LEX_STRING> *files; + char *path; + size_t path_len; + const char *db, *table_name; + handlerton *hton; }; -static my_bool find_files_handlerton(THD *thd, plugin_ref plugin, - void *arg) +static my_bool discover_existence(THD *thd, plugin_ref plugin, + void *arg) { - st_find_files_args *vargs= (st_find_files_args *)arg; - handlerton *hton= plugin_data(plugin, handlerton *); + st_discover_existence_args *args= (st_discover_existence_args*)arg; + handlerton *ht= plugin_hton(plugin); + if (ht->state != SHOW_OPTION_YES || !ht->discover_table_existence) + return FALSE; + args->hton= ht; - if (hton->state == SHOW_OPTION_YES && hton->find_files) - if (hton->find_files(hton, thd, vargs->db, vargs->path, vargs->wild, - vargs->dir, vargs->files)) - return TRUE; + if (ht->discover_table_existence == ext_based_existence) + return file_ext_exists(args->path, args->path_len, + ht->tablefile_extensions[0]); - return FALSE; + return ht->discover_table_existence(ht, args->db, args->table_name); } -int -ha_find_files(THD *thd,const char *db,const char *path, - const char *wild, bool dir, List<LEX_STRING> *files) +class Table_exists_error_handler : public Internal_error_handler { - int error= 0; - DBUG_ENTER("ha_find_files"); - DBUG_PRINT("enter", ("db: '%s' path: '%s' wild: '%s' dir: %d", - db, path, wild, dir)); - st_find_files_args args= {db, path, wild, dir, files}; - - plugin_foreach(thd, find_files_handlerton, - MYSQL_STORAGE_ENGINE_PLUGIN, &args); - /* The return value is not currently used */ - DBUG_RETURN(error); +public: + Table_exists_error_handler() + : m_handled_errors(0), m_unhandled_errors(0) + {} + + bool handle_condition(THD *thd, + uint sql_errno, + const char* sqlstate, + MYSQL_ERROR::enum_warning_level level, + const char* msg, + MYSQL_ERROR ** cond_hdl) + { + *cond_hdl= NULL; + if (sql_errno == ER_NO_SUCH_TABLE || + sql_errno == ER_NO_SUCH_TABLE_IN_ENGINE || + sql_errno == ER_WRONG_OBJECT) + { + m_handled_errors++; + return TRUE; + } + + if (level == MYSQL_ERROR::WARN_LEVEL_ERROR) + m_unhandled_errors++; + return FALSE; + } + + bool safely_trapped_errors() + { + return ((m_handled_errors > 0) && (m_unhandled_errors == 0)); + } + +private: + int m_handled_errors; + int m_unhandled_errors; +}; + +/** + Check if a given table exists, without doing a full discover, if possible + + If the 'hton' is not NULL, it's set to the handlerton of the storage engine + of this table, or to view_pseudo_hton if the frm belongs to a view. + + + @retval true Table exists (even if the error occurred, like bad frm) + @retval false Table does not exist (one can do CREATE TABLE table_name) +*/ +bool ha_table_exists(THD *thd, const char *db, const char *table_name, + handlerton **hton) +{ + DBUG_ENTER("ha_table_exists"); + + if (hton) + *hton= 0; + + if (need_full_discover_for_existence) + { + TABLE_LIST table; + uint flags = GTS_TABLE | GTS_VIEW; + + if (!hton) + flags|= GTS_NOLOCK; + + Table_exists_error_handler no_such_table_handler; + thd->push_internal_handler(&no_such_table_handler); + TABLE_SHARE *share= get_table_share(thd, db, table_name, flags); + thd->pop_internal_handler(); + + if (hton && share) + { + *hton= share->db_type(); + mysql_mutex_lock(&LOCK_open); + release_table_share(share); + mysql_mutex_unlock(&LOCK_open); + } + + // the table doesn't exist if we've caught ER_NO_SUCH_TABLE and nothing else + DBUG_RETURN(!no_such_table_handler.safely_trapped_errors()); + } + + mysql_mutex_lock(&LOCK_open); + TABLE_SHARE *share= get_cached_table_share(db, table_name); + if (hton && share) + *hton= share->db_type(); + mysql_mutex_unlock(&LOCK_open); + + if (share) + DBUG_RETURN(TRUE); + + char path[FN_REFLEN + 1]; + size_t path_len = build_table_filename(path, sizeof(path) - 1, + db, table_name, "", 0); + + if (file_ext_exists(path, path_len, reg_ext)) + { + if (hton) + { + enum legacy_db_type db_type; + if (dd_frm_type(thd, path, &db_type) != FRMTYPE_VIEW) + *hton= ha_resolve_by_legacy_type(thd, db_type); + else + *hton= view_pseudo_hton; + } + DBUG_RETURN(TRUE); + } + + st_discover_existence_args args= {path, path_len, db, table_name, 0}; + + if (plugin_foreach(thd, discover_existence, MYSQL_STORAGE_ENGINE_PLUGIN, + &args)) + { + if (hton) + *hton= args.hton; + DBUG_RETURN(TRUE); + } + + DBUG_RETURN(FALSE); } /** - Ask handler if the table exists in engine. - @retval - HA_ERR_NO_SUCH_TABLE Table does not exist - @retval - HA_ERR_TABLE_EXIST Table exists - @retval - \# Error code + Discover all table names in a given database */ -struct st_table_exists_in_engine_args +extern "C" { + +static int cmp_file_names(const void *a, const void *b) { - const char *db; - const char *name; - int err; -}; + CHARSET_INFO *cs= character_set_filesystem; + char *aa= ((FILEINFO *)a)->name; + char *bb= ((FILEINFO *)b)->name; + return my_strnncoll(cs, (uchar*)aa, strlen(aa), (uchar*)bb, strlen(bb)); +} -static my_bool table_exists_in_engine_handlerton(THD *thd, plugin_ref plugin, - void *arg) +static int cmp_table_names(LEX_STRING * const *a, LEX_STRING * const *b) { - st_table_exists_in_engine_args *vargs= (st_table_exists_in_engine_args *)arg; - handlerton *hton= plugin_data(plugin, handlerton *); + return my_strnncoll(&my_charset_bin, (uchar*)((*a)->str), (*a)->length, + (uchar*)((*b)->str), (*b)->length); +} - int err= HA_ERR_NO_SUCH_TABLE; +} - if (hton->state == SHOW_OPTION_YES && hton->table_exists_in_engine) - err = hton->table_exists_in_engine(hton, thd, vargs->db, vargs->name); +Discovered_table_list::Discovered_table_list(THD *thd_arg, + Dynamic_array<LEX_STRING*> *tables_arg, + const LEX_STRING *wild_arg) : + thd(thd_arg), with_temps(false), tables(tables_arg) +{ + if (wild_arg->str && wild_arg->str[0]) + { + wild= wild_arg->str; + wend= wild + wild_arg->length; + } + else + wild= 0; +} - vargs->err = err; - if (vargs->err == HA_ERR_TABLE_EXIST) - return TRUE; +bool Discovered_table_list::add_table(const char *tname, size_t tlen) +{ + /* + TODO Check with_temps and filter out temp tables. + Implement the check, when we'll have at least one affected engine (with + custom discover_table_names() method, that calls add_table() directly). + Note: avoid comparing the same name twice (here and in add_file). + */ + if (wild && my_wildcmp(files_charset_info, tname, tname + tlen, wild, wend, + wild_prefix, wild_one, wild_many)) + return 0; - return FALSE; + LEX_STRING *name= thd->make_lex_string(tname, tlen); + if (!name || tables->append(name)) + return 1; + return 0; +} + +bool Discovered_table_list::add_file(const char *fname) +{ + bool is_temp= strncmp(fname, STRING_WITH_LEN(tmp_file_prefix)) == 0; + + if (is_temp && !with_temps) + return 0; + + char tname[SAFE_NAME_LEN + 1]; + size_t tlen= filename_to_tablename(fname, tname, sizeof(tname), is_temp); + return add_table(tname, tlen); } -int ha_table_exists_in_engine(THD* thd, const char* db, const char* name) + +void Discovered_table_list::sort() { - DBUG_ENTER("ha_table_exists_in_engine"); - DBUG_PRINT("enter", ("db: %s, name: %s", db, name)); - st_table_exists_in_engine_args args= {db, name, HA_ERR_NO_SUCH_TABLE}; - plugin_foreach(thd, table_exists_in_engine_handlerton, - MYSQL_STORAGE_ENGINE_PLUGIN, &args); - DBUG_PRINT("exit", ("error: %d", args.err)); - DBUG_RETURN(args.err); + tables->sort(cmp_table_names); } +void Discovered_table_list::remove_duplicates() +{ + LEX_STRING **src= tables->front(); + LEX_STRING **dst= src; + while (++dst < tables->back()) + { + LEX_STRING *s= *src, *d= *dst; + DBUG_ASSERT(strncmp(s->str, d->str, min(s->length, d->length)) <= 0); + if ((s->length != d->length || strncmp(s->str, d->str, d->length))) + { + src++; + if (src != dst) + *src= *dst; + } + } + tables->set_elements(src - tables->front() + 1); +} + +struct st_discover_names_args +{ + LEX_STRING *db; + MY_DIR *dirp; + Discovered_table_list *result; + uint possible_duplicates; +}; + +static my_bool discover_names(THD *thd, plugin_ref plugin, + void *arg) +{ + st_discover_names_args *args= (st_discover_names_args *)arg; + handlerton *ht= plugin_hton(plugin); + + if (ht->state == SHOW_OPTION_YES && ht->discover_table_names) + { + uint old_elements= args->result->tables->elements(); + if (ht->discover_table_names(ht, args->db, args->dirp, args->result)) + return 1; + + /* + hton_ext_based_table_discovery never discovers a table that has + a corresponding .frm file; but custom engine discover methods might + */ + if (ht->discover_table_names != hton_ext_based_table_discovery) + args->possible_duplicates+= args->result->tables->elements() - old_elements; + } + + return 0; +} + +/** + Return the list of tables + + @param thd + @param db database to look into + @param dirp list of files in this database (as returned by my_dir()) + @param result the object to return the list of files in + @param reusable if true, on return, 'dirp' will be a valid list of all + non-table files. If false, discovery will work much faster, + but it will leave 'dirp' corrupted and completely unusable, + only good for my_dirend(). + + Normally, reusable=false for SHOW and INFORMATION_SCHEMA, and reusable=true + for DROP DATABASE (as it needs to know and delete non-table files). +*/ + +int ha_discover_table_names(THD *thd, LEX_STRING *db, MY_DIR *dirp, + Discovered_table_list *result, bool reusable) +{ + int error; + DBUG_ENTER("ha_discover_table_names"); + + if (engines_with_discover_table_names == 0 && !reusable) + { + error= ext_table_discovery_simple(dirp, result); + result->sort(); + } + else + { + st_discover_names_args args= {db, dirp, result, 0}; + + /* extension_based_table_discovery relies on dirp being sorted */ + my_qsort(dirp->dir_entry, dirp->number_of_files, + sizeof(FILEINFO), cmp_file_names); + + error= extension_based_table_discovery(dirp, reg_ext, result) || + plugin_foreach(thd, discover_names, + MYSQL_STORAGE_ENGINE_PLUGIN, &args); + result->sort(); + + if (args.possible_duplicates > 0) + result->remove_duplicates(); + } + + DBUG_RETURN(error); +} + + #ifdef HAVE_NDB_BINLOG /* TODO: change this into a dynamic struct @@ -4475,7 +4757,7 @@ struct binlog_func_st static my_bool binlog_func_list(THD *thd, plugin_ref plugin, void *arg) { hton_list_st *hton_list= (hton_list_st *)arg; - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->state == SHOW_OPTION_YES && hton->binlog_func) { uint sz= hton_list->sz; @@ -4565,7 +4847,7 @@ static my_bool binlog_log_query_handlerton(THD *thd, plugin_ref plugin, void *args) { - return binlog_log_query_handlerton2(thd, plugin_data(plugin, handlerton *), args); + return binlog_log_query_handlerton2(thd, plugin_hton(plugin), args); } void ha_binlog_log_query(THD *thd, handlerton *hton, @@ -4796,27 +5078,21 @@ static my_bool exts_handlerton(THD *unused, plugin_ref plugin, void *arg) { List<char> *found_exts= (List<char> *) arg; - handlerton *hton= plugin_data(plugin, handlerton *); - handler *file; - if (hton->state == SHOW_OPTION_YES && hton->create && - (file= hton->create(hton, (TABLE_SHARE*) 0, current_thd->mem_root))) - { - List_iterator_fast<char> it(*found_exts); - const char **ext, *old_ext; + handlerton *hton= plugin_hton(plugin); + List_iterator_fast<char> it(*found_exts); + const char **ext, *old_ext; - for (ext= file->bas_ext(); *ext; ext++) + for (ext= hton->tablefile_extensions; *ext; ext++) + { + while ((old_ext= it++)) { - while ((old_ext= it++)) - { - if (!strcmp(old_ext, *ext)) - break; - } - if (!old_ext) - found_exts->push_back((char *) *ext); - - it.rewind(); + if (!strcmp(old_ext, *ext)) + break; } - delete file; + if (!old_ext) + found_exts->push_back((char *) *ext); + + it.rewind(); } return FALSE; } @@ -4871,7 +5147,7 @@ static my_bool showstat_handlerton(THD *thd, plugin_ref plugin, void *arg) { enum ha_stat_type stat= *(enum ha_stat_type *) arg; - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->state == SHOW_OPTION_YES && hton->show_status && hton->show_status(hton, thd, stat_print, stat)) return TRUE; @@ -4919,7 +5195,7 @@ bool ha_show_status(THD *thd, handlerton *db_type, enum ha_stat_type stat) if (!result && !thd->is_error()) my_eof(thd); else if (!thd->is_error()) - my_error(ER_GET_ERRNO, MYF(0), errno); + my_error(ER_GET_ERRNO, MYF(0), errno, hton_name(db_type)->str); return result; } @@ -5397,7 +5673,7 @@ fl_log_iterator_buffer_init(struct handler_iterator *iterator) /* to be able to make my_free without crash in case of error */ iterator->buffer= 0; - if (!(dirp = my_dir(fl_dir, MYF(0)))) + if (!(dirp = my_dir(fl_dir, MYF(MY_THREAD_SPECIFIC)))) { return HA_ITERATOR_ERROR; } @@ -5406,7 +5682,7 @@ fl_log_iterator_buffer_init(struct handler_iterator *iterator) sizeof(enum log_status) + + FN_REFLEN + 1) * (uint) dirp->number_off_files), - MYF(0))) == 0) + MYF(MY_THREAD_SPECIFIC))) == 0) { return HA_ITERATOR_ERROR; } @@ -5440,6 +5716,7 @@ fl_log_iterator_buffer_init(struct handler_iterator *iterator) iterator->buffer= buff; iterator->next= &fl_log_iterator_next; iterator->destroy= &fl_log_iterator_destroy; + my_dirend(dirp); return HA_ITERATOR_OK; } diff --git a/sql/handler.h b/sql/handler.h index a23e3c2d17e..a3537add781 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -2,7 +2,7 @@ #define HANDLER_INCLUDED /* Copyright (c) 2000, 2011, Oracle and/or its affiliates. - Copyright (c) 2009-2011 Monty Program Ab + Copyright (c) 2009, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -31,6 +31,7 @@ #include "thr_lock.h" /* thr_lock_type, THR_LOCK_DATA */ #include "sql_cache.h" #include "structs.h" /* SHOW_COMP_OPTION */ +#include "sql_array.h" /* Dynamic_array<> */ #include <my_compare.h> #include <ft_global.h> @@ -59,9 +60,9 @@ /* Bits in table_flags() to show what database can do */ -#define HA_NO_TRANSACTIONS (1 << 0) /* Doesn't support transactions */ -#define HA_PARTIAL_COLUMN_READ (1 << 1) /* read may not return all columns */ -#define HA_TABLE_SCAN_ON_INDEX (1 << 2) /* No separate data/index file */ +#define HA_NO_TRANSACTIONS (1ULL << 0) /* Doesn't support transactions */ +#define HA_PARTIAL_COLUMN_READ (1ULL << 1) /* read may not return all columns */ +#define HA_TABLE_SCAN_ON_INDEX (1ULL << 2) /* No separate data/index file */ /* The following should be set if the following is not true when scanning a table with rnd_next() @@ -70,37 +71,37 @@ If this flag is not set, filesort will do a position() call for each matched row to be able to find the row later. */ -#define HA_REC_NOT_IN_SEQ (1 << 3) -#define HA_CAN_GEOMETRY (1 << 4) +#define HA_REC_NOT_IN_SEQ (1ULL << 3) +#define HA_CAN_GEOMETRY (1ULL << 4) /* Reading keys in random order is as fast as reading keys in sort order (Used in records.cc to decide if we should use a record cache and by filesort to decide if we should sort key + data or key + pointer-to-row */ -#define HA_FAST_KEY_READ (1 << 5) +#define HA_FAST_KEY_READ (1ULL << 5) /* Set the following flag if we on delete should force all key to be read and on update read all keys that changes */ -#define HA_REQUIRES_KEY_COLUMNS_FOR_DELETE (1 << 6) -#define HA_NULL_IN_KEY (1 << 7) /* One can have keys with NULL */ -#define HA_DUPLICATE_POS (1 << 8) /* ha_position() gives dup row */ -#define HA_NO_BLOBS (1 << 9) /* Doesn't support blobs */ -#define HA_CAN_INDEX_BLOBS (1 << 10) -#define HA_AUTO_PART_KEY (1 << 11) /* auto-increment in multi-part key */ -#define HA_REQUIRE_PRIMARY_KEY (1 << 12) /* .. and can't create a hidden one */ -#define HA_STATS_RECORDS_IS_EXACT (1 << 13) /* stats.records is exact */ +#define HA_REQUIRES_KEY_COLUMNS_FOR_DELETE (1ULL << 6) +#define HA_NULL_IN_KEY (1ULL << 7) /* One can have keys with NULL */ +#define HA_DUPLICATE_POS (1ULL << 8) /* ha_position() gives dup row */ +#define HA_NO_BLOBS (1ULL << 9) /* Doesn't support blobs */ +#define HA_CAN_INDEX_BLOBS (1ULL << 10) +#define HA_AUTO_PART_KEY (1ULL << 11) /* auto-increment in multi-part key */ +#define HA_REQUIRE_PRIMARY_KEY (1ULL << 12) /* .. and can't create a hidden one */ +#define HA_STATS_RECORDS_IS_EXACT (1ULL << 13) /* stats.records is exact */ /* INSERT_DELAYED only works with handlers that uses MySQL internal table level locks */ -#define HA_CAN_INSERT_DELAYED (1 << 14) +#define HA_CAN_INSERT_DELAYED (1ULL << 14) /* If we get the primary key columns for free when we do an index read It also implies that we have to retrive the primary key when using position() and rnd_pos(). */ -#define HA_PRIMARY_KEY_IN_READ_INDEX (1 << 15) +#define HA_PRIMARY_KEY_IN_READ_INDEX (1ULL << 15) /* If HA_PRIMARY_KEY_REQUIRED_FOR_POSITION is set, it means that to position() uses a primary key given by the record argument. @@ -108,36 +109,36 @@ If not set, the position is returned as the current rows position regardless of what argument is given. */ -#define HA_PRIMARY_KEY_REQUIRED_FOR_POSITION (1 << 16) -#define HA_CAN_RTREEKEYS (1 << 17) -#define HA_NOT_DELETE_WITH_CACHE (1 << 18) +#define HA_PRIMARY_KEY_REQUIRED_FOR_POSITION (1ULL << 16) +#define HA_CAN_RTREEKEYS (1ULL << 17) +#define HA_NOT_DELETE_WITH_CACHE (1ULL << 18) /* The following is we need to a primary key to delete (and update) a row. If there is no primary key, all columns needs to be read on update and delete */ -#define HA_PRIMARY_KEY_REQUIRED_FOR_DELETE (1 << 19) -#define HA_NO_PREFIX_CHAR_KEYS (1 << 20) -#define HA_CAN_FULLTEXT (1 << 21) -#define HA_CAN_SQL_HANDLER (1 << 22) -#define HA_NO_AUTO_INCREMENT (1 << 23) +#define HA_PRIMARY_KEY_REQUIRED_FOR_DELETE (1ULL << 19) +#define HA_NO_PREFIX_CHAR_KEYS (1ULL << 20) +#define HA_CAN_FULLTEXT (1ULL << 21) +#define HA_CAN_SQL_HANDLER (1ULL << 22) +#define HA_NO_AUTO_INCREMENT (1ULL << 23) /* Has automatic checksums and uses the old checksum format */ -#define HA_HAS_OLD_CHECKSUM (1 << 24) +#define HA_HAS_OLD_CHECKSUM (1ULL << 24) /* Table data are stored in separate files (for lower_case_table_names) */ -#define HA_FILE_BASED (1 << 26) -#define HA_NO_VARCHAR (1 << 27) -#define HA_CAN_BIT_FIELD (1 << 28) /* supports bit fields */ -#define HA_NEED_READ_RANGE_BUFFER (1 << 29) /* for read_multi_range */ -#define HA_ANY_INDEX_MAY_BE_UNIQUE (1 << 30) -#define HA_NO_COPY_ON_ALTER (LL(1) << 31) -#define HA_HAS_RECORDS (LL(1) << 32) /* records() gives exact count*/ +#define HA_FILE_BASED (1ULL << 26) +#define HA_NO_VARCHAR (1ULL << 27) +#define HA_CAN_BIT_FIELD (1ULL << 28) /* supports bit fields */ +#define HA_NEED_READ_RANGE_BUFFER (1ULL << 29) /* for read_multi_range */ +#define HA_ANY_INDEX_MAY_BE_UNIQUE (1ULL << 30) +#define HA_NO_COPY_ON_ALTER (1ULL << 31) +#define HA_HAS_RECORDS (1ULL << 32) /* records() gives exact count*/ /* Has it's own method of binlog logging */ -#define HA_HAS_OWN_BINLOGGING (LL(1) << 33) +#define HA_HAS_OWN_BINLOGGING (1ULL << 33) /* Engine is capable of row-format and statement-format logging, respectively */ -#define HA_BINLOG_ROW_CAPABLE (LL(1) << 34) -#define HA_BINLOG_STMT_CAPABLE (LL(1) << 35) +#define HA_BINLOG_ROW_CAPABLE (1ULL << 34) +#define HA_BINLOG_STMT_CAPABLE (1ULL << 35) /* When a multiple key conflict happens in a REPLACE command mysql expects the conflicts to be reported in the ascending order of @@ -160,20 +161,20 @@ This flag helps the underlying SE to inform the server that the keys are not ordered. */ -#define HA_DUPLICATE_KEY_NOT_IN_ORDER (LL(1) << 36) +#define HA_DUPLICATE_KEY_NOT_IN_ORDER (1ULL << 36) /* Engine supports REPAIR TABLE. Used by CHECK TABLE FOR UPGRADE if an incompatible table is detected. If this flag is set, CHECK TABLE FOR UPGRADE will report ER_TABLE_NEEDS_UPGRADE, otherwise ER_TABLE_NEED_REBUILD. */ -#define HA_CAN_REPAIR (LL(1) << 37) +#define HA_CAN_REPAIR (1ULL << 37) /* Has automatic checksums and uses the new checksum format */ -#define HA_HAS_NEW_CHECKSUM (LL(1) << 38) -#define HA_CAN_VIRTUAL_COLUMNS (LL(1) << 39) -#define HA_MRR_CANT_SORT (LL(1) << 40) -#define HA_RECORD_MUST_BE_CLEAN_ON_WRITE (LL(1) << 41) +#define HA_HAS_NEW_CHECKSUM (1ULL << 38) +#define HA_CAN_VIRTUAL_COLUMNS (1ULL << 39) +#define HA_MRR_CANT_SORT (1ULL << 40) +#define HA_RECORD_MUST_BE_CLEAN_ON_WRITE (1ULL << 41) /* Table condition pushdown must be performed regardless of @@ -186,7 +187,7 @@ then the "query=..." condition must be always pushed down into storage engine. */ -#define HA_MUST_USE_TABLE_CONDITION_PUSHDOWN (LL(1) << 42) +#define HA_MUST_USE_TABLE_CONDITION_PUSHDOWN (1ULL << 42) /* Set of all binlog flags. Currently only contain the capabilities @@ -337,25 +338,22 @@ enum legacy_db_type { - DB_TYPE_UNKNOWN=0,DB_TYPE_DIAB_ISAM=1, - DB_TYPE_HASH,DB_TYPE_MISAM,DB_TYPE_PISAM, - DB_TYPE_RMS_ISAM, DB_TYPE_HEAP, DB_TYPE_ISAM, - DB_TYPE_MRG_ISAM, DB_TYPE_MYISAM, DB_TYPE_MRG_MYISAM, - DB_TYPE_BERKELEY_DB, DB_TYPE_INNODB, - DB_TYPE_GEMINI, DB_TYPE_NDBCLUSTER, - DB_TYPE_EXAMPLE_DB, DB_TYPE_ARCHIVE_DB, DB_TYPE_CSV_DB, - DB_TYPE_FEDERATED_DB, - DB_TYPE_BLACKHOLE_DB, - DB_TYPE_PARTITION_DB, - DB_TYPE_BINLOG, - DB_TYPE_SOLID, - DB_TYPE_PBXT, - DB_TYPE_TABLE_FUNCTION, - DB_TYPE_MEMCACHE, - DB_TYPE_FALCON, - DB_TYPE_MARIA, - /** Performance schema engine. */ - DB_TYPE_PERFORMANCE_SCHEMA, + /* note these numerical values are fixed and can *not* be changed */ + DB_TYPE_UNKNOWN=0, + DB_TYPE_HEAP=6, + DB_TYPE_MYISAM=9, + DB_TYPE_MRG_MYISAM=10, + DB_TYPE_INNODB=12, + DB_TYPE_NDBCLUSTER=14, + DB_TYPE_EXAMPLE_DB=15, + DB_TYPE_ARCHIVE_DB=16, + DB_TYPE_CSV_DB=17, + DB_TYPE_FEDERATED_DB=18, + DB_TYPE_BLACKHOLE_DB=19, + DB_TYPE_PARTITION_DB=20, + DB_TYPE_BINLOG=21, + DB_TYPE_PBXT=23, + DB_TYPE_PERFORMANCE_SCHEMA=28, DB_TYPE_ARIA=42, DB_TYPE_TOKUDB=43, DB_TYPE_FIRST_DYNAMIC=44, @@ -595,14 +593,18 @@ struct TABLE; */ enum enum_schema_tables { - SCH_CHARSETS= 0, + SCH_ALL_PLUGINS, + SCH_APPLICABLE_ROLES, + SCH_CHARSETS, SCH_CLIENT_STATS, SCH_COLLATIONS, SCH_COLLATION_CHARACTER_SET_APPLICABILITY, SCH_COLUMNS, SCH_COLUMN_PRIVILEGES, + SCH_ENABLED_ROLES, SCH_ENGINES, SCH_EVENTS, + SCH_EXPLAIN, SCH_FILES, SCH_GLOBAL_STATUS, SCH_GLOBAL_VARIABLES, @@ -637,6 +639,7 @@ enum enum_schema_tables }; struct TABLE_SHARE; +struct HA_CREATE_INFO; struct st_foreign_key_info; typedef struct st_foreign_key_info FOREIGN_KEY_INFO; typedef bool (stat_print_fn)(THD *thd, const char *type, uint type_len, @@ -716,22 +719,26 @@ struct ha_index_option_struct; enum ha_option_type { HA_OPTION_TYPE_ULL, /* unsigned long long */ HA_OPTION_TYPE_STRING, /* char * */ HA_OPTION_TYPE_ENUM, /* uint */ - HA_OPTION_TYPE_BOOL}; /* bool */ + HA_OPTION_TYPE_BOOL, /* bool */ + HA_OPTION_TYPE_SYSVAR};/* type of the sysval */ #define HA_xOPTION_NUMBER(name, struc, field, def, min, max, blk_siz) \ { HA_OPTION_TYPE_ULL, name, sizeof(name)-1, \ - offsetof(struc, field), def, min, max, blk_siz, 0 } + offsetof(struc, field), def, min, max, blk_siz, 0, 0 } #define HA_xOPTION_STRING(name, struc, field) \ { HA_OPTION_TYPE_STRING, name, sizeof(name)-1, \ - offsetof(struc, field), 0, 0, 0, 0, 0 } + offsetof(struc, field), 0, 0, 0, 0, 0, 0} #define HA_xOPTION_ENUM(name, struc, field, values, def) \ { HA_OPTION_TYPE_ENUM, name, sizeof(name)-1, \ offsetof(struc, field), def, 0, \ - sizeof(values)-1, 0, values } + sizeof(values)-1, 0, values, 0 } #define HA_xOPTION_BOOL(name, struc, field, def) \ { HA_OPTION_TYPE_BOOL, name, sizeof(name)-1, \ - offsetof(struc, field), def, 0, 1, 0, 0 } -#define HA_xOPTION_END { HA_OPTION_TYPE_ULL, 0, 0, 0, 0, 0, 0, 0, 0 } + offsetof(struc, field), def, 0, 1, 0, 0, 0 } +#define HA_xOPTION_SYSVAR(name, struc, field, sysvar) \ + { HA_OPTION_TYPE_SYSVAR, name, sizeof(name)-1, \ + offsetof(struc, field), 0, 0, 0, 0, 0, MYSQL_SYSVAR(sysvar) } +#define HA_xOPTION_END { HA_OPTION_TYPE_ULL, 0, 0, 0, 0, 0, 0, 0, 0, 0 } #define HA_TOPTION_NUMBER(name, field, def, min, max, blk_siz) \ HA_xOPTION_NUMBER(name, ha_table_option_struct, field, def, min, max, blk_siz) @@ -741,6 +748,8 @@ enum ha_option_type { HA_OPTION_TYPE_ULL, /* unsigned long long */ HA_xOPTION_ENUM(name, ha_table_option_struct, field, values, def) #define HA_TOPTION_BOOL(name, field, def) \ HA_xOPTION_BOOL(name, ha_table_option_struct, field, def) +#define HA_TOPTION_SYSVAR(name, field, sysvar) \ + HA_xOPTION_SYSVAR(name, ha_table_option_struct, field, sysvar) #define HA_TOPTION_END HA_xOPTION_END #define HA_FOPTION_NUMBER(name, field, def, min, max, blk_siz) \ @@ -751,6 +760,8 @@ enum ha_option_type { HA_OPTION_TYPE_ULL, /* unsigned long long */ HA_xOPTION_ENUM(name, ha_field_option_struct, field, values, def) #define HA_FOPTION_BOOL(name, field, def) \ HA_xOPTION_BOOL(name, ha_field_option_struct, field, def) +#define HA_FOPTION_SYSVAR(name, field, sysvar) \ + HA_xOPTION_SYSVAR(name, ha_field_option_struct, field, sysvar) #define HA_FOPTION_END HA_xOPTION_END #define HA_IOPTION_NUMBER(name, field, def, min, max, blk_siz) \ @@ -761,6 +772,8 @@ enum ha_option_type { HA_OPTION_TYPE_ULL, /* unsigned long long */ HA_xOPTION_ENUM(name, ha_index_option_struct, field, values, def) #define HA_IOPTION_BOOL(name, field, def) \ HA_xOPTION_BOOL(name, ha_index_option_struct, field, def) +#define HA_IOPTION_SYSVAR(name, field, sysvar) \ + HA_xOPTION_SYSVAR(name, ha_index_option_struct, field, sysvar) #define HA_IOPTION_END HA_xOPTION_END typedef struct st_ha_create_table_option { @@ -771,6 +784,7 @@ typedef struct st_ha_create_table_option { ulonglong def_value; ulonglong min_value, max_value, block_size; const char *values; + struct st_mysql_sys_var *var; } ha_create_table_option; enum handler_iterator_type @@ -983,6 +997,46 @@ struct handlerton int (*recover)(handlerton *hton, XID *xid_list, uint len); int (*commit_by_xid)(handlerton *hton, XID *xid); int (*rollback_by_xid)(handlerton *hton, XID *xid); + /* + The commit_checkpoint_request() handlerton method is used to checkpoint + the XA recovery process for storage engines that support two-phase + commit. + + The method is optional - an engine that does not implemented is expected + to work the traditional way, where every commit() durably flushes the + transaction to disk in the engine before completion, so XA recovery will + no longer be needed for that transaction. + + An engine that does implement commit_checkpoint_request() is also + expected to implement commit_ordered(), so that ordering of commits is + consistent between 2pc participants. Such engine is no longer required to + durably flush to disk transactions in commit(), provided that the + transaction has been successfully prepare()d and commit_ordered(); thus + potentionally saving one fsync() call. (Engine must still durably flush + to disk in commit() when no prepare()/commit_ordered() steps took place, + at least if durable commits are wanted; this happens eg. if binlog is + disabled). + + The TC will periodically (eg. once per binlog rotation) call + commit_checkpoint_request(). When this happens, the engine must arrange + for all transaction that have completed commit_ordered() to be durably + flushed to disk (this does not include transactions that might be in the + middle of executing commit_ordered()). When such flush has completed, the + engine must call commit_checkpoint_notify_ha(), passing back the opaque + "cookie". + + The flush and call of commit_checkpoint_notify_ha() need not happen + immediately - it can be scheduled and performed asynchroneously (ie. as + part of next prepare(), or sync every second, or whatever), but should + not be postponed indefinitely. It is however also permissible to do it + immediately, before returning from commit_checkpoint_request(). + + When commit_checkpoint_notify_ha() is called, the TC will know that the + transactions are durably committed, and thus no longer require XA + recovery. It uses that to reduce the work needed for any subsequent XA + recovery process. + */ + void (*commit_checkpoint_request)(handlerton *hton, void *cookie); /* "Disable or enable checkpointing internal to the storage engine. This is used for FLUSH TABLES WITH READ LOCK AND DISABLE CHECKPOINT to ensure that @@ -1039,18 +1093,6 @@ struct handlerton enum handler_create_iterator_result (*create_iterator)(handlerton *hton, enum handler_iterator_type type, struct handler_iterator *fill_this_in); - int (*discover)(handlerton *hton, THD* thd, const char *db, - const char *name, - uchar **frmblob, - size_t *frmlen); - int (*find_files)(handlerton *hton, THD *thd, - const char *db, - const char *path, - const char *wild, bool dir, List<LEX_STRING> *files); - int (*table_exists_in_engine)(handlerton *hton, THD* thd, const char *db, - const char *name); - - uint32 license; /* Flag for Engine License */ /* Optional clauses in the CREATE/ALTER TABLE */ @@ -1058,14 +1100,129 @@ struct handlerton ha_create_table_option *field_options; // these are specified per field ha_create_table_option *index_options; // these are specified per index + /** + The list of extensions of files created for a single table in the + database directory (datadir/db_name/). + + Used by open_table_error(), by the default rename_table and delete_table + handler methods, and by the default discovery implementation. + + For engines that have more than one file name extentions (separate + metadata, index, and/or data files), the order of elements is relevant. + First element of engine file name extentions array should be metadata + file extention. This is implied by the open_table_error() + and the default discovery implementation. + + Second element - data file extention. This is implied + assumed by REPAIR TABLE ... USE_FRM implementation. + */ + const char **tablefile_extensions; // by default - empty list + + /********************************************************************* + Table discovery API. + It allows the server to "discover" tables that exist in the storage + engine, without user issuing an explicit CREATE TABLE statement. + **********************************************************************/ + + /* + This method is required for any engine that supports automatic table + discovery, there is no default implementation. + + Given a TABLE_SHARE discover_table() fills it in with a correct table + structure using one of the TABLE_SHARE::init_from_* methods. + + Returns HA_ERR_NO_SUCH_TABLE if the table did not exist in the engine, + zero if the table was discovered successfully, or any other + HA_ERR_* error code as appropriate if the table existed, but the + discovery failed. + */ + int (*discover_table)(handlerton *hton, THD* thd, TABLE_SHARE *share); + + /* + The discover_table_names method tells the server + about all tables in the specified database that the engine + knows about. Tables (or file names of tables) are added to + the provided discovered_list collector object using + add_table() or add_file() methods. + */ + class discovered_list + { + public: + virtual bool add_table(const char *tname, size_t tlen) = 0; + virtual bool add_file(const char *fname) = 0; + protected: virtual ~discovered_list() {} + }; + + /* + By default (if not implemented by the engine, but the discovery_table() is + implemented) it will perform a file-based discovery: + + - if tablefile_extensions[0] is not null, this will discovers all tables + with the tablefile_extensions[0] extension. + + Returns 0 on success and 1 on error. + */ + int (*discover_table_names)(handlerton *hton, LEX_STRING *db, MY_DIR *dir, + discovered_list *result); + + /* + This is a method that allows to server to check if a table exists without + an overhead of the complete discovery. + + By default (if not implemented by the engine, but the discovery_table() is + implemented) it will try to perform a file-based discovery: + + - if tablefile_extensions[0] is not null this will look for a file name + with the tablefile_extensions[0] extension. + + - if tablefile_extensions[0] is null, this will resort to discover_table(). + + Note that resorting to discover_table() is slow and the engine + should probably implement its own discover_table_existence() method, + if its tablefile_extensions[0] is null. + + Returns 1 if the table exists and 0 if it does not. + */ + int (*discover_table_existence)(handlerton *hton, const char *db, + const char *table_name); + + /* + This is the assisted table discovery method. Unlike the fully + automatic discovery as above, here a user is expected to issue an + explicit CREATE TABLE with the appropriate table attributes to + "assist" the discovery of a table. But this "discovering" CREATE TABLE + statement will not specify the table structure - the engine discovers + it using this method. For example, FederatedX uses it in + + CREATE TABLE t1 ENGINE=FEDERATED CONNECTION="mysql://foo/bar/t1"; + + Given a TABLE_SHARE discover_table_structure() fills it in with a correct + table structure using one of the TABLE_SHARE::init_from_* methods. + + Assisted discovery works independently from the automatic discover. + An engine is allowed to support only assisted discovery and not + support automatic one. Or vice versa. + */ + int (*discover_table_structure)(handlerton *hton, THD* thd, + TABLE_SHARE *share, HA_CREATE_INFO *info); }; -inline LEX_STRING *hton_name(const handlerton *hton) +static inline LEX_STRING *hton_name(const handlerton *hton) { return &(hton2plugin[hton->slot]->name); } +static inline handlerton *plugin_hton(plugin_ref plugin) +{ + return plugin_data(plugin, handlerton *); +} + +static inline sys_var *find_hton_sysvar(handlerton *hton, st_mysql_sys_var *var) +{ + return find_plugin_sysvar(hton2plugin[hton->slot], var); +} + /* Possible flags of a handlerton (there can be 32 of them) */ #define HTON_NO_FLAGS 0 @@ -1247,9 +1404,10 @@ struct st_partition_iter; enum ha_choice { HA_CHOICE_UNDEF, HA_CHOICE_NO, HA_CHOICE_YES }; -typedef struct st_ha_create_information +struct HA_CREATE_INFO { CHARSET_INFO *table_charset, *default_table_charset; + LEX_CUSTRING tabledef_version; LEX_STRING connect_string; const char *password, *tablespace; LEX_STRING comment; @@ -1278,16 +1436,18 @@ typedef struct st_ha_create_information uint merge_insert_method; uint extra_size; /* length of extra data segment */ enum ha_choice transactional; - bool frm_only; ///< 1 if no ha_create_table() bool varchar; ///< 1 if table has a VARCHAR enum ha_storage_media storage_media; ///< DEFAULT, DISK or MEMORY enum ha_choice page_checksum; ///< If we have page_checksums engine_option_value *option_list; ///< list of table create options + /* the following three are only for ALTER TABLE, check_if_incompatible_data() */ ha_table_option_struct *option_struct; ///< structure with parsed table options ha_field_option_struct **fields_option_struct; ///< array of field option structures ha_index_option_struct **indexes_option_struct; ///< array of index option structures -} HA_CREATE_INFO; + + bool tmp_table() { return options & HA_LEX_CREATE_TMP_TABLE; } +}; typedef struct st_key_create_information @@ -1931,11 +2091,11 @@ public: /** to be actually called to get 'check()' functionality*/ int ha_check(THD *thd, HA_CHECK_OPT *check_opt); int ha_repair(THD* thd, HA_CHECK_OPT* check_opt); - void ha_start_bulk_insert(ha_rows rows) + void ha_start_bulk_insert(ha_rows rows, uint flags= 0) { DBUG_ENTER("handler::ha_start_bulk_insert"); estimation_rows_to_insert= rows; - start_bulk_insert(rows); + start_bulk_insert(rows, flags); DBUG_VOID_RETURN; } int ha_end_bulk_insert() @@ -1963,8 +2123,8 @@ public: int ha_create(const char *name, TABLE *form, HA_CREATE_INFO *info); - int ha_create_handler_files(const char *name, const char *old_name, - int action_flag, HA_CREATE_INFO *info); + int ha_create_partitioning_metadata(const char *name, const char *old_name, + int action_flag); int ha_change_partitions(HA_CREATE_INFO *create_info, const char *path, @@ -2387,19 +2547,8 @@ public: { return; } /* prepare InnoDB for HANDLER */ virtual void free_foreign_key_create_info(char* str) {} /** The following can be called without an open handler */ - virtual const char *table_type() const =0; - /** - If frm_error() is called then we will use this to find out what file - extentions exist for the storage engine. This is also used by the default - rename_table and delete_table method in handler.cc. - - For engines that have two file name extentions (separate meta/index file - and data file), the order of elements is relevant. First element of engine - file name extentions array should be meta/index file extention. Second - element - data file extention. This order is assumed by - prepare_for_repair() when REPAIR TABLE ... USE_FRM is issued. - */ - virtual const char **bas_ext() const =0; + const char *table_type() const { return hton_name(ht)->str; } + const char **bas_ext() const { return ht->tablefile_extensions; } virtual int get_default_no_partitions(HA_CREATE_INFO *create_info) { return 1;} @@ -2867,7 +3016,7 @@ private: DBUG_ASSERT(!(ha_table_flags() & HA_CAN_REPAIR)); return HA_ADMIN_NOT_IMPLEMENTED; } - virtual void start_bulk_insert(ha_rows rows) {} + virtual void start_bulk_insert(ha_rows rows, uint flags) {} virtual int end_bulk_insert() { return 0; } virtual int index_read(uchar * buf, const uchar * key, uint key_len, enum ha_rkey_function find_flag) @@ -2941,8 +3090,8 @@ private: virtual void drop_table(const char *name); virtual int create(const char *name, TABLE *form, HA_CREATE_INFO *info)=0; - virtual int create_handler_files(const char *name, const char *old_name, - int action_flag, HA_CREATE_INFO *info) + virtual int create_partitioning_metadata(const char *name, const char *old_name, + int action_flag) { return FALSE; } virtual int change_partitions(HA_CREATE_INFO *create_info, @@ -3012,6 +3161,8 @@ static inline bool ha_storage_engine_is_enabled(const handlerton *db_type) (db_type->state == SHOW_OPTION_YES) : FALSE; } +#define view_pseudo_hton ((handlerton *)1) + /* basic stuff */ int ha_init_errors(void); int ha_init(void); @@ -3026,10 +3177,10 @@ void ha_kill_query(THD* thd, enum thd_kill_levels level); bool ha_flush_logs(handlerton *db_type); void ha_drop_database(char* path); void ha_checkpoint_state(bool disable); +void ha_commit_checkpoint_request(void *cookie, void (*pre_hook)(void *)); int ha_create_table(THD *thd, const char *path, const char *db, const char *table_name, - HA_CREATE_INFO *create_info, - bool update_create_info); + HA_CREATE_INFO *create_info, LEX_CUSTRING *frm); int ha_delete_table(THD *thd, handlerton *db_type, const char *path, const char *db, const char *alias, bool generate_warning); @@ -3037,14 +3188,34 @@ int ha_delete_table(THD *thd, handlerton *db_type, const char *path, bool ha_show_status(THD *thd, handlerton *db_type, enum ha_stat_type stat); /* discovery */ -int ha_create_table_from_engine(THD* thd, const char *db, const char *name); -bool ha_check_if_table_exists(THD* thd, const char *db, const char *name, - bool *exists); -int ha_discover(THD* thd, const char* dbname, const char* name, - uchar** frmblob, size_t* frmlen); -int ha_find_files(THD *thd,const char *db,const char *path, - const char *wild, bool dir, List<LEX_STRING>* files); -int ha_table_exists_in_engine(THD* thd, const char* db, const char* name); +#ifdef MYSQL_SERVER +class Discovered_table_list: public handlerton::discovered_list +{ + THD *thd; + const char *wild, *wend; + bool with_temps; // whether to include temp tables in the result +public: + Dynamic_array<LEX_STRING*> *tables; + + Discovered_table_list(THD *thd_arg, Dynamic_array<LEX_STRING*> *tables_arg, + const LEX_STRING *wild_arg); + Discovered_table_list(THD *thd_arg, Dynamic_array<LEX_STRING*> *tables_arg) + : thd(thd_arg), wild(NULL), with_temps(true), tables(tables_arg) {} + ~Discovered_table_list() {} + + bool add_table(const char *tname, size_t tlen); + bool add_file(const char *fname); + + void sort(); + void remove_duplicates(); // assumes that the list is sorted +}; + +int ha_discover_table(THD *thd, TABLE_SHARE *share); +int ha_discover_table_names(THD *thd, LEX_STRING *db, MY_DIR *dirp, + Discovered_table_list *result, bool reusable); +bool ha_table_exists(THD *thd, const char *db, const char *table_name, + handlerton **hton= 0); +#endif /* key cache */ extern "C" int ha_init_key_cache(const char *name, KEY_CACHE *key_cache, void *); @@ -3106,6 +3277,7 @@ int ha_binlog_end(THD *thd); const char *get_canonical_filename(handler *file, const char *path, char *tmp_path); bool mysql_xa_recover(THD *thd); +void commit_checkpoint_notify_ha(handlerton *hton, void *cookie); inline const char *table_case_name(HA_CREATE_INFO *info, const char *name) { diff --git a/sql/item.cc b/sql/item.cc index 5e4c4b03c16..6542888d511 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2010, 2013, Monty Program Ab. + Copyright (c) 2010, 2013, Monty Program Ab This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -198,7 +198,7 @@ Hybrid_type_traits_integer::fix_length_and_dec(Item *item, Item *arg) const void item_init(void) { - item_user_lock_init(); + item_func_sleep_init(); uuid_short_init(); } @@ -884,10 +884,15 @@ bool Item_ident::remove_dependence_processor(uchar * arg) bool Item_ident::collect_outer_ref_processor(uchar *param) { Collect_deps_prm *prm= (Collect_deps_prm *)param; - if (depended_from && + if (depended_from && depended_from->nest_level_base == prm->nest_level_base && depended_from->nest_level < prm->nest_level) - prm->parameters->add_unique(this, &cmp_items); + { + if (prm->collect) + prm->parameters->add_unique(this, &cmp_items); + else + prm->count++; + } return FALSE; } @@ -4097,8 +4102,8 @@ double Item_copy_string::val_real() longlong Item_copy_string::val_int() { int err; - return null_value ? LL(0) : my_strntoll(str_value.charset(),str_value.ptr(), - str_value.length(),10, (char**) 0, + return null_value ? 0 : my_strntoll(str_value.charset(),str_value.ptr(), + str_value.length(), 10, (char**) 0, &err); } @@ -4268,7 +4273,7 @@ double Item_copy_decimal::val_real() longlong Item_copy_decimal::val_int() { if (null_value) - return LL(0); + return 0; else { longlong result; @@ -6596,6 +6601,13 @@ Item* Item::cache_const_expr_transformer(uchar *arg) return this; } +/** + Find Item by reference in the expression +*/ +bool Item::find_item_processor(uchar *arg) +{ + return (this == ((Item *) arg)); +} bool Item_field::send(Protocol *protocol, String *buffer) { @@ -9818,14 +9830,3 @@ const char *dbug_print_item(Item *item) #endif /*DBUG_OFF*/ -/***************************************************************************** -** Instantiate templates -*****************************************************************************/ - -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class List<Item>; -template class List_iterator<Item>; -template class List_iterator_fast<Item>; -template class List_iterator_fast<Item_field>; -template class List<List_item>; -#endif diff --git a/sql/item.h b/sql/item.h index 0714ea835f8..d73e858d68e 100644 --- a/sql/item.h +++ b/sql/item.h @@ -531,7 +531,7 @@ public: struct st_dyncall_create_def { - Item *num, *value; + Item *key, *value; CHARSET_INFO *cs; uint len, frac; DYNAMIC_COLUMN_TYPE type; @@ -556,11 +556,21 @@ typedef bool (Item::*Item_analyzer) (uchar **argp); typedef Item* (Item::*Item_transformer) (uchar *arg); typedef void (*Cond_traverser) (const Item *item, void *arg); +struct st_cond_statistic; + +struct find_selective_predicates_list_processor_data +{ + TABLE *table; + List<st_cond_statistic> list; +}; + class Item_equal; class COND_EQUAL; class st_select_lex_unit; +class Item_func_not; + class Item { Item(const Item &); /* Prevent use of these */ void operator=(Item &); @@ -1124,6 +1134,11 @@ public: return (this->*processor)(arg); } + virtual bool walk_top_and(Item_processor processor, uchar *arg) + { + return (this->*processor)(arg); + } + virtual Item* transform(Item_transformer transformer, uchar *arg); /* @@ -1171,6 +1186,7 @@ public: virtual bool collect_item_field_processor(uchar * arg) { return 0; } virtual bool add_field_to_set_processor(uchar * arg) { return 0; } virtual bool find_item_in_field_list_processor(uchar *arg) { return 0; } + virtual bool find_item_processor(uchar *arg); virtual bool change_context_processor(uchar *context) { return 0; } virtual bool reset_query_id_processor(uchar *query_id_arg) { return 0; } virtual bool is_expensive_processor(uchar *arg) { return 0; } @@ -1186,9 +1202,12 @@ public: virtual bool is_subquery_processor (uchar *opt_arg) { return 0; } virtual bool count_sargable_conds(uchar *arg) { return 0; } virtual bool limit_index_condition_pushdown_processor(uchar *opt_arg) - { + { return FALSE; } + virtual bool exists2in_processor(uchar *opt_arg) { return 0; } + virtual bool find_selective_predicates_list_processor(uchar *opt_arg) + { return 0; } /* To call bool function for all arguments */ struct bool_func_call_args @@ -1205,6 +1224,7 @@ public: return FALSE; } + /* The next function differs from the previous one that a bitmap to be updated is passed as uchar *arg. @@ -1334,7 +1354,9 @@ public: List<Item> *parameters; /* unit from which we count nest_level */ st_select_lex_unit *nest_level_base; + uint count; int nest_level; + bool collect; }; /** Collect outer references @@ -1503,6 +1525,15 @@ public: virtual void get_cache_parameters(List<Item> ¶meters) { }; virtual void mark_as_condition_AND_part(TABLE_LIST *embedding) {}; + + /* how much position should be reserved for Exists2In transformation */ + virtual uint exists2in_reserved_items() { return 0; }; + + /** + Inform the item that it is located under a NOT, which is a top-level item. + */ + virtual void under_not(Item_func_not * upper + __attribute__((unused))) {}; }; @@ -3094,6 +3125,13 @@ public: alias_name_used_arg) {} + bool fix_fields(THD *thd, Item **it) + { + if ((!(*ref)->fixed && (*ref)->fix_fields(thd, ref)) || + (*ref)->check_cols(1)) + return TRUE; + return Item_ref::fix_fields(thd, it); + } void save_val(Field *to); double val_real(); longlong val_int(); @@ -3315,7 +3353,7 @@ public: bool subst_argument_checker(uchar **arg); Item *equal_fields_propagator(uchar *arg); Item *replace_equal_field(uchar *arg); - table_map used_tables() const; + table_map used_tables() const; table_map not_null_tables() const; bool walk(Item_processor processor, bool walk_subquery, uchar *arg) { @@ -3682,7 +3720,7 @@ public: } virtual longlong val_int() { - return null_value ? LL(0) : cached_value; + return null_value ? 0 : cached_value; } virtual void copy(); }; diff --git a/sql/item_buff.cc b/sql/item_buff.cc index 86e0fd32774..ce396736d6f 100644 --- a/sql/item_buff.cc +++ b/sql/item_buff.cc @@ -173,12 +173,3 @@ bool Cached_item_decimal::cmp() return FALSE; } - -/***************************************************************************** -** Instansiate templates -*****************************************************************************/ - -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class List<Cached_item>; -template class List_iterator<Cached_item>; -#endif diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index fa67b3e6afd..75bb1608a0e 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -1439,7 +1439,7 @@ bool Item_in_optimizer::eval_not_null_tables(uchar *opt_arg) } -bool Item_in_optimizer::fix_left(THD *thd, Item **ref) +bool Item_in_optimizer::fix_left(THD *thd) { DBUG_ENTER("Item_in_optimizer::fix_left"); if ((!args[0]->fixed && args[0]->fix_fields(thd, args)) || @@ -1493,6 +1493,13 @@ bool Item_in_optimizer::fix_left(THD *thd, Item **ref) cache->store(args[0]); cache->cache_value(); } + if (args[1]->fixed) + { + /* to avoid overriding is called to update left expression */ + used_tables_cache|= args[1]->used_tables(); + with_sum_func= with_sum_func || args[1]->with_sum_func; + const_item_cache= const_item_cache && args[1]->const_item(); + } DBUG_RETURN(0); } @@ -1500,15 +1507,17 @@ bool Item_in_optimizer::fix_left(THD *thd, Item **ref) bool Item_in_optimizer::fix_fields(THD *thd, Item **ref) { DBUG_ASSERT(fixed == 0); - if (fix_left(thd, ref)) + if (fix_left(thd)) return TRUE; if (args[0]->maybe_null) maybe_null=1; if (!args[1]->fixed && args[1]->fix_fields(thd, args+1)) return TRUE; + Item_in_subselect * sub= (Item_in_subselect *)args[1]; - if (args[0]->cols() != sub->engine->cols()) + if (!invisible_mode() && + args[0]->cols() != sub->engine->cols()) { my_error(ER_OPERAND_COLUMNS, MYF(0), args[0]->cols()); return TRUE; @@ -1524,6 +1533,30 @@ bool Item_in_optimizer::fix_fields(THD *thd, Item **ref) return FALSE; } +/** + Check if Item_in_optimizer should work as a pass-through item for its + arguments. + + @note + Item_in_optimizer should work as pass-through for + - subqueries that were processed by ALL/ANY->MIN/MAX rewrite + - subqueries taht were originally EXISTS subqueries (and were coverted by + the EXISTS->IN rewrite) + + When Item_in_optimizer is not not working as a pass-through, it + - caches its "left argument", args[0]. + - makes adjustments to subquery item's return value for proper NULL + value handling +*/ + +bool Item_in_optimizer::invisible_mode() +{ + /* MAX/MIN transformed or EXISTS->IN prepared => do nothing */ + return (args[1]->type() != Item::SUBSELECT_ITEM || + ((Item_subselect *)args[1])->substype() == + Item_subselect::EXISTS_SUBS); +} + /** Add an expression cache for this subquery if it is needed @@ -1547,8 +1580,9 @@ Item *Item_in_optimizer::expr_cache_insert_transformer(uchar *thd_arg) { THD *thd= (THD*) thd_arg; DBUG_ENTER("Item_in_optimizer::expr_cache_insert_transformer"); - if (args[1]->type() != Item::SUBSELECT_ITEM) - DBUG_RETURN(this); // MAX/MIN transformed => do nothing + + if (invisible_mode()) + DBUG_RETURN(this); if (expr_cache) DBUG_RETURN(expr_cache); @@ -1571,13 +1605,16 @@ Item *Item_in_optimizer::expr_cache_insert_transformer(uchar *thd_arg) void Item_in_optimizer::get_cache_parameters(List<Item> ¶meters) { /* Add left expression to the list of the parameters of the subquery */ - if (args[0]->cols() == 1) - parameters.add_unique(args[0], &cmp_items); - else + if (!invisible_mode()) { - for (uint i= 0; i < args[0]->cols(); i++) + if (args[0]->cols() == 1) + parameters.add_unique(args[0], &cmp_items); + else { - parameters.add_unique(args[0]->element_index(i), &cmp_items); + for (uint i= 0; i < args[0]->cols(); i++) + { + parameters.add_unique(args[0]->element_index(i), &cmp_items); + } } } args[1]->get_cache_parameters(parameters); @@ -1660,17 +1697,19 @@ longlong Item_in_optimizer::val_int() DBUG_ASSERT(fixed == 1); cache->store(args[0]); cache->cache_value(); + DBUG_ENTER(" Item_in_optimizer::val_int"); - if (args[1]->type() != Item::SUBSELECT_ITEM) + if (invisible_mode()) { - /* MAX/MIN transformed => pass through */ longlong res= args[1]->val_int(); null_value= args[1]->null_value; - return (res); + DBUG_PRINT("info", ("pass trough")); + DBUG_RETURN(res); } if (cache->null_value) { + DBUG_PRINT("info", ("Left NULL...")); /* We're evaluating "<outer_value_list> [NOT] IN (SELECT <inner_value_list>...)" @@ -1742,11 +1781,11 @@ longlong Item_in_optimizer::val_int() for (uint i= 0; i < ncols; i++) item_subs->set_cond_guard_var(i, TRUE); } - return 0; + DBUG_RETURN(0); } tmp= args[1]->val_bool_result(); null_value= args[1]->null_value; - return tmp; + DBUG_RETURN(tmp); } @@ -1797,7 +1836,8 @@ bool Item_in_optimizer::is_null() @retval NULL if an error occurred */ -Item *Item_in_optimizer::transform(Item_transformer transformer, uchar *argument) +Item *Item_in_optimizer::transform(Item_transformer transformer, + uchar *argument) { Item *new_item; @@ -1817,7 +1857,7 @@ Item *Item_in_optimizer::transform(Item_transformer transformer, uchar *argument if ((*args) != new_item) current_thd->change_item_tree(args, new_item); - if (args[1]->type() != Item::SUBSELECT_ITEM) + if (invisible_mode()) { /* MAX/MIN transformed => pass through */ new_item= args[1]->transform(transformer, argument); @@ -4454,6 +4494,16 @@ bool Item_cond::walk(Item_processor processor, bool walk_subquery, uchar *arg) return Item_func::walk(processor, walk_subquery, arg); } +bool Item_cond_and::walk_top_and(Item_processor processor, uchar *arg) +{ + List_iterator_fast<Item> li(list); + Item *item; + while ((item= li++)) + if (item->walk_top_and(processor, arg)) + return 1; + return Item_cond::walk_top_and(processor, arg); +} + /** Transform an Item_cond object with a transformer callback function. @@ -4970,6 +5020,7 @@ bool Item_func_like::fix_fields(THD *thd, Item **ref) turboBM_compute_bad_character_shifts(); DBUG_PRINT("info",("done")); } + use_sampling= (len > 2 && (*first == wild_many || *first == wild_one)); } } return FALSE; @@ -4982,156 +5033,215 @@ void Item_func_like::cleanup() Item_bool_func2::cleanup(); } + +bool Item_func_like::find_selective_predicates_list_processor(uchar *arg) +{ + find_selective_predicates_list_processor_data *data= + (find_selective_predicates_list_processor_data *) arg; + if (use_sampling && used_tables() == data->table->map) + { + COND_STATISTIC *stat= (COND_STATISTIC *)sql_alloc(sizeof(COND_STATISTIC)); + if (!stat) + return TRUE; + stat->cond= this; + Item *arg0= args[0]->real_item(); + if (args[1]->const_item() && arg0->type() == FIELD_ITEM) + stat->field_arg= ((Item_field *)arg0)->field; + else + stat->field_arg= NULL; + data->list.push_back(stat); + } + return FALSE; +} + + + +/** + Convert string to lib_charset, if needed. +*/ +String *Regexp_processor_pcre::convert_if_needed(String *str, String *converter) +{ + if (m_conversion_is_needed) + { + uint dummy_errors; + if (converter->copy(str->ptr(), str->length(), str->charset(), + m_library_charset, &dummy_errors)) + return NULL; + str= converter; + } + return str; +} + + /** @brief Compile regular expression. + @param[in] pattern the pattern to compile from. @param[in] send_error send error message if any. @details Make necessary character set conversion then compile regular expression passed in the args[1]. - @retval 0 success. - @retval 1 error occurred. - @retval -1 given null regular expression. + @retval false success. + @retval true error occurred. */ -int Item_func_regex::regcomp(bool send_error) +bool Regexp_processor_pcre::compile(String *pattern, bool send_error) { - char buff[MAX_FIELD_WIDTH]; - String tmp(buff,sizeof(buff),&my_charset_bin); - String *res= args[1]->val_str(&tmp); - int error; - - if (args[1]->null_value) - return -1; + const char *pcreErrorStr; + int pcreErrorOffset; - if (regex_compiled) + if (is_compiled()) { - if (!stringcmp(res, &prev_regexp)) - return 0; - prev_regexp.copy(*res); - my_regfree(&preg); - regex_compiled= 0; + if (!stringcmp(pattern, &m_prev_pattern)) + return false; + m_prev_pattern.copy(*pattern); + pcre_free(m_pcre); + m_pcre= NULL; } - if (cmp_collation.collation != regex_lib_charset) - { - /* Convert UCS2 strings to UTF8 */ - uint dummy_errors; - if (conv.copy(res->ptr(), res->length(), res->charset(), - regex_lib_charset, &dummy_errors)) - return 1; - res= &conv; - } + if (!(pattern= convert_if_needed(pattern, &pattern_converter))) + return true; - if ((error= my_regcomp(&preg, res->c_ptr_safe(), - regex_lib_flags, regex_lib_charset))) + m_pcre= pcre_compile(pattern->c_ptr_safe(), m_library_flags, + &pcreErrorStr, &pcreErrorOffset, NULL); + + if (m_pcre == NULL) { if (send_error) { - (void) my_regerror(error, &preg, buff, sizeof(buff)); + char buff[MAX_FIELD_WIDTH]; + my_snprintf(buff, sizeof(buff), "%s at offset %d", pcreErrorStr, pcreErrorOffset); my_error(ER_REGEXP_ERROR, MYF(0), buff); } - return 1; + return true; } - regex_compiled= 1; - return 0; + return false; } -bool -Item_func_regex::fix_fields(THD *thd, Item **ref) +bool Regexp_processor_pcre::compile(Item *item, bool send_error) { - DBUG_ASSERT(fixed == 0); - if ((!args[0]->fixed && - args[0]->fix_fields(thd, args)) || args[0]->check_cols(1) || - (!args[1]->fixed && - args[1]->fix_fields(thd, args + 1)) || args[1]->check_cols(1)) - return TRUE; /* purecov: inspected */ - with_sum_func=args[0]->with_sum_func || args[1]->with_sum_func; - with_field= args[0]->with_field || args[1]->with_field; - with_subselect= args[0]->has_subquery() || args[1]->has_subquery(); - max_length= 1; - decimals= 0; + char buff[MAX_FIELD_WIDTH]; + String tmp(buff, sizeof(buff), &my_charset_bin); + String *pattern= item->val_str(&tmp); + if (item->null_value || compile(pattern, send_error)) + return true; + return false; +} - if (agg_arg_charsets_for_comparison(cmp_collation, args, 2)) - return TRUE; - regex_lib_flags= (cmp_collation.collation->state & - (MY_CS_BINSORT | MY_CS_CSSORT)) ? - REG_EXTENDED | REG_NOSUB : - REG_EXTENDED | REG_NOSUB | REG_ICASE; - /* - If the case of UCS2 and other non-ASCII character sets, - we will convert patterns and strings to UTF8. - */ - regex_lib_charset= (cmp_collation.collation->mbminlen > 1) ? - &my_charset_utf8_general_ci : - cmp_collation.collation; +bool Regexp_processor_pcre::exec(const char *str, int length, int offset) +{ + m_pcre_exec_rc= pcre_exec(m_pcre, NULL, str, length, + offset, 0, m_SubStrVec, m_subpatterns_needed * 3); + return false; +} - used_tables_cache=args[0]->used_tables() | args[1]->used_tables(); - not_null_tables_cache= (args[0]->not_null_tables() | - args[1]->not_null_tables()); - const_item_cache=args[0]->const_item() && args[1]->const_item(); - if (!regex_compiled && args[1]->const_item()) + +bool Regexp_processor_pcre::exec(String *str, int offset, + uint n_result_offsets_to_convert) +{ + if (!(str= convert_if_needed(str, &subject_converter))) + return true; + m_pcre_exec_rc= pcre_exec(m_pcre, NULL, str->c_ptr_safe(), str->length(), + offset, 0, m_SubStrVec, m_subpatterns_needed * 3); + if (m_pcre_exec_rc > 0) { - int comp_res= regcomp(TRUE); - if (comp_res == -1) - { // Will always return NULL - maybe_null=1; - fixed= 1; - return FALSE; + uint i; + for (i= 0; i < n_result_offsets_to_convert; i++) + { + /* + Convert byte offset into character offset. + */ + m_SubStrVec[i]= (int) str->charset()->cset->numchars(str->charset(), + str->ptr(), + str->ptr() + + m_SubStrVec[i]); } - else if (comp_res) - return TRUE; - regex_is_const= 1; - maybe_null= args[0]->maybe_null; } - else - maybe_null=1; - fixed= 1; - return FALSE; + return false; } -longlong Item_func_regex::val_int() +bool Regexp_processor_pcre::exec(Item *item, int offset, + uint n_result_offsets_to_convert) { - DBUG_ASSERT(fixed == 1); char buff[MAX_FIELD_WIDTH]; String tmp(buff,sizeof(buff),&my_charset_bin); - String *res= args[0]->val_str(&tmp); + String *res= item->val_str(&tmp); + if (item->null_value) + return true; + return exec(res, offset, n_result_offsets_to_convert); +} - if ((null_value= (args[0]->null_value || - (!regex_is_const && regcomp(FALSE))))) - return 0; - if (cmp_collation.collation != regex_lib_charset) +void Regexp_processor_pcre::fix_owner(Item_func *owner, + Item *subject_arg, + Item *pattern_arg) +{ + if (!is_compiled() && pattern_arg->const_item()) { - /* Convert UCS2 strings to UTF8 */ - uint dummy_errors; - if (conv.copy(res->ptr(), res->length(), res->charset(), - regex_lib_charset, &dummy_errors)) + if (compile(pattern_arg, true)) { - null_value= 1; - return 0; + owner->maybe_null= 1; // Will always return NULL + return; } - res= &conv; + set_const(true); + owner->maybe_null= subject_arg->maybe_null; } - return my_regexec(&preg,res->c_ptr_safe(),0,(my_regmatch_t*) 0,0) ? 0 : 1; + else + owner->maybe_null= 1; } -void Item_func_regex::cleanup() +void +Item_func_regex::fix_length_and_dec() { - DBUG_ENTER("Item_func_regex::cleanup"); - Item_bool_func::cleanup(); - if (regex_compiled) - { - my_regfree(&preg); - regex_compiled=0; - prev_regexp.length(0); - } - DBUG_VOID_RETURN; + Item_bool_func::fix_length_and_dec(); + + if (agg_arg_charsets_for_comparison(cmp_collation, args, 2)) + return; + + re.init(cmp_collation.collation, 0, 0); + re.fix_owner(this, args[0], args[1]); +} + + +longlong Item_func_regex::val_int() +{ + DBUG_ASSERT(fixed == 1); + if ((null_value= re.recompile(args[1]))) + return 0; + + if ((null_value= re.exec(args[0], 0, 0))) + return 0; + + return re.match(); +} + + +void +Item_func_regexp_instr::fix_length_and_dec() +{ + if (agg_arg_charsets_for_comparison(cmp_collation, args, 2)) + return; + + re.init(cmp_collation.collation, 0, 1); + re.fix_owner(this, args[0], args[1]); +} + + +longlong Item_func_regexp_instr::val_int() +{ + DBUG_ASSERT(fixed == 1); + if ((null_value= re.recompile(args[1]))) + return 0; + + if ((null_value= re.exec(args[0], 0, 1))) + return 0; + + return re.match() ? re.subpattern_start(0) + 1 : 0; } @@ -5422,6 +5532,7 @@ Item *Item_func_not::neg_transformer(THD *thd) /* NOT(x) -> x */ bool Item_func_not::fix_fields(THD *thd, Item **ref) { + args[0]->under_not(this); if (args[0]->type() == FIELD_ITEM) { /* replace "NOT <field>" with "<filed> == 0" */ @@ -5599,7 +5710,7 @@ Item *Item_bool_rowready_func2::negated_item() Item_equal::Item_equal(Item *f1, Item *f2, bool with_const_item) : Item_bool_func(), eval_item(0), cond_false(0), cond_true(0), - context_field(NULL) + context_field(NULL), link_equal_fields(FALSE) { const_item_cache= 0; with_const= with_const_item; @@ -5625,7 +5736,7 @@ Item_equal::Item_equal(Item *f1, Item *f2, bool with_const_item) Item_equal::Item_equal(Item_equal *item_equal) : Item_bool_func(), eval_item(0), cond_false(0), cond_true(0), - context_field(NULL) + context_field(NULL), link_equal_fields(FALSE) { const_item_cache= 0; List_iterator_fast<Item> li(item_equal->equal_items); @@ -5985,6 +6096,9 @@ bool Item_equal::fix_fields(THD *thd, Item **ref) DBUG_ASSERT(fixed == 0); Item_equal_fields_iterator it(*this); Item *item; + Field *first_equal_field; + Field *last_equal_field; + Field *prev_equal_field= NULL; not_null_tables_cache= used_tables_cache= 0; const_item_cache= 0; while ((item= it++)) @@ -5998,7 +6112,18 @@ bool Item_equal::fix_fields(THD *thd, Item **ref) maybe_null= 1; if (!item->get_item_equal()) item->set_item_equal(this); + if (link_equal_fields && item->real_item()->type() == FIELD_ITEM) + { + last_equal_field= ((Item_field *) (item->real_item()))->field; + if (!prev_equal_field) + first_equal_field= last_equal_field; + else + prev_equal_field->next_equal_field= last_equal_field; + prev_equal_field= last_equal_field; + } } + if (prev_equal_field && last_equal_field != first_equal_field) + last_equal_field->next_equal_field= first_equal_field; fix_length_and_dec(); fixed= 1; return FALSE; @@ -6273,23 +6398,87 @@ Item* Item_equal::get_first(JOIN_TAB *context, Item *field_item) } -longlong Item_func_dyncol_exists::val_int() +longlong Item_func_dyncol_check::val_int() { char buff[STRING_BUFFER_USUAL_SIZE]; String tmp(buff, sizeof(buff), &my_charset_bin); DYNAMIC_COLUMN col; String *str; - ulonglong num; enum enum_dyncol_func_result rc; - num= args[1]->val_int(); + str= args[0]->val_str(&tmp); + if (args[0]->null_value) + goto null; + col.length= str->length(); + /* We do not change the string, so could do this trick */ + col.str= (char *)str->ptr(); + rc= mariadb_dyncol_check(&col); + if (rc < 0 && rc != ER_DYNCOL_FORMAT) + { + dynamic_column_error_message(rc); + goto null; + } + null_value= FALSE; + return rc == ER_DYNCOL_OK; + +null: + null_value= TRUE; + return 0; +} + +longlong Item_func_dyncol_exists::val_int() +{ + char buff[STRING_BUFFER_USUAL_SIZE], nmstrbuf[11]; + String tmp(buff, sizeof(buff), &my_charset_bin), + nmbuf(nmstrbuf, sizeof(nmstrbuf), system_charset_info); + DYNAMIC_COLUMN col; + String *str; + LEX_STRING buf, *name= NULL; + ulonglong num= 0; + enum enum_dyncol_func_result rc; + + if (args[1]->result_type() == INT_RESULT) + num= args[1]->val_int(); + else + { + String *nm= args[1]->val_str(&nmbuf); + if (!nm || args[1]->null_value) + { + null_value= 1; + return 1; + } + if (my_charset_same(nm->charset(), &my_charset_utf8_general_ci)) + { + buf.str= (char *) nm->ptr(); + buf.length= nm->length(); + } + else + { + uint strlen; + uint dummy_errors; + buf.str= (char *)sql_alloc((strlen= nm->length() * + my_charset_utf8_general_ci.mbmaxlen + 1)); + if (buf.str) + { + buf.length= + copy_and_convert(buf.str, strlen, &my_charset_utf8_general_ci, + nm->ptr(), nm->length(), nm->charset(), + &dummy_errors); + } + else + buf.length= 0; + } + name= &buf; + } str= args[0]->val_str(&tmp); if (args[0]->null_value || args[1]->null_value || num > UINT_MAX16) goto null; col.length= str->length(); /* We do not change the string, so could do this trick */ col.str= (char *)str->ptr(); - rc= dynamic_column_exists(&col, (uint) num); + rc= ((name == NULL) ? + mariadb_dyncol_exists(&col, (uint) num) : + mariadb_dyncol_exists_named(&col, name)); if (rc < 0) { dynamic_column_error_message(rc); diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 9504be57a90..0a26569aafd 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -25,7 +25,8 @@ #include "thr_malloc.h" /* sql_calloc */ #include "item_func.h" /* Item_int_func, Item_bool_func */ -#include "my_regex.h" +#define PCRE_STATIC 1 /* Important on Windows */ +#include "pcre.h" /* pcre header file */ extern Item_result item_cmp_type(Item_result a,Item_result b); class Item_bool_func2; @@ -246,12 +247,12 @@ protected: */ int result_for_null_param; public: - Item_in_optimizer(Item *a, Item_in_subselect *b): - Item_bool_func(a, reinterpret_cast<Item *>(b)), cache(0), expr_cache(0), + Item_in_optimizer(Item *a, Item *b): + Item_bool_func(a, b), cache(0), expr_cache(0), save_cache(0), result_for_null_param(UNKNOWN) { with_subselect= true; } bool fix_fields(THD *, Item **); - bool fix_left(THD *thd, Item **ref); + bool fix_left(THD *thd); table_map not_null_tables() const { return 0; } bool is_null(); longlong val_int(); @@ -269,6 +270,8 @@ public: bool is_top_level_item(); bool eval_not_null_tables(uchar *opt_arg); void fix_after_pullout(st_select_lex *new_parent, Item **ref); + bool invisible_mode(); + void reset_cache() { cache= NULL; } }; class Comp_creator @@ -437,8 +440,11 @@ public: class Item_func_not :public Item_bool_func { + bool abort_on_null; public: - Item_func_not(Item *a) :Item_bool_func(a) {} + Item_func_not(Item *a) :Item_bool_func(a), abort_on_null(FALSE) {} + virtual void top_level_item() { abort_on_null= 1; } + bool is_top_level_item() { return abort_on_null; } longlong val_int(); enum Functype functype() const { return NOT_FUNC; } const char *func_name() const { return "not"; } @@ -496,16 +502,13 @@ class Item_func_not_all :public Item_func_not Item_sum_hybrid *test_sum_item; Item_maxmin_subselect *test_sub_item; - bool abort_on_null; public: bool show; Item_func_not_all(Item *a) - :Item_func_not(a), test_sum_item(0), test_sub_item(0), abort_on_null(0), + :Item_func_not(a), test_sum_item(0), test_sub_item(0), show(0) {} - virtual void top_level_item() { abort_on_null= 1; } - bool is_top_level_item() { return abort_on_null; } table_map not_null_tables() const { return 0; } longlong val_int(); enum Functype functype() const { return NOT_ALL_FUNC; } @@ -551,6 +554,7 @@ public: - Otherwise, UINT_MAX */ uint in_equality_no; + virtual uint exists2in_reserved_items() { return 1; }; }; class Item_func_equal :public Item_bool_rowready_func2 @@ -1462,8 +1466,9 @@ class Item_func_like :public Item_bool_func2 enum { alphabet_size = 256 }; Item *escape_item; - + bool escape_used_in_parsing; + bool use_sampling; public: int escape; @@ -1471,7 +1476,7 @@ public: Item_func_like(Item *a,Item *b, Item *escape_arg, bool escape_used) :Item_bool_func2(a,b), canDoTurboBM(FALSE), pattern(0), pattern_len(0), bmGs(0), bmBc(0), escape_item(escape_arg), - escape_used_in_parsing(escape_used) {} + escape_used_in_parsing(escape_used), use_sampling(0) {} longlong val_int(); enum Functype functype() const { return LIKE_FUNC; } optimize_type select_optimize() const; @@ -1479,26 +1484,107 @@ public: const char *func_name() const { return "like"; } bool fix_fields(THD *thd, Item **ref); void cleanup(); + + bool find_selective_predicates_list_processor(uchar *arg); +}; + + +class Regexp_processor_pcre +{ + pcre *m_pcre; + bool m_conversion_is_needed; + bool m_is_const; + int m_library_flags; + CHARSET_INFO *m_data_charset; + CHARSET_INFO *m_library_charset; + String m_prev_pattern; + int m_pcre_exec_rc; + int m_SubStrVec[30]; + uint m_subpatterns_needed; +public: + String *convert_if_needed(String *src, String *converter); + String subject_converter; + String pattern_converter; + String replace_converter; + Regexp_processor_pcre() : + m_pcre(NULL), m_conversion_is_needed(true), m_is_const(0), + m_library_flags(0), + m_data_charset(&my_charset_utf8_general_ci), + m_library_charset(&my_charset_utf8_general_ci), + m_subpatterns_needed(0) + {} + void init(CHARSET_INFO *data_charset, int extra_flags, uint nsubpatterns) + { + m_library_flags= extra_flags | + (data_charset != &my_charset_bin ? + (PCRE_UTF8 | PCRE_UCP) : 0) | + ((data_charset->state & + (MY_CS_BINSORT | MY_CS_CSSORT)) ? 0 : PCRE_CASELESS); + + // Convert text data to utf-8. + m_library_charset= data_charset == &my_charset_bin ? + &my_charset_bin : &my_charset_utf8_general_ci; + + m_conversion_is_needed= (data_charset != &my_charset_bin) && + !my_charset_same(data_charset, m_library_charset); + m_subpatterns_needed= nsubpatterns; + } + void fix_owner(Item_func *owner, Item *subject_arg, Item *pattern_arg); + bool compile(String *pattern, bool send_error); + bool compile(Item *item, bool send_error); + bool recompile(Item *item) + { + return !m_is_const && compile(item, false); + } + bool exec(const char *str, int length, int offset); + bool exec(String *str, int offset, uint n_result_offsets_to_convert); + bool exec(Item *item, int offset, uint n_result_offsets_to_convert); + bool match() const { return m_pcre_exec_rc < 0 ? 0 : 1; } + int nsubpatterns() const { return m_pcre_exec_rc <= 0 ? 0 : m_pcre_exec_rc; } + int subpattern_start(int n) const + { + return m_pcre_exec_rc <= 0 ? 0 : m_SubStrVec[n * 2]; + } + int subpattern_end(int n) const + { + return m_pcre_exec_rc <= 0 ? 0 : m_SubStrVec[n * 2 + 1]; + } + int subpattern_length(int n) const + { + return subpattern_end(n) - subpattern_start(n); + } + void cleanup() + { + if (m_pcre) + { + pcre_free(m_pcre); + m_pcre= NULL; + } + m_prev_pattern.length(0); + } + bool is_compiled() const { return m_pcre != NULL; } + bool is_const() const { return m_is_const; } + void set_const(bool arg) { m_is_const= arg; } + CHARSET_INFO * library_charset() const { return m_library_charset; } }; class Item_func_regex :public Item_bool_func { - my_regex_t preg; - bool regex_compiled; - bool regex_is_const; - String prev_regexp; + Regexp_processor_pcre re; DTCollation cmp_collation; - CHARSET_INFO *regex_lib_charset; - int regex_lib_flags; - String conv; - int regcomp(bool send_error); public: - Item_func_regex(Item *a,Item *b) :Item_bool_func(a,b), - regex_compiled(0),regex_is_const(0) {} - void cleanup(); + Item_func_regex(Item *a,Item *b) :Item_bool_func(a,b) + {} + void cleanup() + { + DBUG_ENTER("Item_func_regex::cleanup"); + Item_bool_func::cleanup(); + re.cleanup(); + DBUG_VOID_RETURN; + } longlong val_int(); - bool fix_fields(THD *thd, Item **ref); + void fix_length_and_dec(); const char *func_name() const { return "regexp"; } virtual inline void print(String *str, enum_query_type query_type) @@ -1510,6 +1596,26 @@ public: }; +class Item_func_regexp_instr :public Item_int_func +{ + Regexp_processor_pcre re; + DTCollation cmp_collation; +public: + Item_func_regexp_instr(Item *a, Item *b) :Item_int_func(a, b) + {} + void cleanup() + { + DBUG_ENTER("Item_func_regexp_instr::cleanup"); + Item_int_func::cleanup(); + re.cleanup(); + DBUG_VOID_RETURN; + } + longlong val_int(); + void fix_length_and_dec(); + const char *func_name() const { return "regexp_instr"; } +}; + + typedef class Item COND; class Item_cond :public Item_bool_func @@ -1712,6 +1818,8 @@ class Item_equal: public Item_bool_func */ Item_field *context_field; + bool link_equal_fields; + public: COND_EQUAL *upper_levels; /* multiple equalities of upper and levels */ @@ -1750,6 +1858,8 @@ public: CHARSET_INFO *compare_collation(); void set_context_field(Item_field *ctx_field) { context_field= ctx_field; } + void set_link_equal_fields(bool flag) { link_equal_fields= flag; } + friend class Item_equal_fields_iterator; bool count_sargable_conds(uchar *arg); friend class Item_equal_iterator<List_iterator_fast,Item>; friend class Item_equal_iterator<List_iterator,Item>; @@ -1886,6 +1996,8 @@ public: } Item *neg_transformer(THD *thd); void mark_as_condition_AND_part(TABLE_LIST *embedding); + virtual uint exists2in_reserved_items() { return list.elements; }; + bool walk_top_and(Item_processor processor, uchar *arg); }; inline bool is_cond_and(Item *item) @@ -1918,6 +2030,14 @@ public: Item *neg_transformer(THD *thd); }; +class Item_func_dyncol_check :public Item_bool_func +{ +public: + Item_func_dyncol_check(Item *str) :Item_bool_func(str) {} + longlong val_int(); + const char *func_name() const { return "column_check"; } +}; + class Item_func_dyncol_exists :public Item_bool_func { public: diff --git a/sql/item_create.cc b/sql/item_create.cc index 45de3850fcd..c158816bf32 100644 --- a/sql/item_create.cc +++ b/sql/item_create.cc @@ -447,6 +447,19 @@ protected: }; +class Create_func_binlog_gtid_pos : public Create_func_arg2 +{ +public: + virtual Item *create_2_arg(THD *thd, Item *arg1, Item *arg2); + + static Create_func_binlog_gtid_pos s_singleton; + +protected: + Create_func_binlog_gtid_pos() {} + virtual ~Create_func_binlog_gtid_pos() {} +}; + + class Create_func_bit_count : public Create_func_arg1 { public: @@ -526,6 +539,54 @@ protected: virtual ~Create_func_coercibility() {} }; +class Create_func_dyncol_check : public Create_func_arg1 +{ +public: + virtual Item *create_1_arg(THD *thd, Item *arg1); + + static Create_func_dyncol_check s_singleton; + +protected: + Create_func_dyncol_check() {} + virtual ~Create_func_dyncol_check() {} +}; + +class Create_func_dyncol_exists : public Create_func_arg2 +{ +public: + virtual Item *create_2_arg(THD *thd, Item *arg1, Item *arg2); + + static Create_func_dyncol_exists s_singleton; + +protected: + Create_func_dyncol_exists() {} + virtual ~Create_func_dyncol_exists() {} +}; + +class Create_func_dyncol_list : public Create_func_arg1 +{ +public: + virtual Item *create_1_arg(THD *thd, Item *arg1); + + static Create_func_dyncol_list s_singleton; + +protected: + Create_func_dyncol_list() {} + virtual ~Create_func_dyncol_list() {} +}; + +class Create_func_dyncol_json : public Create_func_arg1 +{ +public: + virtual Item *create_1_arg(THD *thd, Item *arg1); + + static Create_func_dyncol_json s_singleton; + +protected: + Create_func_dyncol_json() {} + virtual ~Create_func_dyncol_json() {} +}; + class Create_func_compress : public Create_func_arg1 { @@ -553,6 +614,19 @@ protected: }; +class Create_func_decode_histogram : public Create_func_arg2 +{ +public: + Item *create_2_arg(THD *thd, Item *arg1, Item *arg2); + + static Create_func_decode_histogram s_singleton; + +protected: + Create_func_decode_histogram() {} + virtual ~Create_func_decode_histogram() {} +}; + + class Create_func_concat_ws : public Create_native_func { public: @@ -1709,6 +1783,19 @@ protected: }; +class Create_func_master_gtid_wait : public Create_native_func +{ +public: + virtual Item *create_native(THD *thd, LEX_STRING name, List<Item> *item_list); + + static Create_func_master_gtid_wait s_singleton; + +protected: + Create_func_master_gtid_wait() {} + virtual ~Create_func_master_gtid_wait() {} +}; + + class Create_func_md5 : public Create_func_arg1 { public: @@ -1940,6 +2027,45 @@ protected: }; +class Create_func_regexp_instr : public Create_func_arg2 +{ +public: + virtual Item *create_2_arg(THD *thd, Item *arg1, Item *arg2); + + static Create_func_regexp_instr s_singleton; + +protected: + Create_func_regexp_instr() {} + virtual ~Create_func_regexp_instr() {} +}; + + +class Create_func_regexp_replace : public Create_func_arg3 +{ +public: + virtual Item *create_3_arg(THD *thd, Item *arg1, Item *arg2, Item *arg3); + + static Create_func_regexp_replace s_singleton; + +protected: + Create_func_regexp_replace() {} + virtual ~Create_func_regexp_replace() {} +}; + + +class Create_func_regexp_substr : public Create_func_arg2 +{ +public: + virtual Item *create_2_arg(THD *thd, Item *arg1, Item *arg2); + + static Create_func_regexp_substr s_singleton; + +protected: + Create_func_regexp_substr() {} + virtual ~Create_func_regexp_substr() {} +}; + + class Create_func_radians : public Create_func_arg1 { public: @@ -3052,6 +3178,16 @@ Create_func_bin::create_1_arg(THD *thd, Item *arg1) } +Create_func_binlog_gtid_pos Create_func_binlog_gtid_pos::s_singleton; + +Item* +Create_func_binlog_gtid_pos::create_2_arg(THD *thd, Item *arg1, Item *arg2) +{ + thd->lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_SYSTEM_FUNCTION); + return new (thd->mem_root) Item_func_binlog_gtid_pos(arg1, arg2); +} + + Create_func_bit_count Create_func_bit_count::s_singleton; Item* @@ -3108,6 +3244,38 @@ Create_func_coercibility::create_1_arg(THD *thd, Item *arg1) } +Create_func_dyncol_check Create_func_dyncol_check::s_singleton; + +Item* +Create_func_dyncol_check::create_1_arg(THD *thd, Item *arg1) +{ + return new (thd->mem_root) Item_func_dyncol_check(arg1); +} + +Create_func_dyncol_exists Create_func_dyncol_exists::s_singleton; + +Item* +Create_func_dyncol_exists::create_2_arg(THD *thd, Item *arg1, Item *arg2) +{ + return new (thd->mem_root) Item_func_dyncol_exists(arg1, arg2); +} + +Create_func_dyncol_list Create_func_dyncol_list::s_singleton; + +Item* +Create_func_dyncol_list::create_1_arg(THD *thd, Item *arg1) +{ + return new (thd->mem_root) Item_func_dyncol_list(arg1); +} + +Create_func_dyncol_json Create_func_dyncol_json::s_singleton; + +Item* +Create_func_dyncol_json::create_1_arg(THD *thd, Item *arg1) +{ + return new (thd->mem_root) Item_func_dyncol_json(arg1); +} + Create_func_concat Create_func_concat::s_singleton; Item* @@ -3128,6 +3296,13 @@ Create_func_concat::create_native(THD *thd, LEX_STRING name, return new (thd->mem_root) Item_func_concat(*item_list); } +Create_func_decode_histogram Create_func_decode_histogram::s_singleton; + +Item * +Create_func_decode_histogram::create_2_arg(THD *thd, Item *arg1, Item *arg2) +{ + return new (thd->mem_root) Item_func_decode_histogram(arg1, arg2); +} Create_func_concat_ws Create_func_concat_ws::s_singleton; @@ -4392,27 +4567,75 @@ Create_func_master_pos_wait::create_native(THD *thd, LEX_STRING name, if (item_list != NULL) arg_count= item_list->elements; + if (arg_count < 2 || arg_count > 4) + { + my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name.str); + return func; + } + + thd->lex->safe_to_cache_query= 0; + + Item *param_1= item_list->pop(); + Item *param_2= item_list->pop(); switch (arg_count) { case 2: { - Item *param_1= item_list->pop(); - Item *param_2= item_list->pop(); func= new (thd->mem_root) Item_master_pos_wait(param_1, param_2); - thd->lex->safe_to_cache_query= 0; break; } case 3: { - Item *param_1= item_list->pop(); - Item *param_2= item_list->pop(); Item *param_3= item_list->pop(); func= new (thd->mem_root) Item_master_pos_wait(param_1, param_2, param_3); - thd->lex->safe_to_cache_query= 0; break; } - default: + case 4: + { + Item *param_3= item_list->pop(); + Item *param_4= item_list->pop(); + func= new (thd->mem_root) Item_master_pos_wait(param_1, param_2, param_3, + param_4); + break; + } + } + + return func; +} + + +Create_func_master_gtid_wait Create_func_master_gtid_wait::s_singleton; + +Item* +Create_func_master_gtid_wait::create_native(THD *thd, LEX_STRING name, + List<Item> *item_list) +{ + Item *func= NULL; + int arg_count= 0; + + thd->lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_SYSTEM_FUNCTION); + + if (item_list != NULL) + arg_count= item_list->elements; + + if (arg_count < 1 || arg_count > 2) { my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name.str); + return func; + } + + thd->lex->safe_to_cache_query= 0; + + Item *param_1= item_list->pop(); + switch (arg_count) { + case 1: + { + func= new (thd->mem_root) Item_master_gtid_wait(param_1); + break; + } + case 2: + { + Item *param_2= item_list->pop(); + func= new (thd->mem_root) Item_master_gtid_wait(param_1, param_2); break; } } @@ -4589,6 +4812,33 @@ Create_func_quote::create_1_arg(THD *thd, Item *arg1) } +Create_func_regexp_instr Create_func_regexp_instr::s_singleton; + +Item* +Create_func_regexp_instr::create_2_arg(THD *thd, Item *arg1, Item *arg2) +{ + return new (thd->mem_root) Item_func_regexp_instr(arg1, arg2); +} + + +Create_func_regexp_replace Create_func_regexp_replace::s_singleton; + +Item* +Create_func_regexp_replace::create_3_arg(THD *thd, Item *arg1, Item *arg2, Item *arg3) +{ + return new (thd->mem_root) Item_func_regexp_replace(arg1, arg2, arg3); +} + + +Create_func_regexp_substr Create_func_regexp_substr::s_singleton; + +Item* +Create_func_regexp_substr::create_2_arg(THD *thd, Item *arg1, Item *arg2) +{ + return new (thd->mem_root) Item_func_regexp_substr(arg1, arg2); +} + + Create_func_radians Create_func_radians::s_singleton; Item* @@ -5235,6 +5485,7 @@ static Native_func_registry func_array[] = { { C_STRING_WITH_LEN("ATAN2") }, BUILDER(Create_func_atan)}, { { C_STRING_WITH_LEN("BENCHMARK") }, BUILDER(Create_func_benchmark)}, { { C_STRING_WITH_LEN("BIN") }, BUILDER(Create_func_bin)}, + { { C_STRING_WITH_LEN("BINLOG_GTID_POS") }, BUILDER(Create_func_binlog_gtid_pos)}, { { C_STRING_WITH_LEN("BIT_COUNT") }, BUILDER(Create_func_bit_count)}, { { C_STRING_WITH_LEN("BIT_LENGTH") }, BUILDER(Create_func_bit_length)}, { { C_STRING_WITH_LEN("BUFFER") }, GEOM_BUILDER(Create_func_buffer)}, @@ -5244,6 +5495,10 @@ static Native_func_registry func_array[] = { { C_STRING_WITH_LEN("CHARACTER_LENGTH") }, BUILDER(Create_func_char_length)}, { { C_STRING_WITH_LEN("CHAR_LENGTH") }, BUILDER(Create_func_char_length)}, { { C_STRING_WITH_LEN("COERCIBILITY") }, BUILDER(Create_func_coercibility)}, + { { C_STRING_WITH_LEN("COLUMN_CHECK") }, BUILDER(Create_func_dyncol_check)}, + { { C_STRING_WITH_LEN("COLUMN_EXISTS") }, BUILDER(Create_func_dyncol_exists)}, + { { C_STRING_WITH_LEN("COLUMN_LIST") }, BUILDER(Create_func_dyncol_list)}, + { { C_STRING_WITH_LEN("COLUMN_JSON") }, BUILDER(Create_func_dyncol_json)}, { { C_STRING_WITH_LEN("COMPRESS") }, BUILDER(Create_func_compress)}, { { C_STRING_WITH_LEN("CONCAT") }, BUILDER(Create_func_concat)}, { { C_STRING_WITH_LEN("CONCAT_WS") }, BUILDER(Create_func_concat_ws)}, @@ -5262,6 +5517,7 @@ static Native_func_registry func_array[] = { { C_STRING_WITH_LEN("DAYOFYEAR") }, BUILDER(Create_func_dayofyear)}, { { C_STRING_WITH_LEN("DECODE") }, BUILDER(Create_func_decode)}, { { C_STRING_WITH_LEN("DEGREES") }, BUILDER(Create_func_degrees)}, + { { C_STRING_WITH_LEN("DECODE_HISTOGRAM") }, BUILDER(Create_func_decode_histogram)}, { { C_STRING_WITH_LEN("DES_DECRYPT") }, BUILDER(Create_func_des_decrypt)}, { { C_STRING_WITH_LEN("DES_ENCRYPT") }, BUILDER(Create_func_des_encrypt)}, { { C_STRING_WITH_LEN("DIMENSION") }, GEOM_BUILDER(Create_func_dimension)}, @@ -5334,6 +5590,7 @@ static Native_func_registry func_array[] = { { C_STRING_WITH_LEN("MAKEDATE") }, BUILDER(Create_func_makedate)}, { { C_STRING_WITH_LEN("MAKETIME") }, BUILDER(Create_func_maketime)}, { { C_STRING_WITH_LEN("MAKE_SET") }, BUILDER(Create_func_make_set)}, + { { C_STRING_WITH_LEN("MASTER_GTID_WAIT") }, BUILDER(Create_func_master_gtid_wait)}, { { C_STRING_WITH_LEN("MASTER_POS_WAIT") }, BUILDER(Create_func_master_pos_wait)}, { { C_STRING_WITH_LEN("MBRCONTAINS") }, GEOM_BUILDER(Create_func_mbr_contains)}, { { C_STRING_WITH_LEN("MBRDISJOINT") }, GEOM_BUILDER(Create_func_mbr_disjoint)}, @@ -5378,6 +5635,9 @@ static Native_func_registry func_array[] = { { C_STRING_WITH_LEN("POW") }, BUILDER(Create_func_pow)}, { { C_STRING_WITH_LEN("POWER") }, BUILDER(Create_func_pow)}, { { C_STRING_WITH_LEN("QUOTE") }, BUILDER(Create_func_quote)}, + { { C_STRING_WITH_LEN("REGEXP_INSTR") }, BUILDER(Create_func_regexp_instr)}, + { { C_STRING_WITH_LEN("REGEXP_REPLACE") }, BUILDER(Create_func_regexp_replace)}, + { { C_STRING_WITH_LEN("REGEXP_SUBSTR") }, BUILDER(Create_func_regexp_substr)}, { { C_STRING_WITH_LEN("RADIANS") }, BUILDER(Create_func_radians)}, { { C_STRING_WITH_LEN("RAND") }, BUILDER(Create_func_rand)}, { { C_STRING_WITH_LEN("RELEASE_LOCK") }, BUILDER(Create_func_release_lock)}, @@ -5703,7 +5963,7 @@ static List<Item> *create_func_dyncol_prepare(THD *thd, for (uint i= 0; (def= li++) ;) { dfs[0][i++]= *def; - args->push_back(def->num); + args->push_back(def->key); args->push_back(def->value); } return args; @@ -5719,7 +5979,6 @@ Item *create_func_dyncol_create(THD *thd, List<DYNCALL_CREATE_DEF> &list) return new (thd->mem_root) Item_func_dyncol_create(*args, dfs); } - Item *create_func_dyncol_add(THD *thd, Item *str, List<DYNCALL_CREATE_DEF> &list) { @@ -5739,7 +5998,7 @@ Item *create_func_dyncol_add(THD *thd, Item *str, Item *create_func_dyncol_delete(THD *thd, Item *str, List<Item> &nums) { DYNCALL_CREATE_DEF *dfs; - Item *num; + Item *key; List_iterator_fast<Item> it(nums); List<Item> *args= new (thd->mem_root) List<Item>; @@ -5749,12 +6008,12 @@ Item *create_func_dyncol_delete(THD *thd, Item *str, List<Item> &nums) if (!args || !dfs) return NULL; - for (uint i= 0; (num= it++); i++) + for (uint i= 0; (key= it++); i++) { - dfs[i].num= num; + dfs[i].key= key; dfs[i].value= new Item_null(); dfs[i].type= DYN_COL_INT; - args->push_back(dfs[i].num); + args->push_back(dfs[i].key); args->push_back(dfs[i].value); } diff --git a/sql/item_create.h b/sql/item_create.h index ac6b0f8454f..5ecb45e9eae 100644 --- a/sql/item_create.h +++ b/sql/item_create.h @@ -180,5 +180,6 @@ Item *create_func_dyncol_get(THD *thd, Item *num, Item *str, Cast_target cast_type, const char *c_len, const char *c_dec, CHARSET_INFO *cs); +Item *create_func_dyncol_json(THD *thd, Item *str); #endif diff --git a/sql/item_func.cc b/sql/item_func.cc index 453494b374f..df7ccea3caf 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -2382,7 +2382,7 @@ longlong Item_func_shift_left::val_int() return 0; } null_value=0; - return (shift < sizeof(longlong)*8 ? (longlong) res : LL(0)); + return (shift < sizeof(longlong)*8 ? (longlong) res : 0); } longlong Item_func_shift_right::val_int() @@ -2397,7 +2397,7 @@ longlong Item_func_shift_right::val_int() return 0; } null_value=0; - return (shift < sizeof(longlong)*8 ? (longlong) res : LL(0)); + return (shift < sizeof(longlong)*8 ? (longlong) res : 0); } @@ -3345,7 +3345,7 @@ void Item_func_find_in_set::fix_length_and_dec() find->length(), 0); enum_bit=0; if (enum_value) - enum_bit=LL(1) << (enum_value-1); + enum_bit=1LL << (enum_value-1); } } } @@ -3426,7 +3426,7 @@ longlong Item_func_find_in_set::val_int() wc == (my_wc_t) separator) return (longlong) ++position; else - return LL(0); + return 0; } } return 0; @@ -3932,144 +3932,90 @@ udf_handler::~udf_handler() bool udf_handler::get_arguments() { return 0; } #endif /* HAVE_DLOPEN */ -/* -** User level locks -*/ - -mysql_mutex_t LOCK_user_locks; -static HASH hash_user_locks; -class User_level_lock +longlong Item_master_pos_wait::val_int() { - uchar *key; - size_t key_length; - -public: - int count; - bool locked; - mysql_cond_t cond; - my_thread_id thread_id; - void set_thread(THD *thd) { thread_id= thd->thread_id; } + DBUG_ASSERT(fixed == 1); + THD* thd = current_thd; + String *log_name = args[0]->val_str(&value); + int event_count= 0; - User_level_lock(const uchar *key_arg,uint length, ulong id) - :key_length(length),count(1),locked(1), thread_id(id) + null_value=0; + if (thd->slave_thread || !log_name || !log_name->length()) { - key= (uchar*) my_memdup(key_arg,length,MYF(0)); - mysql_cond_init(key_user_level_lock_cond, &cond, NULL); - if (key) - { - if (my_hash_insert(&hash_user_locks,(uchar*) this)) - { - my_free(key); - key=0; - } - } + null_value = 1; + return 0; } - ~User_level_lock() +#ifdef HAVE_REPLICATION + longlong pos = (ulong)args[1]->val_int(); + longlong timeout = (arg_count==3) ? args[2]->val_int() : 0 ; + String connection_name_buff; + LEX_STRING connection_name; + Master_info *mi; + if (arg_count == 4) { - if (key) + String *con; + if (!(con= args[3]->val_str(&connection_name_buff))) + goto err; + + connection_name.str= (char*) con->ptr(); + connection_name.length= con->length(); + if (check_master_connection_name(&connection_name)) { - my_hash_delete(&hash_user_locks,(uchar*) this); - my_free(key); + my_error(ER_WRONG_ARGUMENTS, MYF(ME_JUST_WARNING), + "MASTER_CONNECTION_NAME"); + goto err; } - mysql_cond_destroy(&cond); } - inline bool initialized() { return key != 0; } - friend void item_user_lock_release(User_level_lock *ull); - friend uchar *ull_get_key(const User_level_lock *ull, size_t *length, - my_bool not_used); -}; - -uchar *ull_get_key(const User_level_lock *ull, size_t *length, - my_bool not_used __attribute__((unused))) -{ - *length= ull->key_length; - return ull->key; -} - -#ifdef HAVE_PSI_INTERFACE -static PSI_mutex_key key_LOCK_user_locks; - -static PSI_mutex_info all_user_mutexes[]= -{ - { &key_LOCK_user_locks, "LOCK_user_locks", PSI_FLAG_GLOBAL} -}; - -static void init_user_lock_psi_keys(void) -{ - const char* category= "sql"; - int count; - - if (PSI_server == NULL) - return; - - count= array_elements(all_user_mutexes); - PSI_server->register_mutex(category, all_user_mutexes, count); -} -#endif - -static bool item_user_lock_inited= 0; + else + connection_name= thd->variables.default_master_connection; -void item_user_lock_init(void) -{ -#ifdef HAVE_PSI_INTERFACE - init_user_lock_psi_keys(); + if (!(mi= master_info_index->get_master_info(&connection_name, + MYSQL_ERROR::WARN_LEVEL_WARN))) + goto err; + if ((event_count = mi->rli.wait_for_pos(thd, log_name, pos, timeout)) == -2) + { + null_value = 1; + event_count=0; + } #endif + return event_count; - mysql_mutex_init(key_LOCK_user_locks, &LOCK_user_locks, MY_MUTEX_INIT_SLOW); - my_hash_init(&hash_user_locks,system_charset_info, - 16,0,0,(my_hash_get_key) ull_get_key,NULL,0); - item_user_lock_inited= 1; -} - -void item_user_lock_free(void) -{ - if (item_user_lock_inited) +#ifdef HAVE_REPLICATION +err: { - item_user_lock_inited= 0; - my_hash_free(&hash_user_locks); - mysql_mutex_destroy(&LOCK_user_locks); + null_value = 1; + return 0; } +#endif } -void item_user_lock_release(User_level_lock *ull) -{ - ull->locked=0; - ull->thread_id= 0; - if (--ull->count) - mysql_cond_signal(&ull->cond); - else - delete ull; -} - -/** - Wait until we are at or past the given position in the master binlog - on the slave. -*/ -longlong Item_master_pos_wait::val_int() +longlong Item_master_gtid_wait::val_int() { DBUG_ASSERT(fixed == 1); - THD* thd = current_thd; - String *log_name = args[0]->val_str(&value); - int event_count= 0; + longlong result= 0; - null_value=0; - if (thd->slave_thread || !log_name || !log_name->length()) + if (args[0]->null_value) { - null_value = 1; + null_value= 1; return 0; } + + null_value=0; #ifdef HAVE_REPLICATION - longlong pos = (ulong)args[1]->val_int(); - longlong timeout = (arg_count==3) ? args[2]->val_int() : 0 ; - if ((event_count = active_mi->rli.wait_for_pos(thd, log_name, pos, timeout)) == -2) - { - null_value = 1; - event_count=0; - } + THD* thd= current_thd; + longlong timeout_us; + String *gtid_pos = args[0]->val_str(&value); + + if (arg_count==2 && !args[1]->null_value) + timeout_us= (longlong)(1e6*args[1]->val_real()); + else + timeout_us= (longlong)-1; + + result= rpl_global_gtid_waiting.wait_for_pos(thd, gtid_pos, timeout_us); #endif - return event_count; + return result; } @@ -4115,7 +4061,7 @@ class Interruptible_wait /** Time to wait before polling the connection status. */ -const ulonglong Interruptible_wait::m_interrupt_interval= 5 * ULL(1000000000); +const ulonglong Interruptible_wait::m_interrupt_interval= 5 * 1000000000ULL; /** @@ -4160,7 +4106,136 @@ int Interruptible_wait::wait(mysql_cond_t *cond, mysql_mutex_t *mutex) /** - Get a user level lock. If the thread has an old lock this is first released. + For locks with EXPLICIT duration, MDL returns a new ticket + every time a lock is granted. This allows to implement recursive + locks without extra allocation or additional data structures, such + as below. However, if there are too many tickets in the same + MDL_context, MDL_context::find_ticket() is getting too slow, + since it's using a linear search. + This is why a separate structure is allocated for a user + level lock, and before requesting a new lock from MDL, + GET_LOCK() checks thd->ull_hash if such lock is already granted, + and if so, simply increments a reference counter. +*/ + +class User_level_lock +{ +public: + MDL_ticket *lock; + int refs; +}; + + +/** Extract a hash key from User_level_lock. */ + +uchar *ull_get_key(const uchar *ptr, size_t *length, + my_bool not_used __attribute__((unused))) +{ + User_level_lock *ull = (User_level_lock*) ptr; + MDL_key *key = ull->lock->get_key(); + *length= key->length(); + return (uchar*) key->ptr(); +} + + +/** + Release all user level locks for this THD. +*/ + +void mysql_ull_cleanup(THD *thd) +{ + User_level_lock *ull; + DBUG_ENTER("mysql_ull_cleanup"); + + for (uint i= 0; i < thd->ull_hash.records; i++) + { + ull = (User_level_lock*) my_hash_element(&thd->ull_hash, i); + thd->mdl_context.release_lock(ull->lock); + my_free(ull); + } + + my_hash_free(&thd->ull_hash); + + DBUG_VOID_RETURN; +} + + +/** + Set explicit duration for metadata locks corresponding to + user level locks to protect them from being released at the end + of transaction. +*/ + +void mysql_ull_set_explicit_lock_duration(THD *thd) +{ + User_level_lock *ull; + DBUG_ENTER("mysql_ull_set_explicit_lock_duration"); + + for (uint i= 0; i < thd->ull_hash.records; i++) + { + ull= (User_level_lock*) my_hash_element(&thd->ull_hash, i); + thd->mdl_context.set_lock_duration(ull->lock, MDL_EXPLICIT); + } + DBUG_VOID_RETURN; +} + + +/** + When MDL detects a lock wait timeout, it pushes + an error into the statement diagnostics area. + For GET_LOCK(), lock wait timeout is not an error, + but a special return value (0). NULL is returned in + case of error. + Capture and suppress lock wait timeout. +*/ + +class Lock_wait_timeout_handler: public Internal_error_handler +{ +public: + Lock_wait_timeout_handler() :m_lock_wait_timeout(false) {} + + bool m_lock_wait_timeout; + + bool handle_condition(THD * /* thd */, uint sql_errno, + const char * /* sqlstate */, + MYSQL_ERROR::enum_warning_level /* level */, + const char *message, + MYSQL_ERROR ** /* cond_hdl */); +}; + +bool +Lock_wait_timeout_handler:: +handle_condition(THD * /* thd */, uint sql_errno, + const char * /* sqlstate */, + MYSQL_ERROR::enum_warning_level /* level */, + const char *message, + MYSQL_ERROR ** /* cond_hdl */) +{ + if (sql_errno == ER_LOCK_WAIT_TIMEOUT) + { + m_lock_wait_timeout= true; + return true; /* condition handled */ + } + return false; +} + + +static int ull_name_ok(String *name) +{ + if (!name || !name->length()) + return 0; + + if (name->length() > NAME_LEN) + { + my_error(ER_TOO_LONG_IDENT, MYF(0), name->c_ptr_safe()); + return 0; + } + return 1; +} + + +/** + Get a user level lock. @retval 1 : Got lock @@ -4173,14 +4248,13 @@ int Interruptible_wait::wait(mysql_cond_t *cond, mysql_mutex_t *mutex) longlong Item_func_get_lock::val_int() { DBUG_ASSERT(fixed == 1); - String *res=args[0]->val_str(&value); + String *res= args[0]->val_str(&value); ulonglong timeout= args[1]->val_int(); - THD *thd=current_thd; + THD *thd= current_thd; User_level_lock *ull; - int error; - Interruptible_wait timed_cond(thd); DBUG_ENTER("Item_func_get_lock::val_int"); + null_value= 1; /* In slave thread no need to get locks, everything is serialized. Anyway there is no way to make GET_LOCK() work on slave like it did on master @@ -4189,104 +4263,69 @@ longlong Item_func_get_lock::val_int() it's not guaranteed to be same as on master. */ if (thd->slave_thread) + { + null_value= 0; DBUG_RETURN(1); + } - mysql_mutex_lock(&LOCK_user_locks); - - if (!res || !res->length()) + if (!ull_name_ok(res)) + DBUG_RETURN(0); + DBUG_PRINT("enter", ("lock: %.*s", res->length(), res->ptr())); + /* HASH entries are of type User_level_lock. */ + if (! my_hash_inited(&thd->ull_hash) && + my_hash_init(&thd->ull_hash, &my_charset_bin, + 16 /* small hash */, 0, 0, ull_get_key, NULL, 0)) { - mysql_mutex_unlock(&LOCK_user_locks); - null_value=1; DBUG_RETURN(0); } - DBUG_PRINT("info", ("lock %.*s, thd=%ld", res->length(), res->ptr(), - (long) thd->real_id)); - null_value=0; - if (thd->ull) - { - item_user_lock_release(thd->ull); - thd->ull=0; - } + MDL_request ull_request; + ull_request.init(MDL_key::USER_LOCK, res->c_ptr_safe(), "", + MDL_SHARED_NO_WRITE, MDL_EXPLICIT); + MDL_key *ull_key = &ull_request.key; + - if (!(ull= ((User_level_lock *) my_hash_search(&hash_user_locks, - (uchar*) res->ptr(), - (size_t) res->length())))) + if ((ull= (User_level_lock*) + my_hash_search(&thd->ull_hash, ull_key->ptr(), ull_key->length()))) { - ull= new User_level_lock((uchar*) res->ptr(), (size_t) res->length(), - thd->thread_id); - if (!ull || !ull->initialized()) - { - delete ull; - mysql_mutex_unlock(&LOCK_user_locks); - null_value=1; // Probably out of memory - DBUG_RETURN(0); - } - ull->set_thread(thd); - thd->ull=ull; - mysql_mutex_unlock(&LOCK_user_locks); - DBUG_PRINT("info", ("made new lock")); - DBUG_RETURN(1); // Got new lock + /* Recursive lock */ + ull->refs++; + null_value = 0; + DBUG_PRINT("info", ("recursive lock, ref-count: %d", (int) ull->refs)); + DBUG_RETURN(1); } - ull->count++; - DBUG_PRINT("info", ("ull->count=%d", ull->count)); - - /* - Structure is now initialized. Try to get the lock. - Set up control struct to allow others to abort locks. - */ - thd_proc_info(thd, "User lock"); - thd->mysys_var->current_mutex= &LOCK_user_locks; - thd->mysys_var->current_cond= &ull->cond; - timed_cond.set_timeout(timeout * ULL(1000000000)); - - error= 0; - thd_wait_begin(thd, THD_WAIT_USER_LOCK); - while (ull->locked && !thd->killed) + Lock_wait_timeout_handler lock_wait_timeout_handler; + thd->push_internal_handler(&lock_wait_timeout_handler); + bool error= thd->mdl_context.acquire_lock(&ull_request, timeout); + (void) thd->pop_internal_handler(); + if (error) { - DBUG_PRINT("info", ("waiting on lock")); - error= timed_cond.wait(&ull->cond, &LOCK_user_locks); - if (error == ETIMEDOUT || error == ETIME) - { - DBUG_PRINT("info", ("lock wait timeout")); - break; - } - error= 0; + if (lock_wait_timeout_handler.m_lock_wait_timeout) + null_value= 0; + DBUG_RETURN(0); } - thd_wait_end(thd); - if (ull->locked) + ull= (User_level_lock*) my_malloc(sizeof(User_level_lock), + MYF(MY_WME|MY_THREAD_SPECIFIC)); + if (ull == NULL) { - if (!--ull->count) - { - DBUG_ASSERT(0); - delete ull; // Should never happen - } - if (!error) // Killed (thd->killed != 0) - { - error=1; - null_value=1; // Return NULL - } + thd->mdl_context.release_lock(ull_request.ticket); + DBUG_RETURN(0); } - else // We got the lock + + ull->lock= ull_request.ticket; + ull->refs= 1; + + if (my_hash_insert(&thd->ull_hash, (uchar*) ull)) { - ull->locked=1; - ull->set_thread(thd); - ull->thread_id= thd->thread_id; - thd->ull=ull; - error=0; - DBUG_PRINT("info", ("got the lock")); + thd->mdl_context.release_lock(ull->lock); + my_free(ull); + DBUG_RETURN(0); } - mysql_mutex_unlock(&LOCK_user_locks); - - mysql_mutex_lock(&thd->mysys_var->mutex); - thd_proc_info(thd, 0); - thd->mysys_var->current_mutex= 0; - thd->mysys_var->current_cond= 0; - mysql_mutex_unlock(&thd->mysys_var->mutex); + null_value= 0; - DBUG_RETURN(!error ? 1 : 0); + DBUG_RETURN(1); } @@ -4301,43 +4340,87 @@ longlong Item_func_get_lock::val_int() longlong Item_func_release_lock::val_int() { DBUG_ASSERT(fixed == 1); - String *res=args[0]->val_str(&value); - User_level_lock *ull; - longlong result; - THD *thd=current_thd; + String *res= args[0]->val_str(&value); + THD *thd= current_thd; DBUG_ENTER("Item_func_release_lock::val_int"); - if (!res || !res->length()) - { - null_value=1; + null_value= 1; + + if (!ull_name_ok(res)) DBUG_RETURN(0); - } - DBUG_PRINT("info", ("lock %.*s", res->length(), res->ptr())); - null_value=0; - result=0; - mysql_mutex_lock(&LOCK_user_locks); - if (!(ull= ((User_level_lock*) my_hash_search(&hash_user_locks, - (const uchar*) res->ptr(), - (size_t) res->length())))) + DBUG_PRINT("enter", ("lock: %.*s", res->length(), res->ptr())); + + MDL_key ull_key; + ull_key.mdl_key_init(MDL_key::USER_LOCK, res->c_ptr_safe(), ""); + + User_level_lock *ull; + + if (!(ull= + (User_level_lock*) my_hash_search(&thd->ull_hash, + ull_key.ptr(), ull_key.length()))) { - null_value=1; + null_value= thd->mdl_context.get_lock_owner(&ull_key) == 0; + DBUG_RETURN(0); } - else + DBUG_PRINT("info", ("ref count: %d", (int) ull->refs)); + null_value= 0; + if (--ull->refs == 0) { - DBUG_PRINT("info", ("ull->locked=%d ull->thread=%lu thd=%lu", - (int) ull->locked, - (long)ull->thread_id, - (long)thd->thread_id)); - if (ull->locked && current_thd->thread_id == ull->thread_id) - { - DBUG_PRINT("info", ("release lock")); - result=1; // Release is ok - item_user_lock_release(ull); - thd->ull=0; - } + my_hash_delete(&thd->ull_hash, (uchar*) ull); + thd->mdl_context.release_lock(ull->lock); + my_free(ull); } - mysql_mutex_unlock(&LOCK_user_locks); - DBUG_RETURN(result); + DBUG_RETURN(1); +} + + +/** + Check a user level lock. + + Sets null_value=TRUE on error. + + @retval + 1 Available + @retval + 0 Already taken, or error +*/ + +longlong Item_func_is_free_lock::val_int() +{ + DBUG_ASSERT(fixed == 1); + String *res= args[0]->val_str(&value); + THD *thd= current_thd; + null_value= 1; + + if (!ull_name_ok(res)) + return 0; + + MDL_key ull_key; + ull_key.mdl_key_init(MDL_key::USER_LOCK, res->c_ptr_safe(), ""); + + null_value= 0; + return thd->mdl_context.get_lock_owner(&ull_key) == 0; +} + + +longlong Item_func_is_used_lock::val_int() +{ + DBUG_ASSERT(fixed == 1); + String *res= args[0]->val_str(&value); + THD *thd= current_thd; + null_value= 1; + + if (!ull_name_ok(res)) + return 0; + + MDL_key ull_key; + ull_key.mdl_key_init(MDL_key::USER_LOCK, res->c_ptr_safe(), ""); + ulong thread_id = thd->mdl_context.get_lock_owner(&ull_key); + if (thread_id == 0) + return 0; + + null_value= 0; + return thread_id; } @@ -4438,6 +4521,54 @@ void Item_func_benchmark::print(String *str, enum_query_type query_type) } +mysql_mutex_t LOCK_item_func_sleep; + +#ifdef HAVE_PSI_INTERFACE +static PSI_mutex_key key_LOCK_item_func_sleep; + +static PSI_mutex_info item_func_sleep_mutexes[]= +{ + { &key_LOCK_item_func_sleep, "LOCK_user_locks", PSI_FLAG_GLOBAL} +}; + + +static void init_item_func_sleep_psi_keys(void) +{ + const char* category= "sql"; + int count; + + if (PSI_server == NULL) + return; + + count= array_elements(item_func_sleep_mutexes); + PSI_server->register_mutex(category, item_func_sleep_mutexes, count); +} +#endif + +static bool item_func_sleep_inited= 0; + + +void item_func_sleep_init(void) +{ +#ifdef HAVE_PSI_INTERFACE + init_item_func_sleep_psi_keys(); +#endif + + mysql_mutex_init(key_LOCK_item_func_sleep, &LOCK_item_func_sleep, MY_MUTEX_INIT_SLOW); + item_func_sleep_inited= 1; +} + + +void item_func_sleep_free(void) +{ + if (item_func_sleep_inited) + { + item_func_sleep_inited= 0; + mysql_mutex_destroy(&LOCK_item_func_sleep); + } +} + + /** This function is just used to create tests with time gaps. */ longlong Item_func_sleep::val_int() @@ -4466,24 +4597,24 @@ longlong Item_func_sleep::val_int() timed_cond.set_timeout((ulonglong) (timeout * 1000000000.0)); mysql_cond_init(key_item_func_sleep_cond, &cond, NULL); - mysql_mutex_lock(&LOCK_user_locks); + mysql_mutex_lock(&LOCK_item_func_sleep); thd_proc_info(thd, "User sleep"); - thd->mysys_var->current_mutex= &LOCK_user_locks; + thd->mysys_var->current_mutex= &LOCK_item_func_sleep; thd->mysys_var->current_cond= &cond; error= 0; thd_wait_begin(thd, THD_WAIT_SLEEP); while (!thd->killed) { - error= timed_cond.wait(&cond, &LOCK_user_locks); + error= timed_cond.wait(&cond, &LOCK_item_func_sleep); if (error == ETIMEDOUT || error == ETIME) break; error= 0; } thd_wait_end(thd); thd_proc_info(thd, 0); - mysql_mutex_unlock(&LOCK_user_locks); + mysql_mutex_unlock(&LOCK_item_func_sleep); mysql_mutex_lock(&thd->mysys_var->mutex); thd->mysys_var->current_mutex= 0; thd->mysys_var->current_cond= 0; @@ -4497,7 +4628,7 @@ longlong Item_func_sleep::val_int() #define extra_size sizeof(double) -static user_var_entry *get_variable(HASH *hash, LEX_STRING &name, +user_var_entry *get_variable(HASH *hash, LEX_STRING &name, bool create_if_not_exists) { user_var_entry *entry; @@ -4509,7 +4640,9 @@ static user_var_entry *get_variable(HASH *hash, LEX_STRING &name, uint size=ALIGN_SIZE(sizeof(user_var_entry))+name.length+1+extra_size; if (!my_hash_inited(hash)) return 0; - if (!(entry = (user_var_entry*) my_malloc(size,MYF(MY_WME | ME_FATALERROR)))) + if (!(entry = (user_var_entry*) my_malloc(size, + MYF(MY_WME | ME_FATALERROR | + MY_THREAD_SPECIFIC)))) return 0; entry->name.str=(char*) entry+ ALIGN_SIZE(sizeof(user_var_entry))+ extra_size; @@ -4737,7 +4870,8 @@ update_hash(user_var_entry *entry, bool set_null, void *ptr, uint length, entry->value=0; entry->value= (char*) my_realloc(entry->value, length, MYF(MY_ALLOW_ZERO_PTR | MY_WME | - ME_FATALERROR)); + ME_FATALERROR | + MY_THREAD_SPECIFIC)); if (!entry->value) return 1; } @@ -4816,7 +4950,7 @@ double user_var_entry::val_real(bool *null_value) longlong user_var_entry::val_int(bool *null_value) const { if ((*null_value= (value == 0))) - return LL(0); + return 0; switch (type) { case REAL_RESULT: @@ -4840,7 +4974,7 @@ longlong user_var_entry::val_int(bool *null_value) const DBUG_ASSERT(0); // Impossible break; } - return LL(0); // Impossible + return 0; // Impossible } @@ -5333,7 +5467,7 @@ longlong Item_func_get_user_var::val_int() { DBUG_ASSERT(fixed == 1); if (!var_entry) - return LL(0); // No such variable + return 0; // No such variable return (var_entry->val_int(&null_value)); } @@ -5815,28 +5949,18 @@ enum_field_types Item_func_get_system_var::field_type() const } -/* - Uses var, var_type, component, cache_present, used_query_id, thd, - cached_llval, null_value, cached_null_value -*/ -#define get_sys_var_safe(type) \ -do { \ - type value; \ - mysql_mutex_lock(&LOCK_global_system_variables); \ - value= *(type*) var->value_ptr(thd, var_type, &component); \ - mysql_mutex_unlock(&LOCK_global_system_variables); \ - cache_present |= GET_SYS_VAR_CACHE_LONG; \ - used_query_id= thd->query_id; \ - cached_llval= null_value ? 0 : (longlong) value; \ - cached_null_value= null_value; \ - return cached_llval; \ -} while (0) - - longlong Item_func_get_system_var::val_int() { THD *thd= current_thd; + DBUG_EXECUTE_IF("simulate_non_gtid_aware_master", + { + if (0 == strcmp("gtid_domain_id", var->name.str)) + { + my_error(ER_VAR_CANT_BE_READ, MYF(0), var->name.str); + return 0; + } + }); if (cache_present && thd->query_id == used_query_id) { if (cache_present & GET_SYS_VAR_CACHE_LONG) @@ -5866,51 +5990,11 @@ longlong Item_func_get_system_var::val_int() } } - switch (var->show_type()) - { - case SHOW_SINT: get_sys_var_safe (int); - case SHOW_SLONG: get_sys_var_safe (long); - case SHOW_SLONGLONG:get_sys_var_safe (longlong); - case SHOW_UINT: get_sys_var_safe (uint); - case SHOW_ULONG: get_sys_var_safe (ulong); - case SHOW_ULONGLONG:get_sys_var_safe (ulonglong); - case SHOW_HA_ROWS: get_sys_var_safe (ha_rows); - case SHOW_BOOL: get_sys_var_safe (bool); - case SHOW_MY_BOOL: get_sys_var_safe (my_bool); - case SHOW_DOUBLE: - { - double dval= val_real(); - - used_query_id= thd->query_id; - cached_llval= (longlong) dval; - cache_present|= GET_SYS_VAR_CACHE_LONG; - return cached_llval; - } - case SHOW_CHAR: - case SHOW_CHAR_PTR: - case SHOW_LEX_STRING: - { - String *str_val= val_str(NULL); - - if (str_val && str_val->length()) - cached_llval= longlong_from_string_with_check (system_charset_info, - str_val->c_ptr(), - str_val->c_ptr() + - str_val->length()); - else - { - null_value= TRUE; - cached_llval= 0; - } - - cache_present|= GET_SYS_VAR_CACHE_LONG; - return cached_llval; - } - - default: - my_error(ER_VAR_CANT_BE_READ, MYF(0), var->name.str); - return 0; // keep the compiler happy - } + cached_llval= var->val_int(&null_value, thd, var_type, &component); + cache_present |= GET_SYS_VAR_CACHE_LONG; + used_query_id= thd->query_id; + cached_null_value= null_value; + return cached_llval; } @@ -5943,61 +6027,10 @@ String* Item_func_get_system_var::val_str(String* str) } } - str= &cached_strval; - switch (var->show_type()) - { - case SHOW_CHAR: - case SHOW_CHAR_PTR: - case SHOW_LEX_STRING: - { - mysql_mutex_lock(&LOCK_global_system_variables); - char *cptr= var->show_type() == SHOW_CHAR ? - (char*) var->value_ptr(thd, var_type, &component) : - *(char**) var->value_ptr(thd, var_type, &component); - if (cptr) - { - size_t len= var->show_type() == SHOW_LEX_STRING ? - ((LEX_STRING*)(var->value_ptr(thd, var_type, &component)))->length : - strlen(cptr); - if (str->copy(cptr, len, collation.collation)) - { - null_value= TRUE; - str= NULL; - } - } - else - { - null_value= TRUE; - str= NULL; - } - mysql_mutex_unlock(&LOCK_global_system_variables); - break; - } - - case SHOW_SINT: - case SHOW_SLONG: - case SHOW_SLONGLONG: - case SHOW_UINT: - case SHOW_ULONG: - case SHOW_ULONGLONG: - case SHOW_HA_ROWS: - case SHOW_BOOL: - case SHOW_MY_BOOL: - str->set (val_int(), collation.collation); - break; - case SHOW_DOUBLE: - str->set_real (val_real(), decimals, collation.collation); - break; - - default: - my_error(ER_VAR_CANT_BE_READ, MYF(0), var->name.str); - str= NULL; - break; - } - + str= var->val_str(&cached_strval, thd, var_type, &component); cache_present|= GET_SYS_VAR_CACHE_STRING; used_query_id= thd->query_id; - cached_null_value= null_value; + cached_null_value= null_value= !str; return str; } @@ -6035,58 +6068,11 @@ double Item_func_get_system_var::val_real() } } - switch (var->show_type()) - { - case SHOW_DOUBLE: - mysql_mutex_lock(&LOCK_global_system_variables); - cached_dval= *(double*) var->value_ptr(thd, var_type, &component); - mysql_mutex_unlock(&LOCK_global_system_variables); - used_query_id= thd->query_id; - cached_null_value= null_value; - if (null_value) - cached_dval= 0; - cache_present|= GET_SYS_VAR_CACHE_DOUBLE; - return cached_dval; - case SHOW_CHAR: - case SHOW_LEX_STRING: - case SHOW_CHAR_PTR: - { - mysql_mutex_lock(&LOCK_global_system_variables); - char *cptr= var->show_type() == SHOW_CHAR ? - (char*) var->value_ptr(thd, var_type, &component) : - *(char**) var->value_ptr(thd, var_type, &component); - if (cptr) - cached_dval= double_from_string_with_check (system_charset_info, - cptr, cptr + strlen (cptr)); - else - { - null_value= TRUE; - cached_dval= 0; - } - mysql_mutex_unlock(&LOCK_global_system_variables); - used_query_id= thd->query_id; - cached_null_value= null_value; - cache_present|= GET_SYS_VAR_CACHE_DOUBLE; - return cached_dval; - } - case SHOW_SINT: - case SHOW_SLONG: - case SHOW_SLONGLONG: - case SHOW_UINT: - case SHOW_ULONG: - case SHOW_ULONGLONG: - case SHOW_HA_ROWS: - case SHOW_BOOL: - case SHOW_MY_BOOL: - cached_dval= (double) val_int(); - cache_present|= GET_SYS_VAR_CACHE_DOUBLE; - used_query_id= thd->query_id; - cached_null_value= null_value; - return cached_dval; - default: - my_error(ER_VAR_CANT_BE_READ, MYF(0), var->name.str); - return 0; - } + cached_dval= var->val_real(&null_value, thd, var_type, &component); + cache_present |= GET_SYS_VAR_CACHE_DOUBLE; + used_query_id= thd->query_id; + cached_null_value= null_value; + return cached_dval; } @@ -6291,7 +6277,7 @@ bool Item_func_match::fix_fields(THD *thd, Item **ref) table=((Item_field *)item)->field->table; if (!(table->file->ha_table_flags() & HA_CAN_FULLTEXT)) { - my_error(ER_TABLE_CANT_HANDLE_FT, MYF(0)); + my_error(ER_TABLE_CANT_HANDLE_FT, MYF(0), table->file->table_type()); return 1; } table->fulltext_searched=1; @@ -6519,61 +6505,6 @@ Item *get_system_var(THD *thd, enum_var_type var_type, LEX_STRING name, } -/** - Check a user level lock. - - Sets null_value=TRUE on error. - - @retval - 1 Available - @retval - 0 Already taken, or error -*/ - -longlong Item_func_is_free_lock::val_int() -{ - DBUG_ASSERT(fixed == 1); - String *res=args[0]->val_str(&value); - User_level_lock *ull; - - null_value=0; - if (!res || !res->length()) - { - null_value=1; - return 0; - } - - mysql_mutex_lock(&LOCK_user_locks); - ull= (User_level_lock *) my_hash_search(&hash_user_locks, (uchar*) res->ptr(), - (size_t) res->length()); - mysql_mutex_unlock(&LOCK_user_locks); - if (!ull || !ull->locked) - return 1; - return 0; -} - -longlong Item_func_is_used_lock::val_int() -{ - DBUG_ASSERT(fixed == 1); - String *res=args[0]->val_str(&value); - User_level_lock *ull; - - null_value=1; - if (!res || !res->length()) - return 0; - - mysql_mutex_lock(&LOCK_user_locks); - ull= (User_level_lock *) my_hash_search(&hash_user_locks, (uchar*) res->ptr(), - (size_t) res->length()); - mysql_mutex_unlock(&LOCK_user_locks); - if (!ull || !ull->locked) - return 0; - - null_value=0; - return ull->thread_id; -} - - longlong Item_func_row_count::val_int() { DBUG_ASSERT(fixed == 1); @@ -7024,7 +6955,7 @@ ulonglong uuid_value; void uuid_short_init() { - uuid_value= ((((ulonglong) server_id) << 56) + + uuid_value= ((((ulonglong) global_system_variables.server_id) << 56) + (((ulonglong) server_start_time) << 24)); } diff --git a/sql/item_func.h b/sql/item_func.h index 0d9901d90b2..3c15ca0094d 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -41,7 +41,14 @@ protected: uint allowed_arg_cols; public: uint arg_count; - table_map used_tables_cache, not_null_tables_cache; + /* + In some cases used_tables_cache is not what used_tables() return + so the method should be used where one need used tables bit map + (even internally in Item_func_* code). + */ + table_map used_tables_cache; + table_map not_null_tables_cache; + bool const_item_cache; enum Functype { UNKNOWN_FUNC,EQ_FUNC,EQUAL_FUNC,NE_FUNC,LT_FUNC,LE_FUNC, GE_FUNC,GT_FUNC,FT_FUNC, @@ -58,7 +65,7 @@ public: NOW_FUNC, TRIG_COND_FUNC, SUSERVAR_FUNC, GUSERVAR_FUNC, COLLATE_FUNC, EXTRACT_FUNC, CHAR_TYPECAST_FUNC, FUNC_SP, UDF_FUNC, - NEG_FUNC, GSYSVAR_FUNC }; + NEG_FUNC, GSYSVAR_FUNC, DYNCOL_FUNC }; enum optimize_type { OPTIMIZE_NONE,OPTIMIZE_KEY,OPTIMIZE_OP, OPTIMIZE_NULL, OPTIMIZE_EQUAL }; enum Type type() const { return FUNC_ITEM; } @@ -582,6 +589,8 @@ public: { collation.set_numeric(); fix_char_length(21); sargable= false; } Item_int_func(Item *a,Item *b,Item *c) :Item_func(a,b,c) { collation.set_numeric(); fix_char_length(21); sargable= false; } + Item_int_func(Item *a,Item *b,Item *c, Item *d) :Item_func(a,b,c,d) + { collation.set_numeric(); fix_char_length(21); sargable= false; } Item_int_func(List<Item> &list) :Item_func(list) { collation.set_numeric(); fix_char_length(21); sargable= false; } Item_int_func(THD *thd, Item_int_func *item) :Item_func(thd, item) @@ -1319,17 +1328,20 @@ public: }; +void item_func_sleep_init(void); +void item_func_sleep_free(void); + class Item_func_sleep :public Item_int_func { public: Item_func_sleep(Item *a) :Item_int_func(a) {} bool const_item() const { return 0; } const char *func_name() const { return "sleep"; } - void update_used_tables() + table_map used_tables() const { - Item_int_func::update_used_tables(); - used_tables_cache|= RAND_TABLE_BIT; + return Item_int_func::used_tables() | RAND_TABLE_BIT; } + bool is_expensive() { return 1; } longlong val_int(); bool check_vcol_func_processor(uchar *int_arg) { @@ -1568,14 +1580,8 @@ public: #endif /* HAVE_DLOPEN */ -/* -** User level locks -*/ - -class User_level_lock; -void item_user_lock_init(void); -void item_user_lock_release(User_level_lock *ull); -void item_user_lock_free(void); +void mysql_ull_cleanup(THD *thd); +void mysql_ull_set_explicit_lock_duration(THD *thd); class Item_func_get_lock :public Item_int_func { @@ -1585,6 +1591,12 @@ class Item_func_get_lock :public Item_int_func longlong val_int(); const char *func_name() const { return "get_lock"; } void fix_length_and_dec() { max_length=1; maybe_null=1;} + table_map used_tables() const + { + return Item_int_func::used_tables() | RAND_TABLE_BIT; + } + bool const_item() const { return 0; } + bool is_expensive() { return 1; } bool check_vcol_func_processor(uchar *int_arg) { return trace_unsupported_by_check_vcol_func_processor(func_name()); @@ -1598,7 +1610,13 @@ public: Item_func_release_lock(Item *a) :Item_int_func(a) {} longlong val_int(); const char *func_name() const { return "release_lock"; } - void fix_length_and_dec() { max_length=1; maybe_null=1;} + void fix_length_and_dec() { max_length= 1; maybe_null= 1;} + table_map used_tables() const + { + return Item_int_func::used_tables() | RAND_TABLE_BIT; + } + bool const_item() const { return 0; } + bool is_expensive() { return 1; } bool check_vcol_func_processor(uchar *int_arg) { return trace_unsupported_by_check_vcol_func_processor(func_name()); @@ -1613,6 +1631,7 @@ class Item_master_pos_wait :public Item_int_func public: Item_master_pos_wait(Item *a,Item *b) :Item_int_func(a,b) {} Item_master_pos_wait(Item *a,Item *b,Item *c) :Item_int_func(a,b,c) {} + Item_master_pos_wait(Item *a,Item *b, Item *c, Item *d) :Item_int_func(a,b,c,d) {} longlong val_int(); const char *func_name() const { return "master_pos_wait"; } void fix_length_and_dec() { max_length=21; maybe_null=1;} @@ -1623,6 +1642,22 @@ public: }; +class Item_master_gtid_wait :public Item_int_func +{ + String value; +public: + Item_master_gtid_wait(Item *a) :Item_int_func(a) {} + Item_master_gtid_wait(Item *a,Item *b) :Item_int_func(a,b) {} + longlong val_int(); + const char *func_name() const { return "master_gtid_wait"; } + void fix_length_and_dec() { max_length=10+1+10+1+20+1; maybe_null=0;} + bool check_vcol_func_processor(uchar *int_arg) + { + return trace_unsupported_by_check_vcol_func_processor(func_name()); + } +}; + + /* Handling of user definable variables */ class user_var_entry; @@ -1689,6 +1724,12 @@ public: enum Item_result result_type () const { return cached_result_type; } bool fix_fields(THD *thd, Item **ref); void fix_length_and_dec(); + table_map used_tables() const + { + return Item_func::used_tables() | RAND_TABLE_BIT; + } + bool const_item() const { return 0; } + bool is_expensive() { return 1; } virtual void print(String *str, enum_query_type query_type); void print_as_stmt(String *str, enum_query_type query_type); const char *func_name() const { return "set_user_var"; } diff --git a/sql/item_row.cc b/sql/item_row.cc index ee1d17213ee..6345eaa864b 100644 --- a/sql/item_row.cc +++ b/sql/item_row.cc @@ -47,13 +47,13 @@ Item_row::Item_row(List<Item> &arg): items= (Item**) sql_alloc(sizeof(Item*)*arg_count); else items= 0; - List_iterator<Item> li(arg); + List_iterator_fast<Item> li(arg); uint i= 0; Item *item; while ((item= li++)) { items[i]= item; - i++; + i++; } } diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 469e8d2ac1c..09a976b5edd 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -58,8 +58,11 @@ C_MODE_START #include "../mysys/my_static.h" // For soundex_map C_MODE_END +#include "sql_show.h" // append_identifier +#include <sql_repl.h> +#include "sql_statistics.h" -size_t username_char_length= 16; +size_t username_char_length= 80; /* For the Items which have only val_str_ascii() method @@ -466,6 +469,82 @@ void Item_func_aes_decrypt::fix_length_and_dec() maybe_null= 1; } +/////////////////////////////////////////////////////////////////////////////// + + +const char *histogram_types[] = + {"SINGLE_PREC_HB", "DOUBLE_PREC_HB", 0}; +static TYPELIB hystorgam_types_typelib= + { array_elements(histogram_types), + "histogram_types", + histogram_types, NULL}; +const char *representation_by_type[]= {"%.3f", "%.5f"}; + +String *Item_func_decode_histogram::val_str(String *str) +{ + DBUG_ASSERT(fixed == 1); + char buff[STRING_BUFFER_USUAL_SIZE]; + String *res, tmp(buff, sizeof(buff), &my_charset_bin); + int type; + + tmp.length(0); + if (!(res= args[1]->val_str(&tmp)) || + (type= find_type(res->c_ptr_safe(), + &hystorgam_types_typelib, MYF(0))) <= 0) + { + null_value= 1; + return 0; + } + type--; + + tmp.length(0); + if (!(res= args[0]->val_str(&tmp))) + { + null_value= 1; + return 0; + } + if (type == DOUBLE_PREC_HB && res->length() % 2 != 0) + res->length(res->length() - 1); // one byte is unused + + double prev= 0.0; + uint i; + str->length(0); + char numbuf[32]; + const uchar *p= (uchar*)res->c_ptr(); + for (i= 0; i < res->length(); i++) + { + double val; + switch (type) + { + case SINGLE_PREC_HB: + val= p[i] / ((double)((1 << 8) - 1)); + break; + case DOUBLE_PREC_HB: + val= ((uint16 *)(p + i))[0] / ((double)((1 << 16) - 1)); + i++; + break; + default: + val= 0; + DBUG_ASSERT(0); + } + /* show delta with previous value */ + int size= my_snprintf(numbuf, sizeof(numbuf), + representation_by_type[type], val - prev); + str->append(numbuf, size); + str->append(","); + prev= val; + } + /* show delta with max */ + int size= my_snprintf(numbuf, sizeof(numbuf), + representation_by_type[type], 1.0 - prev); + str->append(numbuf, size); + + null_value=0; + return str; +} + + +/////////////////////////////////////////////////////////////////////////////// /** Concatenate args with the following premises: @@ -1176,6 +1255,187 @@ void Item_func_replace::fix_length_and_dec() } +/*********************************************************************/ +void Item_func_regexp_replace::fix_length_and_dec() +{ + if (agg_arg_charsets_for_string_result_with_comparison(collation, args, 3)) + return; + max_length= MAX_BLOB_WIDTH; + re.init(collation.collation, 0, 10); + re.fix_owner(this, args[0], args[1]); +} + + +/* + Traverse through the replacement string and append to "str". + Sub-pattern references \0 .. \9 are recognized, which are replaced + to the chunks of the source string. +*/ +bool Item_func_regexp_replace::append_replacement(String *str, + const LEX_CSTRING *source, + const LEX_CSTRING *replace) +{ + const char *beg= replace->str; + const char *end= beg + replace->length; + CHARSET_INFO *cs= re.library_charset(); + + for ( ; ; ) + { + my_wc_t wc; + int cnv, n; + + if ((cnv= cs->cset->mb_wc(cs, &wc, (const uchar *) beg, + (const uchar *) end)) < 1) + break; /* End of line */ + beg+= cnv; + + if (wc != '\\') + { + if (str->append(beg - cnv, cnv, cs)) + return true; + continue; + } + + if ((cnv= cs->cset->mb_wc(cs, &wc, (const uchar *) beg, + (const uchar *) end)) < 1) + break; /* End of line */ + beg+= cnv; + + if ((n= ((int) wc) - '0') >= 0 && n <= 9 && n < re.nsubpatterns()) + { + /* A valid sub-pattern reference found */ + int pbeg= re.subpattern_start(n), plength= re.subpattern_end(n) - pbeg; + if (str->append(source->str + pbeg, plength, cs)) + return true; + } + else + { + /* + A non-digit character following after '\'. + Just add the character itself. + */ + if (str->append(beg - cnv, cnv, cs)) + return false; + } + } + return false; +} + + +String *Item_func_regexp_replace::val_str(String *str) +{ + DBUG_ASSERT(fixed == 1); + char buff0[MAX_FIELD_WIDTH]; + char buff2[MAX_FIELD_WIDTH]; + String tmp0(buff0,sizeof(buff0),&my_charset_bin); + String tmp2(buff2,sizeof(buff2),&my_charset_bin); + String *source= args[0]->val_str(&tmp0); + String *replace= args[2]->val_str(&tmp2); + LEX_CSTRING src, rpl; + int startoffset= 0; + + if ((null_value= (args[0]->null_value || args[2]->null_value || + re.recompile(args[1])))) + return (String *) 0; + + if (!(source= re.convert_if_needed(source, &re.subject_converter)) || + !(replace= re.convert_if_needed(replace, &re.replace_converter))) + goto err; + + src= source->lex_cstring(); + rpl= replace->lex_cstring(); + + str->length(0); + str->set_charset(collation.collation); + + for ( ; ; ) // Iterate through all matches + { + + if (re.exec(src.str, src.length, startoffset)) + goto err; + + if (!re.match() || re.subpattern_length(0) == 0) + { + /* + No match or an empty match. + Append the rest of the source string + starting from startoffset until the end of the source. + */ + if (str->append(src.str + startoffset, src.length - startoffset, re.library_charset())) + goto err; + return str; + } + + /* + Append prefix, the part before the matching pattern. + starting from startoffset until the next match + */ + if (str->append(src.str + startoffset, re.subpattern_start(0) - startoffset, re.library_charset())) + goto err; + + // Append replacement + if (append_replacement(str, &src, &rpl)) + goto err; + + // Set the new start point as the end of previous match + startoffset= re.subpattern_end(0); + } + return str; + +err: + null_value= true; + return (String *) 0; +} + + +void Item_func_regexp_substr::fix_length_and_dec() +{ + if (agg_arg_charsets_for_string_result_with_comparison(collation, args, 2)) + return; + fix_char_length(args[0]->max_char_length()); + re.init(collation.collation, 0, 10); + re.fix_owner(this, args[0], args[1]); +} + + +String *Item_func_regexp_substr::val_str(String *str) +{ + DBUG_ASSERT(fixed == 1); + char buff0[MAX_FIELD_WIDTH]; + String tmp0(buff0,sizeof(buff0),&my_charset_bin); + String *source= args[0]->val_str(&tmp0); + + if ((null_value= (args[0]->null_value || re.recompile(args[1])))) + return (String *) 0; + + if (!(source= re.convert_if_needed(source, &re.subject_converter))) + goto err; + + str->length(0); + str->set_charset(collation.collation); + + if (re.exec(source->ptr(), source->length(), 0)) + goto err; + + if (!re.match()) + return str; + + if (str->append(source->ptr() + re.subpattern_start(0), + re.subpattern_end(0) - re.subpattern_start(0), + re.library_charset())) + goto err; + + return str; + +err: + null_value= true; + return (String *) 0; +} + + +/************************************************************************/ + + String *Item_func_insert::val_str(String *str) { DBUG_ASSERT(fixed == 1); @@ -2086,16 +2346,32 @@ bool Item_func_current_user::fix_fields(THD *thd, Item **ref) if (Item_func_sysconst::fix_fields(thd, ref)) return TRUE; - Security_context *ctx= -#ifndef NO_EMBEDDED_ACCESS_CHECKS - (context->security_ctx - ? context->security_ctx : thd->security_ctx); -#else - thd->security_ctx; -#endif /*NO_EMBEDDED_ACCESS_CHECKS*/ + Security_context *ctx= context->security_ctx + ? context->security_ctx : thd->security_ctx; return init(ctx->priv_user, ctx->priv_host); } +bool Item_func_current_role::fix_fields(THD *thd, Item **ref) +{ + if (Item_func_sysconst::fix_fields(thd, ref)) + return 1; + + Security_context *ctx= context->security_ctx + ? context->security_ctx : thd->security_ctx; + + if (ctx->priv_role[0]) + { + if (str_value.copy(ctx->priv_role, strlen(ctx->priv_role), + system_charset_info)) + return 1; + + str_value.mark_as_const(); + return 0; + } + null_value= maybe_null= 1; + return 0; +} + void Item_func_soundex::fix_length_and_dec() { @@ -2683,6 +2959,46 @@ err: } +void Item_func_binlog_gtid_pos::fix_length_and_dec() +{ + collation.set(system_charset_info); + max_length= MAX_BLOB_WIDTH; + maybe_null= 1; +} + + +String *Item_func_binlog_gtid_pos::val_str(String *str) +{ + DBUG_ASSERT(fixed == 1); +#ifndef HAVE_REPLICATION + null_value= 0; + str->copy("", 0, system_charset_info); + return str; +#else + String name_str, *name; + longlong pos; + + if (args[0]->null_value || args[1]->null_value) + goto err; + + name= args[0]->val_str(&name_str); + pos= args[1]->val_int(); + + if (pos < 0 || pos > UINT_MAX32) + goto err; + + if (gtid_state_from_binlog_pos(name->c_ptr_safe(), (uint32)pos, str)) + goto err; + null_value= 0; + return str; + +err: + null_value= 1; + return NULL; +#endif /* !HAVE_REPLICATION */ +} + + void Item_func_rpad::fix_length_and_dec() { // Handle character set for args[0] and args[2]. @@ -3377,7 +3693,7 @@ String* Item_func_inet_ntoa::val_str(String* str) Also return null if n > 255.255.255.255 */ - if ((null_value= (args[0]->null_value || n > (ulonglong) LL(4294967295)))) + if ((null_value= (args[0]->null_value || n > 0xffffffff))) return 0; // Null value str->set_charset(collation.collation); @@ -3749,7 +4065,8 @@ String *Item_func_uuid::val_str(String *str) Item_func_dyncol_create::Item_func_dyncol_create(List<Item> &args, DYNCALL_CREATE_DEF *dfs) - : Item_str_func(args), defs(dfs), vals(0), nums(0) + : Item_str_func(args), defs(dfs), vals(0), keys_num(NULL), keys_str(NULL), + names(FALSE), force_names(FALSE) { DBUG_ASSERT((args.elements & 0x1) == 0); // even number of arguments } @@ -3757,14 +4074,31 @@ Item_func_dyncol_create::Item_func_dyncol_create(List<Item> &args, bool Item_func_dyncol_create::fix_fields(THD *thd, Item **ref) { + uint i; bool res= Item_func::fix_fields(thd, ref); // no need Item_str_func here - vals= (DYNAMIC_COLUMN_VALUE *) alloc_root(thd->mem_root, - sizeof(DYNAMIC_COLUMN_VALUE) * - (arg_count / 2)); - nums= (uint *) alloc_root(thd->mem_root, - sizeof(uint) * (arg_count / 2)); - status_var_increment(thd->status_var.feature_dynamic_columns); - return res || vals == 0 || nums == 0; + if (!res) + { + vals= (DYNAMIC_COLUMN_VALUE *) alloc_root(thd->mem_root, + sizeof(DYNAMIC_COLUMN_VALUE) * + (arg_count / 2)); + for (i= 0; + i + 1 < arg_count && args[i]->result_type() == INT_RESULT; + i+= 2) + ; + if (i + 1 < arg_count) + { + names= TRUE; + } + + keys_num= (uint *) alloc_root(thd->mem_root, + (sizeof(LEX_STRING) > sizeof(uint) ? + sizeof(LEX_STRING) : + sizeof(uint)) * + (arg_count / 2)); + keys_str= (LEX_STRING *) keys_num; + status_var_increment(thd->status_var.feature_dynamic_columns); + } + return res || vals == 0 || keys_num == 0; } @@ -3775,13 +4109,49 @@ void Item_func_dyncol_create::fix_length_and_dec() decimals= 0; } -void Item_func_dyncol_create::prepare_arguments() +bool Item_func_dyncol_create::prepare_arguments(bool force_names_arg) { char buff[STRING_BUFFER_USUAL_SIZE]; String *res, tmp(buff, sizeof(buff), &my_charset_bin); uint column_count= (arg_count / 2); uint i; my_decimal dtmp, *dres; + force_names= force_names_arg; + + if (!(names || force_names)) + { + for (i= 0; i < column_count; i++) + { + uint valpos= i * 2 + 1; + DYNAMIC_COLUMN_TYPE type= defs[i].type; + if (type == DYN_COL_NULL) + switch (args[valpos]->field_type()) + { + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_ENUM: + case MYSQL_TYPE_SET: + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_BLOB: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_STRING: + case MYSQL_TYPE_GEOMETRY: + type= DYN_COL_STRING; + break; + default: + break; + } + + if (type == DYN_COL_STRING && + args[valpos]->type() == Item::FUNC_ITEM && + ((Item_func *)args[valpos])->functype() == DYNCOL_FUNC) + { + force_names= 1; + break; + } + } + } /* get values */ for (i= 0; i < column_count; i++) @@ -3840,7 +4210,59 @@ void Item_func_dyncol_create::prepare_arguments() break; } } - nums[i]= (uint) args[i * 2]->val_int(); + if (type == DYN_COL_STRING && + args[valpos]->type() == Item::FUNC_ITEM && + ((Item_func *)args[valpos])->functype() == DYNCOL_FUNC) + { + DBUG_ASSERT(names || force_names); + type= DYN_COL_DYNCOL; + } + if (names || force_names) + { + res= args[i * 2]->val_str(&tmp); + if (res) + { + // guaranty UTF-8 string for names + if (my_charset_same(res->charset(), &my_charset_utf8_general_ci)) + { + keys_str[i].length= res->length(); + keys_str[i].str= sql_strmake(res->ptr(), res->length()); + } + else + { + uint strlen; + uint dummy_errors; + char *str= + (char *)sql_alloc((strlen= res->length() * + my_charset_utf8_general_ci.mbmaxlen + 1)); + if (str) + { + keys_str[i].length= + copy_and_convert(str, strlen, &my_charset_utf8_general_ci, + res->ptr(), res->length(), res->charset(), + &dummy_errors); + keys_str[i].str= str; + } + else + keys_str[i].length= 0; + + } + } + else + { + keys_str[i].length= 0; + keys_str[i].str= NULL; + } + } + else + keys_num[i]= (uint) args[i * 2]->val_int(); + if (args[i * 2]->null_value) + { + /* to make cleanup possible */ + for (; i < column_count; i++) + vals[i].type= DYN_COL_NULL; + return 1; + } vals[i].type= type; switch (type) { case DYN_COL_NULL: @@ -3855,11 +4277,11 @@ void Item_func_dyncol_create::prepare_arguments() case DYN_COL_DOUBLE: vals[i].x.double_value= args[valpos]->val_real(); break; + case DYN_COL_DYNCOL: case DYN_COL_STRING: res= args[valpos]->val_str(&tmp); if (res && - (vals[i].x.string.value.str= my_strndup(res->ptr(), res->length(), - MYF(MY_WME)))) + (vals[i].x.string.value.str= sql_strmake(res->ptr(), res->length()))) { vals[i].x.string.value.length= res->length(); vals[i].x.string.charset= res->charset(); @@ -3874,7 +4296,7 @@ void Item_func_dyncol_create::prepare_arguments() case DYN_COL_DECIMAL: if ((dres= args[valpos]->val_decimal(&dtmp))) { - dynamic_column_prepare_decimal(&vals[i]); + mariadb_dyncol_prepare_decimal(&vals[i]); DBUG_ASSERT(vals[i].x.decimal.value.len == dres->len); vals[i].x.decimal.value.intg= dres->intg; vals[i].x.decimal.value.frac= dres->frac; @@ -3884,7 +4306,7 @@ void Item_func_dyncol_create::prepare_arguments() } else { - dynamic_column_prepare_decimal(&vals[i]); // just to be safe + mariadb_dyncol_prepare_decimal(&vals[i]); // just to be safe DBUG_ASSERT(args[valpos]->null_value); } break; @@ -3903,24 +4325,12 @@ void Item_func_dyncol_create::prepare_arguments() } if (vals[i].type != DYN_COL_NULL && args[valpos]->null_value) { - if (vals[i].type == DYN_COL_STRING) - my_free(vals[i].x.string.value.str); vals[i].type= DYN_COL_NULL; } } + return FALSE; } -void Item_func_dyncol_create::cleanup_arguments() -{ - uint column_count= (arg_count / 2); - uint i; - - for (i= 0; i < column_count; i++) - { - if (vals[i].type == DYN_COL_STRING) - my_free(vals[i].x.string.value.str); - } -} String *Item_func_dyncol_create::val_str(String *str) { @@ -3930,30 +4340,37 @@ String *Item_func_dyncol_create::val_str(String *str) enum enum_dyncol_func_result rc; DBUG_ASSERT((arg_count & 0x1) == 0); // even number of arguments - prepare_arguments(); - - if ((rc= dynamic_column_create_many(&col, column_count, nums, vals))) + if (prepare_arguments(FALSE)) { - dynamic_column_error_message(rc); - dynamic_column_column_free(&col); res= NULL; - null_value= TRUE; + null_value= 1; } else { - /* Move result from DYNAMIC_COLUMN to str_value */ - char *ptr; - size_t length, alloc_length; - dynamic_column_reassociate(&col, &ptr, &length, &alloc_length); - str_value.reassociate(ptr, (uint32) length, (uint32) alloc_length, - &my_charset_bin); - res= &str_value; - null_value= FALSE; + if ((rc= ((names || force_names) ? + mariadb_dyncol_create_many_named(&col, column_count, keys_str, + vals, TRUE) : + mariadb_dyncol_create_many(&col, column_count, keys_num, + vals, TRUE)))) + { + dynamic_column_error_message(rc); + dynamic_column_column_free(&col); + res= NULL; + null_value= TRUE; + } + else + { + /* Move result from DYNAMIC_COLUMN to str_value */ + char *ptr; + size_t length, alloc_length; + dynstr_reassociate(&col, &ptr, &length, &alloc_length); + str_value.reassociate(ptr, (uint32) length, (uint32) alloc_length, + &my_charset_bin); + res= &str_value; + null_value= FALSE; + } } - /* cleanup */ - cleanup_arguments(); - return res; } @@ -3979,6 +4396,7 @@ void Item_func_dyncol_create::print_arguments(String *str, case DYN_COL_DOUBLE: str->append(STRING_WITH_LEN(" AS double")); break; + case DYN_COL_DYNCOL: case DYN_COL_STRING: str->append(STRING_WITH_LEN(" AS char")); if (defs[i].cs) @@ -4016,6 +4434,40 @@ void Item_func_dyncol_create::print(String *str, str->append(')'); } +String *Item_func_dyncol_json::val_str(String *str) +{ + DYNAMIC_STRING json, col; + String *res; + enum enum_dyncol_func_result rc; + + res= args[0]->val_str(str); + if (args[0]->null_value) + goto null; + + col.str= (char *)res->ptr(); + col.length= res->length(); + if ((rc= mariadb_dyncol_json(&col, &json))) + { + dynamic_column_error_message(rc); + goto null; + } + bzero(&col, sizeof(col)); + { + /* Move result from DYNAMIC_COLUMN to str */ + char *ptr; + size_t length, alloc_length; + dynstr_reassociate(&json, &ptr, &length, &alloc_length); + str->reassociate(ptr, (uint32) length, (uint32) alloc_length, + &my_charset_utf8_general_ci); + null_value= FALSE; + } + return str; + +null: + bzero(&col, sizeof(col)); + null_value= TRUE; + return NULL; +} String *Item_func_dyncol_add::val_str(String *str) { @@ -4027,21 +4479,25 @@ String *Item_func_dyncol_add::val_str(String *str) /* We store the packed data last */ res= args[arg_count - 1]->val_str(str); - if (args[arg_count - 1]->null_value) + if (args[arg_count - 1]->null_value || + init_dynamic_string(&col, NULL, res->length() + STRING_BUFFER_USUAL_SIZE, + STRING_BUFFER_USUAL_SIZE)) goto null; - init_dynamic_string(&col, NULL, res->length() + STRING_BUFFER_USUAL_SIZE, - STRING_BUFFER_USUAL_SIZE); col.length= res->length(); memcpy(col.str, res->ptr(), col.length); - prepare_arguments(); + if (prepare_arguments(mariadb_dyncol_has_names(&col))) + goto null; - if ((rc= dynamic_column_update_many(&col, column_count, nums, vals))) + if ((rc= ((names || force_names) ? + mariadb_dyncol_update_many_named(&col, column_count, + keys_str, vals) : + mariadb_dyncol_update_many(&col, column_count, + keys_num, vals)))) { dynamic_column_error_message(rc); dynamic_column_column_free(&col); - cleanup_arguments(); goto null; } @@ -4049,16 +4505,12 @@ String *Item_func_dyncol_add::val_str(String *str) /* Move result from DYNAMIC_COLUMN to str */ char *ptr; size_t length, alloc_length; - dynamic_column_reassociate(&col, &ptr, &length, &alloc_length); + dynstr_reassociate(&col, &ptr, &length, &alloc_length); str->reassociate(ptr, (uint32) length, (uint32) alloc_length, &my_charset_bin); null_value= FALSE; } - /* cleanup */ - dynamic_column_column_free(&col); - cleanup_arguments(); - return str; null: @@ -4090,10 +4542,48 @@ bool Item_dyncol_get::get_dyn_value(DYNAMIC_COLUMN_VALUE *val, String *tmp) { DYNAMIC_COLUMN dyn_str; String *res; - longlong num; + longlong num= 0; + LEX_STRING buf, *name= NULL; + char nmstrbuf[11]; + String nmbuf(nmstrbuf, sizeof(nmstrbuf), system_charset_info); enum enum_dyncol_func_result rc; - num= args[1]->val_int(); + if (args[1]->result_type() == INT_RESULT) + num= args[1]->val_int(); + else + { + String *nm= args[1]->val_str(&nmbuf); + if (!nm || args[1]->null_value) + { + null_value= 1; + return 1; + } + + if (my_charset_same(nm->charset(), &my_charset_utf8_general_ci)) + { + buf.str= (char *) nm->ptr(); + buf.length= nm->length(); + } + else + { + uint strlen; + uint dummy_errors; + buf.str= (char *)sql_alloc((strlen= nm->length() * + my_charset_utf8_general_ci.mbmaxlen + 1)); + if (buf.str) + { + buf.length= + copy_and_convert(buf.str, strlen, &my_charset_utf8_general_ci, + nm->ptr(), nm->length(), nm->charset(), + &dummy_errors); + } + else + buf.length= 0; + } + name= &buf; + } + + if (args[1]->null_value || num < 0 || num > INT_MAX) { null_value= 1; @@ -4109,7 +4599,9 @@ bool Item_dyncol_get::get_dyn_value(DYNAMIC_COLUMN_VALUE *val, String *tmp) dyn_str.str= (char*) res->ptr(); dyn_str.length= res->length(); - if ((rc= dynamic_column_get(&dyn_str, (uint) num, val))) + if ((rc= ((name == NULL) ? + mariadb_dyncol_get(&dyn_str, (uint) num, val) : + mariadb_dyncol_get_named(&dyn_str, name, val)))) { dynamic_column_error_message(rc); null_value= 1; @@ -4141,6 +4633,7 @@ String *Item_dyncol_get::val_str(String *str_result) case DYN_COL_DOUBLE: str_result->set_real(val.x.double_value, NOT_FIXED_DEC, &my_charset_latin1); break; + case DYN_COL_DYNCOL: case DYN_COL_STRING: if ((char*) tmp.ptr() <= val.x.string.value.str && (char*) tmp.ptr() + tmp.length() >= val.x.string.value.str) @@ -4216,6 +4709,7 @@ longlong Item_dyncol_get::val_int() return 0; switch (val.type) { + case DYN_COL_DYNCOL: case DYN_COL_NULL: goto null; case DYN_COL_UINT: @@ -4296,6 +4790,7 @@ double Item_dyncol_get::val_real() return 0.0; switch (val.type) { + case DYN_COL_DYNCOL: case DYN_COL_NULL: goto null; case DYN_COL_UINT: @@ -4353,6 +4848,7 @@ my_decimal *Item_dyncol_get::val_decimal(my_decimal *decimal_value) return NULL; switch (val.type) { + case DYN_COL_DYNCOL: case DYN_COL_NULL: goto null; case DYN_COL_UINT: @@ -4412,6 +4908,7 @@ bool Item_dyncol_get::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) return 1; // Error switch (val.type) { + case DYN_COL_DYNCOL: case DYN_COL_NULL: goto null; case DYN_COL_INT: @@ -4457,7 +4954,6 @@ null: return 1; } - void Item_dyncol_get::print(String *str, enum_query_type query_type) { /* see create_func_dyncol_get */ @@ -4477,7 +4973,8 @@ String *Item_func_dyncol_list::val_str(String *str) { uint i; enum enum_dyncol_func_result rc; - DYNAMIC_ARRAY arr; + LEX_STRING *names= 0; + uint count; DYNAMIC_COLUMN col; String *res= args[0]->val_str(str); @@ -4486,33 +4983,37 @@ String *Item_func_dyncol_list::val_str(String *str) col.length= res->length(); /* We do not change the string, so could do this trick */ col.str= (char *)res->ptr(); - if ((rc= dynamic_column_list(&col, &arr))) + if ((rc= mariadb_dyncol_list_named(&col, &count, &names))) { + bzero(&col, sizeof(col)); dynamic_column_error_message(rc); - delete_dynamic(&arr); goto null; } + bzero(&col, sizeof(col)); /* - We support elements from 0 - 65536, so max size for one element is - 6 (including ,). + We estimate average name length as 10 */ - if (str->alloc(arr.elements * 6)) + if (str->alloc(count * 13)) goto null; str->length(0); - for (i= 0; i < arr.elements; i++) + str->set_charset(&my_charset_utf8_general_ci); + for (i= 0; i < count; i++) { - str->qs_append(*dynamic_element(&arr, i, uint*)); - if (i < arr.elements - 1) + append_identifier(current_thd, str, names[i].str, names[i].length); + if (i < count - 1) str->qs_append(','); } - null_value= FALSE; - delete_dynamic(&arr); + if (names) + my_free(names); return str; null: null_value= TRUE; + if (names) + my_free(names); return NULL; } + diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 1731fcf7e99..edc4ce5e151 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -145,6 +145,22 @@ public: const char *func_name() const { return "concat"; } }; +class Item_func_decode_histogram :public Item_str_func +{ + String tmp_value; +public: + Item_func_decode_histogram(Item *a, Item *b) + :Item_str_func(a, b) {} + String *val_str(String *); + void fix_length_and_dec() + { + collation.set(system_charset_info); + max_length= MAX_BLOB_WIDTH; + maybe_null= 1; + } + const char *func_name() const { return "decode_histogram"; } +}; + class Item_func_concat_ws :public Item_str_func { String tmp_value; @@ -179,6 +195,49 @@ public: }; +class Item_func_regexp_replace :public Item_str_func +{ + Regexp_processor_pcre re; + bool append_replacement(String *str, + const LEX_CSTRING *source, + const LEX_CSTRING *replace); +public: + Item_func_regexp_replace(Item *a, Item *b, Item *c) + :Item_str_func(a, b, c) + {} + void cleanup() + { + DBUG_ENTER("Item_func_regex::cleanup"); + Item_str_func::cleanup(); + re.cleanup(); + DBUG_VOID_RETURN; + } + String *val_str(String *str); + void fix_length_and_dec(); + const char *func_name() const { return "regexp_replace"; } +}; + + +class Item_func_regexp_substr :public Item_str_func +{ + Regexp_processor_pcre re; +public: + Item_func_regexp_substr(Item *a, Item *b) + :Item_str_func(a, b) + {} + void cleanup() + { + DBUG_ENTER("Item_func_regex::cleanup"); + Item_str_func::cleanup(); + re.cleanup(); + DBUG_VOID_RETURN; + } + String *val_str(String *str); + void fix_length_and_dec(); + const char *func_name() const { return "regexp_substr"; } +}; + + class Item_func_insert :public Item_str_func { String tmp_value; @@ -521,6 +580,28 @@ public: }; +class Item_func_current_role :public Item_func_sysconst +{ + Name_resolution_context *context; + +public: + Item_func_current_role(Name_resolution_context *context_arg) + : context(context_arg) {} + bool fix_fields(THD *thd, Item **ref); + void fix_length_and_dec() + { max_length= username_char_length * SYSTEM_CHARSET_MBMAXLEN; } + int save_in_field(Field *field, bool no_conversions) + { return save_str_value_in_field(field, &str_value); } + const char *func_name() const { return "current_role"; } + const char *fully_qualified_func_name() const { return "current_role()"; } + String *val_str(String *) + { + DBUG_ASSERT(fixed == 1); + return (null_value ? 0 : &str_value); + } +}; + + class Item_func_soundex :public Item_str_func { String tmp_value; @@ -600,6 +681,17 @@ public: }; +class Item_func_binlog_gtid_pos :public Item_str_func +{ + String tmp_value; +public: + Item_func_binlog_gtid_pos(Item *arg1,Item *arg2) :Item_str_func(arg1,arg2) {} + String *val_str(String *); + void fix_length_and_dec(); + const char *func_name() const { return "binlog_gtid_pos"; } +}; + + class Item_func_rpad :public Item_str_func { String tmp_value, rpad_str; @@ -998,9 +1090,10 @@ class Item_func_dyncol_create: public Item_str_func protected: DYNCALL_CREATE_DEF *defs; DYNAMIC_COLUMN_VALUE *vals; - uint *nums; - void prepare_arguments(); - void cleanup_arguments(); + uint *keys_num; + LEX_STRING *keys_str; + bool names, force_names; + bool prepare_arguments(bool force_names); void print_arguments(String *str, enum_query_type query_type); public: Item_func_dyncol_create(List<Item> &args, DYNCALL_CREATE_DEF *dfs); @@ -1009,6 +1102,7 @@ public: const char *func_name() const{ return "column_create"; } String *val_str(String *); virtual void print(String *str, enum_query_type query_type); + virtual enum Functype functype() const { return DYNCOL_FUNC; } }; @@ -1023,6 +1117,19 @@ public: virtual void print(String *str, enum_query_type query_type); }; +class Item_func_dyncol_json: public Item_str_func +{ +public: + Item_func_dyncol_json(Item *str) :Item_str_func(str) {} + const char *func_name() const{ return "column_json"; } + String *val_str(String *); + void fix_length_and_dec() + { + maybe_null= 1; + collation.set(&my_charset_bin); + decimals= 0; + } +}; /* The following functions is always called from an Item_cast function @@ -1033,11 +1140,9 @@ class Item_dyncol_get: public Item_str_func public: Item_dyncol_get(Item *str, Item *num) :Item_str_func(str, num) - { - max_length= MAX_DYNAMIC_COLUMN_LENGTH; - } + {} void fix_length_and_dec() - { maybe_null= 1; } + { maybe_null= 1;; max_length= MAX_BLOB_WIDTH; } /* Mark that collation can change between calls */ bool dynamic_result() { return 1; } @@ -1062,3 +1167,4 @@ public: }; #endif /* ITEM_STRFUNC_INCLUDED */ + diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 65ce50c1e3c..3986e488dab 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -43,6 +43,9 @@ double get_post_group_estimate(JOIN* join, double join_op_rows); +const char *exists_outer_expr_name= "<exists outer expr>"; + +int check_and_do_in_subquery_rewrites(JOIN *join); Item_subselect::Item_subselect(): Item_result_field(), value_assigned(0), own_engine(0), thd(0), old_engine(0), @@ -83,15 +86,24 @@ void Item_subselect::init(st_select_lex *select_lex, if (unit->item) { - /* - Item can be changed in JOIN::prepare while engine in JOIN::optimize - => we do not copy old_engine here - */ engine= unit->item->engine; - own_engine= FALSE; parsing_place= unit->item->parsing_place; - thd->change_item_tree((Item**)&unit->item, this); - engine->change_result(this, result, TRUE); + if (unit->item->substype() == EXISTS_SUBS && + ((Item_exists_subselect *)unit->item)->exists_transformed) + { + /* it is permanent transformation of EXISTS to IN */ + unit->item= this; + engine->change_result(this, result, FALSE); + } + else + { + /* + Item can be changed in JOIN::prepare while engine in JOIN::optimize + => we do not copy old_engine here + */ + thd->change_item_tree((Item**)&unit->item, this); + engine->change_result(this, result, TRUE); + } } else { @@ -462,7 +474,7 @@ public: void Item_subselect::recalc_used_tables(st_select_lex *new_parent, bool after_pullout) { - List_iterator<Ref_to_outside> it(upper_refs); + List_iterator_fast<Ref_to_outside> it(upper_refs); Ref_to_outside *upper; used_tables_cache= 0; @@ -672,9 +684,12 @@ bool Item_subselect::exec() void Item_subselect::get_cache_parameters(List<Item> ¶meters) { - Collect_deps_prm prm= {¶meters, - unit->first_select()->nest_level_base, - unit->first_select()->nest_level}; + Collect_deps_prm prm= {¶meters, // parameters + unit->first_select()->nest_level_base, // nest_level_base + 0, // count + unit->first_select()->nest_level, // nest_level + TRUE // collect + }; walk(&Item::collect_outer_ref_processor, TRUE, (uchar*)&prm); } @@ -1314,10 +1329,12 @@ bool Item_singlerow_subselect::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) Item_exists_subselect::Item_exists_subselect(st_select_lex *select_lex): - Item_subselect() + Item_subselect(), upper_not(NULL), abort_on_null(0), + emb_on_expr_nest(NULL), optimizer(0), exists_transformed(0) { DBUG_ENTER("Item_exists_subselect::Item_exists_subselect"); bool val_bool(); + init(select_lex, new select_exists_subselect(this)); max_columns= UINT_MAX; null_value= FALSE; //can't be NULL @@ -1351,21 +1368,19 @@ bool Item_in_subselect::test_limit(st_select_lex_unit *unit_arg) Item_in_subselect::Item_in_subselect(Item * left_exp, st_select_lex *select_lex): - Item_exists_subselect(), - left_expr_cache(0), first_execution(TRUE), in_strategy(SUBS_NOT_TRANSFORMED), - optimizer(0), pushed_cond_guards(NULL), emb_on_expr_nest(NULL), - is_jtbm_merged(FALSE), is_jtbm_const_tab(FALSE), - is_flattenable_semijoin(FALSE), - is_registered_semijoin(FALSE), + Item_exists_subselect(), left_expr_cache(0), first_execution(TRUE), + in_strategy(SUBS_NOT_TRANSFORMED), + pushed_cond_guards(NULL), is_jtbm_merged(FALSE), is_jtbm_const_tab(FALSE), + is_flattenable_semijoin(FALSE), is_registered_semijoin(FALSE), upper_item(0) { DBUG_ENTER("Item_in_subselect::Item_in_subselect"); + DBUG_PRINT("info", ("in_strategy: %u", (uint)in_strategy)); left_expr= left_exp; func= &eq_creator; init(select_lex, new select_exists_subselect(this)); max_columns= UINT_MAX; maybe_null= 1; - abort_on_null= 0; reset(); //if test_limit will fail then error will be reported to client test_limit(select_lex->master_unit()); @@ -1770,8 +1785,7 @@ Item_in_subselect::single_value_transformer(JOIN *join) SELECT_LEX *current= thd->lex->current_select; thd->lex->current_select= current->return_after_parsing(); - //optimizer never use Item **ref => we can pass 0 as parameter - if (!optimizer || optimizer->fix_left(thd, 0)) + if (!optimizer || optimizer->fix_left(thd)) { thd->lex->current_select= current; DBUG_RETURN(true); @@ -1948,7 +1962,7 @@ bool Item_allany_subselect::is_maxmin_applicable(JOIN *join) WHERE condition. */ return (abort_on_null || (upper_item && upper_item->is_top_level_item())) && - !join->select_lex->master_unit()->uncacheable && !func->eqne_op(); + !(join->select_lex->master_unit()->uncacheable & ~UNCACHEABLE_EXPLAIN) && !func->eqne_op(); } @@ -2151,8 +2165,7 @@ Item_in_subselect::row_value_transformer(JOIN *join) SELECT_LEX *current= thd->lex->current_select; thd->lex->current_select= current->return_after_parsing(); - //optimizer never use Item **ref => we can pass 0 as parameter - if (!optimizer || optimizer->fix_left(thd, 0)) + if (!optimizer || optimizer->fix_left(thd)) { thd->lex->current_select= current; DBUG_RETURN(true); @@ -2332,7 +2345,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, ref_pointer_array+i, (char *)"<no matter>", (char *)"<list ref>")); - if (!abort_on_null) + if (!abort_on_null && select_lex->ref_pointer_array[i]->maybe_null) { Item *having_col_item= new Item_is_not_null_test(this, @@ -2351,10 +2364,6 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, (char *)"<no matter>", (char *)"<list ref>")); item= new Item_cond_or(item, item_isnull); - /* - TODO: why we create the above for cases where the right part - cant be NULL? - */ if (left_expr->element_index(i)->maybe_null) { if (!(item= new Item_func_trig_cond(item, get_cond_guard(i)))) @@ -2365,6 +2374,11 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, } *having_item= and_items(*having_item, having_col_item); } + if (!abort_on_null && left_expr->element_index(i)->maybe_null) + { + if (!(item= new Item_func_trig_cond(item, get_cond_guard(i)))) + DBUG_RETURN(true); + } *where_item= and_items(*where_item, item); } } @@ -2395,6 +2409,12 @@ Item_in_subselect::select_transformer(JOIN *join) return select_in_like_transformer(join); } +bool +Item_exists_subselect::select_transformer(JOIN *join) +{ + return select_prepare_to_be_in(); +} + /** Create the predicates needed to transform an IN/ALL/ANY subselect into a @@ -2530,6 +2550,437 @@ bool Item_in_subselect::inject_in_to_exists_cond(JOIN *join_arg) } +/* + If this select can potentially be converted by EXISTS->IN conversion, wrap it + in an Item_in_optimizer object. Final decision whether to do the conversion + is done at a later phase. +*/ + +bool Item_exists_subselect::select_prepare_to_be_in() +{ + bool trans_res= FALSE; + DBUG_ENTER("Item_exists_subselect::select_prepare_to_be_in"); + if (!optimizer && + thd->lex->sql_command == SQLCOM_SELECT && + !unit->first_select()->is_part_of_union() && + optimizer_flag(thd, OPTIMIZER_SWITCH_EXISTS_TO_IN) && + (is_top_level_item() || + (upper_not && upper_not->is_top_level_item()))) + { + Query_arena *arena, backup; + bool result; + arena= thd->activate_stmt_arena_if_needed(&backup); + result= (!(optimizer= new Item_in_optimizer(new Item_int(1), this))); + if (arena) + thd->restore_active_arena(arena, &backup); + if (result) + trans_res= TRUE; + else + substitution= optimizer; + } + DBUG_RETURN(trans_res); +} + +/** + Check if 'func' is an equality in form "inner_table.column = outer_expr" + + @param func Expression to check + @param local_field OUT Return "inner_table.column" here + @param outer_expr OUT Return outer_expr here + + @return true - 'func' is an Equality. +*/ + +static bool check_equality_for_exist2in(Item_func *func, + Item_ident **local_field, + Item **outer_exp) +{ + Item **args; + if (func->functype() != Item_func::EQ_FUNC) + return FALSE; + DBUG_ASSERT(func->arg_count == 2); + args= func->arguments(); + if (args[0]->real_type() == Item::FIELD_ITEM && + args[0]->all_used_tables() != OUTER_REF_TABLE_BIT && + args[1]->all_used_tables() == OUTER_REF_TABLE_BIT) + { + /* It is Item_field or Item_direct_view_ref) */ + DBUG_ASSERT(args[0]->type() == Item::FIELD_ITEM || + args[0]->type() == Item::REF_ITEM); + *local_field= (Item_ident *)args[0]; + *outer_exp= args[1]; + return TRUE; + } + else if (args[1]->real_type() == Item::FIELD_ITEM && + args[1]->all_used_tables() != OUTER_REF_TABLE_BIT && + args[0]->all_used_tables() == OUTER_REF_TABLE_BIT) + { + /* It is Item_field or Item_direct_view_ref) */ + DBUG_ASSERT(args[0]->type() == Item::FIELD_ITEM || + args[0]->type() == Item::REF_ITEM); + *local_field= (Item_ident *)args[1]; + *outer_exp= args[0]; + return TRUE; + } + + return FALSE; +} + +typedef struct st_eq_field_outer +{ + Item_func **eq_ref; + Item_ident *local_field; + Item *outer_exp; +} EQ_FIELD_OUTER; + + +/** + Check if 'conds' is a set of AND-ed outer_expr=inner_table.col equalities + + @detail + Check if 'conds' has form + + outer1=inner_tbl1.col1 AND ... AND outer2=inner_tbl1.col2 AND remainder_cond + + @param conds Condition to be checked + @parm result Array to collect EQ_FIELD_OUTER elements describing + inner-vs-outer equalities the function has found. + @return + false - some inner-vs-outer equalities were found + true - otherwise. +*/ + +static bool find_inner_outer_equalities(Item **conds, + Dynamic_array<EQ_FIELD_OUTER> &result) +{ + bool found= FALSE; + EQ_FIELD_OUTER element; + if (is_cond_and(*conds)) + { + List_iterator<Item> li(*((Item_cond*)*conds)->argument_list()); + Item *item; + while ((item= li++)) + { + if (item->type() == Item::FUNC_ITEM && + check_equality_for_exist2in((Item_func *)item, + &element.local_field, + &element.outer_exp)) + { + found= TRUE; + element.eq_ref= (Item_func **)li.ref(); + if (result.append(element)) + goto alloc_err; + } + } + } + else if ((*conds)->type() == Item::FUNC_ITEM && + check_equality_for_exist2in((Item_func *)*conds, + &element.local_field, + &element.outer_exp)) + { + found= TRUE; + element.eq_ref= (Item_func **)conds; + if (result.append(element)) + goto alloc_err; + } + + return !found; +alloc_err: + return TRUE; +} + +/** + Converts EXISTS subquery to IN subquery if it is possible and has sense + + @param opt_arg Pointer on THD + + @return TRUE in case of error and FALSE otherwise. +*/ + +bool Item_exists_subselect::exists2in_processor(uchar *opt_arg) +{ + THD *thd= (THD *)opt_arg; + SELECT_LEX *first_select=unit->first_select(), *save_select; + JOIN *join= first_select->join; + Item_func *eq= NULL, **eq_ref= NULL; + Item_ident *local_field= NULL; + Item *outer_exp= NULL; + Item *left_exp= NULL; Item_in_subselect *in_subs; + Query_arena *arena= NULL, backup; + int res= FALSE; + List<Item> outer; + Dynamic_array<EQ_FIELD_OUTER> eqs(5, 5); + bool will_be_correlated; + DBUG_ENTER("Item_exists_subselect::exists2in_processor"); + + if (!optimizer || + !optimizer_flag(thd, OPTIMIZER_SWITCH_EXISTS_TO_IN) || + (!is_top_level_item() && (!upper_not || + !upper_not->is_top_level_item())) || + first_select->is_part_of_union() || + first_select->group_list.elements || + first_select->order_list.elements || + join->having || + first_select->with_sum_func || + !first_select->leaf_tables.elements|| + !join->conds) + DBUG_RETURN(FALSE); + + DBUG_ASSERT(first_select->order_list.elements == 0 && + first_select->group_list.elements == 0 && + first_select->having == NULL); + + if (find_inner_outer_equalities(&join->conds, eqs)) + DBUG_RETURN(FALSE); + + DBUG_ASSERT(eqs.elements() != 0); + + save_select= thd->lex->current_select; + thd->lex->current_select= first_select; + + /* check that the subquery has only dependencies we are going pull out */ + { + List<Item> unused; + Collect_deps_prm prm= {&unused, // parameters + unit->first_select()->nest_level_base, // nest_level_base + 0, // count + unit->first_select()->nest_level, // nest_level + FALSE // collect + }; + walk(&Item::collect_outer_ref_processor, TRUE, (uchar*)&prm); + DBUG_ASSERT(prm.count > 0); + DBUG_ASSERT(prm.count >= (uint)eqs.elements()); + will_be_correlated= prm.count > (uint)eqs.elements(); + if (upper_not && will_be_correlated) + goto out; + } + + if ((uint)eqs.elements() > (first_select->item_list.elements + + first_select->select_n_reserved)) + goto out; + /* It is simple query */ + DBUG_ASSERT(first_select->join->all_fields.elements == + first_select->item_list.elements); + + arena= thd->activate_stmt_arena_if_needed(&backup); + + while (first_select->item_list.elements > (uint)eqs.elements()) + { + first_select->item_list.pop(); + first_select->join->all_fields.elements--; + } + { + List_iterator<Item> it(first_select->item_list); + + for (uint i= 0; i < (uint)eqs.elements(); i++) + { + Item *item= it++; + eq_ref= eqs.at(i).eq_ref; + eq= *eq_ref; + local_field= eqs.at(i).local_field; + outer_exp= eqs.at(i).outer_exp; + /* Add the field to the SELECT_LIST */ + if (item) + it.replace(local_field); + else + { + first_select->item_list.push_back(local_field); + first_select->join->all_fields.elements++; + } + first_select->ref_pointer_array[i]= (Item *)local_field; + + /* remove the parts from condition */ + if (!upper_not || !local_field->maybe_null) + { + eq->arguments()[0]= new Item_int(1); + eq->arguments()[1]= new Item_int(1); + } + else + { + *eq_ref= new Item_func_isnotnull( + new Item_field(thd, + ((Item_field*)(local_field->real_item()))->context, + ((Item_field*)(local_field->real_item()))->field)); + if((*eq_ref)->fix_fields(thd, (Item **)eq_ref)) + { + res= TRUE; + goto out; + } + } + outer_exp->fix_after_pullout(unit->outer_select(), &outer_exp); + outer_exp->update_used_tables(); + outer.push_back(outer_exp); + } + } + + join->conds->update_used_tables(); + + /* make IN SUBQUERY and put outer_exp as left part */ + if (eqs.elements() == 1) + left_exp= outer_exp; + else + { + if (!(left_exp= new Item_row(outer))) + { + res= TRUE; + goto out; + } + } + + /* make EXISTS->IN permanet (see Item_subselect::init()) */ + set_exists_transformed(); + + first_select->select_limit= NULL; + if (!(in_subs= new Item_in_subselect(left_exp, first_select))) + { + res= TRUE; + goto out; + } + in_subs->set_exists_transformed(); + optimizer->arguments()[0]= left_exp; + optimizer->arguments()[1]= in_subs; + in_subs->optimizer= optimizer; + DBUG_ASSERT(is_top_level_item() || + (upper_not && upper_not->is_top_level_item())); + in_subs->top_level_item(); + { + SELECT_LEX *current= thd->lex->current_select; + optimizer->reset_cache(); // renew cache, and we will not keep it + thd->lex->current_select= unit->outer_select(); + DBUG_ASSERT(optimizer); + if (optimizer->fix_left(thd)) + { + res= TRUE; + /* + We should not restore thd->lex->current_select because it will be + reset on exit from this procedure + */ + goto out; + } + /* + As far as Item_ref_in_optimizer do not substitute itself on fix_fields + we can use same item for all selects. + */ + in_subs->expr= new Item_direct_ref(&first_select->context, + (Item**)optimizer->get_cache(), + (char *)"<no matter>", + (char *)in_left_expr_name); + if (in_subs->fix_fields(thd, optimizer->arguments() + 1)) + { + res= TRUE; + /* + We should not restore thd->lex->current_select because it will be + reset on exit from this procedure + */ + goto out; + } + { + /* Move dependence list */ + List_iterator_fast<Ref_to_outside> it(upper_refs); + Ref_to_outside *upper; + while ((upper= it++)) + { + uint i; + for (i= 0; i < (uint)eqs.elements(); i++) + if (eqs.at(i).outer_exp-> + walk(&Item::find_item_processor, TRUE, (uchar*)upper->item)) + break; + if (i == (uint)eqs.elements() && + (in_subs->upper_refs.push_back(upper, thd->stmt_arena->mem_root))) + goto out; + } + } + in_subs->update_used_tables(); + /* + The engine of the subquery is fixed so above fix_fields() is not + complete and should be fixed + */ + in_subs->upper_refs= upper_refs; + upper_refs.empty(); + thd->lex->current_select= current; + } + + DBUG_ASSERT(unit->item == in_subs); + DBUG_ASSERT(join == first_select->join); + /* + Fix dependency info + */ + in_subs->is_correlated= will_be_correlated; + if (!will_be_correlated) + { + first_select->uncacheable&= ~UNCACHEABLE_DEPENDENT_GENERATED; + unit->uncacheable&= ~UNCACHEABLE_DEPENDENT_GENERATED; + } + /* + set possible optimization strategies + */ + in_subs->emb_on_expr_nest= emb_on_expr_nest; + res= check_and_do_in_subquery_rewrites(join); + first_select->join->prepare_stage2(); + + first_select->fix_prepare_information(thd, &join->conds, &join->having); + + if (upper_not) + { + Item *exp; + if (eqs.elements() == 1) + { + exp= (optimizer->arguments()[0]->maybe_null ? + (Item*) + new Item_cond_and( + new Item_func_isnotnull( + new Item_direct_ref(&unit->outer_select()->context, + optimizer->arguments(), + (char *)"<no matter>", + (char *)exists_outer_expr_name)), + optimizer) : + (Item *)optimizer); + } + else + { + List<Item> *and_list= new List<Item>; + if (!and_list) + { + res= TRUE; + goto out; + } + for (size_t i= 0; i < eqs.elements(); i++) + { + if (optimizer->arguments()[0]->maybe_null) + { + and_list-> + push_front( + new Item_func_isnotnull( + new Item_direct_ref(&unit->outer_select()->context, + optimizer->arguments()[0]->addr(i), + (char *)"<no matter>", + (char *)exists_outer_expr_name))); + } + } + if (and_list->elements > 0) + { + and_list->push_front(optimizer); + exp= new Item_cond_and(*and_list); + } + else + exp= optimizer; + } + upper_not->arguments()[0]= exp; + if (!exp->fixed && exp->fix_fields(thd, upper_not->arguments())) + { + res= TRUE; + goto out; + } + } + +out: + thd->lex->current_select= save_select; + if (arena) + thd->restore_active_arena(arena, &backup); + DBUG_RETURN(res); +} + + /** Prepare IN/ALL/ANY/SOME subquery transformation and call the appropriate transformation function. @@ -2590,7 +3041,7 @@ Item_in_subselect::select_in_like_transformer(JOIN *join) } thd->lex->current_select= current->return_after_parsing(); - result= optimizer->fix_left(thd, optimizer->arguments()); + result= optimizer->fix_left(thd); /* fix_fields can change reference to left_expr, we need reassign it */ left_expr= optimizer->arguments()[0]; thd->lex->current_select= current; @@ -2646,14 +3097,23 @@ void Item_in_subselect::print(String *str, enum_query_type query_type) Item_subselect::print(str, query_type); } +bool Item_exists_subselect::fix_fields(THD *thd_arg, Item **ref) +{ + DBUG_ENTER("Item_exists_subselect::fix_fields"); + if (exists_transformed) + DBUG_RETURN( !( (*ref)= new Item_int(1))); + DBUG_RETURN(Item_subselect::fix_fields(thd_arg, ref)); +} + bool Item_in_subselect::fix_fields(THD *thd_arg, Item **ref) { uint outer_cols_num; List<Item> *inner_cols; + DBUG_ENTER("Item_in_subselect::fix_fields"); if (test_strategy(SUBS_SEMI_JOIN)) - return !( (*ref)= new Item_int(1)); + DBUG_RETURN( !( (*ref)= new Item_int(1)) ); /* Check if the outer and inner IN operands match in those cases when we @@ -2685,7 +3145,7 @@ bool Item_in_subselect::fix_fields(THD *thd_arg, Item **ref) if (outer_cols_num != inner_cols->elements) { my_error(ER_OPERAND_COLUMNS, MYF(0), outer_cols_num); - return TRUE; + DBUG_RETURN(TRUE); } if (outer_cols_num > 1) { @@ -2695,7 +3155,7 @@ bool Item_in_subselect::fix_fields(THD *thd_arg, Item **ref) { inner_col= inner_col_it++; if (inner_col->check_cols(left_expr->element_index(i)->cols())) - return TRUE; + DBUG_RETURN(TRUE); } } } @@ -2703,12 +3163,12 @@ bool Item_in_subselect::fix_fields(THD *thd_arg, Item **ref) if (thd_arg->lex->is_view_context_analysis() && left_expr && !left_expr->fixed && left_expr->fix_fields(thd_arg, &left_expr)) - return TRUE; + DBUG_RETURN(TRUE); else if (Item_subselect::fix_fields(thd_arg, ref)) - return TRUE; + DBUG_RETURN(TRUE); fixed= TRUE; - return FALSE; + DBUG_RETURN(FALSE); } @@ -3835,6 +4295,7 @@ subselect_single_select_engine::change_result(Item_subselect *si, select_result_interceptor *res, bool temp) { + DBUG_ENTER("subselect_single_select_engine::change_result"); item= si; if (temp) { @@ -3855,7 +4316,7 @@ subselect_single_select_engine::change_result(Item_subselect *si, that would not require a lot of extra code that would be harder to manage than the current code. */ - return select_lex->join->change_result(res); + DBUG_RETURN(select_lex->join->change_result(res)); } @@ -5091,7 +5552,7 @@ bool Ordered_key::alloc_keys_buffers() DBUG_ASSERT(key_buff_elements > 0); if (!(key_buff= (rownum_t*) my_malloc((size_t)(key_buff_elements * - sizeof(rownum_t)), MYF(MY_WME)))) + sizeof(rownum_t)), MYF(MY_WME | MY_THREAD_SPECIFIC)))) return TRUE; /* @@ -5122,7 +5583,8 @@ int Ordered_key::cmp_keys_by_row_data(ha_rows a, ha_rows b) { uchar *rowid_a, *rowid_b; - int error, cmp_res; + int __attribute__((unused)) error; + int cmp_res; /* The length in bytes of the rowids (positions) of tmp_table. */ uint rowid_length= tbl->file->ref_length; @@ -5218,7 +5680,8 @@ int Ordered_key::cmp_key_with_search_key(rownum_t row_num) /* The length in bytes of the rowids (positions) of tmp_table. */ uint rowid_length= tbl->file->ref_length; uchar *cur_rowid= row_num_to_rowid + row_num * rowid_length; - int error, cmp_res; + int __attribute__((unused)) error; + int cmp_res; if ((error= tbl->file->ha_rnd_pos(tbl->record[0], cur_rowid))) { @@ -5516,7 +5979,7 @@ subselect_rowid_merge_engine::init(MY_BITMAP *non_null_key_parts, !(null_bitmaps= (MY_BITMAP**) thd->alloc(merge_keys_count * sizeof(MY_BITMAP*))) || !(row_num_to_rowid= (uchar*) my_malloc((size_t)(row_count * rowid_length), - MYF(MY_WME)))) + MYF(MY_WME | MY_THREAD_SPECIFIC)))) return TRUE; /* Create the only non-NULL key if there is any. */ diff --git a/sql/item_subselect.h b/sql/item_subselect.h index 592e7711a10..f83f2557e07 100644 --- a/sql/item_subselect.h +++ b/sql/item_subselect.h @@ -244,6 +244,7 @@ public: virtual bool expr_cache_is_needed(THD *); virtual void get_cache_parameters(List<Item> ¶meters); virtual bool is_subquery_processor (uchar *opt_arg) { return 1; } + bool exists2in_processor(uchar *opt_arg) { return 0; } bool limit_index_condition_pushdown_processor(uchar *opt_arg) { return TRUE; @@ -339,13 +340,35 @@ public: class Item_exists_subselect :public Item_subselect { protected: + Item_func_not *upper_not; bool value; /* value of this item (boolean: exists/not-exists) */ + bool abort_on_null; void init_length_and_dec(); + bool select_prepare_to_be_in(); public: + /* + Used by subquery optimizations to keep track about in which clause this + subquery predicate is located: + NO_JOIN_NEST - the predicate is an AND-part of the WHERE + join nest pointer - the predicate is an AND-part of ON expression + of a join nest + NULL - for all other locations + */ + TABLE_LIST *emb_on_expr_nest; + /** + Reference on the Item_in_optimizer wrapper of this subquery + */ + Item_in_optimizer *optimizer; + /* true if we got this from EXISTS or to IN */ + bool exists_transformed; + Item_exists_subselect(st_select_lex *select_lex); - Item_exists_subselect(): Item_subselect() {} + Item_exists_subselect() + :Item_subselect(), upper_not(NULL),abort_on_null(0), + emb_on_expr_nest(NULL), optimizer(0), exists_transformed(0) + {} subs_type substype() { return EXISTS_SUBS; } void reset() @@ -361,11 +384,24 @@ public: String *val_str(String*); my_decimal *val_decimal(my_decimal *); bool val_bool(); + bool fix_fields(THD *thd, Item **ref); void fix_length_and_dec(); virtual void print(String *str, enum_query_type query_type); + bool select_transformer(JOIN *join); + void top_level_item() { abort_on_null=1; } + inline bool is_top_level_item() { return abort_on_null; } + bool exists2in_processor(uchar *opt_arg); Item* expr_cache_insert_transformer(uchar *thd_arg); + void mark_as_condition_AND_part(TABLE_LIST *embedding) + { + emb_on_expr_nest= embedding; + } + virtual void under_not(Item_func_not *upper) { upper_not= upper; }; + + void set_exists_transformed() { exists_transformed= TRUE; } + friend class select_exists_subselect; friend class subselect_uniquesubquery_engine; friend class subselect_indexsubquery_engine; @@ -425,11 +461,8 @@ protected: */ Item *expr; bool was_null; - bool abort_on_null; /* A bitmap of possible execution strategies for an IN predicate. */ uchar in_strategy; -public: - Item_in_optimizer *optimizer; protected: /* Used to trigger on/off conditions that were pushed down to subselect */ bool *pushed_cond_guards; @@ -452,15 +485,6 @@ public: /* Priority of this predicate in the convert-to-semi-join-nest process. */ int sj_convert_priority; /* - Used by subquery optimizations to keep track about in which clause this - subquery predicate is located: - NO_JOIN_NEST - the predicate is an AND-part of the WHERE - join nest pointer - the predicate is an AND-part of ON expression - of a join nest - NULL - for all other locations - */ - TABLE_LIST *emb_on_expr_nest; - /* Types of left_expr and subquery's select list allow to perform subquery materialization. Currently, we set this to FALSE when it as well could be TRUE. This is to be properly addressed with fix for BUG#36752. @@ -528,7 +552,9 @@ public: */ Item *original_item() { - return is_flattenable_semijoin ? (Item*)this : (Item*)optimizer; + return (is_flattenable_semijoin && !exists_transformed ? + (Item*)this : + (Item*)optimizer); } bool *get_cond_guard(int i) @@ -547,11 +573,9 @@ public: Item_in_subselect(Item * left_expr, st_select_lex *select_lex); Item_in_subselect() :Item_exists_subselect(), left_expr_cache(0), first_execution(TRUE), - abort_on_null(0), in_strategy(SUBS_NOT_TRANSFORMED), optimizer(0), - pushed_cond_guards(NULL), func(NULL), emb_on_expr_nest(NULL), - is_jtbm_merged(FALSE), is_jtbm_const_tab(FALSE), - upper_item(0) - {} + in_strategy(SUBS_NOT_TRANSFORMED), + pushed_cond_guards(NULL), func(NULL), is_jtbm_merged(FALSE), + is_jtbm_const_tab(FALSE), upper_item(0) {} void cleanup(); subs_type substype() { return IN_SUBS; } void reset() @@ -572,8 +596,6 @@ public: my_decimal *val_decimal(my_decimal *); void update_null_value () { (void) val_bool(); } bool val_bool(); - void top_level_item() { abort_on_null=1; } - inline bool is_top_level_item() { return abort_on_null; } bool test_limit(st_select_lex_unit *unit); virtual void print(String *str, enum_query_type query_type); bool fix_fields(THD *thd, Item **ref); @@ -590,19 +612,14 @@ public: void set_first_execution() { if (first_execution) first_execution= FALSE; } bool expr_cache_is_needed(THD *thd); inline bool left_expr_has_null(); - + int optimize(double *out_rows, double *cost); - /* + /* Return the identifier that we could use to identify the subquery for the user. */ int get_identifier(); - void mark_as_condition_AND_part(TABLE_LIST *embedding) - { - emb_on_expr_nest= embedding; - } - bool test_strategy(uchar strategy) { return test(in_strategy & strategy); } @@ -631,6 +648,9 @@ public: void add_strategy (uchar strategy) { + DBUG_ENTER("Item_in_subselect::add_strategy"); + DBUG_PRINT("enter", ("current: %u add: %u", + (uint) in_strategy, (uint) strategy)); DBUG_ASSERT(strategy != SUBS_NOT_TRANSFORMED); DBUG_ASSERT(!(strategy & SUBS_STRATEGY_CHOSEN)); /* @@ -640,16 +660,25 @@ public: DBUG_ASSERT(!(in_strategy & SUBS_STRATEGY_CHOSEN)); */ in_strategy|= strategy; + DBUG_VOID_RETURN; } void reset_strategy(uchar strategy) { + DBUG_ENTER("Item_in_subselect::reset_strategy"); + DBUG_PRINT("enter", ("current: %u new: %u", + (uint) in_strategy, (uint) strategy)); DBUG_ASSERT(strategy != SUBS_NOT_TRANSFORMED); in_strategy= strategy; + DBUG_VOID_RETURN; } void set_strategy(uchar strategy) { + DBUG_ENTER("Item_in_subselect::set_strategy"); + DBUG_PRINT("enter", ("current: %u set: %u", + (uint) in_strategy, + (uint) (SUBS_STRATEGY_CHOSEN | strategy))); /* Check that only one strategy is set for execution. */ DBUG_ASSERT(strategy == SUBS_SEMI_JOIN || strategy == SUBS_IN_TO_EXISTS || @@ -659,7 +688,12 @@ public: strategy == SUBS_MAXMIN_INJECTED || strategy == SUBS_MAXMIN_ENGINE); in_strategy= (SUBS_STRATEGY_CHOSEN | strategy); + DBUG_VOID_RETURN; } + bool exists2in_processor(uchar *opt_arg __attribute__((unused))) + { + return 0; + }; friend class Item_ref_null_helper; friend class Item_is_not_null_test; @@ -667,6 +701,7 @@ public: friend class subselect_indexsubquery_engine; friend class subselect_hash_sj_engine; friend class subselect_partial_match_engine; + friend class Item_exists_subselect; }; diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 0395c856817..c824cc70617 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -389,7 +389,12 @@ bool Item_sum::collect_outer_ref_processor(uchar *param) if ((ds= depended_from()) && ds->nest_level_base == prm->nest_level_base && ds->nest_level < prm->nest_level) - prm->parameters->add_unique(this, &cmp_items); + { + if (prm->collect) + prm->parameters->add_unique(this, &cmp_items); + else + prm->count++; + } return FALSE; } @@ -650,13 +655,24 @@ void Item_sum::cleanup() @retval > 0 if key1 > key2 */ -static int simple_str_key_cmp(void* arg, uchar* key1, uchar* key2) +int simple_str_key_cmp(void* arg, uchar* key1, uchar* key2) { Field *f= (Field*) arg; return f->cmp(key1, key2); } +C_MODE_START + +int count_distinct_walk(void *elem, element_count count, void *arg) +{ + (*((ulonglong*)arg))++; + return 0; +} + +C_MODE_END + + /** Correctly compare composite keys. @@ -724,7 +740,7 @@ C_MODE_START /* Declarations for auxilary C-callbacks */ -static int simple_raw_key_cmp(void* arg, const void* key1, const void* key2) +int simple_raw_key_cmp(void* arg, const void* key1, const void* key2) { return memcmp(key1, key2, *(uint *) arg); } @@ -3373,12 +3389,8 @@ bool Item_func_group_concat::add() TREE_ELEMENT *el= 0; // Only for safety if (row_eligible && tree) { - DBUG_EXECUTE_IF("trigger_OOM_in_gconcat_add", - DBUG_SET("+d,simulate_persistent_out_of_memory");); el= tree_insert(tree, table->record[0] + table->s->null_bytes, 0, tree->custom_arg); - DBUG_EXECUTE_IF("trigger_OOM_in_gconcat_add", - DBUG_SET("-d,simulate_persistent_out_of_memory");); /* check if there was enough memory to insert the row */ if (!el) return 1; @@ -3573,7 +3585,8 @@ bool Item_func_group_concat::setup(THD *thd) init_tree(tree, (uint) min(thd->variables.max_heap_table_size, thd->variables.sortbuff_size/16), 0, tree_key_length, - group_concat_key_cmp_with_order , 0, NULL, (void*) this); + group_concat_key_cmp_with_order, NULL, (void*) this, + MYF(MY_THREAD_SPECIFIC)); } if (distinct) diff --git a/sql/item_sum.h b/sql/item_sum.h index f074cc6c822..d28c654c438 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -1125,7 +1125,7 @@ public: class Item_sum_or :public Item_sum_bit { public: - Item_sum_or(Item *item_par) :Item_sum_bit(item_par,LL(0)) {} + Item_sum_or(Item *item_par) :Item_sum_bit(item_par, 0) {} Item_sum_or(THD *thd, Item_sum_or *item) :Item_sum_bit(thd, item) {} bool add(); const char *func_name() const { return "bit_or("; } @@ -1146,7 +1146,7 @@ class Item_sum_and :public Item_sum_bit class Item_sum_xor :public Item_sum_bit { public: - Item_sum_xor(Item *item_par) :Item_sum_bit(item_par,LL(0)) {} + Item_sum_xor(Item *item_par) :Item_sum_bit(item_par, 0) {} Item_sum_xor(THD *thd, Item_sum_xor *item) :Item_sum_bit(thd, item) {} bool add(); const char *func_name() const { return "bit_xor("; } diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 9758d86213d..63ad633cd2e 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -709,8 +709,8 @@ static bool get_interval_info(const char *str,uint length,CHARSET_INFO *cs, { longlong value; const char *start= str; - for (value=0; str != end && my_isdigit(cs,*str) ; str++) - value= value*LL(10) + (longlong) (*str - '0'); + for (value=0; str != end && my_isdigit(cs, *str) ; str++) + value= value*10 + *str - '0'; msec_length= 6 - (str - start); values[i]= value; while (str != end && !my_isdigit(cs,*str)) diff --git a/sql/keycaches.cc b/sql/keycaches.cc index 26a39808c56..120aa7e1029 100644 --- a/sql/keycaches.cc +++ b/sql/keycaches.cc @@ -20,6 +20,7 @@ ****************************************************************************/ NAMED_ILIST key_caches; +NAMED_ILIST rpl_filters; /** ilink (intrusive list element) with a name @@ -66,6 +67,23 @@ uchar* find_named(I_List<NAMED_ILINK> *list, const char *name, uint length, } +bool NAMED_ILIST::delete_element(const char *name, uint length, void (*free_element)(const char *name, uchar*)) +{ + I_List_iterator<NAMED_ILINK> it(*this); + NAMED_ILINK *element; + DBUG_ENTER("NAMED_ILIST::delete_element"); + while ((element= it++)) + { + if (element->cmp(name, length)) + { + (*free_element)(element->name, element->data); + delete element; + DBUG_RETURN(0); + } + } + DBUG_RETURN(1); +} + void NAMED_ILIST::delete_elements(void (*free_element)(const char *name, uchar*)) { NAMED_ILINK *element; @@ -159,7 +177,55 @@ bool process_key_caches(process_key_cache_t func, void *param) return res != 0; } -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class I_List_iterator<NAMED_ILINK>; -#endif +/* Rpl_filter functions */ + +LEX_STRING default_rpl_filter_base= {C_STRING_WITH_LEN("")}; + +Rpl_filter *get_rpl_filter(LEX_STRING *filter_name) +{ + if (!filter_name->length) + filter_name= &default_rpl_filter_base; + return ((Rpl_filter*) find_named(&rpl_filters, + filter_name->str, filter_name->length, 0)); +} + +Rpl_filter *create_rpl_filter(const char *name, uint length) +{ + Rpl_filter *filter; + DBUG_ENTER("create_rpl_filter"); + DBUG_PRINT("enter",("name: %.*s", length, name)); + + filter= new Rpl_filter; + if (filter) + { + if (!new NAMED_ILINK(&rpl_filters, name, length, (uchar*) filter)) + { + delete filter; + filter= 0; + } + } + DBUG_RETURN(filter); +} + +Rpl_filter *get_or_create_rpl_filter(const char *name, uint length) +{ + LEX_STRING rpl_filter_name; + Rpl_filter *filter; + + rpl_filter_name.str= (char *) name; + rpl_filter_name.length= length; + if (!(filter= get_rpl_filter(&rpl_filter_name))) + filter= create_rpl_filter(name, length); + return filter; +} + +void free_rpl_filter(const char *name, Rpl_filter *filter) +{ + delete filter; +} + +void free_all_rpl_filters() +{ + rpl_filters.delete_elements((void (*)(const char*, uchar*)) free_rpl_filter); +} diff --git a/sql/keycaches.h b/sql/keycaches.h index 04d3f6145e7..32537339e2e 100644 --- a/sql/keycaches.h +++ b/sql/keycaches.h @@ -18,6 +18,7 @@ #include "sql_list.h" #include <keycache.h> +#include <rpl_filter.h> extern "C" { @@ -30,8 +31,10 @@ class NAMED_ILIST: public I_List<NAMED_ILINK> { public: void delete_elements(void (*free_element)(const char*, uchar*)); + bool delete_element(const char *name, uint length, void (*free_element)(const char*, uchar*)); }; +/* For key cache */ extern LEX_STRING default_key_cache_base; extern KEY_CACHE zero_key_cache; extern NAMED_ILIST key_caches; @@ -42,4 +45,14 @@ KEY_CACHE *get_or_create_key_cache(const char *name, uint length); void free_key_cache(const char *name, KEY_CACHE *key_cache); bool process_key_caches(process_key_cache_t func, void *param); +/* For Rpl_filter */ +extern LEX_STRING default_rpl_filter_base; +extern NAMED_ILIST rpl_filters; + +Rpl_filter *create_rpl_filter(const char *name, uint length); +Rpl_filter *get_rpl_filter(LEX_STRING *filter_name); +Rpl_filter *get_or_create_rpl_filter(const char *name, uint length); +void free_rpl_filter(const char *name, Rpl_filter *filter); +void free_all_rpl_filters(void); + #endif /* KEYCACHES_INCLUDED */ diff --git a/sql/lex.h b/sql/lex.h index 9bf4c439cb6..e5f570e5526 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -59,6 +59,7 @@ static SYMBOL symbols[] = { { "ACCESSIBLE", SYM(ACCESSIBLE_SYM)}, { "ACTION", SYM(ACTION)}, { "ADD", SYM(ADD)}, + { "ADMIN", SYM(ADMIN_SYM)}, { "AFTER", SYM(AFTER_SYM)}, { "AGAINST", SYM(AGAINST)}, { "AGGREGATE", SYM(AGGREGATE_SYM)}, @@ -77,6 +78,7 @@ static SYMBOL symbols[] = { { "AUTHORS", SYM(AUTHORS_SYM)}, { "AUTO_INCREMENT", SYM(AUTO_INC)}, { "AUTOEXTEND_SIZE", SYM(AUTOEXTEND_SIZE_SYM)}, + { "AUTO", SYM(AUTO_SYM)}, { "AVG", SYM(AVG_SYM)}, { "AVG_ROW_LENGTH", SYM(AVG_ROW_LENGTH)}, { "BACKUP", SYM(BACKUP_SYM)}, @@ -123,11 +125,10 @@ static SYMBOL symbols[] = { { "COLUMN_NAME", SYM(COLUMN_NAME_SYM)}, { "COLUMNS", SYM(COLUMNS)}, { "COLUMN_ADD", SYM(COLUMN_ADD_SYM)}, + { "COLUMN_CHECK", SYM(COLUMN_CHECK_SYM)}, { "COLUMN_CREATE", SYM(COLUMN_CREATE_SYM)}, { "COLUMN_DELETE", SYM(COLUMN_DELETE_SYM)}, - { "COLUMN_EXISTS", SYM(COLUMN_EXISTS_SYM)}, { "COLUMN_GET", SYM(COLUMN_GET_SYM)}, - { "COLUMN_LIST", SYM(COLUMN_LIST_SYM)}, { "COMMENT", SYM(COMMENT_SYM)}, { "COMMIT", SYM(COMMIT_SYM)}, { "COMMITTED", SYM(COMMITTED_SYM)}, @@ -152,6 +153,8 @@ static SYMBOL symbols[] = { { "CROSS", SYM(CROSS)}, { "CUBE", SYM(CUBE_SYM)}, { "CURRENT_DATE", SYM(CURDATE)}, + { "CURRENT_POS", SYM(CURRENT_POS_SYM)}, + { "CURRENT_ROLE", SYM(CURRENT_ROLE)}, { "CURRENT_TIME", SYM(CURTIME)}, { "CURRENT_TIMESTAMP", SYM(NOW_SYM)}, { "CURRENT_USER", SYM(CURRENT_USER)}, @@ -261,6 +264,7 @@ static SYMBOL symbols[] = { { "HOUR_MICROSECOND", SYM(HOUR_MICROSECOND_SYM)}, { "HOUR_MINUTE", SYM(HOUR_MINUTE_SYM)}, { "HOUR_SECOND", SYM(HOUR_SECOND_SYM)}, + { "ID", SYM(ID_SYM)}, { "IDENTIFIED", SYM(IDENTIFIED_SYM)}, { "IF", SYM(IF)}, { "IGNORE", SYM(IGNORE_SYM)}, @@ -330,6 +334,7 @@ static SYMBOL symbols[] = { { "LOW_PRIORITY", SYM(LOW_PRIORITY)}, { "MASTER", SYM(MASTER_SYM)}, { "MASTER_CONNECT_RETRY", SYM(MASTER_CONNECT_RETRY_SYM)}, + { "MASTER_GTID_POS", SYM(MASTER_GTID_POS_SYM)}, { "MASTER_HOST", SYM(MASTER_HOST_SYM)}, { "MASTER_LOG_FILE", SYM(MASTER_LOG_FILE_SYM)}, { "MASTER_LOG_POS", SYM(MASTER_LOG_POS_SYM)}, @@ -344,6 +349,7 @@ static SYMBOL symbols[] = { { "MASTER_SSL_KEY", SYM(MASTER_SSL_KEY_SYM)}, { "MASTER_SSL_VERIFY_SERVER_CERT", SYM(MASTER_SSL_VERIFY_SERVER_CERT_SYM)}, { "MASTER_USER", SYM(MASTER_USER_SYM)}, + { "MASTER_USE_GTID", SYM(MASTER_USE_GTID_SYM)}, { "MASTER_HEARTBEAT_PERIOD", SYM(MASTER_HEARTBEAT_PERIOD_SYM)}, { "MATCH", SYM(MATCH)}, { "MAX_CONNECTIONS_PER_HOUR", SYM(MAX_CONNECTIONS_PER_HOUR)}, @@ -480,10 +486,12 @@ static SYMBOL symbols[] = { { "RESTRICT", SYM(RESTRICT)}, { "RESUME", SYM(RESUME_SYM)}, { "RETURN", SYM(RETURN_SYM)}, + { "RETURNING", SYM(RETURNING_SYM)}, { "RETURNS", SYM(RETURNS_SYM)}, { "REVOKE", SYM(REVOKE)}, { "RIGHT", SYM(RIGHT)}, { "RLIKE", SYM(REGEXP)}, /* Like in mSQL2 */ + { "ROLE", SYM(ROLE_SYM)}, { "ROLLBACK", SYM(ROLLBACK_SYM)}, { "ROLLUP", SYM(ROLLUP_SYM)}, { "ROUTINE", SYM(ROUTINE_SYM)}, @@ -514,6 +522,8 @@ static SYMBOL symbols[] = { { "SIGNED", SYM(SIGNED_SYM)}, { "SIMPLE", SYM(SIMPLE_SYM)}, { "SLAVE", SYM(SLAVE)}, + { "SLAVES", SYM(SLAVES)}, + { "SLAVE_POS", SYM(SLAVE_POS_SYM)}, { "SLOW", SYM(SLOW)}, { "SNAPSHOT", SYM(SNAPSHOT_SYM)}, { "SMALLINT", SYM(SMALLINT)}, diff --git a/sql/lock.cc b/sql/lock.cc index 3a1a6d41ce3..5bb42185d90 100644 --- a/sql/lock.cc +++ b/sql/lock.cc @@ -93,7 +93,7 @@ extern HASH open_cache; static int lock_external(THD *thd, TABLE **table,uint count); static int unlock_external(THD *thd, TABLE **table,uint count); -static void print_lock_error(int error, const char *); +static void print_lock_error(int error, TABLE *); /* Map the return value of thr_lock to an error from errmsg.txt */ static int thr_lock_errno_to_mysql[]= @@ -358,7 +358,7 @@ static int lock_external(THD *thd, TABLE **tables, uint count) if ((error=(*tables)->file->ha_external_lock(thd,lock_type))) { - print_lock_error(error, (*tables)->file->table_type()); + print_lock_error(error, *tables); while (--i) { tables--; @@ -673,7 +673,7 @@ static int unlock_external(THD *thd, TABLE **table,uint count) if ((error=(*table)->file->ha_external_lock(thd, F_UNLCK))) { error_code=error; - print_lock_error(error_code, (*table)->file->table_type()); + print_lock_error(error_code, *table); } } table++; @@ -895,7 +895,7 @@ bool lock_object_name(THD *thd, MDL_key::enum_mdl_namespace mdl_type, } -static void print_lock_error(int error, const char *table) +static void print_lock_error(int error, TABLE *table) { int textno; DBUG_ENTER("print_lock_error"); @@ -911,17 +911,15 @@ static void print_lock_error(int error, const char *table) textno=ER_LOCK_DEADLOCK; break; case HA_ERR_WRONG_COMMAND: - textno=ER_ILLEGAL_HA; - break; + my_error(ER_ILLEGAL_HA, MYF(0), table->file->table_type(), + table->s->db.str, table->s->table_name.str); + DBUG_VOID_RETURN; default: textno=ER_CANT_LOCK; break; } - if ( textno == ER_ILLEGAL_HA ) - my_error(textno, MYF(ME_BELL+ME_OLDWIN+ME_WAITTANG), table); - else - my_error(textno, MYF(ME_BELL+ME_OLDWIN+ME_WAITTANG), error); + my_error(textno, MYF(0), error); DBUG_VOID_RETURN; } diff --git a/sql/log.cc b/sql/log.cc index d129fcbe8a9..155b50946a3 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -40,6 +40,7 @@ #include "rpl_rli.h" #include "sql_audit.h" #include "log_slow.h" +#include "mysqld.h" #include <my_dir.h> #include <stdarg.h> @@ -53,6 +54,7 @@ #include "rpl_handler.h" #include "debug_sync.h" #include "sql_show.h" +#include "my_pthread.h" /* max size of the log message */ #define MAX_LOG_BUFFER_SIZE 1024 @@ -86,6 +88,7 @@ ulong opt_binlog_dbug_fsync_sleep= 0; #endif mysql_mutex_t LOCK_prepare_ordered; +mysql_cond_t COND_prepare_ordered; mysql_mutex_t LOCK_commit_ordered; static ulonglong binlog_status_var_num_commits; @@ -106,6 +109,18 @@ static SHOW_VAR binlog_status_vars_detail[]= {NullS, NullS, SHOW_LONG} }; +/* + Variables for the binlog background thread. + Protected by the MYSQL_BIN_LOG::LOCK_binlog_background_thread mutex. + */ +static bool binlog_background_thread_started= false; +static bool binlog_background_thread_stop= false; +static MYSQL_BIN_LOG::xid_count_per_binlog * + binlog_background_thread_queue= NULL; + +static bool start_binlog_background_thread(); + +static rpl_binlog_state rpl_global_gtid_binlog_state; /** purge logs, master and slave sides both, related error code @@ -479,6 +494,14 @@ public: */ bool using_xa; my_xid xa_xid; + bool need_unlog; + /* + Id of binlog that transaction was written to; only needed if need_unlog is + true. + */ + ulong binlog_id; + /* Set if we get an error during commit that must be returned from unlog(). */ + bool delayed_error; private: @@ -665,7 +688,8 @@ bool Log_to_csv_event_handler:: /* do a write */ if (table->field[1]->store(user_host, user_host_len, client_cs) || table->field[2]->store((longlong) thread_id, TRUE) || - table->field[3]->store((longlong) server_id, TRUE) || + table->field[3]->store((longlong) global_system_variables.server_id, + TRUE) || table->field[4]->store(command_type, command_type_len, client_cs)) goto err; @@ -862,7 +886,7 @@ bool Log_to_csv_event_handler:: table->field[8]->set_notnull(); } - if (table->field[9]->store((longlong) server_id, TRUE)) + if (table->field[9]->store((longlong)global_system_variables.server_id, TRUE)) goto err; table->field[9]->set_notnull(); @@ -1660,6 +1684,20 @@ binlog_flush_cache(THD *thd, binlog_cache_mngr *cache_mngr, end_ev, all, using_stmt, using_trx); } + else + { + /* + This can happen in row-format binlog with something like + BEGIN; INSERT INTO nontrans_table; INSERT IGNORE INTO trans_table; + The nontrans_table is written directly into the binlog before commit, + and if the trans_table is ignored there will be no rows to write when + we get here. + + So there is no work to do. Therefore, we will not increment any XID + count, so we must not decrement any XID count in unlog(). + */ + cache_mngr->need_unlog= 0; + } cache_mngr->reset(using_stmt, using_trx); DBUG_ASSERT((!using_stmt || cache_mngr->stmt_cache.empty()) && @@ -2242,7 +2280,7 @@ static int find_uniq_filename(char *name) DBUG_RETURN(1); } file_info= dir_info->dir_entry; - for (i= dir_info->number_off_files ; i-- ; file_info++) + for (i= dir_info->number_of_files ; i-- ; file_info++) { if (memcmp(file_info->name, start, length) == 0 && test_if_number(file_info->name+length, &number,0)) @@ -2788,6 +2826,15 @@ bool MYSQL_QUERY_LOG::write(THD *thd, time_t current_time, "Yes" : "No"), thd->query_plan_fsort_passes) == (size_t) -1) tmp_errno= errno; + if (thd->variables.log_slow_verbosity & LOG_SLOW_VERBOSITY_EXPLAIN && + thd->lex->explain) + { + StringBuffer<128> buf; + DBUG_ASSERT(!thd->free_list); + if (!print_explain_query(thd->lex, thd, &buf)) + my_b_printf(&log_file, "%s", buf.c_ptr_safe()); + thd->free_items(); + } if (thd->db && strcmp(thd->db, db)) { // Database changed if (my_b_printf(&log_file,"use %s;\n",thd->db) == (size_t) -1) @@ -2886,15 +2933,17 @@ const char *MYSQL_LOG::generate_name(const char *log_name, MYSQL_BIN_LOG::MYSQL_BIN_LOG(uint *sync_period) - :bytes_written(0), prepared_xids(0), file_id(1), open_count(1), - need_start_event(TRUE), + :reset_master_pending(false), mark_xid_done_waiting(0), + bytes_written(0), file_id(1), open_count(1), group_commit_queue(0), group_commit_queue_busy(FALSE), num_commits(0), num_group_commits(0), sync_period_ptr(sync_period), sync_counter(0), + state_file_deleted(false), binlog_state_recover_done(false), is_relay_log(0), signal_cnt(0), checksum_alg_reset(BINLOG_CHECKSUM_ALG_UNDEF), relay_log_checksum_alg(BINLOG_CHECKSUM_ALG_UNDEF), - description_event_for_exec(0), description_event_for_queue(0) + description_event_for_exec(0), description_event_for_queue(0), + current_binlog_id(0) { /* We don't want to initialize locks here as such initialization depends on @@ -2914,23 +2963,63 @@ void MYSQL_BIN_LOG::cleanup() DBUG_ENTER("cleanup"); if (inited) { + xid_count_per_binlog *b; + + /* Wait for the binlog background thread to stop. */ + if (!is_relay_log && binlog_background_thread_started) + { + mysql_mutex_lock(&LOCK_binlog_background_thread); + binlog_background_thread_stop= true; + mysql_cond_signal(&COND_binlog_background_thread); + while (binlog_background_thread_stop) + mysql_cond_wait(&COND_binlog_background_thread_end, + &LOCK_binlog_background_thread); + mysql_mutex_unlock(&LOCK_binlog_background_thread); + binlog_background_thread_started= false; + } + inited= 0; close(LOG_CLOSE_INDEX|LOG_CLOSE_STOP_EVENT); delete description_event_for_queue; delete description_event_for_exec; + + while ((b= binlog_xid_count_list.get())) + { + /* + There should be no pending XIDs at shutdown, and only one entry (for + the active binlog file) in the list. + */ + DBUG_ASSERT(b->xid_count == 0); + DBUG_ASSERT(!binlog_xid_count_list.head()); + my_free(b); + } + mysql_mutex_destroy(&LOCK_log); mysql_mutex_destroy(&LOCK_index); + mysql_mutex_destroy(&LOCK_xid_list); + mysql_mutex_destroy(&LOCK_binlog_background_thread); mysql_cond_destroy(&update_cond); + mysql_cond_destroy(&COND_queue_busy); + mysql_cond_destroy(&COND_xid_list); + mysql_cond_destroy(&COND_binlog_background_thread); + mysql_cond_destroy(&COND_binlog_background_thread_end); } + + /* + Free data for global binlog state. + We can't do that automaticly as we need to do this before + safemalloc is shut down + */ + if (!is_relay_log) + rpl_global_gtid_binlog_state.free(); DBUG_VOID_RETURN; } /* Init binlog-specific vars */ -void MYSQL_BIN_LOG::init(bool no_auto_events_arg, ulong max_size_arg) +void MYSQL_BIN_LOG::init(ulong max_size_arg) { DBUG_ENTER("MYSQL_BIN_LOG::init"); - no_auto_events= no_auto_events_arg; max_size= max_size_arg; DBUG_PRINT("info",("max_size: %lu", max_size)); DBUG_VOID_RETURN; @@ -2942,8 +3031,18 @@ void MYSQL_BIN_LOG::init_pthread_objects() MYSQL_LOG::init_pthread_objects(); mysql_mutex_init(m_key_LOCK_index, &LOCK_index, MY_MUTEX_INIT_SLOW); mysql_mutex_setflags(&LOCK_index, MYF_NO_DEADLOCK_DETECTION); + mysql_mutex_init(key_BINLOG_LOCK_xid_list, + &LOCK_xid_list, MY_MUTEX_INIT_FAST); mysql_cond_init(m_key_update_cond, &update_cond, 0); mysql_cond_init(m_key_COND_queue_busy, &COND_queue_busy, 0); + mysql_cond_init(key_BINLOG_COND_xid_list, &COND_xid_list, 0); + + mysql_mutex_init(key_BINLOG_LOCK_binlog_background_thread, + &LOCK_binlog_background_thread, MY_MUTEX_INIT_FAST); + mysql_cond_init(key_BINLOG_COND_binlog_background_thread, + &COND_binlog_background_thread, 0); + mysql_cond_init(key_BINLOG_COND_binlog_background_thread_end, + &COND_binlog_background_thread_end, 0); } @@ -3031,16 +3130,30 @@ bool MYSQL_BIN_LOG::open(const char *log_name, enum_log_type log_type_arg, const char *new_name, enum cache_type io_cache_type_arg, - bool no_auto_events_arg, ulong max_size_arg, bool null_created_arg, bool need_mutex) { File file= -1; + xid_count_per_binlog *new_xid_list_entry= NULL, *b; DBUG_ENTER("MYSQL_BIN_LOG::open"); DBUG_PRINT("enter",("log_type: %d",(int) log_type_arg)); + if (!is_relay_log) + { + if (!binlog_state_recover_done) + { + binlog_state_recover_done= true; + if (do_binlog_recovery(opt_bin_logname, false)) + DBUG_RETURN(1); + } + + if (!binlog_background_thread_started && + start_binlog_background_thread()) + DBUG_RETURN(1); + } + if (init_and_set_log_file_name(log_name, new_name, log_type_arg, io_cache_type_arg)) { @@ -3092,7 +3205,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, DBUG_RETURN(1); /* all warnings issued */ } - init(no_auto_events_arg, max_size_arg); + init(max_size_arg); open_count++; @@ -3116,11 +3229,10 @@ bool MYSQL_BIN_LOG::open(const char *log_name, write_file_name_to_index_file= 1; } - if (need_start_event && !no_auto_events) { /* - In 4.x we set need_start_event=0 here, but in 5.0 we want a Start event - even if this is not the very first binlog. + In 4.x we put Start event only in the first binlog. But from 5.0 we + want a Start event even if this is not the very first binlog. */ Format_description_log_event s(BINLOG_VERSION); /* @@ -3147,6 +3259,101 @@ bool MYSQL_BIN_LOG::open(const char *log_name, if (s.write(&log_file)) goto err; bytes_written+= s.data_written; + + if (!is_relay_log) + { + char buf[FN_REFLEN]; + + /* + Output a Gtid_list_log_event at the start of the binlog file. + + This is used to quickly determine which GTIDs are found in binlog + files earlier than this one, and which are found in this (or later) + binlogs. + + The list gives a mapping from (domain_id, server_id) -> seq_no (so + this means that there is at most one entry for every unique pair + (domain_id, server_id) in the list). It indicates that this seq_no is + the last one found in an earlier binlog file for this (domain_id, + server_id) combination - so any higher seq_no should be search for + from this binlog file, or a later one. + + This allows to locate the binlog file containing a given GTID by + scanning backwards, reading just the Gtid_list_log_event at the + start of each file, and scanning only the relevant binlog file when + found, not all binlog files. + + The existence of a given entry (domain_id, server_id, seq_no) + guarantees only that this seq_no will not be found in this or any + later binlog file. It does not guarantee that it can be found it an + earlier binlog file, for example the file may have been purged. + + If there is no entry for a given (domain_id, server_id) pair, then + it means that no such GTID exists in any earlier binlog. It is + permissible to remove such pair from future Gtid_list_log_events + if all previous binlog files containing such GTIDs have been purged + (though such optimization is not performed at the time of this + writing). So if there is no entry for given GTID it means that such + GTID should be search for in this or later binlog file, same as if + there had been an entry (domain_id, server_id, 0). + */ + + Gtid_list_log_event gl_ev(&rpl_global_gtid_binlog_state, 0); + if (gl_ev.write(&log_file)) + goto err; + + /* Output a binlog checkpoint event at the start of the binlog file. */ + + /* + Construct an entry in the binlog_xid_count_list for the new binlog + file (we will not link it into the list until we know the new file + is successfully created; otherwise we would have to remove it again + if creation failed, which gets tricky since other threads may have + seen the entry in the meantime - and we do not want to hold + LOCK_xid_list for long periods of time). + + Write the current binlog checkpoint into the log, so XA recovery will + know from where to start recovery. + */ + uint off= dirname_length(log_file_name); + uint len= strlen(log_file_name) - off; + char *entry_mem, *name_mem; + if (!(new_xid_list_entry = (xid_count_per_binlog *) + my_multi_malloc(MYF(MY_WME), + &entry_mem, sizeof(xid_count_per_binlog), + &name_mem, len, + NULL))) + goto err; + memcpy(name_mem, log_file_name+off, len); + new_xid_list_entry->binlog_name= name_mem; + new_xid_list_entry->binlog_name_len= len; + new_xid_list_entry->xid_count= 0; + + /* + Find the name for the Initial binlog checkpoint. + + Normally this will just be the first entry, as we delete entries + when their count drops to zero. But we scan the list to handle any + corner case, eg. for the first binlog file opened after startup, the + list will be empty. + */ + mysql_mutex_lock(&LOCK_xid_list); + I_List_iterator<xid_count_per_binlog> it(binlog_xid_count_list); + while ((b= it++) && b->xid_count == 0) + ; + mysql_mutex_unlock(&LOCK_xid_list); + if (!b) + b= new_xid_list_entry; + strmake(buf, b->binlog_name, b->binlog_name_len); + Binlog_checkpoint_log_event ev(buf, len); + DBUG_EXECUTE_IF("crash_before_write_checkpoint_event", + flush_io_cache(&log_file); + mysql_file_sync(log_file.file, MYF(MY_WME)); + DBUG_SUICIDE();); + if (ev.write(&log_file)) + goto err; + bytes_written+= ev.data_written; + } } if (description_event_for_queue && description_event_for_queue->binlog_version>=4) @@ -3217,6 +3424,42 @@ bool MYSQL_BIN_LOG::open(const char *log_name, #endif } } + + if (!is_relay_log) + { + /* + Now the file was created successfully, so we can link in the entry for + the new binlog file in binlog_xid_count_list. + */ + mysql_mutex_lock(&LOCK_xid_list); + ++current_binlog_id; + new_xid_list_entry->binlog_id= current_binlog_id; + /* Remove any initial entries with no pending XIDs. */ + while ((b= binlog_xid_count_list.head()) && b->xid_count == 0) + my_free(binlog_xid_count_list.get()); + binlog_xid_count_list.push_back(new_xid_list_entry); + mysql_mutex_unlock(&LOCK_xid_list); + + /* + Now that we have synced a new binlog file with an initial Gtid_list + event, it is safe to delete the binlog state file. We will write out + a new, updated file at shutdown, and if we crash before we can recover + the state from the newly written binlog file. + + Since the state file will contain out-of-date data as soon as the first + new GTID is binlogged, it is better to remove it, to avoid any risk of + accidentally reading incorrect data later. + */ + if (!state_file_deleted) + { + char buf[FN_REFLEN]; + fn_format(buf, opt_bin_logname, mysql_data_home, ".state", + MY_UNPACK_FILENAME); + my_delete(buf, MY_SYNC_DIR); + state_file_deleted= true; + } + } + log_state= LOG_OPENED; #ifdef HAVE_REPLICATION @@ -3235,6 +3478,8 @@ err: Turning logging off for the whole duration of the MySQL server process. \ To turn it on again: fix the cause, \ shutdown the MySQL server and restart it.", name, errno); + if (new_xid_list_entry) + my_free(new_xid_list_entry); if (file >= 0) mysql_file_close(file, MYF(0)); close(LOG_CLOSE_INDEX); @@ -3480,11 +3725,11 @@ err: /** Delete all logs refered to in the index file. - Start writing to a new log file. The new index file will only contain this file. - @param thd Thread + @param thd Thread + @param create_new_log 1 if we should start writing to a new log file @note If not called from slave thread, write start event to new log @@ -3495,7 +3740,8 @@ err: 1 error */ -bool MYSQL_BIN_LOG::reset_logs(THD* thd) +bool MYSQL_BIN_LOG::reset_logs(THD* thd, bool create_new_log, + rpl_gtid *init_state, uint32 init_state_len) { LOG_INFO linfo; bool error=0; @@ -3503,7 +3749,33 @@ bool MYSQL_BIN_LOG::reset_logs(THD* thd) const char* save_name; DBUG_ENTER("reset_logs"); - ha_reset_logs(thd); + if (!is_relay_log) + { + if (init_state && !is_empty_state()) + { + my_error(ER_BINLOG_MUST_BE_EMPTY, MYF(0)); + DBUG_RETURN(1); + } + + /* + Mark that a RESET MASTER is in progress. + This ensures that a binlog checkpoint will not try to write binlog + checkpoint events, which would be useless (as we are deleting the binlog + anyway) and could deadlock, as we are holding LOCK_log. + + Wait for any mark_xid_done() calls that might be already running to + complete (mark_xid_done_waiting counter to drop to zero); we need to + do this before we take the LOCK_log to not deadlock. + */ + mysql_mutex_lock(&LOCK_xid_list); + reset_master_pending= true; + while (mark_xid_done_waiting > 0) + mysql_cond_wait(&COND_xid_list, &LOCK_xid_list); + mysql_mutex_unlock(&LOCK_xid_list); + } + + if (thd) + ha_reset_logs(thd); /* We need to get both locks to be sure that no one is trying to write to the index log file. @@ -3511,6 +3783,50 @@ bool MYSQL_BIN_LOG::reset_logs(THD* thd) mysql_mutex_lock(&LOCK_log); mysql_mutex_lock(&LOCK_index); + if (!is_relay_log) + { + /* + We are going to nuke all binary log files. + Without binlog, we cannot XA recover prepared-but-not-committed + transactions in engines. So force a commit checkpoint first. + + Note that we take and immediately release LOCK_commit_ordered. This has + the effect to ensure that any on-going group commit (in + trx_group_commit_leader()) has completed before we request the checkpoint, + due to the chaining of LOCK_log and LOCK_commit_ordered in that function. + (We are holding LOCK_log, so no new group commit can start). + + Without this, it is possible (though perhaps unlikely) that the RESET + MASTER could run in-between the write to the binlog and the + commit_ordered() in the engine of some transaction, and then a crash + later would leave such transaction not recoverable. + */ + mysql_mutex_lock(&LOCK_commit_ordered); + mysql_mutex_unlock(&LOCK_commit_ordered); + + mark_xids_active(current_binlog_id, 1); + do_checkpoint_request(current_binlog_id); + + /* Now wait for all checkpoint requests and pending unlog() to complete. */ + mysql_mutex_lock(&LOCK_xid_list); + for (;;) + { + if (is_xidlist_idle_nolock()) + break; + /* + Wait until signalled that one more binlog dropped to zero, then check + again. + */ + mysql_cond_wait(&COND_xid_list, &LOCK_xid_list); + } + + /* + Now all XIDs are fully flushed to disk, and we are holding LOCK_log so + no new ones will be written. So we can proceed to delete the logs. + */ + mysql_mutex_unlock(&LOCK_xid_list); + } + /* The following mutex is needed to ensure that no threads call 'delete thd' as we would then risk missing a 'rollback' from this @@ -3574,6 +3890,14 @@ bool MYSQL_BIN_LOG::reset_logs(THD* thd) break; } + if (!is_relay_log) + { + if (init_state) + rpl_global_gtid_binlog_state.load(init_state, init_state_len); + else + rpl_global_gtid_binlog_state.reset(); + } + /* Start logging with a new file */ close(LOG_CLOSE_INDEX | LOG_CLOSE_TO_BE_OPENED); if ((error= my_delete(index_file_name, MYF(0)))) // Reset (open will update) @@ -3601,10 +3925,8 @@ bool MYSQL_BIN_LOG::reset_logs(THD* thd) goto err; } } - if (!thd->slave_thread) - need_start_event=1; - if (!open_index_file(index_file_name, 0, FALSE)) - if ((error= open(save_name, log_type, 0, io_cache_type, no_auto_events, max_size, 0, FALSE))) + if (create_new_log && !open_index_file(index_file_name, 0, FALSE)) + if ((error= open(save_name, log_type, 0, io_cache_type, max_size, 0, FALSE))) goto err; my_free((void *) save_name); @@ -3612,6 +3934,31 @@ err: if (error == 1) name= const_cast<char*>(save_name); mysql_mutex_unlock(&LOCK_thread_count); + + if (!is_relay_log) + { + xid_count_per_binlog *b; + /* + Remove all entries in the xid_count list except the last. + Normally we will just be deleting all the entries that we waited for to + drop to zero above. But if we fail during RESET MASTER for some reason + then we will not have created any new log file, and we may keep the last + of the old entries. + */ + mysql_mutex_lock(&LOCK_xid_list); + for (;;) + { + b= binlog_xid_count_list.head(); + DBUG_ASSERT(b /* List can never become empty. */); + if (b->binlog_id == current_binlog_id) + break; + DBUG_ASSERT(b->xid_count == 0); + my_free(binlog_xid_count_list.get()); + } + reset_master_pending= false; + mysql_mutex_unlock(&LOCK_xid_list); + } + mysql_mutex_unlock(&LOCK_index); mysql_mutex_unlock(&LOCK_log); DBUG_RETURN(error); @@ -3664,7 +4011,7 @@ int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) DBUG_ENTER("purge_first_log"); DBUG_ASSERT(is_open()); - DBUG_ASSERT(rli->slave_running == 1); + DBUG_ASSERT(rli->slave_running == MYSQL_SLAVE_RUN_NOT_CONNECT); DBUG_ASSERT(!strcmp(rli->linfo.log_file_name,rli->event_relay_log_name)); mysql_mutex_lock(&LOCK_index); @@ -3819,8 +4166,7 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, if ((error=find_log_pos(&log_info, NullS, 0 /*no mutex*/))) goto err; while ((strcmp(to_log,log_info.log_file_name) || (exit_loop=included)) && - !is_active(log_info.log_file_name) && - !log_in_use(log_info.log_file_name)) + can_purge_log(log_info.log_file_name)) { if ((error= register_purge_index_entry(log_info.log_file_name))) { @@ -4170,8 +4516,7 @@ int MYSQL_BIN_LOG::purge_logs_before_date(time_t purge_time) goto err; while (strcmp(log_file_name, log_info.log_file_name) && - !is_active(log_info.log_file_name) && - !log_in_use(log_info.log_file_name)) + can_purge_log(log_info.log_file_name)) { if (!mysql_file_stat(m_key_file_log, log_info.log_file_name, &stat_area, MYF(0))) @@ -4224,9 +4569,57 @@ err: mysql_mutex_unlock(&LOCK_index); DBUG_RETURN(error); } + + +bool +MYSQL_BIN_LOG::can_purge_log(const char *log_file_name) +{ + xid_count_per_binlog *b; + + if (is_active(log_file_name)) + return false; + mysql_mutex_lock(&LOCK_xid_list); + { + I_List_iterator<xid_count_per_binlog> it(binlog_xid_count_list); + while ((b= it++) && + 0 != strncmp(log_file_name+dirname_length(log_file_name), + b->binlog_name, b->binlog_name_len)) + ; + } + mysql_mutex_unlock(&LOCK_xid_list); + if (b) + return false; + return !log_in_use(log_file_name); +} #endif /* HAVE_REPLICATION */ +bool +MYSQL_BIN_LOG::is_xidlist_idle() +{ + bool res; + mysql_mutex_lock(&LOCK_xid_list); + res= is_xidlist_idle_nolock(); + mysql_mutex_unlock(&LOCK_xid_list); + return res; +} + + +bool +MYSQL_BIN_LOG::is_xidlist_idle_nolock() +{ + xid_count_per_binlog *b; + + I_List_iterator<xid_count_per_binlog> it(binlog_xid_count_list); + while ((b= it++)) + { + if (b->xid_count > 0) + return false; + } + return true; +} + + /** Create a new log file name. @@ -4317,26 +4710,6 @@ int MYSQL_BIN_LOG::new_file_impl(bool need_lock) mysql_mutex_assert_owner(&LOCK_log); mysql_mutex_assert_owner(&LOCK_index); - /* - if binlog is used as tc log, be sure all xids are "unlogged", - so that on recover we only need to scan one - latest - binlog file - for prepared xids. As this is expected to be a rare event, - simple wait strategy is enough. We're locking LOCK_log to be sure no - new Xid_log_event's are added to the log (and prepared_xids is not - increased), and waiting on COND_prep_xids for late threads to - catch up. - */ - if (prepared_xids) - { - tc_log_page_waits++; - mysql_mutex_lock(&LOCK_prep_xids); - while (prepared_xids) { - DBUG_PRINT("info", ("prepared_xids=%lu", prepared_xids)); - mysql_cond_wait(&COND_prep_xids, &LOCK_prep_xids); - } - mysql_mutex_unlock(&LOCK_prep_xids); - } - /* Reuse old name if not binlog and not update log */ new_name_ptr= name; @@ -4351,7 +4724,6 @@ int MYSQL_BIN_LOG::new_file_impl(bool need_lock) if (log_type == LOG_BIN) { - if (!no_auto_events) { /* We log the whole file name for log file as the user may decide @@ -4426,7 +4798,7 @@ int MYSQL_BIN_LOG::new_file_impl(bool need_lock) /* reopen the binary log file. */ file_to_open= new_name_ptr; error= open(old_name, log_type, new_name_ptr, io_cache_type, - no_auto_events, max_size, 1, FALSE); + max_size, 1, FALSE); } /* handle reopening errors */ @@ -4478,12 +4850,23 @@ end: } -bool MYSQL_BIN_LOG::append(Log_event* ev) +bool +MYSQL_BIN_LOG::append(Log_event *ev) { - bool error = 0; + bool res; mysql_mutex_lock(&LOCK_log); + res= append_no_lock(ev); + mysql_mutex_unlock(&LOCK_log); + return res; +} + + +bool MYSQL_BIN_LOG::append_no_lock(Log_event* ev) +{ + bool error = 0; DBUG_ENTER("MYSQL_BIN_LOG::append"); + mysql_mutex_assert_owner(&LOCK_log); DBUG_ASSERT(log_file.type == SEQ_READ_APPEND); /* Log_event::write() is smart enough to use my_b_write() or @@ -4498,10 +4881,9 @@ bool MYSQL_BIN_LOG::append(Log_event* ev) DBUG_PRINT("info",("max_size: %lu",max_size)); if (flush_and_sync(0)) goto err; - if ((uint) my_b_append_tell(&log_file) > max_size) + if (my_b_append_tell(&log_file) > max_size) error= new_file_without_locking(); err: - mysql_mutex_unlock(&LOCK_log); signal_update(); // Safe as we don't call close DBUG_RETURN(error); } @@ -4529,7 +4911,7 @@ bool MYSQL_BIN_LOG::appendv(const char* buf, uint len,...) DBUG_PRINT("info",("max_size: %lu",max_size)); if (flush_and_sync(0)) goto err; - if ((uint) my_b_append_tell(&log_file) > max_size) + if (my_b_append_tell(&log_file) > max_size) error= new_file_without_locking(); err: if (!error) @@ -5035,6 +5417,225 @@ MYSQL_BIN_LOG::flush_and_set_pending_rows_event(THD *thd, DBUG_RETURN(error); } + +/* Generate a new global transaction ID, and write it to the binlog */ +bool +MYSQL_BIN_LOG::write_gtid_event(THD *thd, bool standalone, + bool is_transactional, uint64 commit_id) +{ + rpl_gtid gtid; + uint32 domain_id= thd->variables.gtid_domain_id; + uint32 server_id= thd->variables.server_id; + uint64 seq_no= thd->variables.gtid_seq_no; + int err; + + /* + Reset the session variable gtid_seq_no, to reduce the risk of accidentally + producing a duplicate GTID. + */ + thd->variables.gtid_seq_no= 0; + if (seq_no != 0) + { + /* Use the specified sequence number. */ + gtid.domain_id= domain_id; + gtid.server_id= server_id; + gtid.seq_no= seq_no; + err= rpl_global_gtid_binlog_state.update(>id, opt_gtid_strict_mode); + if (err && thd->stmt_da->sql_errno()==ER_GTID_STRICT_OUT_OF_ORDER) + errno= ER_GTID_STRICT_OUT_OF_ORDER; + } + else + { + /* Allocate the next sequence number for the GTID. */ + err= rpl_global_gtid_binlog_state.update_with_next_gtid(domain_id, + server_id, >id); + seq_no= gtid.seq_no; + } + if (err) + return true; + thd->last_commit_gtid= gtid; + + Gtid_log_event gtid_event(thd, seq_no, domain_id, standalone, + LOG_EVENT_SUPPRESS_USE_F, is_transactional, + commit_id); + + /* Write the event to the binary log. */ + if (gtid_event.write(&mysql_bin_log.log_file)) + return true; + status_var_add(thd->status_var.binlog_bytes_written, gtid_event.data_written); + + return false; +} + + +int +MYSQL_BIN_LOG::write_state_to_file() +{ + File file_no; + IO_CACHE cache; + char buf[FN_REFLEN]; + int err; + bool opened= false; + bool inited= false; + + fn_format(buf, opt_bin_logname, mysql_data_home, ".state", + MY_UNPACK_FILENAME); + if ((file_no= mysql_file_open(key_file_binlog_state, buf, + O_RDWR|O_CREAT|O_TRUNC|O_BINARY, + MYF(MY_WME))) < 0) + { + err= 1; + goto err; + } + opened= true; + if ((err= init_io_cache(&cache, file_no, IO_SIZE, WRITE_CACHE, 0, 0, + MYF(MY_WME|MY_WAIT_IF_FULL)))) + goto err; + inited= true; + if ((err= rpl_global_gtid_binlog_state.write_to_iocache(&cache))) + goto err; + inited= false; + if ((err= end_io_cache(&cache))) + goto err; + if ((err= mysql_file_sync(file_no, MYF(MY_WME|MY_SYNC_FILESIZE)))) + goto err; + goto end; + +err: + sql_print_error("Error writing binlog state to file '%s'.\n", buf); + if (inited) + end_io_cache(&cache); +end: + if (opened) + mysql_file_close(file_no, MYF(0)); + + return err; +} + + +int +MYSQL_BIN_LOG::read_state_from_file() +{ + File file_no; + IO_CACHE cache; + char buf[FN_REFLEN]; + int err; + bool opened= false; + bool inited= false; + + fn_format(buf, opt_bin_logname, mysql_data_home, ".state", + MY_UNPACK_FILENAME); + if ((file_no= mysql_file_open(key_file_binlog_state, buf, + O_RDONLY|O_BINARY, MYF(0))) < 0) + { + if (my_errno != ENOENT) + { + err= 1; + goto err; + } + else + { + /* + If the state file does not exist, this is the first server startup + with GTID enabled. So initialize to empty state. + */ + rpl_global_gtid_binlog_state.reset(); + err= 0; + goto end; + } + } + opened= true; + if ((err= init_io_cache(&cache, file_no, IO_SIZE, READ_CACHE, 0, 0, + MYF(MY_WME|MY_WAIT_IF_FULL)))) + goto err; + inited= true; + if ((err= rpl_global_gtid_binlog_state.read_from_iocache(&cache))) + goto err; + goto end; + +err: + sql_print_error("Error reading binlog GTID state from file '%s'.\n", buf); +end: + if (inited) + end_io_cache(&cache); + if (opened) + mysql_file_close(file_no, MYF(0)); + + return err; +} + + +int +MYSQL_BIN_LOG::get_most_recent_gtid_list(rpl_gtid **list, uint32 *size) +{ + return rpl_global_gtid_binlog_state.get_most_recent_gtid_list(list, size); +} + + +bool +MYSQL_BIN_LOG::append_state_pos(String *str) +{ + return rpl_global_gtid_binlog_state.append_pos(str); +} + + +bool +MYSQL_BIN_LOG::append_state(String *str) +{ + return rpl_global_gtid_binlog_state.append_state(str); +} + + +bool +MYSQL_BIN_LOG::is_empty_state() +{ + return (rpl_global_gtid_binlog_state.count() == 0); +} + + +bool +MYSQL_BIN_LOG::find_in_binlog_state(uint32 domain_id, uint32 server_id, + rpl_gtid *out_gtid) +{ + rpl_gtid *gtid; + if ((gtid= rpl_global_gtid_binlog_state.find(domain_id, server_id))) + *out_gtid= *gtid; + return gtid != NULL; +} + + +bool +MYSQL_BIN_LOG::lookup_domain_in_binlog_state(uint32 domain_id, + rpl_gtid *out_gtid) +{ + rpl_gtid *found_gtid; + + if ((found_gtid= rpl_global_gtid_binlog_state.find_most_recent(domain_id))) + { + *out_gtid= *found_gtid; + return true; + } + + return false; +} + + +int +MYSQL_BIN_LOG::bump_seq_no_counter_if_needed(uint32 domain_id, uint64 seq_no) +{ + return rpl_global_gtid_binlog_state.bump_seq_no_if_needed(domain_id, seq_no); +} + + +bool +MYSQL_BIN_LOG::check_strict_gtid_sequence(uint32 domain_id, uint32 server_id, + uint64 seq_no) +{ + return rpl_global_gtid_binlog_state.check_strict_sequence(domain_id, + server_id, seq_no); +} + + /** Write an event to the binary log. If with_annotate != NULL and *with_annotate = TRUE write also Annotate_rows before the event @@ -5050,6 +5651,8 @@ bool MYSQL_BIN_LOG::write(Log_event *event_info, my_bool *with_annotate) bool is_trans_cache= FALSE; bool using_trans= event_info->use_trans_cache(); bool direct= event_info->use_direct_logging(); + ulong prev_binlog_id; + LINT_INIT(prev_binlog_id); if (thd->binlog_evt_union.do_union) { @@ -5101,6 +5704,9 @@ bool MYSQL_BIN_LOG::write(Log_event *event_info, my_bool *with_annotate) file= &log_file; my_org_b_tell= my_b_tell(file); mysql_mutex_lock(&LOCK_log); + prev_binlog_id= current_binlog_id; + if (write_gtid_event(thd, true, using_trans, 0)) + goto err; } else { @@ -5244,7 +5850,7 @@ err: mysql_mutex_unlock(&LOCK_log); if (check_purge) - purge(); + checkpoint_and_purge(prev_binlog_id); } if (error) @@ -5329,6 +5935,60 @@ bool general_log_write(THD *thd, enum enum_server_command command, return FALSE; } + +static void +binlog_checkpoint_callback(void *cookie) +{ + MYSQL_BIN_LOG::xid_count_per_binlog *entry= + (MYSQL_BIN_LOG::xid_count_per_binlog *)cookie; + /* + For every supporting engine, we increment the xid_count and issue a + commit_checkpoint_request(). Then we can count when all + commit_checkpoint_notify() callbacks have occured, and then log a new + binlog checkpoint event. + */ + mysql_bin_log.mark_xids_active(entry->binlog_id, 1); +} + + +/* + Request a commit checkpoint from each supporting engine. + This must be called after each binlog rotate, and after LOCK_log has been + released. The xid_count value in the xid_count_per_binlog entry was + incremented by 1 and will be decremented in this function; this ensures + that the entry will not go away early despite LOCK_log not being held. +*/ +void +MYSQL_BIN_LOG::do_checkpoint_request(ulong binlog_id) +{ + xid_count_per_binlog *entry; + + /* + Find the binlog entry, and invoke commit_checkpoint_request() on it in + each supporting storage engine. + */ + mysql_mutex_lock(&LOCK_xid_list); + I_List_iterator<xid_count_per_binlog> it(binlog_xid_count_list); + do { + entry= it++; + DBUG_ASSERT(entry /* binlog_id is always somewhere in the list. */); + } while (entry->binlog_id != binlog_id); + mysql_mutex_unlock(&LOCK_xid_list); + + ha_commit_checkpoint_request(entry, binlog_checkpoint_callback); + /* + When we rotated the binlog, we incremented xid_count to make sure the + entry would not go away until this point, where we have done all necessary + commit_checkpoint_request() calls. + So now we can (and must) decrease the count - when it reaches zero, we + will know that both all pending unlog() and all pending + commit_checkpoint_notify() calls are done, and we can log a new binlog + checkpoint. + */ + mark_xid_done(binlog_id, true); +} + + /** The method executes rotation when LOCK_log is already acquired by the caller. @@ -5337,6 +5997,15 @@ bool general_log_write(THD *thd, enum enum_server_command command, @param check_purge is set to true if rotation took place @note + Caller _must_ check the check_purge variable. If this is set, it means + that the binlog was rotated, and caller _must_ ensure that + do_checkpoint_request() is called later with the binlog_id of the rotated + binlog file. The call to do_checkpoint_request() must happen after + LOCK_log is released (which is why we cannot simply do it here). + Usually, checkpoint_and_purge() is appropriate, as it will both handle + the checkpointing and any needed purging of old logs. + + @note If rotation fails, for instance the server was unable to create a new log file, we still try to write an incident event to the current log. @@ -5354,7 +6023,27 @@ int MYSQL_BIN_LOG::rotate(bool force_rotate, bool* check_purge) if (force_rotate || (my_b_tell(&log_file) >= (my_off_t) max_size)) { + ulong binlog_id= current_binlog_id; + /* + We rotate the binlog, so we need to start a commit checkpoint in all + supporting engines - when it finishes, we can log a new binlog checkpoint + event. + + But we cannot start the checkpoint here - there could be a group commit + still in progress which needs to be included in the checkpoint, and + besides we do not want to do the (possibly expensive) checkpoint while + LOCK_log is held. + + On the other hand, we must be sure that the xid_count entry for the + previous log does not go away until we start the checkpoint - which it + could do as it is no longer the most recent. So we increment xid_count + (to count the pending checkpoint request) - this will fix the entry in + place until we decrement again in do_checkpoint_request(). + */ + mark_xids_active(binlog_id, 1); + if ((error= new_file_without_locking())) + { /** Be conservative... There are possible lost events (eg, failing to log the Execute_load_query_log_event @@ -5367,7 +6056,14 @@ int MYSQL_BIN_LOG::rotate(bool force_rotate, bool* check_purge) if (!write_incident_already_locked(current_thd)) flush_and_sync(0); - *check_purge= true; + /* + We failed to rotate - so we have to decrement the xid_count back that + we incremented before attempting the rotate. + */ + mark_xid_done(binlog_id, false); + } + else + *check_purge= true; } DBUG_RETURN(error); } @@ -5395,6 +6091,13 @@ void MYSQL_BIN_LOG::purge() #endif } + +void MYSQL_BIN_LOG::checkpoint_and_purge(ulong binlog_id) +{ + do_checkpoint_request(binlog_id); + purge(); +} + /** The method is a shortcut of @c rotate() and @c purge(). LOCK_log is acquired prior to rotate and is released after it. @@ -5407,11 +6110,13 @@ void MYSQL_BIN_LOG::purge() int MYSQL_BIN_LOG::rotate_and_purge(bool force_rotate) { int error= 0; + ulong prev_binlog_id; DBUG_ENTER("MYSQL_BIN_LOG::rotate_and_purge"); bool check_purge= false; //todo: fix the macro def and restore safe_mutex_assert_not_owner(&LOCK_log); mysql_mutex_lock(&LOCK_log); + prev_binlog_id= current_binlog_id; if ((error= rotate(force_rotate, &check_purge))) check_purge= false; /* @@ -5421,7 +6126,7 @@ int MYSQL_BIN_LOG::rotate_and_purge(bool force_rotate) mysql_mutex_unlock(&LOCK_log); if (check_purge) - purge(); + checkpoint_and_purge(prev_binlog_id); DBUG_RETURN(error); } @@ -5752,11 +6457,13 @@ bool MYSQL_BIN_LOG::write_incident(THD *thd) uint error= 0; my_off_t offset; bool check_purge= false; + ulong prev_binlog_id; DBUG_ENTER("MYSQL_BIN_LOG::write_incident"); mysql_mutex_lock(&LOCK_log); if (likely(is_open())) { + prev_binlog_id= current_binlog_id; if (!(error= write_incident_already_locked(thd)) && !(error= flush_and_sync(0))) { @@ -5776,12 +6483,51 @@ bool MYSQL_BIN_LOG::write_incident(THD *thd) mysql_mutex_unlock(&LOCK_log); if (check_purge) - purge(); + checkpoint_and_purge(prev_binlog_id); } DBUG_RETURN(error); } +void +MYSQL_BIN_LOG::write_binlog_checkpoint_event_already_locked(const char *name, + uint len) +{ + my_off_t offset; + Binlog_checkpoint_log_event ev(name, len); + /* + Note that we must sync the binlog checkpoint to disk. + Otherwise a subsequent log purge could delete binlogs that XA recovery + thinks are needed (even though they are not really). + */ + if (!ev.write(&log_file) && !flush_and_sync(0)) + { + signal_update(); + } + else + { + /* + If we fail to write the checkpoint event, something is probably really + bad with the binlog. We complain in the error log. + + Note that failure to write binlog checkpoint does not compromise the + ability to do crash recovery - crash recovery will just have to scan a + bit more of the binlog than strictly necessary. + */ + sql_print_error("Failed to write binlog checkpoint event to binary log\n"); + } + + offset= my_b_tell(&log_file); + /* + Take mutex to protect against a reader seeing partial writes of 64-bit + offset on 32-bit CPUs. + */ + mysql_mutex_lock(&LOCK_commit_ordered); + last_commit_pos_offset= offset; + mysql_mutex_unlock(&LOCK_commit_ordered); +} + + /** Write a cached log entry to the binary log. - To support transaction over replication, we wrap the transaction @@ -5814,6 +6560,7 @@ MYSQL_BIN_LOG::write_transaction_to_binlog(THD *thd, bool using_trx_cache) { group_commit_entry entry; + Ha_trx_info *ha_info; DBUG_ENTER("MYSQL_BIN_LOG::write_transaction_to_binlog"); entry.thd= thd; @@ -5822,20 +6569,16 @@ MYSQL_BIN_LOG::write_transaction_to_binlog(THD *thd, entry.all= all; entry.using_stmt_cache= using_stmt_cache; entry.using_trx_cache= using_trx_cache; + entry.need_unlog= false; + ha_info= all ? thd->transaction.all.ha_list : thd->transaction.stmt.ha_list; + for (; ha_info; ha_info= ha_info->next()) + { + if (ha_info->is_started() && ha_info->ht() != binlog_hton && + !ha_info->ht()->commit_checkpoint_request) + entry.need_unlog= true; + break; + } - /* - Log "BEGIN" at the beginning of every transaction. Here, a transaction is - either a BEGIN..COMMIT block or a single statement in autocommit mode. - - Create the necessary events here, where we have the correct THD (and - thread context). - - Due to group commit the actual writing to binlog may happen in a different - thread. - */ - Query_log_event qinfo(thd, STRING_WITH_LEN("BEGIN"), using_trx_cache, TRUE, - TRUE, 0); - entry.begin_event= &qinfo; entry.end_event= end_ev; if (cache_mngr->stmt_cache.has_incident() || cache_mngr->trx_cache.has_incident()) @@ -5851,45 +6594,356 @@ MYSQL_BIN_LOG::write_transaction_to_binlog(THD *thd, } } -bool -MYSQL_BIN_LOG::write_transaction_to_binlog_events(group_commit_entry *entry) + +/* + Put a transaction that is ready to commit in the group commit queue. + The transaction is identified by the ENTRY object passed into this function. + + To facilitate group commit for the binlog, we first queue up ourselves in + this function. Then later the first thread to enter the queue waits for + the LOCK_log mutex, and commits for everyone in the queue once it gets the + lock. Any other threads in the queue just wait for the first one to finish + the commit and wake them up. This way, all transactions in the queue get + committed in a single disk operation. + + The main work in this function is when the commit in one transaction has + been marked to wait for the commit of another transaction to happen + first. This is used to support in-order parallel replication, where + transactions can execute out-of-order but need to be committed in-order with + how they happened on the master. The waiting of one commit on another needs + to be integrated with the group commit queue, to ensure that the waiting + transaction can participate in the same group commit as the waited-for + transaction. + + So when we put a transaction in the queue, we check if there were other + transactions already prepared to commit but just waiting for the first one + to commit. If so, we add those to the queue as well, transitively for all + waiters. + + @retval < 0 Error + @retval > 0 If queued as the first entry in the queue (meaning this + is the leader) + @retval 0 Otherwise (queued as participant, leader handles the commit) +*/ + +int +MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) { + group_commit_entry *entry, *orig_queue; + wait_for_commit *cur, *last; + wait_for_commit *wfc; + DBUG_ENTER("MYSQL_BIN_LOG::queue_for_group_commit"); + + /* + Check if we need to wait for another transaction to commit before us. + + It is safe to do a quick check without lock first in the case where we do + not have to wait. But if the quick check shows we need to wait, we must do + another safe check under lock, to avoid the race where the other + transaction wakes us up between the check and the wait. + */ + wfc= orig_entry->thd->wait_for_commit_ptr; + orig_entry->queued_by_other= false; + if (wfc && wfc->waiting_for_commit) + { + mysql_mutex_lock(&wfc->LOCK_wait_commit); + /* Do an extra check here, this time safely under lock. */ + if (wfc->waiting_for_commit) + { + const char *old_msg; + /* + By setting wfc->opaque_pointer to our own entry, we mark that we are + ready to commit, but waiting for another transaction to commit before + us. + + This other transaction may then take over the commit process for us to + get us included in its own group commit. If this happens, the + queued_by_other flag is set. + */ + wfc->opaque_pointer= orig_entry; + old_msg= + orig_entry->thd->enter_cond(&wfc->COND_wait_commit, + &wfc->LOCK_wait_commit, + "Waiting for prior transaction to commit"); + DEBUG_SYNC(orig_entry->thd, "group_commit_waiting_for_prior"); + while (wfc->waiting_for_commit && !orig_entry->thd->check_killed()) + mysql_cond_wait(&wfc->COND_wait_commit, &wfc->LOCK_wait_commit); + wfc->opaque_pointer= NULL; + DBUG_PRINT("info", ("After waiting for prior commit, queued_by_other=%d", + orig_entry->queued_by_other)); + + if (wfc->waiting_for_commit) + { + /* Wait terminated due to kill. */ + wait_for_commit *loc_waitee= wfc->waitee; + mysql_mutex_lock(&loc_waitee->LOCK_wait_commit); + if (loc_waitee->wakeup_subsequent_commits_running || + orig_entry->queued_by_other) + { + /* Our waitee is already waking us up, so ignore the kill. */ + mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); + do + { + mysql_cond_wait(&wfc->COND_wait_commit, &wfc->LOCK_wait_commit); + } while (wfc->waiting_for_commit); + } + else + { + /* We were killed, so remove us from the list of waitee. */ + wfc->remove_from_list(&loc_waitee->subsequent_commits_list); + mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); + + orig_entry->thd->exit_cond(old_msg); + /* Interrupted by kill. */ + DEBUG_SYNC(orig_entry->thd, "group_commit_waiting_for_prior_killed"); + wfc->wakeup_error= orig_entry->thd->killed_errno(); + if (wfc->wakeup_error) + wfc->wakeup_error= ER_QUERY_INTERRUPTED; + my_message(wfc->wakeup_error, ER(wfc->wakeup_error), MYF(0)); + DBUG_RETURN(-1); + } + } + orig_entry->thd->exit_cond(old_msg); + } + else + mysql_mutex_unlock(&wfc->LOCK_wait_commit); + + if (wfc->wakeup_error) + { + my_error(ER_PRIOR_COMMIT_FAILED, MYF(0)); + DBUG_RETURN(-1); + } + } + /* - To facilitate group commit for the binlog, we first queue up ourselves in - the group commit queue. Then the first thread to enter the queue waits for - the LOCK_log mutex, and commits for everyone in the queue once it gets the - lock. Any other threads in the queue just wait for the first one to finish - the commit and wake them up. + If the transaction we were waiting for has already put us into the group + commit queue (and possibly already done the entire binlog commit for us), + then there is nothing else to do. */ + if (orig_entry->queued_by_other) + DBUG_RETURN(0); - entry->thd->clear_wakeup_ready(); + /* Now enqueue ourselves in the group commit queue. */ + DEBUG_SYNC(orig_entry->thd, "commit_before_enqueue"); + orig_entry->thd->clear_wakeup_ready(); mysql_mutex_lock(&LOCK_prepare_ordered); - group_commit_entry *orig_queue= group_commit_queue; - entry->next= orig_queue; - group_commit_queue= entry; + orig_queue= group_commit_queue; + + /* + Iteratively process everything added to the queue, looking for waiters, + and their waiters, and so on. If a waiter is ready to commit, we + immediately add it to the queue; if not we just wake it up. + + This would be natural to do with recursion, but we want to avoid + potentially unbounded recursion blowing the C stack, so we use the list + approach instead. + + We keep a list of all the waiters that need to be processed in `list', + linked through the next_subsequent_commit pointer. Initially this list + contains only the entry passed into this function. + + We process entries in the list one by one. The element currently being + processed is pointed to by `cur`, and the element at the end of the list + is pointed to by `last` (we do not use NULL to terminate the list). + + As we process an element, it is first added to the group_commit_queue. + Then any waiters for that element are added at the end of the list, to + be processed in subsequent iterations. This continues until the list + is exhausted, with all elements ever added eventually processed. - if (entry->cache_mngr->using_xa) + The end result is a breath-first traversal of the tree of waiters, + re-using the next_subsequent_commit pointers in place of extra stack + space in a recursive traversal. + + The temporary list created in next_subsequent_commit is not + used by the caller or any other function. + */ + + cur= wfc; + last= wfc; + entry= orig_entry; + for (;;) { - DEBUG_SYNC(entry->thd, "commit_before_prepare_ordered"); - run_prepare_ordered(entry->thd, entry->all); - DEBUG_SYNC(entry->thd, "commit_after_prepare_ordered"); + /* Add the entry to the group commit queue. */ + entry->next= group_commit_queue; + group_commit_queue= entry; + + if (entry->cache_mngr->using_xa) + { + DEBUG_SYNC(entry->thd, "commit_before_prepare_ordered"); + run_prepare_ordered(entry->thd, entry->all); + DEBUG_SYNC(entry->thd, "commit_after_prepare_ordered"); + } + + if (!cur) + break; // Can happen if initial entry has no wait_for_commit + + /* + Check if this transaction has other transaction waiting for it to commit. + + If so, process the waiting transactions, and their waiters and so on, + transitively. + */ + if (cur->subsequent_commits_list) + { + wait_for_commit *waiter; + wait_for_commit *wakeup_list= NULL; + wait_for_commit **wakeup_next_ptr= &wakeup_list; + + mysql_mutex_lock(&cur->LOCK_wait_commit); + /* + Grab the list, now safely under lock, and process it if still + non-empty. + */ + waiter= cur->subsequent_commits_list; + cur->subsequent_commits_list= NULL; + while (waiter) + { + wait_for_commit *next= waiter->next_subsequent_commit; + group_commit_entry *entry2= + (group_commit_entry *)waiter->opaque_pointer; + if (entry2) + { + /* + This is another transaction ready to be written to the binary + log. We can put it into the queue directly, without needing a + separate context switch to the other thread. We just set a flag + so that the other thread will know when it wakes up that it was + already processed. + + So put it at the end of the list to be processed in a subsequent + iteration of the outer loop. + */ + entry2->queued_by_other= true; + last->next_subsequent_commit= waiter; + last= waiter; + /* + As a small optimisation, we do not actually need to set + waiter->next_subsequent_commit to NULL, as we can use the + pointer `last' to check for end-of-list. + */ + } + else + { + /* + Wake up the waiting transaction. + + For this, we need to set the "wakeup running" flag and release + the waitee lock to avoid a deadlock, see comments on + THD::wakeup_subsequent_commits2() for details. + + So we need to put these on a list and delay the wakeup until we + have released the lock. + */ + *wakeup_next_ptr= waiter; + wakeup_next_ptr= &waiter->next_subsequent_commit; + } + waiter= next; + } + if (wakeup_list) + { + /* Now release our lock and do the wakeups that were delayed above. */ + cur->wakeup_subsequent_commits_running= true; + mysql_mutex_unlock(&cur->LOCK_wait_commit); + for (;;) + { + wait_for_commit *next; + + /* + ToDo: We wakeup the waiter here, so that it can have the chance to + reach its own commit state and queue up for this same group commit, + if it is still pending. + + One problem with this is that if the waiter does not reach its own + commit state before this group commit starts, and then the group + commit fails (binlog write failure), we do not get to propagate + the error to the waiter. + + A solution for this could be to delay the wakeup until commit is + successful. But then we need to set a flag in the waitee that it is + already queued for group commit, so that the waiter can check this + flag and queue itself if it _does_ reach the commit state in time. + + (But error handling in case of binlog write failure is currently + broken in other ways, as well). + */ + if (&wakeup_list->next_subsequent_commit == wakeup_next_ptr) + { + /* The last one in the list. */ + wakeup_list->wakeup(0); + break; + } + /* + Important: don't access wakeup_list->next after the wakeup() call, + it may be invalidated by the other thread. + */ + next= wakeup_list->next_subsequent_commit; + wakeup_list->wakeup(0); + wakeup_list= next; + } + /* + We need a full memory barrier between walking the list and clearing + the flag wakeup_subsequent_commits_running. This barrier is needed + to ensure that no other thread will start to modify the list + pointers before we are done traversing the list. + + But wait_for_commit::wakeup(), which was called above, does a full + memory barrier already (it locks a mutex). + */ + cur->wakeup_subsequent_commits_running= false; + } + else + mysql_mutex_unlock(&cur->LOCK_wait_commit); + } + if (cur == last) + break; + /* + Move to the next entry in the flattened list of waiting transactions + that still need to be processed transitively. + */ + cur= cur->next_subsequent_commit; + entry= (group_commit_entry *)cur->opaque_pointer; + DBUG_ASSERT(entry != NULL); } + + if (opt_binlog_commit_wait_count > 0) + mysql_cond_signal(&COND_prepare_ordered); mysql_mutex_unlock(&LOCK_prepare_ordered); - DEBUG_SYNC(entry->thd, "commit_after_release_LOCK_prepare_ordered"); + DEBUG_SYNC(orig_entry->thd, "commit_after_release_LOCK_prepare_ordered"); + + DBUG_PRINT("info", ("Queued for group commit as %s\n", + (orig_queue == NULL) ? "leader" : "participant")); + DBUG_RETURN(orig_queue == NULL); +} + +bool +MYSQL_BIN_LOG::write_transaction_to_binlog_events(group_commit_entry *entry) +{ + int is_leader= queue_for_group_commit(entry); /* - The first in the queue handle group commit for all; the others just wait + The first in the queue handles group commit for all; the others just wait to be signalled when group commit is done. */ - if (orig_queue != NULL) + if (is_leader < 0) + return true; /* Error */ + else if (is_leader) + trx_group_commit_leader(entry); + else if (!entry->queued_by_other) entry->thd->wait_for_wakeup_ready(); else - trx_group_commit_leader(entry); + { + /* + If we were queued by another prior commit, then we are woken up + only when the leader has already completed the commit for us. + So nothing to do here then. + */ + } if (!opt_optimize_thread_scheduling) { /* For the leader, trx_group_commit_leader() already took the lock. */ - if (orig_queue != NULL) + if (!is_leader) mysql_mutex_lock(&LOCK_commit_ordered); DEBUG_SYNC(entry->thd, "commit_loop_entry_commit_ordered"); @@ -5908,8 +6962,33 @@ MYSQL_BIN_LOG::write_transaction_to_binlog_events(group_commit_entry *entry) if (next) { - next->thd->signal_wakeup_ready(); + /* + Wake up the next thread in the group commit. + + The next thread can be waiting in two different ways, depending on + whether it put itself in the queue, or if it was put in queue by us + because it had to wait for us to commit first. + + So execute the appropriate wakeup, identified by the queued_by_other + field. + */ + if (next->queued_by_other) + next->thd->wait_for_commit_ptr->wakeup(entry->error); + else + next->thd->signal_wakeup_ready(); + } + else + { + /* + If we rotated the binlog, and if we are using the unoptimized thread + scheduling where every thread runs its own commit_ordered(), then we + must do the commit checkpoint and log purge here, after all + commit_ordered() calls have finished, and locks have been released. + */ + if (entry->check_purge) + checkpoint_and_purge(entry->binlog_id); } + } if (likely(!entry->error)) @@ -5940,8 +7019,9 @@ MYSQL_BIN_LOG::write_transaction_to_binlog_events(group_commit_entry *entry) we need to mark it as not needed for recovery (unlog() is not called for a transaction if log_xid() fails). */ - if (entry->cache_mngr->using_xa && entry->cache_mngr->xa_xid) - mark_xid_done(); + if (entry->cache_mngr->using_xa && entry->cache_mngr->xa_xid && + entry->cache_mngr->need_unlog) + mark_xid_done(entry->cache_mngr->binlog_id, true); return 1; } @@ -5961,14 +7041,14 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) { uint xid_count= 0; my_off_t UNINIT_VAR(commit_offset); - group_commit_entry *current; - group_commit_entry *last_in_queue; + group_commit_entry *current, *last_in_queue; group_commit_entry *queue= NULL; bool check_purge= false; + ulong binlog_id; + uint64 commit_id; DBUG_ENTER("MYSQL_BIN_LOG::trx_group_commit_leader"); + LINT_INIT(binlog_id); - DBUG_ASSERT(is_open()); - if (likely(is_open())) // Should always be true { /* Lock the LOCK_log(), and once we get it, collect any additional writes @@ -5978,9 +7058,16 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) DEBUG_SYNC(leader->thd, "commit_after_get_LOCK_log"); mysql_mutex_lock(&LOCK_prepare_ordered); + if (opt_binlog_commit_wait_count) + wait_for_sufficient_commits(); + /* + Note that wait_for_sufficient_commits() may have released and + re-acquired the LOCK_log and LOCK_prepare_ordered if it needed to wait. + */ current= group_commit_queue; group_commit_queue= NULL; mysql_mutex_unlock(&LOCK_prepare_ordered); + binlog_id= current_binlog_id; /* As the queue is in reverse order of entering, reverse it. */ last_in_queue= current; @@ -5994,7 +7081,12 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) DBUG_ASSERT(leader == queue /* the leader should be first in queue */); /* Now we have in queue the list of transactions to be committed in order. */ + } + DBUG_ASSERT(is_open()); + if (likely(is_open())) // Should always be true + { + commit_id= (last_in_queue == leader ? 0 : (uint64)leader->thd->query_id); /* Commit every transaction in the queue. @@ -6015,13 +7107,31 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) */ DBUG_ASSERT(!cache_mngr->stmt_cache.empty() || !cache_mngr->trx_cache.empty()); - current->error= write_transaction_or_stmt(current); + if ((current->error= write_transaction_or_stmt(current, commit_id))) + current->commit_errno= errno; strmake_buf(cache_mngr->last_commit_pos_file, log_file_name); commit_offset= my_b_write_tell(&log_file); cache_mngr->last_commit_pos_offset= commit_offset; if (cache_mngr->using_xa && cache_mngr->xa_xid) - xid_count++; + { + /* + If all storage engines support commit_checkpoint_request(), then we + do not need to keep track of when this XID is durably committed. + Instead we will just ask the storage engine to durably commit all its + XIDs when we rotate a binlog file. + */ + if (current->need_unlog) + { + xid_count++; + cache_mngr->need_unlog= true; + cache_mngr->binlog_id= binlog_id; + } + else + cache_mngr->need_unlog= false; + + cache_mngr->delayed_error= false; + } } bool synced= 0; @@ -6064,30 +7174,34 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) } /* - if any commit_events are Xid_log_event, increase the number of - prepared_xids (it's decreased in ::unlog()). Binlog cannot be rotated - if there're prepared xids in it - see the comment in new_file() for - an explanation. - If no Xid_log_events (then it's all Query_log_event) rotate binlog, - if necessary. + If any commit_events are Xid_log_event, increase the number of pending + XIDs in current binlog (it's decreased in ::unlog()). When the count in + a (not active) binlog file reaches zero, we know that it is no longer + needed in XA recovery, and we can log a new binlog checkpoint event. */ if (xid_count > 0) { - mark_xids_active(xid_count); + mark_xids_active(binlog_id, xid_count); } - else + + if (rotate(false, &check_purge)) { - if (rotate(false, &check_purge)) - { - /* - If we fail to rotate, which thread should get the error? - We give the error to the *last* transaction thread; that seems to - make the most sense, as it was the last to write to the log. - */ - last_in_queue->error= ER_ERROR_ON_WRITE; - last_in_queue->commit_errno= errno; - check_purge= false; - } + /* + If we fail to rotate, which thread should get the error? + We give the error to the leader, as any my_error() thrown inside + rotate() will have been registered for the leader THD. + + However we must not return error from here - that would cause + ha_commit_trans() to abort and rollback the transaction, which would + leave an inconsistent state with the transaction committed in the + binlog but rolled back in the engine. + + Instead set a flag so that we can return error later, from unlog(), + when the transaction has been safely committed in the engine. + */ + leader->cache_mngr->delayed_error= true; + my_error(ER_ERROR_ON_WRITE, MYF(ME_NOREFRESH), name, errno); + check_purge= false; } } @@ -6102,9 +7216,6 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) */ mysql_mutex_unlock(&LOCK_log); - if (check_purge) - purge(); - DEBUG_SYNC(leader->thd, "commit_after_release_LOCK_log"); ++num_group_commits; @@ -6122,6 +7233,15 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) mysql_cond_wait(&COND_queue_busy, &LOCK_commit_ordered); group_commit_queue_busy= TRUE; + /* + Set these so parent can run checkpoint_and_purge() in last thread. + (When using optimized thread scheduling, we run checkpoint_and_purge() + in this function, so parent does not need to and we need not set these + values). + */ + last_in_queue->check_purge= check_purge; + last_in_queue->binlog_id= binlog_id; + /* Note that we return with LOCK_commit_ordered locked! */ DBUG_VOID_RETURN; } @@ -6137,7 +7257,8 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) DEBUG_SYNC(leader->thd, "commit_loop_entry_commit_ordered"); ++num_commits; - if (current->cache_mngr->using_xa && !current->error) + if (current->cache_mngr->using_xa && !current->error && + DBUG_EVALUATE_IF("skip_commit_ordered", 0, 1)) run_commit_ordered(current->thd, current->all); /* @@ -6146,25 +7267,33 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) */ next= current->next; if (current != leader) // Don't wake up ourself - current->thd->signal_wakeup_ready(); + { + if (current->queued_by_other) + current->thd->wait_for_commit_ptr->wakeup(current->error); + else + current->thd->signal_wakeup_ready(); + } current= next; } DEBUG_SYNC(leader->thd, "commit_after_group_run_commit_ordered"); mysql_mutex_unlock(&LOCK_commit_ordered); + DEBUG_SYNC(leader->thd, "commit_after_group_release_commit_ordered"); + + if (check_purge) + checkpoint_and_purge(binlog_id); DBUG_VOID_RETURN; } int -MYSQL_BIN_LOG::write_transaction_or_stmt(group_commit_entry *entry) +MYSQL_BIN_LOG::write_transaction_or_stmt(group_commit_entry *entry, + uint64 commit_id) { binlog_cache_mngr *mngr= entry->cache_mngr; - if (entry->begin_event->write(&log_file)) + if (write_gtid_event(entry->thd, false, entry->using_trx_cache, commit_id)) return ER_ERROR_ON_WRITE; - status_var_add(entry->thd->status_var.binlog_bytes_written, - entry->begin_event->data_written); if (entry->using_stmt_cache && !mngr->stmt_cache.empty() && write_cache(entry->thd, mngr->get_binlog_cache_log(FALSE))) @@ -6231,6 +7360,72 @@ MYSQL_BIN_LOG::write_transaction_or_stmt(group_commit_entry *entry) return 0; } + +/* + Wait for sufficient commits to queue up for group commit, according to the + values of binlog_commit_wait_count and binlog_commit_wait_usec. + + Note that this function may release and re-acquire LOCK_log and + LOCK_prepare_ordered if it needs to wait. +*/ + +void +MYSQL_BIN_LOG::wait_for_sufficient_commits() +{ + size_t count; + group_commit_entry *e; + group_commit_entry *last_head; + struct timespec wait_until; + + mysql_mutex_assert_owner(&LOCK_log); + mysql_mutex_assert_owner(&LOCK_prepare_ordered); + + for (e= last_head= group_commit_queue, count= 0; e; e= e->next) + if (++count >= opt_binlog_commit_wait_count) + return; + + mysql_mutex_unlock(&LOCK_log); + set_timespec_nsec(wait_until, (ulonglong)1000*opt_binlog_commit_wait_usec); + + for (;;) + { + int err; + group_commit_entry *head; + + err= mysql_cond_timedwait(&COND_prepare_ordered, &LOCK_prepare_ordered, + &wait_until); + if (err == ETIMEDOUT) + break; + head= group_commit_queue; + for (e= head; e && e != last_head; e= e->next) + ++count; + if (count >= opt_binlog_commit_wait_count) + break; + last_head= head; + } + + /* + We must not wait for LOCK_log while holding LOCK_prepare_ordered. + LOCK_log can be held for long periods (eg. we do I/O under it), while + LOCK_prepare_ordered must only be held for short periods. + + In addition, waiting for LOCK_log while holding LOCK_prepare_ordered would + violate locking order of LOCK_log-before-LOCK_prepare_ordered. This could + cause SAFEMUTEX warnings (even if it cannot actually deadlock with current + code, as there can be at most one group commit leader thread at a time). + + So release and re-acquire LOCK_prepare_ordered if we need to wait for the + LOCK_log. + */ + if (mysql_mutex_trylock(&LOCK_log)) + { + mysql_mutex_unlock(&LOCK_prepare_ordered); + mysql_mutex_lock(&LOCK_log); + mysql_mutex_lock(&LOCK_prepare_ordered); + } +} + + /** Wait until we get a signal that the relay log has been updated. @@ -6305,12 +7500,14 @@ int MYSQL_BIN_LOG::wait_for_update_bin_log(THD* thd, void MYSQL_BIN_LOG::close(uint exiting) { // One can't set log_type here! + bool failed_to_save_state= false; + DBUG_ENTER("MYSQL_BIN_LOG::close"); DBUG_PRINT("enter",("exiting: %d", (int) exiting)); if (log_state == LOG_OPENED) { #ifdef HAVE_REPLICATION - if (log_type == LOG_BIN && !no_auto_events && + if (log_type == LOG_BIN && (exiting & LOG_CLOSE_STOP_EVENT)) { Stop_log_event s; @@ -6322,6 +7519,27 @@ void MYSQL_BIN_LOG::close(uint exiting) s.write(&log_file); bytes_written+= s.data_written; signal_update(); + + /* + When we shut down server, write out the binlog state to a separate + file so we do not have to scan an entire binlog file to recover it + at next server start. + + Note that this must be written and synced to disk before marking the + last binlog file as "not crashed". + */ + if (!is_relay_log && write_state_to_file()) + { + sql_print_error("Failed to save binlog GTID state during shutdown. " + "Binlog will be marked as crashed, so that crash " + "recovery can recover the state at next server " + "startup."); + /* + Leave binlog file marked as crashed, so we can recover state by + scanning it now that we failed to write out the state properly. + */ + failed_to_save_state= true; + } } #endif /* HAVE_REPLICATION */ @@ -6330,7 +7548,8 @@ void MYSQL_BIN_LOG::close(uint exiting) && !(exiting & LOG_CLOSE_DELAYED_CLOSE)) { my_off_t org_position= mysql_file_tell(log_file.file, MYF(0)); - clear_inuse_flag_when_closing(log_file.file); + if (!failed_to_save_state) + clear_inuse_flag_when_closing(log_file.file); /* Restore position so that anything we have in the IO_cache is written to the correct position. @@ -6568,16 +7787,33 @@ static void print_buffer_to_file(enum loglevel level, const char *buffer, time_t skr; struct tm tm_tmp; struct tm *start; + THD *thd; + int tag_length= 0; + char tag[NAME_LEN]; DBUG_ENTER("print_buffer_to_file"); DBUG_PRINT("enter",("buffer: %s", buffer)); + if (mysqld_server_initialized && (thd= current_thd)) + { + if (thd->connection_name.length) + { + /* + Add tag for slaves so that the user can see from which connection + the error originates. + */ + tag_length= my_snprintf(tag, sizeof(tag), ER(ER_MASTER_LOG_PREFIX), + (int) thd->connection_name.length, + thd->connection_name.str); + } + } + mysql_mutex_lock(&LOCK_error_log); skr= my_time(0); localtime_r(&skr, &tm_tmp); start=&tm_tmp; - fprintf(stderr, "%02d%02d%02d %2d:%02d:%02d [%s] %.*s\n", + fprintf(stderr, "%02d%02d%02d %2d:%02d:%02d [%s] %.*s%.*s\n", start->tm_year % 100, start->tm_mon+1, start->tm_mday, @@ -6586,6 +7822,7 @@ static void print_buffer_to_file(enum loglevel level, const char *buffer, start->tm_sec, (level == ERROR_LEVEL ? "ERROR" : level == WARNING_LEVEL ? "Warning" : "Note"), + tag_length, tag, (int) length, buffer); fflush(stderr); @@ -6731,6 +7968,9 @@ int TC_LOG_MMAP::log_and_order(THD *thd, my_xid xid, bool all, mysql_mutex_unlock(&LOCK_prepare_ordered); } + if (thd->wait_for_prior_commit()) + return 0; + cookie= 0; if (xid) cookie= log_one_transaction(xid); @@ -6944,6 +8184,8 @@ int TC_LOG_MMAP::open(const char *opt_name) mysql_mutex_init(key_LOCK_sync, &LOCK_sync, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_active, &LOCK_active, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_pool, &LOCK_pool, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_LOCK_pending_checkpoint, &LOCK_pending_checkpoint, + MY_MUTEX_INIT_FAST); mysql_cond_init(key_COND_active, &COND_active, 0); mysql_cond_init(key_COND_pool, &COND_pool, 0); mysql_cond_init(key_TC_LOG_MMAP_COND_queue_busy, &COND_queue_busy, 0); @@ -7194,17 +8436,93 @@ int TC_LOG_MMAP::sync() return err; } +static void +mmap_do_checkpoint_callback(void *data) +{ + TC_LOG_MMAP::pending_cookies *pending= + static_cast<TC_LOG_MMAP::pending_cookies *>(data); + ++pending->pending_count; +} + +int TC_LOG_MMAP::unlog(ulong cookie, my_xid xid) +{ + pending_cookies *full_buffer= NULL; + DBUG_ASSERT(*(my_xid *)(data+cookie) == xid); + + /* + Do not delete the entry immediately, as there may be participating storage + engines which implement commit_checkpoint_request(), and thus have not yet + flushed the commit durably to disk. + + Instead put it in a queue - and periodically, we will request a checkpoint + from all engines and delete a whole batch at once. + */ + mysql_mutex_lock(&LOCK_pending_checkpoint); + if (pending_checkpoint == NULL) + { + uint32 size= sizeof(*pending_checkpoint); + if (!(pending_checkpoint= + (pending_cookies *)my_malloc(size, MYF(MY_ZEROFILL)))) + { + my_error(ER_OUTOFMEMORY, MYF(0), size); + mysql_mutex_unlock(&LOCK_pending_checkpoint); + return 1; + } + } + + pending_checkpoint->cookies[pending_checkpoint->count++]= cookie; + if (pending_checkpoint->count == sizeof(pending_checkpoint->cookies) / + sizeof(pending_checkpoint->cookies[0])) + { + full_buffer= pending_checkpoint; + pending_checkpoint= NULL; + } + mysql_mutex_unlock(&LOCK_pending_checkpoint); + + if (full_buffer) + { + /* + We do an extra increment and notify here - this ensures that + things work also if there are no engines at all that support + commit_checkpoint_request. + */ + ++full_buffer->pending_count; + ha_commit_checkpoint_request(full_buffer, mmap_do_checkpoint_callback); + commit_checkpoint_notify(full_buffer); + } + return 0; +} + + +void +TC_LOG_MMAP::commit_checkpoint_notify(void *cookie) +{ + uint count; + pending_cookies *pending= static_cast<pending_cookies *>(cookie); + mysql_mutex_lock(&LOCK_pending_checkpoint); + DBUG_ASSERT(pending->pending_count > 0); + count= --pending->pending_count; + mysql_mutex_unlock(&LOCK_pending_checkpoint); + if (count == 0) + { + uint i; + for (i= 0; i < sizeof(pending->cookies)/sizeof(pending->cookies[0]); ++i) + delete_entry(pending->cookies[i]); + my_free(pending); + } +} + + /** erase xid from the page, update page free space counters/pointers. cookie points directly to the memory where xid was logged. */ -int TC_LOG_MMAP::unlog(ulong cookie, my_xid xid) +int TC_LOG_MMAP::delete_entry(ulong cookie) { PAGE *p=pages+(cookie/tc_log_page_size); my_xid *x=(my_xid *)(data+cookie); - DBUG_ASSERT(*x == xid); DBUG_ASSERT(x >= p->start && x < p->end); mysql_mutex_lock(&p->lock); @@ -7228,6 +8546,7 @@ void TC_LOG_MMAP::close() mysql_mutex_destroy(&LOCK_sync); mysql_mutex_destroy(&LOCK_active); mysql_mutex_destroy(&LOCK_pool); + mysql_mutex_destroy(&LOCK_pending_checkpoint); mysql_cond_destroy(&COND_pool); mysql_cond_destroy(&COND_active); mysql_cond_destroy(&COND_queue_busy); @@ -7250,9 +8569,12 @@ void TC_LOG_MMAP::close() } if (inited>=5) // cannot do in the switch because of Windows mysql_file_delete(key_file_tclog, logname, MYF(MY_WME)); + if (pending_checkpoint) + my_free(pending_checkpoint); inited=0; } + int TC_LOG_MMAP::recover() { HASH xids; @@ -7338,26 +8660,13 @@ int TC_LOG::using_heuristic_recover() /****** transaction coordinator log for 2pc - binlog() based solution ******/ #define TC_LOG_BINLOG MYSQL_BIN_LOG -/** - @todo - keep in-memory list of prepared transactions - (add to list in log(), remove on unlog()) - and copy it to the new binlog if rotated - but let's check the behaviour of tc_log_page_waits first! -*/ - int TC_LOG_BINLOG::open(const char *opt_name) { - LOG_INFO log_info; int error= 1; DBUG_ASSERT(total_ha_2pc > 1); DBUG_ASSERT(opt_name && opt_name[0]); - mysql_mutex_init(key_BINLOG_LOCK_prep_xids, - &LOCK_prep_xids, MY_MUTEX_INIT_FAST); - mysql_cond_init(key_BINLOG_COND_prep_xids, &COND_prep_xids, 0); - if (!my_b_inited(&index_file)) { /* There was a failure to open the index file, can't open the binlog */ @@ -7368,77 +8677,19 @@ int TC_LOG_BINLOG::open(const char *opt_name) if (using_heuristic_recover()) { /* generate a new binlog to mask a corrupted one */ - open(opt_name, LOG_BIN, 0, WRITE_CACHE, 0, max_binlog_size, 0, TRUE); + open(opt_name, LOG_BIN, 0, WRITE_CACHE, max_binlog_size, 0, TRUE); cleanup(); return 1; } - if ((error= find_log_pos(&log_info, NullS, 1))) - { - if (error != LOG_INFO_EOF) - sql_print_error("find_log_pos() failed (error: %d)", error); - else - error= 0; - goto err; - } - - { - const char *errmsg; - IO_CACHE log; - File file; - Log_event *ev=0; - Format_description_log_event fdle(BINLOG_VERSION); - char log_name[FN_REFLEN]; - - if (! fdle.is_valid()) - goto err; - - do - { - strmake_buf(log_name, log_info.log_file_name); - } while (!(error= find_next_log(&log_info, 1))); - - if (error != LOG_INFO_EOF) - { - sql_print_error("find_log_pos() failed (error: %d)", error); - goto err; - } - - if ((file= open_binlog(&log, log_name, &errmsg)) < 0) - { - sql_print_error("%s", errmsg); - goto err; - } - - if ((ev= Log_event::read_log_event(&log, 0, &fdle, - opt_master_verify_checksum)) && - ev->get_type_code() == FORMAT_DESCRIPTION_EVENT && - ev->flags & LOG_EVENT_BINLOG_IN_USE_F) - { - sql_print_information("Recovering after a crash using %s", opt_name); - error= recover(&log, (Format_description_log_event *)ev); - } - else - error=0; - - delete ev; - end_io_cache(&log); - mysql_file_close(file, MYF(MY_WME)); - - if (error) - goto err; - } - -err: + error= do_binlog_recovery(opt_name, true); + binlog_state_recover_done= true; return error; } /** This is called on shutdown, after ha_panic. */ void TC_LOG_BINLOG::close() { - DBUG_ASSERT(prepared_xids==0); - mysql_mutex_destroy(&LOCK_prep_xids); - mysql_cond_destroy(&COND_prep_xids); } /* @@ -7463,7 +8714,17 @@ TC_LOG_BINLOG::log_and_order(THD *thd, my_xid xid, bool all, DEBUG_SYNC(thd, "binlog_after_log_and_order"); - DBUG_RETURN(!err); + if (err) + DBUG_RETURN(0); + /* + If using explicit user XA, we will not have XID. We must still return a + non-zero cookie (as zero cookie signals error). + */ + if (!xid || !cache_mngr->need_unlog) + DBUG_RETURN(BINLOG_COOKIE_DUMMY(cache_mngr->delayed_error)); + else + DBUG_RETURN(BINLOG_COOKIE_MAKE(cache_mngr->binlog_id, + cache_mngr->delayed_error)); } /* @@ -7478,92 +8739,537 @@ TC_LOG_BINLOG::log_and_order(THD *thd, my_xid xid, bool all, binary log. */ void -TC_LOG_BINLOG::mark_xids_active(uint xid_count) +TC_LOG_BINLOG::mark_xids_active(ulong binlog_id, uint xid_count) { + xid_count_per_binlog *b; + DBUG_ENTER("TC_LOG_BINLOG::mark_xids_active"); - DBUG_PRINT("info", ("xid_count=%u", xid_count)); - mysql_mutex_lock(&LOCK_prep_xids); - prepared_xids+= xid_count; - mysql_mutex_unlock(&LOCK_prep_xids); + DBUG_PRINT("info", ("binlog_id=%lu xid_count=%u", binlog_id, xid_count)); + + mysql_mutex_lock(&LOCK_xid_list); + I_List_iterator<xid_count_per_binlog> it(binlog_xid_count_list); + while ((b= it++)) + { + if (b->binlog_id == binlog_id) + { + b->xid_count += xid_count; + break; + } + } + /* + As we do not delete elements until count reach zero, elements should always + be found. + */ + DBUG_ASSERT(b); + mysql_mutex_unlock(&LOCK_xid_list); DBUG_VOID_RETURN; } /* - Once an XID is committed, it is safe to rotate the binary log, as it can no - longer be needed during crash recovery. + Once an XID is committed, it can no longer be needed during crash recovery, + as it has been durably recorded on disk as "committed". This function is called to mark an XID this way. It needs to decrease the - count of pending XIDs, and signal the log rotator thread when it reaches zero. + count of pending XIDs in the corresponding binlog. When the count reaches + zero (for an "old" binlog that is not the active one), that binlog file no + longer need to be scanned during crash recovery, so we can log a new binlog + checkpoint. */ void -TC_LOG_BINLOG::mark_xid_done() +TC_LOG_BINLOG::mark_xid_done(ulong binlog_id, bool write_checkpoint) { - my_bool send_signal; + xid_count_per_binlog *b; + bool first; + ulong current; DBUG_ENTER("TC_LOG_BINLOG::mark_xid_done"); - mysql_mutex_lock(&LOCK_prep_xids); - // prepared_xids can be 0 if the transaction had ignorable errors. - DBUG_ASSERT(prepared_xids >= 0); - if (prepared_xids > 0) - prepared_xids--; - send_signal= (prepared_xids == 0); - mysql_mutex_unlock(&LOCK_prep_xids); - if (send_signal) { - DBUG_PRINT("info", ("prepared_xids=%lu", prepared_xids)); - mysql_cond_signal(&COND_prep_xids); + + mysql_mutex_lock(&LOCK_xid_list); + current= current_binlog_id; + I_List_iterator<xid_count_per_binlog> it(binlog_xid_count_list); + first= true; + while ((b= it++)) + { + if (b->binlog_id == binlog_id) + { + --b->xid_count; + break; + } + first= false; + } + /* Binlog is always found, as we do not remove until count reaches 0 */ + DBUG_ASSERT(b); + /* + If a RESET MASTER is pending, we are about to remove all log files, and + the RESET MASTER thread is waiting for all pending unlog() calls to + complete while holding LOCK_log. In this case we should not log a binlog + checkpoint event (it would be deleted immediately anyway and we would + deadlock on LOCK_log) but just signal the thread. + */ + if (unlikely(reset_master_pending)) + { + mysql_cond_signal(&COND_xid_list); + mysql_mutex_unlock(&LOCK_xid_list); + DBUG_VOID_RETURN; + } + + if (likely(binlog_id == current) || b->xid_count != 0 || !first || + !write_checkpoint) + { + /* No new binlog checkpoint reached yet. */ + mysql_mutex_unlock(&LOCK_xid_list); + DBUG_VOID_RETURN; } + + /* + Now log a binlog checkpoint for the first binlog file with a non-zero count. + + Note that it is possible (though perhaps unlikely) that when count of + binlog (N-2) drops to zero, binlog (N-1) is already at zero. So we may + need to skip several entries before we find the one to log in the binlog + checkpoint event. + + We chain the locking of LOCK_xid_list and LOCK_log, so that we ensure that + Binlog_checkpoint_events are logged in order. This simplifies recovery a + bit, as it can just take the last binlog checkpoint in the log, rather + than compare all found against each other to find the one pointing to the + most recent binlog. + + Note also that we need to first release LOCK_xid_list, then aquire + LOCK_log, then re-aquire LOCK_xid_list. If we were to take LOCK_log while + holding LOCK_xid_list, we might deadlock with other threads that take the + locks in the opposite order. + */ + + ++mark_xid_done_waiting; + mysql_mutex_unlock(&LOCK_xid_list); + mysql_mutex_lock(&LOCK_log); + mysql_mutex_lock(&LOCK_xid_list); + --mark_xid_done_waiting; + if (unlikely(reset_master_pending)) + mysql_cond_signal(&COND_xid_list); + /* We need to reload current_binlog_id due to release/re-take of lock. */ + current= current_binlog_id; + + for (;;) + { + /* Remove initial element(s) with zero count. */ + b= binlog_xid_count_list.head(); + /* + We must not remove all elements in the list - the entry for the current + binlog must be present always. + */ + DBUG_ASSERT(b); + if (b->binlog_id == current || b->xid_count > 0) + break; + my_free(binlog_xid_count_list.get()); + } + + mysql_mutex_unlock(&LOCK_xid_list); + write_binlog_checkpoint_event_already_locked(b->binlog_name, + b->binlog_name_len); + mysql_mutex_unlock(&LOCK_log); DBUG_VOID_RETURN; } int TC_LOG_BINLOG::unlog(ulong cookie, my_xid xid) { DBUG_ENTER("TC_LOG_BINLOG::unlog"); - if (xid) - mark_xid_done(); - /* As ::write_transaction_to_binlog() did not rotate, do it here. */ - DBUG_RETURN(rotate_and_purge(0)); + if (!xid) + DBUG_RETURN(0); + + if (!BINLOG_COOKIE_IS_DUMMY(cookie)) + mark_xid_done(BINLOG_COOKIE_GET_ID(cookie), true); + /* + See comment in trx_group_commit_leader() - if rotate() gave a failure, + we delay the return of error code to here. + */ + DBUG_RETURN(BINLOG_COOKIE_GET_ERROR_FLAG(cookie)); } -int TC_LOG_BINLOG::recover(IO_CACHE *log, Format_description_log_event *fdle) +void +TC_LOG_BINLOG::commit_checkpoint_notify(void *cookie) { - Log_event *ev; + xid_count_per_binlog *entry= static_cast<xid_count_per_binlog *>(cookie); + mysql_mutex_lock(&LOCK_binlog_background_thread); + entry->next_in_queue= binlog_background_thread_queue; + binlog_background_thread_queue= entry; + mysql_cond_signal(&COND_binlog_background_thread); + mysql_mutex_unlock(&LOCK_binlog_background_thread); +} + +/* + Binlog background thread. + + This thread is used to log binlog checkpoints in the background, rather than + in the context of random storage engine threads that happen to call + commit_checkpoint_notify_ha() and may not like the delays while syncing + binlog to disk or may not be setup with all my_thread_init() and other + necessary stuff. + + In the future, this thread could also be used to do log rotation in the + background, which could elimiate all stalls around binlog rotations. +*/ +pthread_handler_t +binlog_background_thread(void *arg __attribute__((unused))) +{ + bool stop; + MYSQL_BIN_LOG::xid_count_per_binlog *queue, *next; + THD *thd; + my_thread_init(); + DBUG_ENTER("binlog_background_thread"); + + thd= new THD; + thd->system_thread= SYSTEM_THREAD_BINLOG_BACKGROUND; + thd->thread_stack= (char*) &thd; /* Set approximate stack start */ + mysql_mutex_lock(&LOCK_thread_count); + thd->thread_id= thread_id++; + mysql_mutex_unlock(&LOCK_thread_count); + thd->store_globals(); + + /* + Load the slave replication GTID state from the mysql.gtid_slave_pos + table. + + This is mostly so that we can start our seq_no counter from the highest + seq_no seen by a slave. This way, we have a way to tell if a transaction + logged by ourselves as master is newer or older than a replicated + transaction. + */ +#ifdef HAVE_REPLICATION + if (rpl_load_gtid_slave_state(thd)) + sql_print_warning("Failed to load slave replication state from table " + "%s.%s: %u: %s", "mysql", + rpl_gtid_slave_state_table_name.str, + thd->stmt_da->sql_errno(), thd->stmt_da->message()); +#endif + + mysql_mutex_lock(&mysql_bin_log.LOCK_binlog_background_thread); + binlog_background_thread_started= true; + mysql_cond_signal(&mysql_bin_log.COND_binlog_background_thread_end); + mysql_mutex_unlock(&mysql_bin_log.LOCK_binlog_background_thread); + + for (;;) + { + /* + Wait until there is something in the queue to process, or we are asked + to shut down. + */ + thd_proc_info(thd, "Waiting for background binlog tasks"); + mysql_mutex_lock(&mysql_bin_log.LOCK_binlog_background_thread); + for (;;) + { + stop= binlog_background_thread_stop; + queue= binlog_background_thread_queue; + if (stop && !mysql_bin_log.is_xidlist_idle()) + { + /* + Delay stop until all pending binlog checkpoints have been processed. + */ + stop= false; + } + if (stop || queue) + break; + mysql_cond_wait(&mysql_bin_log.COND_binlog_background_thread, + &mysql_bin_log.LOCK_binlog_background_thread); + } + /* Grab the queue, if any. */ + binlog_background_thread_queue= NULL; + mysql_mutex_unlock(&mysql_bin_log.LOCK_binlog_background_thread); + + /* Process any incoming commit_checkpoint_notify() calls. */ + DBUG_EXECUTE_IF("inject_binlog_background_thread_before_mark_xid_done", + DBUG_ASSERT(!debug_sync_set_action( + thd, + STRING_WITH_LEN("binlog_background_thread_before_mark_xid_done " + "SIGNAL injected_binlog_background_thread " + "WAIT_FOR something_that_will_never_happen " + "TIMEOUT 2"))); + ); + while (queue) + { + thd_proc_info(thd, "Processing binlog checkpoint notification"); + DEBUG_SYNC(current_thd, "binlog_background_thread_before_mark_xid_done"); + /* Grab next pointer first, as mark_xid_done() may free the element. */ + next= queue->next_in_queue; + mysql_bin_log.mark_xid_done(queue->binlog_id, true); + queue= next; + + DBUG_EXECUTE_IF("binlog_background_checkpoint_processed", + DBUG_ASSERT(!debug_sync_set_action( + thd, + STRING_WITH_LEN("now SIGNAL binlog_background_checkpoint_processed"))); + ); + } + + if (stop) + break; + } + + thd_proc_info(thd, "Stopping binlog background thread"); + + mysql_mutex_lock(&LOCK_thread_count); + delete thd; + mysql_mutex_unlock(&LOCK_thread_count); + + my_thread_end(); + + /* Signal that we are (almost) stopped. */ + mysql_mutex_lock(&mysql_bin_log.LOCK_binlog_background_thread); + binlog_background_thread_stop= false; + mysql_cond_signal(&mysql_bin_log.COND_binlog_background_thread_end); + mysql_mutex_unlock(&mysql_bin_log.LOCK_binlog_background_thread); + + DBUG_RETURN(0); +} + +#ifdef HAVE_PSI_INTERFACE +static PSI_thread_key key_thread_binlog; + +static PSI_thread_info all_binlog_threads[]= +{ + { &key_thread_binlog, "binlog_background", PSI_FLAG_GLOBAL}, +}; +#endif /* HAVE_PSI_INTERFACE */ + +static bool +start_binlog_background_thread() +{ + pthread_t th; + +#ifdef HAVE_PSI_INTERFACE + if (PSI_server) + PSI_server->register_thread("sql", all_binlog_threads, + array_elements(all_binlog_threads)); +#endif + + if (mysql_thread_create(key_thread_binlog, &th, NULL, + binlog_background_thread, NULL)) + return 1; + + /* + Wait for the thread to have started (so we know that the slave replication + state is loaded and we have correct global_gtid_counter). + */ + mysql_mutex_lock(&mysql_bin_log.LOCK_binlog_background_thread); + while (!binlog_background_thread_started) + mysql_cond_wait(&mysql_bin_log.COND_binlog_background_thread_end, + &mysql_bin_log.LOCK_binlog_background_thread); + mysql_mutex_unlock(&mysql_bin_log.LOCK_binlog_background_thread); + + return 0; +} + + +int TC_LOG_BINLOG::recover(LOG_INFO *linfo, const char *last_log_name, + IO_CACHE *first_log, + Format_description_log_event *fdle, bool do_xa) +{ + Log_event *ev= NULL; HASH xids; MEM_ROOT mem_root; + char binlog_checkpoint_name[FN_REFLEN]; + bool binlog_checkpoint_found; + bool first_round; + IO_CACHE log; + File file= -1; + const char *errmsg; +#ifdef HAVE_REPLICATION + rpl_gtid last_gtid; + bool last_gtid_standalone= false; + bool last_gtid_valid= false; +#endif if (! fdle->is_valid() || - my_hash_init(&xids, &my_charset_bin, TC_LOG_PAGE_SIZE/3, 0, - sizeof(my_xid), 0, 0, MYF(0))) + (do_xa && my_hash_init(&xids, &my_charset_bin, TC_LOG_PAGE_SIZE/3, 0, + sizeof(my_xid), 0, 0, MYF(0)))) goto err1; - init_alloc_root(&mem_root, TC_LOG_PAGE_SIZE, TC_LOG_PAGE_SIZE); + if (do_xa) + init_alloc_root(&mem_root, TC_LOG_PAGE_SIZE, TC_LOG_PAGE_SIZE, MYF(0)); fdle->flags&= ~LOG_EVENT_BINLOG_IN_USE_F; // abort on the first error - while ((ev= Log_event::read_log_event(log, 0, fdle, - opt_master_verify_checksum)) - && ev->is_valid()) + /* + Scan the binlog for XIDs that need to be committed if still in the + prepared stage. + + Start with the latest binlog file, then continue with any other binlog + files if the last found binlog checkpoint indicates it is needed. + */ + + binlog_checkpoint_found= false; + first_round= true; + for (;;) { - if (ev->get_type_code() == XID_EVENT) + while ((ev= Log_event::read_log_event(first_round ? first_log : &log, + 0, fdle, opt_master_verify_checksum)) + && ev->is_valid()) + { + enum Log_event_type typ= ev->get_type_code(); + switch (typ) + { + case XID_EVENT: + { + if (do_xa) + { + Xid_log_event *xev=(Xid_log_event *)ev; + uchar *x= (uchar *) memdup_root(&mem_root, (uchar*) &xev->xid, + sizeof(xev->xid)); + if (!x || my_hash_insert(&xids, x)) + goto err2; + break; + } + } + case BINLOG_CHECKPOINT_EVENT: + if (first_round && do_xa) + { + uint dir_len; + Binlog_checkpoint_log_event *cev= (Binlog_checkpoint_log_event *)ev; + if (cev->binlog_file_len >= FN_REFLEN) + sql_print_warning("Incorrect binlog checkpoint event with too " + "long file name found."); + else + { + /* + Note that we cannot use make_log_name() here, as we have not yet + initialised MYSQL_BIN_LOG::log_file_name. + */ + dir_len= dirname_length(last_log_name); + strmake(strnmov(binlog_checkpoint_name, last_log_name, dir_len), + cev->binlog_file_name, FN_REFLEN - 1 - dir_len); + binlog_checkpoint_found= true; + } + } + break; + case GTID_LIST_EVENT: + if (first_round) + { + Gtid_list_log_event *glev= (Gtid_list_log_event *)ev; + + /* Initialise the binlog state from the Gtid_list event. */ + if (rpl_global_gtid_binlog_state.load(glev->list, glev->count)) + goto err2; + } + break; + +#ifdef HAVE_REPLICATION + case GTID_EVENT: + if (first_round) + { + Gtid_log_event *gev= (Gtid_log_event *)ev; + + /* Update the binlog state with any GTID logged after Gtid_list. */ + last_gtid.domain_id= gev->domain_id; + last_gtid.server_id= gev->server_id; + last_gtid.seq_no= gev->seq_no; + last_gtid_standalone= + ((gev->flags2 & Gtid_log_event::FL_STANDALONE) ? true : false); + last_gtid_valid= true; + } + break; +#endif + + default: + /* Nothing. */ + break; + } + +#ifdef HAVE_REPLICATION + if (last_gtid_valid && + ((last_gtid_standalone && !ev->is_part_of_group(typ)) || + (!last_gtid_standalone && + (typ == XID_EVENT || + (typ == QUERY_EVENT && + (((Query_log_event *)ev)->is_commit() || + ((Query_log_event *)ev)->is_rollback())))))) + { + if (rpl_global_gtid_binlog_state.update_nolock(&last_gtid, false)) + goto err2; + last_gtid_valid= false; + } +#endif + + delete ev; + ev= NULL; + } + + if (!do_xa) + break; + /* + If the last binlog checkpoint event points to an older log, we have to + scan all logs from there also, to get all possible XIDs to recover. + + If there was no binlog checkpoint event at all, this means the log was + written by an older version of MariaDB (or MySQL) - these always have an + (implicit) binlog checkpoint event at the start of the last binlog file. + */ + if (first_round) { - Xid_log_event *xev=(Xid_log_event *)ev; - uchar *x= (uchar *) memdup_root(&mem_root, (uchar*) &xev->xid, - sizeof(xev->xid)); - if (!x || my_hash_insert(&xids, x)) + if (!binlog_checkpoint_found) + break; + first_round= false; + DBUG_EXECUTE_IF("xa_recover_expect_master_bin_000004", + if (0 != strcmp("./master-bin.000004", binlog_checkpoint_name) && + 0 != strcmp(".\\master-bin.000004", binlog_checkpoint_name)) + DBUG_SUICIDE(); + ); + if (find_log_pos(linfo, binlog_checkpoint_name, 1)) + { + sql_print_error("Binlog file '%s' not found in binlog index, needed " + "for recovery. Aborting.", binlog_checkpoint_name); goto err2; + } + } + else + { + end_io_cache(&log); + mysql_file_close(file, MYF(MY_WME)); + file= -1; + } + + if (0 == strcmp(linfo->log_file_name, last_log_name)) + break; // No more files to do + if ((file= open_binlog(&log, linfo->log_file_name, &errmsg)) < 0) + { + sql_print_error("%s", errmsg); + goto err2; + } + /* + We do not need to read the Format_description_log_event of other binlog + files. It is not possible for a binlog checkpoint to span multiple + binlog files written by different versions of the server. So we can use + the first one read for reading from all binlog files. + */ + if (find_next_log(linfo, 1)) + { + sql_print_error("Error reading binlog files during recovery. Aborting."); + goto err2; } - delete ev; } - if (ha_recover(&xids)) - goto err2; + if (do_xa) + { + if (ha_recover(&xids)) + goto err2; - free_root(&mem_root, MYF(0)); - my_hash_free(&xids); + free_root(&mem_root, MYF(0)); + my_hash_free(&xids); + } return 0; err2: - free_root(&mem_root, MYF(0)); - my_hash_free(&xids); + delete ev; + if (file >= 0) + { + end_io_cache(&log); + mysql_file_close(file, MYF(MY_WME)); + } + if (do_xa) + { + free_root(&mem_root, MYF(0)); + my_hash_free(&xids); + } err1: sql_print_error("Crash recovery failed. Either correct the problem " "(if it's, for example, out of memory error) and restart, " @@ -7573,6 +9279,73 @@ err1: } +int +MYSQL_BIN_LOG::do_binlog_recovery(const char *opt_name, bool do_xa_recovery) +{ + LOG_INFO log_info; + const char *errmsg; + IO_CACHE log; + File file; + Log_event *ev= 0; + Format_description_log_event fdle(BINLOG_VERSION); + char log_name[FN_REFLEN]; + int error; + + if ((error= find_log_pos(&log_info, NullS, 1))) + { + /* + If there are no binlog files (LOG_INFO_EOF), then we still try to read + the .state file to restore the binlog state. This allows to copy a server + to provision a new one without copying the binlog files (except the + master-bin.state file) and still preserve the correct binlog state. + */ + if (error != LOG_INFO_EOF) + sql_print_error("find_log_pos() failed (error: %d)", error); + else + error= read_state_from_file(); + return error; + } + + if (! fdle.is_valid()) + return 1; + + do + { + strmake_buf(log_name, log_info.log_file_name); + } while (!(error= find_next_log(&log_info, 1))); + + if (error != LOG_INFO_EOF) + { + sql_print_error("find_log_pos() failed (error: %d)", error); + return error; + } + + if ((file= open_binlog(&log, log_name, &errmsg)) < 0) + { + sql_print_error("%s", errmsg); + return 1; + } + + if ((ev= Log_event::read_log_event(&log, 0, &fdle, + opt_master_verify_checksum)) && + ev->get_type_code() == FORMAT_DESCRIPTION_EVENT && + ev->flags & LOG_EVENT_BINLOG_IN_USE_F) + { + sql_print_information("Recovering after a crash using %s", opt_name); + error= recover(&log_info, log_name, &log, + (Format_description_log_event *)ev, do_xa_recovery); + } + else + error= read_state_from_file(); + + delete ev; + end_io_cache(&log); + mysql_file_close(file, MYF(MY_WME)); + + return error; +} + + #ifdef INNODB_COMPATIBILITY_HOOKS /** Get the file name of the MySQL binlog. @@ -7627,10 +9400,13 @@ binlog_checksum_update(MYSQL_THD thd, struct st_mysql_sys_var *var, { ulong value= *((ulong *)save); bool check_purge= false; + ulong prev_binlog_id; + LINT_INIT(prev_binlog_id); mysql_mutex_lock(mysql_bin_log.get_log_lock()); if(mysql_bin_log.is_open()) { + prev_binlog_id= mysql_bin_log.current_binlog_id; if (binlog_checksum_options != value) mysql_bin_log.checksum_alg_reset= (uint8) value; if (mysql_bin_log.rotate(true, &check_purge)) @@ -7644,7 +9420,7 @@ binlog_checksum_update(MYSQL_THD thd, struct st_mysql_sys_var *var, mysql_bin_log.checksum_alg_reset= BINLOG_CHECKSUM_ALG_UNDEF; mysql_mutex_unlock(mysql_bin_log.get_log_lock()); if (check_purge) - mysql_bin_log.purge(); + mysql_bin_log.checkpoint_and_purge(prev_binlog_id); } diff --git a/sql/log.h b/sql/log.h index e388df61b38..4249246277f 100644 --- a/sql/log.h +++ b/sql/log.h @@ -45,10 +45,20 @@ class TC_LOG virtual int open(const char *opt_name)=0; virtual void close()=0; + /* + Transaction coordinator 2-phase commit. + + Must invoke the run_prepare_ordered and run_commit_ordered methods, as + described below for these methods. + + In addition, must invoke THD::wait_for_prior_commit(), or equivalent + wait, to ensure that one commit waits for another if registered to do so. + */ virtual int log_and_order(THD *thd, my_xid xid, bool all, bool need_prepare_ordered, bool need_commit_ordered) = 0; virtual int unlog(ulong cookie, my_xid xid)=0; + virtual void commit_checkpoint_notify(void *cookie)= 0; protected: /* @@ -75,9 +85,11 @@ protected: prepare_ordered() or commit_ordered() methods. */ extern mysql_mutex_t LOCK_prepare_ordered; +extern mysql_cond_t COND_prepare_ordered; extern mysql_mutex_t LOCK_commit_ordered; #ifdef HAVE_PSI_INTERFACE extern PSI_mutex_key key_LOCK_prepare_ordered, key_LOCK_commit_ordered; +extern PSI_cond_key key_COND_prepare_ordered; #endif class TC_LOG_DUMMY: public TC_LOG // use it to disable the logging @@ -98,8 +110,12 @@ public: return 1; } int unlog(ulong cookie, my_xid xid) { return 0; } + void commit_checkpoint_notify(void *cookie) { DBUG_ASSERT(0); }; }; +#define TC_LOG_PAGE_SIZE 8192 +#define TC_LOG_MIN_SIZE (3*TC_LOG_PAGE_SIZE) + #ifdef HAVE_MMAP class TC_LOG_MMAP: public TC_LOG { @@ -110,6 +126,12 @@ class TC_LOG_MMAP: public TC_LOG PS_DIRTY // new xids added since last sync } PAGE_STATE; + struct pending_cookies { + uint count; + uint pending_count; + ulong cookies[TC_LOG_PAGE_SIZE/sizeof(my_xid)]; + }; + private: typedef struct st_page { struct st_page *next; // page a linked in a fifo queue @@ -141,7 +163,7 @@ class TC_LOG_MMAP: public TC_LOG one has to use active->lock. Same for LOCK_pool and LOCK_sync */ - mysql_mutex_t LOCK_active, LOCK_pool, LOCK_sync; + mysql_mutex_t LOCK_active, LOCK_pool, LOCK_sync, LOCK_pending_checkpoint; mysql_cond_t COND_pool, COND_active; /* Queue of threads that need to call commit_ordered(). @@ -163,14 +185,16 @@ class TC_LOG_MMAP: public TC_LOG */ mysql_cond_t COND_queue_busy; my_bool commit_ordered_queue_busy; + pending_cookies* pending_checkpoint; public: - TC_LOG_MMAP(): inited(0) {} + TC_LOG_MMAP(): inited(0), pending_checkpoint(0) {} int open(const char *opt_name); void close(); int log_and_order(THD *thd, my_xid xid, bool all, bool need_prepare_ordered, bool need_commit_ordered); int unlog(ulong cookie, my_xid xid); + void commit_checkpoint_notify(void *cookie); int recover(); private: @@ -178,6 +202,7 @@ class TC_LOG_MMAP: public TC_LOG void get_active_from_pool(); int sync(); int overflow(); + int delete_entry(ulong cookie); }; #else #define TC_LOG_MMAP TC_LOG_DUMMY @@ -354,7 +379,36 @@ private: time_t last_time; }; +/* + We assign each binlog file an internal ID, used to identify them for unlog(). + The IDs start from 0 and increment for each new binlog created. + + In unlog() we need to know the ID of the binlog file that the corresponding + transaction was written into. We also need a special value for a corner + case where there is no corresponding binlog id (since nothing was logged). + And we need an error flag to mark that unlog() must return failure. + + We use the following macros to pack all of this information into the single + ulong available with log_and_order() / unlog(). + + Note that we cannot use the value 0 for cookie, as that is reserved as error + return value from log_and_order(). + */ +#define BINLOG_COOKIE_ERROR_RETURN 0 +#define BINLOG_COOKIE_DUMMY_ID 1 +#define BINLOG_COOKIE_BASE 2 +#define BINLOG_COOKIE_DUMMY(error_flag) \ + ( (BINLOG_COOKIE_DUMMY_ID<<1) | ((error_flag)&1) ) +#define BINLOG_COOKIE_MAKE(id, error_flag) \ + ( (((id)+BINLOG_COOKIE_BASE)<<1) | ((error_flag)&1) ) +#define BINLOG_COOKIE_GET_ERROR_FLAG(c) ((c) & 1) +#define BINLOG_COOKIE_GET_ID(c) ( ((ulong)(c)>>1) - BINLOG_COOKIE_BASE ) +#define BINLOG_COOKIE_IS_DUMMY(c) \ + ( ((ulong)(c)>>1) == BINLOG_COOKIE_DUMMY_ID ) + class binlog_cache_mngr; +struct rpl_gtid; +struct wait_for_commit; class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG { private: @@ -379,11 +433,10 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG bool using_stmt_cache; bool using_trx_cache; /* - Extra events (BEGIN, COMMIT/ROLLBACK/XID, and possibly INCIDENT) to be + Extra events (COMMIT/ROLLBACK/XID, and possibly INCIDENT) to be written during group commit. The incident_event is only valid if trx_data->has_incident() is true. */ - Log_event *begin_event; Log_event *end_event; Log_event *incident_event; /* Set during group commit to record any per-thread error. */ @@ -392,12 +445,38 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG IO_CACHE *error_cache; /* This is the `all' parameter for ha_commit_ordered(). */ bool all; + /* + True if we need to increment xid_count in trx_group_commit_leader() and + decrement in unlog() (this is needed if there is a participating engine + that does not implement the commit_checkpoint_request() handlerton + method). + */ + bool need_unlog; + /* + Fields used to pass the necessary information to the last thread in a + group commit, only used when opt_optimize_thread_scheduling is not set. + */ + bool check_purge; + /* Flag used to optimise around wait_for_prior_commit. */ + bool queued_by_other; + ulong binlog_id; }; + /* + When this is set, a RESET MASTER is in progress. + + Then we should not write any binlog checkpoints into the binlog (that + could result in deadlock on LOCK_log, and we will delete all binlog files + anyway). Instead we should signal COND_xid_list whenever a new binlog + checkpoint arrives - when all have arrived, RESET MASTER will complete. + */ + bool reset_master_pending; + ulong mark_xid_done_waiting; + /* LOCK_log and LOCK_index are inited by init_pthread_objects() */ mysql_mutex_t LOCK_index; - mysql_mutex_t LOCK_prep_xids; - mysql_cond_t COND_prep_xids; + mysql_mutex_t LOCK_xid_list; + mysql_cond_t COND_xid_list; mysql_cond_t update_cond; ulonglong bytes_written; IO_CACHE index_file; @@ -414,27 +493,14 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG The max size before rotation (usable only if log_type == LOG_BIN: binary logs and relay logs). For a binlog, max_size should be max_binlog_size. - For a relay log, it should be max_relay_log_size if this is non-zero, - max_binlog_size otherwise. max_size is set in init(), and dynamically changed (when one does SET - GLOBAL MAX_BINLOG_SIZE|MAX_RELAY_LOG_SIZE) by fix_max_binlog_size and - fix_max_relay_log_size). + GLOBAL MAX_BINLOG_SIZE|MAX_RELAY_LOG_SIZE) from sys_vars.cc */ ulong max_size; - long prepared_xids; /* for tc log - number of xids to remember */ // current file sequence number for load data infile binary logging uint file_id; uint open_count; // For replication int readers_count; - bool need_start_event; - /* - no_auto_events means we don't want any of these automatic events : - Start/Rotate/Stop. That is, in 4.x when we rotate a relay log, we don't - want a Rotate_log event to be written to the relay log. When we start a - relay log etc. So in 4.x this is 1 for relay logs, 0 for binlogs. - In 5.0 it's 0 for relay logs too! - */ - bool no_auto_events; /* Queue of transactions queued up to participate in group commit. */ group_commit_entry *group_commit_queue; /* @@ -456,6 +522,8 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG */ uint *sync_period_ptr; uint sync_counter; + bool state_file_deleted; + bool binlog_state_recover_done; inline uint get_sync_period() { @@ -470,13 +538,42 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG */ int new_file_without_locking(); int new_file_impl(bool need_lock); - int write_transaction_or_stmt(group_commit_entry *entry); + void do_checkpoint_request(ulong binlog_id); + void purge(); + int write_transaction_or_stmt(group_commit_entry *entry, uint64 commit_id); + int queue_for_group_commit(group_commit_entry *entry); bool write_transaction_to_binlog_events(group_commit_entry *entry); void trx_group_commit_leader(group_commit_entry *leader); - void mark_xid_done(); - void mark_xids_active(uint xid_count); + bool is_xidlist_idle_nolock(); public: + /* + A list of struct xid_count_per_binlog is used to keep track of how many + XIDs are in prepared, but not committed, state in each binlog. And how + many commit_checkpoint_request()'s are pending. + + When count drops to zero in a binlog after rotation, it means that there + are no more XIDs in prepared state, so that binlog is no longer needed + for XA crash recovery, and we can log a new binlog checkpoint event. + + The list is protected against simultaneous access from multiple + threads by LOCK_xid_list. + */ + struct xid_count_per_binlog : public ilink { + char *binlog_name; + uint binlog_name_len; + ulong binlog_id; + /* Total prepared XIDs and pending checkpoint requests in this binlog. */ + long xid_count; + /* For linking in requests to the binlog background thread. */ + xid_count_per_binlog *next_in_queue; + xid_count_per_binlog(); /* Give link error if constructor used. */ + }; + I_List<xid_count_per_binlog> binlog_xid_count_list; + mysql_mutex_t LOCK_binlog_background_thread; + mysql_cond_t COND_binlog_background_thread; + mysql_cond_t COND_binlog_background_thread_end; + using MYSQL_LOG::generate_name; using MYSQL_LOG::is_open; @@ -534,6 +631,7 @@ public: */ char last_commit_pos_file[FN_REFLEN]; my_off_t last_commit_pos_offset; + ulong current_binlog_id; MYSQL_BIN_LOG(uint *sync_period); /* @@ -562,7 +660,10 @@ public: int log_and_order(THD *thd, my_xid xid, bool all, bool need_prepare_ordered, bool need_commit_ordered); int unlog(ulong cookie, my_xid xid); - int recover(IO_CACHE *log, Format_description_log_event *fdle); + void commit_checkpoint_notify(void *cookie); + int recover(LOG_INFO *linfo, const char *last_log_name, IO_CACHE *first_log, + Format_description_log_event *fdle, bool do_xa); + int do_binlog_recovery(const char *opt_name, bool do_xa_recovery); #if !defined(MYSQL_CLIENT) int flush_and_set_pending_rows_event(THD *thd, Rows_log_event* event, @@ -588,17 +689,17 @@ public: } void set_max_size(ulong max_size_arg); void signal_update(); + void wait_for_sufficient_commits(); void wait_for_update_relay_log(THD* thd); int wait_for_update_bin_log(THD* thd, const struct timespec * timeout); - void set_need_start_event() { need_start_event = 1; } - void init(bool no_auto_events_arg, ulong max_size); + void init(ulong max_size); void init_pthread_objects(); void cleanup(); bool open(const char *log_name, enum_log_type log_type, const char *new_name, enum cache_type io_cache_type_arg, - bool no_auto_events_arg, ulong max_size, + ulong max_size, bool null_created, bool need_mutex); bool open_index_file(const char *index_file_name_arg, @@ -614,6 +715,7 @@ public: bool write_incident_already_locked(THD *thd); bool write_incident(THD *thd); + void write_binlog_checkpoint_event_already_locked(const char *name, uint len); int write_cache(THD *thd, IO_CACHE *cache); void set_write_error(THD *thd, bool is_transactional); bool check_write_error(THD *thd); @@ -628,12 +730,16 @@ public: */ bool appendv(const char* buf,uint len,...); bool append(Log_event* ev); + bool append_no_lock(Log_event* ev); + void mark_xids_active(ulong cookie, uint xid_count); + void mark_xid_done(ulong cookie, bool write_checkpoint); void make_log_name(char* buf, const char* log_ident); bool is_active(const char* log_file_name); + bool can_purge_log(const char *log_file_name); int update_log_index(LOG_INFO* linfo, bool need_update_threads); int rotate(bool force_rotate, bool* check_purge); - void purge(); + void checkpoint_and_purge(ulong binlog_id); int rotate_and_purge(bool force_rotate); /** Flush binlog cache and synchronize to disk. @@ -664,7 +770,8 @@ public: int register_create_index_entry(const char* entry); int purge_index_entry(THD *thd, ulonglong *decrease_log_space, bool need_mutex); - bool reset_logs(THD* thd); + bool reset_logs(THD* thd, bool create_new_log, + rpl_gtid *init_state, uint32 init_state_len); void close(uint exiting); void clear_inuse_flag_when_closing(File file); @@ -687,6 +794,21 @@ public: inline IO_CACHE *get_index_file() { return &index_file;} inline uint32 get_open_count() { return open_count; } void set_status_variables(THD *thd); + bool is_xidlist_idle(); + bool write_gtid_event(THD *thd, bool standalone, bool is_transactional, + uint64 commit_id); + int read_state_from_file(); + int write_state_to_file(); + int get_most_recent_gtid_list(rpl_gtid **list, uint32 *size); + bool append_state_pos(String *str); + bool append_state(String *str); + bool is_empty_state(); + bool find_in_binlog_state(uint32 domain_id, uint32 server_id, + rpl_gtid *out_gtid); + bool lookup_domain_in_binlog_state(uint32 domain_id, rpl_gtid *out_gtid); + int bump_seq_no_counter_if_needed(uint32 domain_id, uint64 seq_no); + bool check_strict_gtid_sequence(uint32 domain_id, uint32 server_id, + uint64 seq_no); }; class Log_event_handler diff --git a/sql/log_event.cc b/sql/log_event.cc index 826a45f6da8..0aa53a6f2bd 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -48,6 +48,7 @@ #include "transaction.h" #include <my_dir.h> #include "sql_show.h" // append_identifier +#include <strfunc.h> #endif /* MYSQL_CLIENT */ @@ -130,7 +131,7 @@ const ulong checksum_version_product_mariadb= checksum_version_split_mariadb[2]; #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) -static int rows_event_stmt_cleanup(Relay_log_info const *rli, THD* thd); +static int rows_event_stmt_cleanup(rpl_group_info *rgi, THD* thd); static const char *HA_ERR(int i) { @@ -362,7 +363,6 @@ static void clear_all_errors(THD *thd, Relay_log_info *rli) { thd->is_slave_error = 0; thd->clear_error(); - rli->clear_error(); } inline int idempotent_error_code(int err_code) @@ -519,11 +519,59 @@ pretty_print_str(String *packet, const char *str, int len) #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) /** - Creates a temporary name for load data infile:. + Create a prefix for the temporary files that is to be used for + load data file name for this master + + @param name Store prefix of name here + @param connection_name Connection name + + @return pointer to end of name + + @description + We assume that FN_REFLEN is big enough to hold + MAX_CONNECTION_NAME * MAX_FILENAME_MBWIDTH characters + 2 numbers + + a short extension. + + The resulting file name has the following parts, each separated with a '-' + - PREFIX_SQL_LOAD (SQL_LOAD-) + - If a connection name is given (multi-master setup): + - Add an extra '-' to mark that this is a multi-master file + - connection name in lower case, converted to safe file characters. + (see create_logfile_name_with_suffix()). + - server_id + - A last '-' (after server_id). +*/ + +static char *load_data_tmp_prefix(char *name, + LEX_STRING *connection_name) +{ + name= strmov(name, PREFIX_SQL_LOAD); + if (connection_name->length) + { + uint buf_length; + uint errors; + /* Add marker that this is a multi-master-file */ + *name++='-'; + /* Convert connection_name to a safe filename */ + buf_length= strconvert(system_charset_info, connection_name->str, + &my_charset_filename, name, FN_REFLEN, + &errors); + name+= buf_length; + *name++= '-'; + } + name= int10_to_str(global_system_variables.server_id, name, 10); + *name++ = '-'; + *name= '\0'; // For testing prefixes + return name; +} + + +/** + Creates a temporary name for LOAD DATA INFILE @param buf Store new filename here @param file_id File_id (part of file name) - @param event_server_id Event_id (part of file name) + @param event_server_id Event_id (part of file name) @param ext Extension for file name @return @@ -531,16 +579,14 @@ pretty_print_str(String *packet, const char *str, int len) */ static char *slave_load_file_stem(char *buf, uint file_id, - int event_server_id, const char *ext) + int event_server_id, const char *ext, + LEX_STRING *connection_name) { char *res; - fn_format(buf,PREFIX_SQL_LOAD,slave_load_tmpdir, "", MY_UNPACK_FILENAME); + res= buf+ unpack_dirname(buf, slave_load_tmpdir); to_unix_path(buf); - - buf = strend(buf); - buf = int10_to_str(::server_id, buf, 10); - *buf++ = '-'; - buf = int10_to_str(event_server_id, buf, 10); + buf= load_data_tmp_prefix(res, connection_name); + buf= int10_to_str(event_server_id, buf, 10); *buf++ = '-'; res= int10_to_str(file_id, buf, 10); strmov(res, ext); // Add extension last @@ -555,14 +601,17 @@ static char *slave_load_file_stem(char *buf, uint file_id, Delete all temporary files used for SQL_LOAD. */ -static void cleanup_load_tmpdir() +static void cleanup_load_tmpdir(LEX_STRING *connection_name) { MY_DIR *dirp; FILEINFO *file; uint i; - char fname[FN_REFLEN], prefbuf[31], *p; + char dir[FN_REFLEN], fname[FN_REFLEN]; + char prefbuf[31 + MAX_CONNECTION_NAME* MAX_FILENAME_MBWIDTH + 1]; + DBUG_ENTER("cleanup_load_tmpdir"); - if (!(dirp=my_dir(slave_load_tmpdir,MYF(0)))) + unpack_dirname(dir, slave_load_tmpdir); + if (!(dirp=my_dir(dir, MYF(MY_WME)))) return; /* @@ -573,12 +622,11 @@ static void cleanup_load_tmpdir() we cannot meet Start_log event in the middle of events from one LOAD DATA. */ - p= strmake(prefbuf, STRING_WITH_LEN(PREFIX_SQL_LOAD)); - p= int10_to_str(::server_id, p, 10); - *(p++)= '-'; - *p= 0; - for (i=0 ; i < (uint)dirp->number_off_files; i++) + load_data_tmp_prefix(prefbuf, connection_name); + DBUG_PRINT("enter", ("dir: '%s' prefix: '%s'", dir, prefbuf)); + + for (i=0 ; i < (uint)dirp->number_of_files; i++) { file=dirp->dir_entry+i; if (is_prefix(file->name, prefbuf)) @@ -589,6 +637,7 @@ static void cleanup_load_tmpdir() } my_dirend(dirp); + DBUG_VOID_RETURN; } #endif @@ -751,6 +800,9 @@ const char* Log_event::get_type_str(Log_event_type type) case EXECUTE_LOAD_QUERY_EVENT: return "Execute_load_query"; case INCIDENT_EVENT: return "Incident"; case ANNOTATE_ROWS_EVENT: return "Annotate_rows"; + case BINLOG_CHECKPOINT_EVENT: return "Binlog_checkpoint"; + case GTID_EVENT: return "Gtid"; + case GTID_LIST_EVENT: return "Gtid_list"; default: return "Unknown"; /* impossible */ } } @@ -771,7 +823,7 @@ Log_event::Log_event(THD* thd_arg, uint16 flags_arg, bool using_trans) crc(0), thd(thd_arg), checksum_alg(BINLOG_CHECKSUM_ALG_UNDEF) { - server_id= thd->server_id; + server_id= thd->variables.server_id; when= thd->start_time; when_sec_part=thd->start_time_sec_part; @@ -796,7 +848,7 @@ Log_event::Log_event() cache_type(Log_event::EVENT_INVALID_CACHE), crc(0), thd(0), checksum_alg(BINLOG_CHECKSUM_ALG_UNDEF) { - server_id= ::server_id; + server_id= global_system_variables.server_id; /* We can't call my_time() here as this would cause a call before my_init() is called @@ -885,8 +937,11 @@ Log_event::Log_event(const char* buf, #ifndef MYSQL_CLIENT #ifdef HAVE_REPLICATION -int Log_event::do_update_pos(Relay_log_info *rli) +int Log_event::do_update_pos(rpl_group_info *rgi) { + Relay_log_info *rli= rgi->rli; + DBUG_ENTER("Log_event::do_update_pos"); + /* rli is null when (as far as I (Guilhem) know) the caller is Load_log_event::do_apply_event *and* that one is called from @@ -911,27 +966,37 @@ int Log_event::do_update_pos(Relay_log_info *rli) if (debug_not_change_ts_if_art_event == 1 && is_artificial_event()) debug_not_change_ts_if_art_event= 0; ); - rli->stmt_done(log_pos, is_artificial_event() && - IF_DBUG(debug_not_change_ts_if_art_event > 0, 1) ? - 0 : when); + /* + In parallel execution, delay position update for the events that are + not part of event groups (format description, rotate, and such) until + the actual event execution reaches that point. + */ + if (!rgi->is_parallel_exec || is_group_event(get_type_code())) + rli->stmt_done(log_pos, + (is_artificial_event() && + IF_DBUG(debug_not_change_ts_if_art_event > 0, 1) ? + 0 : when), + thd, rgi); DBUG_EXECUTE_IF("let_first_flush_log_change_timestamp", if (debug_not_change_ts_if_art_event == 0) debug_not_change_ts_if_art_event= 2; ); } - return 0; // Cannot fail currently + DBUG_RETURN(0); // Cannot fail currently } Log_event::enum_skip_reason -Log_event::do_shall_skip(Relay_log_info *rli) +Log_event::do_shall_skip(rpl_group_info *rgi) { - DBUG_PRINT("info", ("ev->server_id=%lu, ::server_id=%lu," - " rli->replicate_same_server_id=%d," - " rli->slave_skip_counter=%d", - (ulong) server_id, (ulong) ::server_id, + Relay_log_info *rli= rgi->rli; + DBUG_PRINT("info", ("ev->server_id: %lu, ::server_id: %lu," + " rli->replicate_same_server_id: %d," + " rli->slave_skip_counter: %lu", + (ulong) server_id, (ulong) global_system_variables.server_id, rli->replicate_same_server_id, rli->slave_skip_counter)); - if ((server_id == ::server_id && !rli->replicate_same_server_id) || + if ((server_id == global_system_variables.server_id && + !rli->replicate_same_server_id) || (rli->slave_skip_counter == 1 && rli->is_in_group()) || (flags & LOG_EVENT_SKIP_REPLICATION_F && opt_replicate_events_marked_for_skip != RPL_SKIP_REPLICATE)) @@ -1416,7 +1481,7 @@ err: DBUG_ASSERT(error != 0); sql_print_error("Error in Log_event::read_log_event(): " "'%s', data_len: %d, event_type: %d", - error,data_len,head[EVENT_TYPE_OFFSET]); + error,data_len,(uchar)(head[EVENT_TYPE_OFFSET])); my_free(buf); /* The SQL slave thread will check if file->error<0 to know @@ -1565,6 +1630,15 @@ Log_event* Log_event::read_log_event(const char* buf, uint event_len, case ROTATE_EVENT: ev = new Rotate_log_event(buf, event_len, description_event); break; + case BINLOG_CHECKPOINT_EVENT: + ev = new Binlog_checkpoint_log_event(buf, event_len, description_event); + break; + case GTID_EVENT: + ev = new Gtid_log_event(buf, event_len, description_event); + break; + case GTID_LIST_EVENT: + ev = new Gtid_list_log_event(buf, event_len, description_event); + break; #ifdef HAVE_REPLICATION case SLAVE_EVENT: /* can never happen (unused event) */ ev = new Slave_log_event(buf, event_len, description_event); @@ -1691,6 +1765,165 @@ Log_event* Log_event::read_log_event(const char* buf, uint event_len, #ifdef MYSQL_CLIENT +static void hexdump_minimal_header_to_io_cache(IO_CACHE *file, + my_off_t offset, + uchar *ptr) +{ + DBUG_ASSERT(LOG_EVENT_MINIMAL_HEADER_LEN == 19); + + /* + Pretty-print the first LOG_EVENT_MINIMAL_HEADER_LEN (19) bytes of the + common header, which contains the basic information about the log event. + Every event will have at least this much header, but events could contain + more headers (which must be printed by other methods, if desired). + */ + char emit_buf[120]; // Enough for storing one line + my_b_printf(file, + "# " + "|Timestamp " + "|Type " + "|Master ID " + "|Size " + "|Master Pos " + "|Flags\n"); + size_t const emit_buf_written= + my_snprintf(emit_buf, sizeof(emit_buf), + "# %8llx " /* Position */ + "|%02x %02x %02x %02x " /* Timestamp */ + "|%02x " /* Type */ + "|%02x %02x %02x %02x " /* Master ID */ + "|%02x %02x %02x %02x " /* Size */ + "|%02x %02x %02x %02x " /* Master Pos */ + "|%02x %02x\n", /* Flags */ + (ulonglong) offset, /* Position */ + ptr[0], ptr[1], ptr[2], ptr[3], /* Timestamp */ + ptr[4], /* Type */ + ptr[5], ptr[6], ptr[7], ptr[8], /* Master ID */ + ptr[9], ptr[10], ptr[11], ptr[12], /* Size */ + ptr[13], ptr[14], ptr[15], ptr[16], /* Master Pos */ + ptr[17], ptr[18]); /* Flags */ + + DBUG_ASSERT(static_cast<size_t>(emit_buf_written) < sizeof(emit_buf)); + my_b_write(file, reinterpret_cast<uchar*>(emit_buf), emit_buf_written); + my_b_write(file, "#\n", 2); +} + + +/* + The number of bytes to print per line. Should be an even number, + and "hexdump -C" uses 16, so we'll duplicate that here. +*/ +#define HEXDUMP_BYTES_PER_LINE 16 + +static void format_hex_line(char *emit_buff) +{ + memset(emit_buff + 1, ' ', + 1 + 8 + 2 + (HEXDUMP_BYTES_PER_LINE * 3 + 1) + 2 + + HEXDUMP_BYTES_PER_LINE); + emit_buff[0]= '#'; + emit_buff[2 + 8 + 2 + (HEXDUMP_BYTES_PER_LINE * 3 + 1) + 1]= '|'; + emit_buff[2 + 8 + 2 + (HEXDUMP_BYTES_PER_LINE * 3 + 1) + 2 + + HEXDUMP_BYTES_PER_LINE]= '|'; + emit_buff[2 + 8 + 2 + (HEXDUMP_BYTES_PER_LINE * 3 + 1) + 2 + + HEXDUMP_BYTES_PER_LINE + 1]= '\n'; + emit_buff[2 + 8 + 2 + (HEXDUMP_BYTES_PER_LINE * 3 + 1) + 2 + + HEXDUMP_BYTES_PER_LINE + 2]= '\0'; +} + +static void hexdump_data_to_io_cache(IO_CACHE *file, + my_off_t offset, + uchar *ptr, + my_off_t size) +{ + /* + 2 = '# ' + 8 = address + 2 = ' ' + (HEXDUMP_BYTES_PER_LINE * 3 + 1) = Each byte prints as two hex digits, + plus a space + 2 = ' |' + HEXDUMP_BYTES_PER_LINE = text representation + 2 = '|\n' + 1 = '\0' + */ + char emit_buffer[2 + 8 + 2 + (HEXDUMP_BYTES_PER_LINE * 3 + 1) + 2 + + HEXDUMP_BYTES_PER_LINE + 2 + 1 ]; + char *h,*c; + my_off_t i; + + if (size == 0) + return; + + format_hex_line(emit_buffer); + /* + Print the rest of the event (without common header) + */ + my_off_t starting_offset = offset; + for (i= 0, + c= emit_buffer + 2 + 8 + 2 + (HEXDUMP_BYTES_PER_LINE * 3 + 1) + 2, + h= emit_buffer + 2 + 8 + 2; + i < size; + i++, ptr++) + { + my_snprintf(h, 4, "%02x ", *ptr); + h+= 3; + + *c++= my_isprint(&my_charset_bin, *ptr) ? *ptr : '.'; + + /* Print in groups of HEXDUMP_BYTES_PER_LINE characters. */ + if ((i % HEXDUMP_BYTES_PER_LINE) == (HEXDUMP_BYTES_PER_LINE - 1)) + { + /* remove \0 left after printing hex byte representation */ + *h= ' '; + /* prepare space to print address */ + memset(emit_buffer + 2, ' ', 8); + /* print address */ + size_t const emit_buf_written= my_snprintf(emit_buffer + 2, 9, "%8llx", + (ulonglong) starting_offset); + /* remove \0 left after printing address */ + emit_buffer[2 + emit_buf_written]= ' '; + my_b_write(file, reinterpret_cast<uchar*>(emit_buffer), + sizeof(emit_buffer) - 1); + c= emit_buffer + 2 + 8 + 2 + (HEXDUMP_BYTES_PER_LINE * 3 + 1) + 2; + h= emit_buffer + 2 + 8 + 2; + format_hex_line(emit_buffer); + starting_offset+= HEXDUMP_BYTES_PER_LINE; + } + else if ((i % (HEXDUMP_BYTES_PER_LINE / 2)) + == ((HEXDUMP_BYTES_PER_LINE / 2) - 1)) + { + /* + In the middle of the group of HEXDUMP_BYTES_PER_LINE, emit an extra + space in the hex string, to make two groups. + */ + *h++= ' '; + } + + } + + /* + There is still data left in our buffer, which means that the previous + line was not perfectly HEXDUMP_BYTES_PER_LINE characters, so write an + incomplete line, with spaces to pad out to the same length as a full + line would be, to make things more readable. + */ + if (h != emit_buffer + 2 + 8 + 2) + { + *h= ' '; + *c++= '|'; *c++= '\n'; + memset(emit_buffer + 2, ' ', 8); + size_t const emit_buf_written= my_snprintf(emit_buffer + 2, 9, "%8llx", + (ulonglong) starting_offset); + emit_buffer[2 + emit_buf_written]= ' '; + /* pad unprinted area */ + memset(h, ' ', + (HEXDUMP_BYTES_PER_LINE * 3 + 1) - (h - (emit_buffer + 2 + 8 + 2))); + my_b_write(file, reinterpret_cast<uchar*>(emit_buffer), + c - emit_buffer); + } + my_b_write(file, "#\n", 2); +} + /* Log_event::print_header() */ @@ -1725,86 +1958,27 @@ void Log_event::print_header(IO_CACHE* file, { my_b_printf(file, "\n"); uchar *ptr= (uchar*)temp_buf; - my_off_t size= - uint4korr(ptr + EVENT_LEN_OFFSET) - LOG_EVENT_MINIMAL_HEADER_LEN; - my_off_t i; - - /* Header len * 4 >= header len * (2 chars + space + extra space) */ - char *h, hex_string[LOG_EVENT_MINIMAL_HEADER_LEN*4]= {0}; - char *c, char_string[16+1]= {0}; - - /* Pretty-print event common header if header is exactly 19 bytes */ - if (print_event_info->common_header_len == LOG_EVENT_MINIMAL_HEADER_LEN) - { - char emit_buf[256]; // Enough for storing one line - my_b_printf(file, "# Position Timestamp Type Master ID " - "Size Master Pos Flags \n"); - size_t const bytes_written= - my_snprintf(emit_buf, sizeof(emit_buf), - "# %8.8lx %02x %02x %02x %02x %02x " - "%02x %02x %02x %02x %02x %02x %02x %02x " - "%02x %02x %02x %02x %02x %02x\n", - (unsigned long) hexdump_from, - ptr[0], ptr[1], ptr[2], ptr[3], ptr[4], ptr[5], ptr[6], - ptr[7], ptr[8], ptr[9], ptr[10], ptr[11], ptr[12], ptr[13], - ptr[14], ptr[15], ptr[16], ptr[17], ptr[18]); - DBUG_ASSERT(static_cast<size_t>(bytes_written) < sizeof(emit_buf)); - my_b_write(file, (uchar*) emit_buf, bytes_written); - ptr += LOG_EVENT_MINIMAL_HEADER_LEN; - hexdump_from += LOG_EVENT_MINIMAL_HEADER_LEN; - } - - /* Rest of event (without common header) */ - for (i= 0, c= char_string, h=hex_string; - i < size; - i++, ptr++) - { - my_snprintf(h, 4, "%02x ", *ptr); - h += 3; - - *c++= my_isalnum(&my_charset_bin, *ptr) ? *ptr : '.'; - - if (i % 16 == 15) - { - /* - my_b_printf() does not support full printf() formats, so we - have to do it this way. + my_off_t size= uint4korr(ptr + EVENT_LEN_OFFSET); + my_off_t hdr_len= get_header_len(print_event_info->common_header_len); - TODO: Rewrite my_b_printf() to support full printf() syntax. - */ - char emit_buf[256]; - size_t const bytes_written= - my_snprintf(emit_buf, sizeof(emit_buf), - "# %8.8lx %-48.48s |%16s|\n", - (unsigned long) (hexdump_from + (i & 0xfffffff0)), - hex_string, char_string); - DBUG_ASSERT(static_cast<size_t>(bytes_written) < sizeof(emit_buf)); - my_b_write(file, (uchar*) emit_buf, bytes_written); - hex_string[0]= 0; - char_string[0]= 0; - c= char_string; - h= hex_string; - } - else if (i % 8 == 7) *h++ = ' '; - } - *c= '\0'; + size-= hdr_len; + + my_b_printf(file, "# Position\n"); + + /* Write the header, nicely formatted by field. */ + hexdump_minimal_header_to_io_cache(file, hexdump_from, ptr); + + ptr+= hdr_len; + hexdump_from+= hdr_len; + + /* Print the rest of the data, mimicking "hexdump -C" output. */ + hexdump_data_to_io_cache(file, hexdump_from, ptr, size); - if (hex_string[0]) - { - char emit_buf[256]; - size_t const bytes_written= - my_snprintf(emit_buf, sizeof(emit_buf), - "# %8.8lx %-48.48s |%s|\n", - (unsigned long) (hexdump_from + (i & 0xfffffff0)), - hex_string, char_string); - DBUG_ASSERT(static_cast<size_t>(bytes_written) < sizeof(emit_buf)); - my_b_write(file, (uchar*) emit_buf, bytes_written); - } /* - need a # to prefix the rest of printouts for example those of - Rows_log_event::print_helper(). + Prefix the next line so that the output from print_helper() + will appear as a comment. */ - my_b_write(file, reinterpret_cast<const uchar*>("# "), 2); + my_b_write(file, "# Event: ", 9); } DBUG_VOID_RETURN; } @@ -2468,11 +2642,11 @@ void Log_event::print_timestamp(IO_CACHE* file, time_t* ts) #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) inline Log_event::enum_skip_reason -Log_event::continue_group(Relay_log_info *rli) +Log_event::continue_group(rpl_group_info *rgi) { - if (rli->slave_skip_counter == 1) + if (rgi->rli->slave_skip_counter == 1) return Log_event::EVENT_SKIP_IGNORE; - return Log_event::do_shall_skip(rli); + return Log_event::do_shall_skip(rgi); } #endif @@ -2697,17 +2871,22 @@ bool Query_log_event::write(IO_CACHE* file) user= thd->get_invoker_user(); host= thd->get_invoker_host(); } - else if (thd->security_ctx->priv_user) + else { Security_context *ctx= thd->security_ctx; - user.length= strlen(ctx->priv_user); - user.str= ctx->priv_user; - if (ctx->priv_host[0] != '\0') + if (thd->need_binlog_invoker() == THD::INVOKER_USER) { + user.str= ctx->priv_user; host.str= ctx->priv_host; - host.length= strlen(ctx->priv_host); + host.length= strlen(host.str); } + else + { + user.str= ctx->priv_role; + host= empty_lex_str; + } + user.length= strlen(user.str); } if (user.length > 0) @@ -3342,6 +3521,162 @@ Query_log_event::Query_log_event(const char* buf, uint event_len, } +/* + Replace a binlog event read into a packet with a dummy event. Either a + Query_log_event that has just a comment, or if that will not fit in the + space used for the event to be replaced, then a NULL user_var event. + + This is used when sending binlog data to a slave which does not understand + this particular event and which is too old to support informational events + or holes in the event stream. + + This allows to write such events into the binlog on the master and still be + able to replicate against old slaves without them breaking. + + Clears the flag LOG_EVENT_THREAD_SPECIFIC_F and set LOG_EVENT_SUPPRESS_USE_F. + Overwrites the type with QUERY_EVENT (or USER_VAR_EVENT), and replaces the + body with a minimal query / NULL user var. + + Returns zero on success, -1 if error due to too little space in original + event. A minimum of 25 bytes (19 bytes fixed header + 6 bytes in the body) + is needed in any event to be replaced with a dummy event. +*/ +int +Query_log_event::dummy_event(String *packet, ulong ev_offset, + uint8 checksum_alg) +{ + uchar *p= (uchar *)packet->ptr() + ev_offset; + size_t data_len= packet->length() - ev_offset; + uint16 flags; + static const size_t min_user_var_event_len= + LOG_EVENT_HEADER_LEN + UV_NAME_LEN_SIZE + 1 + UV_VAL_IS_NULL; // 25 + static const size_t min_query_event_len= + LOG_EVENT_HEADER_LEN + QUERY_HEADER_LEN + 1 + 1; // 34 + + if (checksum_alg == BINLOG_CHECKSUM_ALG_CRC32) + data_len-= BINLOG_CHECKSUM_LEN; + else + DBUG_ASSERT(checksum_alg == BINLOG_CHECKSUM_ALG_UNDEF || + checksum_alg == BINLOG_CHECKSUM_ALG_OFF); + + if (data_len < min_user_var_event_len) + /* Cannot replace with dummy, event too short. */ + return -1; + + flags= uint2korr(p + FLAGS_OFFSET); + flags&= ~LOG_EVENT_THREAD_SPECIFIC_F; + flags|= LOG_EVENT_SUPPRESS_USE_F; + int2store(p + FLAGS_OFFSET, flags); + + if (data_len < min_query_event_len) + { + /* + Have to use dummy user_var event for such a short packet. + + This works, but the event will be considered part of an event group with + the following event. So for example @@global.sql_slave_skip_counter=1 + will skip not only the dummy event, but also the immediately following + event. + + We write a NULL user var with the name @`!dummyvar` (or as much + as that as will fit within the size of the original event - so + possibly just @`!`). + */ + static const char var_name[]= "!dummyvar"; + uint name_len= data_len - (min_user_var_event_len - 1); + + p[EVENT_TYPE_OFFSET]= USER_VAR_EVENT; + int4store(p + LOG_EVENT_HEADER_LEN, name_len); + memcpy(p + LOG_EVENT_HEADER_LEN + UV_NAME_LEN_SIZE, var_name, name_len); + p[LOG_EVENT_HEADER_LEN + UV_NAME_LEN_SIZE + name_len]= 1; // indicates NULL + } + else + { + /* + Use a dummy query event, just a comment. + */ + static const char message[]= + "# Dummy event replacing event type %u that slave cannot handle."; + char buf[sizeof(message)+1]; /* +1, as %u can expand to 3 digits. */ + uchar old_type= p[EVENT_TYPE_OFFSET]; + uchar *q= p + LOG_EVENT_HEADER_LEN; + size_t comment_len, len; + + p[EVENT_TYPE_OFFSET]= QUERY_EVENT; + int4store(q + Q_THREAD_ID_OFFSET, 0); + int4store(q + Q_EXEC_TIME_OFFSET, 0); + q[Q_DB_LEN_OFFSET]= 0; + int2store(q + Q_ERR_CODE_OFFSET, 0); + int2store(q + Q_STATUS_VARS_LEN_OFFSET, 0); + q[Q_DATA_OFFSET]= 0; /* Zero terminator for empty db */ + q+= Q_DATA_OFFSET + 1; + len= my_snprintf(buf, sizeof(buf), message, old_type); + comment_len= data_len - (min_query_event_len - 1); + if (comment_len <= len) + memcpy(q, buf, comment_len); + else + { + memcpy(q, buf, len); + memset(q+len, ' ', comment_len - len); + } + } + + if (checksum_alg == BINLOG_CHECKSUM_ALG_CRC32) + { + ha_checksum crc= my_checksum(0L, p, data_len); + int4store(p + data_len, crc); + } + return 0; +} + +/* + Replace an event (GTID event) with a BEGIN query event, to be compatible + with an old slave. +*/ +int +Query_log_event::begin_event(String *packet, ulong ev_offset, + uint8 checksum_alg) +{ + uchar *p= (uchar *)packet->ptr() + ev_offset; + uchar *q= p + LOG_EVENT_HEADER_LEN; + size_t data_len= packet->length() - ev_offset; + uint16 flags; + + if (checksum_alg == BINLOG_CHECKSUM_ALG_CRC32) + data_len-= BINLOG_CHECKSUM_LEN; + else + DBUG_ASSERT(checksum_alg == BINLOG_CHECKSUM_ALG_UNDEF || + checksum_alg == BINLOG_CHECKSUM_ALG_OFF); + + /* Currently we only need to replace GTID event. */ + DBUG_ASSERT(data_len == LOG_EVENT_HEADER_LEN + GTID_HEADER_LEN); + if (data_len != LOG_EVENT_HEADER_LEN + GTID_HEADER_LEN) + return 1; + + flags= uint2korr(p + FLAGS_OFFSET); + flags&= ~LOG_EVENT_THREAD_SPECIFIC_F; + flags|= LOG_EVENT_SUPPRESS_USE_F; + int2store(p + FLAGS_OFFSET, flags); + + p[EVENT_TYPE_OFFSET]= QUERY_EVENT; + int4store(q + Q_THREAD_ID_OFFSET, 0); + int4store(q + Q_EXEC_TIME_OFFSET, 0); + q[Q_DB_LEN_OFFSET]= 0; + int2store(q + Q_ERR_CODE_OFFSET, 0); + int2store(q + Q_STATUS_VARS_LEN_OFFSET, 0); + q[Q_DATA_OFFSET]= 0; /* Zero terminator for empty db */ + q+= Q_DATA_OFFSET + 1; + memcpy(q, "BEGIN", 5); + + if (checksum_alg == BINLOG_CHECKSUM_ALG_CRC32) + { + ha_checksum crc= my_checksum(0L, p, data_len); + int4store(p + data_len, crc); + } + return 0; +} + + #ifdef MYSQL_CLIENT /** Query_log_event::print(). @@ -3543,9 +3878,9 @@ void Query_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) -int Query_log_event::do_apply_event(Relay_log_info const *rli) +int Query_log_event::do_apply_event(rpl_group_info *rgi) { - return do_apply_event(rli, query, q_len); + return do_apply_event(rgi, query, q_len); } /** @@ -3594,12 +3929,16 @@ bool test_if_equal_repl_errors(int expected_error, int actual_error) mismatch. This mismatch could be implemented with a new ER_ code, and to ignore it you would use --slave-skip-errors... */ -int Query_log_event::do_apply_event(Relay_log_info const *rli, - const char *query_arg, uint32 q_len_arg) +int Query_log_event::do_apply_event(rpl_group_info *rgi, + const char *query_arg, uint32 q_len_arg) { LEX_STRING new_db; int expected_error,actual_error= 0; HA_CREATE_INFO db_options; + uint64 sub_id= 0; + rpl_gtid gtid; + Relay_log_info const *rli= rgi->rli; + Rpl_filter *rpl_filter= rli->mi->rpl_filter; DBUG_ENTER("Query_log_event::do_apply_event"); /* @@ -3623,21 +3962,10 @@ int Query_log_event::do_apply_event(Relay_log_info const *rli, thd->variables.auto_increment_increment= auto_increment_increment; thd->variables.auto_increment_offset= auto_increment_offset; - /* - InnoDB internally stores the master log position it has executed so far, - i.e. the position just after the COMMIT event. - When InnoDB will want to store, the positions in rli won't have - been updated yet, so group_master_log_* will point to old BEGIN - and event_master_log* will point to the beginning of current COMMIT. - But log_pos of the COMMIT Query event is what we want, i.e. the pos of the - END of the current log event (COMMIT). We save it in rli so that InnoDB can - access it. - */ - const_cast<Relay_log_info*>(rli)->future_group_master_log_pos= log_pos; DBUG_PRINT("info", ("log_pos: %lu", (ulong) log_pos)); clear_all_errors(thd, const_cast<Relay_log_info*>(rli)); - if (strcmp("COMMIT", query) == 0 && rli->tables_to_lock) + if (strcmp("COMMIT", query) == 0 && rgi->tables_to_lock) { /* Cleaning-up the last statement context: @@ -3646,7 +3974,7 @@ int Query_log_event::do_apply_event(Relay_log_info const *rli, */ int error; char llbuff[22]; - if ((error= rows_event_stmt_cleanup(const_cast<Relay_log_info*>(rli), thd))) + if ((error= rows_event_stmt_cleanup(rgi, thd))) { const_cast<Relay_log_info*>(rli)->report(ERROR_LEVEL, error, "Error in cleaning up after an event preceeding the commit; " @@ -3661,12 +3989,11 @@ int Query_log_event::do_apply_event(Relay_log_info const *rli, future-change-proof addon, e.g if COMMIT handling will start checking invariants like IN_STMT flag must be off at committing the transaction. */ - const_cast<Relay_log_info*>(rli)->inc_event_relay_log_pos(); - const_cast<Relay_log_info*>(rli)->clear_flag(Relay_log_info::IN_STMT); + rgi->inc_event_relay_log_pos(); } else { - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); } /* @@ -3787,6 +4114,30 @@ int Query_log_event::do_apply_event(Relay_log_info const *rli, else thd->variables.collation_database= thd->db_charset; + /* + Record any GTID in the same transaction, so slave state is + transactionally consistent. + */ + if (strcmp("COMMIT", query) == 0 && (sub_id= rgi->gtid_sub_id)) + { + /* Clear the GTID from the RLI so we don't accidentally reuse it. */ + rgi->gtid_sub_id= 0; + + gtid= rgi->current_gtid; + if (rpl_global_gtid_slave_state.record_gtid(thd, >id, sub_id, true, false)) + { + rli->report(ERROR_LEVEL, ER_CANNOT_UPDATE_GTID_STATE, + "Error during COMMIT: failed to update GTID state in " + "%s.%s: %d: %s", + "mysql", rpl_gtid_slave_state_table_name.str, + thd->stmt_da->sql_errno(), thd->stmt_da->message()); + trans_rollback(thd); + sub_id= 0; + thd->is_slave_error= 1; + goto end; + } + } + thd->table_map_for_update= (table_map)table_map_for_update; thd->set_invoker(&user, &host); /* @@ -3972,6 +4323,9 @@ Default database: '%s'. Query: '%s'", } end: + if (sub_id && !thd->is_slave_error) + rpl_global_gtid_slave_state.update_state_hash(sub_id, >id); + /* Probably we have set thd->query, thd->db, thd->catalog to point to places in the data_buf of this event. Now the event is going to be deleted @@ -4000,7 +4354,7 @@ end: DBUG_RETURN(thd->is_slave_error); } -int Query_log_event::do_update_pos(Relay_log_info *rli) +int Query_log_event::do_update_pos(rpl_group_info *rgi) { /* Note that we will not increment group* positions if we are just @@ -4009,20 +4363,22 @@ int Query_log_event::do_update_pos(Relay_log_info *rli) */ if (thd->one_shot_set) { - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); return 0; } else - return Log_event::do_update_pos(rli); + return Log_event::do_update_pos(rgi); } Log_event::enum_skip_reason -Query_log_event::do_shall_skip(Relay_log_info *rli) +Query_log_event::do_shall_skip(rpl_group_info *rgi) { + Relay_log_info *rli= rgi->rli; DBUG_ENTER("Query_log_event::do_shall_skip"); DBUG_PRINT("debug", ("query: %s; q_len: %d", query, q_len)); DBUG_ASSERT(query && q_len > 0); + DBUG_ASSERT(thd == rgi->thd); /* An event skipped due to @@skip_replication must not be counted towards the @@ -4034,19 +4390,41 @@ Query_log_event::do_shall_skip(Relay_log_info *rli) if (rli->slave_skip_counter > 0) { - if (strcmp("BEGIN", query) == 0) + if (is_begin()) { thd->variables.option_bits|= OPTION_BEGIN; - DBUG_RETURN(Log_event::continue_group(rli)); + DBUG_RETURN(Log_event::continue_group(rgi)); } - if (strcmp("COMMIT", query) == 0 || strcmp("ROLLBACK", query) == 0) + if (is_commit() || is_rollback()) { thd->variables.option_bits&= ~OPTION_BEGIN; DBUG_RETURN(Log_event::EVENT_SKIP_COUNT); } } - DBUG_RETURN(Log_event::do_shall_skip(rli)); + DBUG_RETURN(Log_event::do_shall_skip(rgi)); +} + + +bool +Query_log_event::peek_is_commit_rollback(const char *event_start, + size_t event_len, uint8 checksum_alg) +{ + if (checksum_alg == BINLOG_CHECKSUM_ALG_CRC32) + { + if (event_len > BINLOG_CHECKSUM_LEN) + event_len-= BINLOG_CHECKSUM_LEN; + else + event_len= 0; + } + else + DBUG_ASSERT(checksum_alg == BINLOG_CHECKSUM_ALG_UNDEF || + checksum_alg == BINLOG_CHECKSUM_ALG_OFF); + + if (event_len < LOG_EVENT_HEADER_LEN + QUERY_HEADER_LEN || event_len < 9) + return false; + return !memcmp(event_start + (event_len-7), "\0COMMIT", 7) || + !memcmp(event_start + (event_len-9), "\0ROLLBACK", 9); } #endif @@ -4143,7 +4521,7 @@ Start_log_event_v3::Start_log_event_v3(const char* buf, *description_event) :Log_event(buf, description_event) { - buf+= description_event->common_header_len; + buf+= LOG_EVENT_MINIMAL_HEADER_LEN; binlog_version= uint2korr(buf+ST_BINLOG_VER_OFFSET); memcpy(server_version, buf+ST_SERVER_VER_OFFSET, ST_SERVER_VER_LEN); @@ -4194,10 +4572,12 @@ bool Start_log_event_v3::write(IO_CACHE* file) other words, no deadlock problem. */ -int Start_log_event_v3::do_apply_event(Relay_log_info const *rli) +int Start_log_event_v3::do_apply_event(rpl_group_info *rgi) { DBUG_ENTER("Start_log_event_v3::do_apply_event"); int error= 0; + Relay_log_info *rli= rgi->rli; + switch (binlog_version) { case 3: @@ -4210,19 +4590,13 @@ int Start_log_event_v3::do_apply_event(Relay_log_info const *rli) */ if (created) { - error= close_temporary_tables(thd); - cleanup_load_tmpdir(); - } - else - { + rli->close_temporary_tables(); + /* - Set all temporary tables thread references to the current thread - as they may point to the "old" SQL slave thread in case of its - restart. + The following is only false if we get here with a BINLOG statement */ - TABLE *table; - for (table= thd->temporary_tables; table; table= table->next) - table->in_use= thd; + if (rli->mi) + cleanup_load_tmpdir(&rli->mi->cmp_connection_name); } break; @@ -4238,7 +4612,7 @@ int Start_log_event_v3::do_apply_event(Relay_log_info const *rli) Can distinguish, based on the value of 'created': this event was generated at master startup. */ - error= close_temporary_tables(thd); + rli->close_temporary_tables(); } /* Otherwise, can't distinguish a Start_log_event generated at @@ -4361,6 +4735,10 @@ Format_description_log_event(uint8 binlog_ver, const char* server_ver) // Set header lengths of Maria events post_header_len[ANNOTATE_ROWS_EVENT-1]= ANNOTATE_ROWS_HEADER_LEN; + post_header_len[BINLOG_CHECKPOINT_EVENT-1]= + BINLOG_CHECKPOINT_HEADER_LEN; + post_header_len[GTID_EVENT-1]= GTID_HEADER_LEN; + post_header_len[GTID_LIST_EVENT-1]= GTID_LIST_HEADER_LEN; // Sanity-check that all post header lengths are initialized. int i; @@ -4470,109 +4848,6 @@ Format_description_log_event(const char* buf, checksum_alg= (uint8) BINLOG_CHECKSUM_ALG_UNDEF; } - /* - In some previous versions, the events were given other event type - id numbers than in the present version. When replicating from such - a version, we therefore set up an array that maps those id numbers - to the id numbers of the present server. - - If post_header_len is null, it means malloc failed, and is_valid - will fail, so there is no need to do anything. - - The trees in which events have wrong id's are: - - mysql-5.1-wl1012.old mysql-5.1-wl2325-5.0-drop6p13-alpha - mysql-5.1-wl2325-5.0-drop6 mysql-5.1-wl2325-5.0 - mysql-5.1-wl2325-no-dd - - (this was found by grepping for two lines in sequence where the - first matches "FORMAT_DESCRIPTION_EVENT," and the second matches - "TABLE_MAP_EVENT," in log_event.h in all trees) - - In these trees, the following server_versions existed since - TABLE_MAP_EVENT was introduced: - - 5.1.1-a_drop5p3 5.1.1-a_drop5p4 5.1.1-alpha - 5.1.2-a_drop5p10 5.1.2-a_drop5p11 5.1.2-a_drop5p12 - 5.1.2-a_drop5p13 5.1.2-a_drop5p14 5.1.2-a_drop5p15 - 5.1.2-a_drop5p16 5.1.2-a_drop5p16b 5.1.2-a_drop5p16c - 5.1.2-a_drop5p17 5.1.2-a_drop5p4 5.1.2-a_drop5p5 - 5.1.2-a_drop5p6 5.1.2-a_drop5p7 5.1.2-a_drop5p8 - 5.1.2-a_drop5p9 5.1.3-a_drop5p17 5.1.3-a_drop5p17b - 5.1.3-a_drop5p17c 5.1.4-a_drop5p18 5.1.4-a_drop5p19 - 5.1.4-a_drop5p20 5.1.4-a_drop6p0 5.1.4-a_drop6p1 - 5.1.4-a_drop6p2 5.1.5-a_drop5p20 5.2.0-a_drop6p3 - 5.2.0-a_drop6p4 5.2.0-a_drop6p5 5.2.0-a_drop6p6 - 5.2.1-a_drop6p10 5.2.1-a_drop6p11 5.2.1-a_drop6p12 - 5.2.1-a_drop6p6 5.2.1-a_drop6p7 5.2.1-a_drop6p8 - 5.2.2-a_drop6p13 5.2.2-a_drop6p13-alpha 5.2.2-a_drop6p13b - 5.2.2-a_drop6p13c - - (this was found by grepping for "mysql," in all historical - versions of configure.in in the trees listed above). - - There are 5.1.1-alpha versions that use the new event id's, so we - do not test that version string. So replication from 5.1.1-alpha - with the other event id's to a new version does not work. - Moreover, we can safely ignore the part after drop[56]. This - allows us to simplify the big list above to the following regexes: - - 5\.1\.[1-5]-a_drop5.* - 5\.1\.4-a_drop6.* - 5\.2\.[0-2]-a_drop6.* - - This is what we test for in the 'if' below. - */ - if (post_header_len && - server_version[0] == '5' && server_version[1] == '.' && - server_version[3] == '.' && - strncmp(server_version + 5, "-a_drop", 7) == 0 && - ((server_version[2] == '1' && - server_version[4] >= '1' && server_version[4] <= '5' && - server_version[12] == '5') || - (server_version[2] == '1' && - server_version[4] == '4' && - server_version[12] == '6') || - (server_version[2] == '2' && - server_version[4] >= '0' && server_version[4] <= '2' && - server_version[12] == '6'))) - { - if (number_of_event_types != 22) - { - DBUG_PRINT("info", (" number_of_event_types=%d", - number_of_event_types)); - /* this makes is_valid() return false. */ - my_free(post_header_len); - post_header_len= NULL; - DBUG_VOID_RETURN; - } - static const uint8 perm[23]= - { - UNKNOWN_EVENT, START_EVENT_V3, QUERY_EVENT, STOP_EVENT, ROTATE_EVENT, - INTVAR_EVENT, LOAD_EVENT, SLAVE_EVENT, CREATE_FILE_EVENT, - APPEND_BLOCK_EVENT, EXEC_LOAD_EVENT, DELETE_FILE_EVENT, - NEW_LOAD_EVENT, - RAND_EVENT, USER_VAR_EVENT, - FORMAT_DESCRIPTION_EVENT, - TABLE_MAP_EVENT, - PRE_GA_WRITE_ROWS_EVENT, - PRE_GA_UPDATE_ROWS_EVENT, - PRE_GA_DELETE_ROWS_EVENT, - XID_EVENT, - BEGIN_LOAD_QUERY_EVENT, - EXECUTE_LOAD_QUERY_EVENT, - }; - event_type_permutation= perm; - /* - Since we use (permuted) event id's to index the post_header_len - array, we need to permute the post_header_len array too. - */ - uint8 post_header_len_temp[23]; - for (int i= 1; i < 23; i++) - post_header_len_temp[perm[i] - 1]= post_header_len[i - 1]; - for (int i= 0; i < 22; i++) - post_header_len[i] = post_header_len_temp[i]; - } DBUG_VOID_RETURN; } @@ -4585,16 +4860,15 @@ bool Format_description_log_event::write(IO_CACHE* file) We don't call Start_log_event_v3::write() because this would make 2 my_b_safe_write(). */ - uchar buff[FORMAT_DESCRIPTION_HEADER_LEN + BINLOG_CHECKSUM_ALG_DESC_LEN]; - size_t rec_size= sizeof(buff); + uchar buff[START_V3_HEADER_LEN+1]; + size_t rec_size= sizeof(buff) + BINLOG_CHECKSUM_ALG_DESC_LEN + + number_of_event_types; int2store(buff + ST_BINLOG_VER_OFFSET,binlog_version); memcpy((char*) buff + ST_SERVER_VER_OFFSET,server_version,ST_SERVER_VER_LEN); if (!dont_set_created) created= get_time(); int4store(buff + ST_CREATED_OFFSET,created); - buff[ST_COMMON_HEADER_LEN_OFFSET]= LOG_EVENT_HEADER_LEN; - memcpy((char*) buff+ST_COMMON_HEADER_LEN_OFFSET + 1, (uchar*) post_header_len, - LOG_EVENT_TYPES); + buff[ST_COMMON_HEADER_LEN_OFFSET]= common_header_len; /* if checksum is requested record the checksum-algorithm descriptor next to @@ -4607,7 +4881,7 @@ bool Format_description_log_event::write(IO_CACHE* file) #ifndef DBUG_OFF data_written= 0; // to prepare for need_checksum assert #endif - buff[FORMAT_DESCRIPTION_HEADER_LEN]= need_checksum() ? + uchar checksum_byte= need_checksum() ? checksum_alg : (uint8) BINLOG_CHECKSUM_ALG_OFF; /* FD of checksum-aware server is always checksum-equipped, (V) is in, @@ -4627,7 +4901,10 @@ bool Format_description_log_event::write(IO_CACHE* file) checksum_alg= BINLOG_CHECKSUM_ALG_CRC32; // Forcing (V) room to fill anyway } ret= (write_header(file, rec_size) || - wrapper_my_b_safe_write(file, buff, rec_size) || + wrapper_my_b_safe_write(file, buff, sizeof(buff)) || + wrapper_my_b_safe_write(file, (uchar*)post_header_len, + number_of_event_types) || + wrapper_my_b_safe_write(file, &checksum_byte, sizeof(checksum_byte)) || write_footer(file)); if (no_checksum) checksum_alg= BINLOG_CHECKSUM_ALG_OFF; @@ -4636,9 +4913,10 @@ bool Format_description_log_event::write(IO_CACHE* file) #endif #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) -int Format_description_log_event::do_apply_event(Relay_log_info const *rli) +int Format_description_log_event::do_apply_event(rpl_group_info *rgi) { int ret= 0; + Relay_log_info const *rli= rgi->rli; DBUG_ENTER("Format_description_log_event::do_apply_event"); /* @@ -4660,7 +4938,7 @@ int Format_description_log_event::do_apply_event(Relay_log_info const *rli) "or ROLLBACK in relay log). A probable cause is that " "the master died while writing the transaction to " "its binary log, thus rolled back too."); - const_cast<Relay_log_info*>(rli)->cleanup_context(thd, 1); + rgi->cleanup_context(thd, 1); } /* @@ -4668,7 +4946,7 @@ int Format_description_log_event::do_apply_event(Relay_log_info const *rli) perform, we don't call Start_log_event_v3::do_apply_event() (this was just to update the log's description event). */ - if (server_id != (uint32) ::server_id) + if (server_id != (uint32) global_system_variables.server_id) { /* If the event was not requested by the slave i.e. the master sent @@ -4679,7 +4957,7 @@ int Format_description_log_event::do_apply_event(Relay_log_info const *rli) 0, then 96, then jump to first really asked event (which is >96). So this is ok. */ - ret= Start_log_event_v3::do_apply_event(rli); + ret= Start_log_event_v3::do_apply_event(rgi); } if (!ret) @@ -4692,9 +4970,9 @@ int Format_description_log_event::do_apply_event(Relay_log_info const *rli) DBUG_RETURN(ret); } -int Format_description_log_event::do_update_pos(Relay_log_info *rli) +int Format_description_log_event::do_update_pos(rpl_group_info *rgi) { - if (server_id == (uint32) ::server_id) + if (server_id == (uint32) global_system_variables.server_id) { /* We only increase the relay log position if we are skipping @@ -4709,17 +4987,17 @@ int Format_description_log_event::do_update_pos(Relay_log_info *rli) Intvar_log_event instead of starting at a Table_map_log_event or the Intvar_log_event respectively. */ - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); return 0; } else { - return Log_event::do_update_pos(rli); + return Log_event::do_update_pos(rgi); } } Log_event::enum_skip_reason -Format_description_log_event::do_shall_skip(Relay_log_info *rli) +Format_description_log_event::do_shall_skip(rpl_group_info *rgi) { return Log_event::EVENT_SKIP_NOT; } @@ -4818,9 +5096,9 @@ uint8 get_checksum_alg(const char* buf, ulong len) DBUG_ENTER("get_checksum_alg"); DBUG_ASSERT(buf[EVENT_TYPE_OFFSET] == FORMAT_DESCRIPTION_EVENT); - memcpy(version, buf + - buf[LOG_EVENT_MINIMAL_HEADER_LEN + ST_COMMON_HEADER_LEN_OFFSET] - + ST_SERVER_VER_OFFSET, ST_SERVER_VER_LEN); + memcpy(version, + buf + LOG_EVENT_MINIMAL_HEADER_LEN + ST_SERVER_VER_OFFSET, + ST_SERVER_VER_LEN); version[ST_SERVER_VER_LEN - 1]= 0; do_server_version_split(version, &version_split); @@ -5351,10 +5629,12 @@ void Load_log_event::set_fields(const char* affected_db, 1 Failure */ -int Load_log_event::do_apply_event(NET* net, Relay_log_info const *rli, +int Load_log_event::do_apply_event(NET* net, rpl_group_info *rgi, bool use_rli_only_for_errors) { LEX_STRING new_db; + Relay_log_info const *rli= rgi->rli; + Rpl_filter *rpl_filter= rli->mi->rpl_filter; DBUG_ENTER("Load_log_event::do_apply_event"); new_db.length= db_len; @@ -5366,7 +5646,7 @@ int Load_log_event::do_apply_event(NET* net, Relay_log_info const *rli, clear_all_errors(thd, const_cast<Relay_log_info*>(rli)); /* see Query_log_event::do_apply_event() and BUG#13360 */ - DBUG_ASSERT(!rli->m_table_map.count()); + DBUG_ASSERT(!rgi->m_table_map.count()); /* Usually lex_start() is called by mysql_parse(), but we need it here as the present method does not call mysql_parse(). @@ -5375,16 +5655,6 @@ int Load_log_event::do_apply_event(NET* net, Relay_log_info const *rli, thd->lex->local_file= local_fname; mysql_reset_thd_for_next_command(thd); - if (!use_rli_only_for_errors) - { - /* - Saved for InnoDB, see comment in - Query_log_event::do_apply_event() - */ - const_cast<Relay_log_info*>(rli)->future_group_master_log_pos= log_pos; - DBUG_PRINT("info", ("log_pos: %lu", (ulong) log_pos)); - } - /* We test replicate_*_db rules. Note that we have already prepared the file to load, even if we are going to ignore and delete it @@ -5624,7 +5894,7 @@ Error '%s' running LOAD DATA INFILE on table '%s'. Default database: '%s'", DBUG_RETURN(1); } - DBUG_RETURN( use_rli_only_for_errors ? 0 : Log_event::do_apply_event(rli) ); + DBUG_RETURN( use_rli_only_for_errors ? 0 : Log_event::do_apply_event(rgi) ); } #endif @@ -5709,16 +5979,14 @@ Rotate_log_event::Rotate_log_event(const char* buf, uint event_len, { DBUG_ENTER("Rotate_log_event::Rotate_log_event(char*,...)"); // The caller will ensure that event_len is what we have at EVENT_LEN_OFFSET - uint8 header_size= description_event->common_header_len; uint8 post_header_len= description_event->post_header_len[ROTATE_EVENT-1]; uint ident_offset; - if (event_len < header_size) + if (event_len < LOG_EVENT_MINIMAL_HEADER_LEN) DBUG_VOID_RETURN; - buf += header_size; - pos = post_header_len ? uint8korr(buf + R_POS_OFFSET) : 4; - ident_len = (uint)(event_len - - (header_size+post_header_len)); - ident_offset = post_header_len; + buf+= LOG_EVENT_MINIMAL_HEADER_LEN; + pos= post_header_len ? uint8korr(buf + R_POS_OFFSET) : 4; + ident_len= (uint)(event_len - (LOG_EVENT_MINIMAL_HEADER_LEN + post_header_len)); + ident_offset= post_header_len; set_if_smaller(ident_len,FN_REFLEN-1); new_log_ident= my_strndup(buf + ident_offset, (uint) ident_len, MYF(MY_WME)); DBUG_PRINT("debug", ("new_log_ident: '%s'", new_log_ident)); @@ -5759,15 +6027,16 @@ bool Rotate_log_event::write(IO_CACHE* file) @retval 0 ok */ -int Rotate_log_event::do_update_pos(Relay_log_info *rli) +int Rotate_log_event::do_update_pos(rpl_group_info *rgi) { + Relay_log_info *rli= rgi->rli; DBUG_ENTER("Rotate_log_event::do_update_pos"); #ifndef DBUG_OFF char buf[32]; #endif DBUG_PRINT("info", ("server_id=%lu; ::server_id=%lu", - (ulong) this->server_id, (ulong) ::server_id)); + (ulong) this->server_id, (ulong) global_system_variables.server_id)); DBUG_PRINT("info", ("new_log_ident: %s", this->new_log_ident)); DBUG_PRINT("info", ("pos: %s", llstr(this->pos, buf))); @@ -5786,10 +6055,16 @@ int Rotate_log_event::do_update_pos(Relay_log_info *rli) correspond to the beginning of the transaction. Starting from 5.0.0, there also are some rotates from the slave itself, in the relay log, which shall not change the group positions. + + In parallel replication, rotate event is executed out-of-band with normal + events, so we cannot update group_master_log_name or _pos here, it will + be updated with the next normal event instead. */ - if ((server_id != ::server_id || rli->replicate_same_server_id) && + if ((server_id != global_system_variables.server_id || + rli->replicate_same_server_id) && !is_relay_log_event() && - !rli->is_in_group()) + !rli->is_in_group() && + !rgi->is_parallel_exec) { mysql_mutex_lock(&rli->data_lock); DBUG_PRINT("info", ("old group_master_log_name: '%s' " @@ -5798,17 +6073,18 @@ int Rotate_log_event::do_update_pos(Relay_log_info *rli) (ulong) rli->group_master_log_pos)); memcpy(rli->group_master_log_name, new_log_ident, ident_len+1); rli->notify_group_master_log_name_update(); - rli->inc_group_relay_log_pos(pos, TRUE /* skip_lock */); + rli->inc_group_relay_log_pos(pos, rgi, TRUE /* skip_lock */); DBUG_PRINT("info", ("new group_master_log_name: '%s' " "new group_master_log_pos: %lu", rli->group_master_log_name, (ulong) rli->group_master_log_pos)); mysql_mutex_unlock(&rli->data_lock); + rpl_global_gtid_slave_state.record_and_update_gtid(thd, rgi); flush_relay_log_info(rli); /* - Reset thd->variables.option_bits and sql_mode etc, because this could be the signal of - a master's downgrade from 5.0 to 4.0. + Reset thd->variables.option_bits and sql_mode etc, because this could + be the signal of a master's downgrade from 5.0 to 4.0. However, no need to reset description_event_for_exec: indeed, if the next master is 5.0 (even 5.0.1) we will soon get a Format_desc; if the next master is 4.0 then the events are in the slave's format (conversion). @@ -5820,7 +6096,7 @@ int Rotate_log_event::do_update_pos(Relay_log_info *rli) thd->variables.auto_increment_offset= 1; } else - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); DBUG_RETURN(0); @@ -5828,9 +6104,9 @@ int Rotate_log_event::do_update_pos(Relay_log_info *rli) Log_event::enum_skip_reason -Rotate_log_event::do_shall_skip(Relay_log_info *rli) +Rotate_log_event::do_shall_skip(rpl_group_info *rgi) { - enum_skip_reason reason= Log_event::do_shall_skip(rli); + enum_skip_reason reason= Log_event::do_shall_skip(rgi); switch (reason) { case Log_event::EVENT_SKIP_NOT: @@ -5848,6 +6124,693 @@ Rotate_log_event::do_shall_skip(Relay_log_info *rli) /************************************************************************** + Binlog_checkpoint_log_event methods +**************************************************************************/ + +#if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) +void Binlog_checkpoint_log_event::pack_info(THD *thd, Protocol *protocol) +{ + protocol->store(binlog_file_name, binlog_file_len, &my_charset_bin); +} + + +Log_event::enum_skip_reason +Binlog_checkpoint_log_event::do_shall_skip(rpl_group_info *rgi) +{ + enum_skip_reason reason= Log_event::do_shall_skip(rgi); + if (reason == EVENT_SKIP_COUNT) + reason= EVENT_SKIP_NOT; + return reason; +} +#endif + + +#ifdef MYSQL_CLIENT +void Binlog_checkpoint_log_event::print(FILE *file, + PRINT_EVENT_INFO *print_event_info) +{ + Write_on_release_cache cache(&print_event_info->head_cache, file, + Write_on_release_cache::FLUSH_F); + + if (print_event_info->short_form) + return; + print_header(&cache, print_event_info, FALSE); + my_b_printf(&cache, "\tBinlog checkpoint "); + my_b_write(&cache, (uchar*)binlog_file_name, binlog_file_len); + my_b_printf(&cache, "\n"); +} +#endif /* MYSQL_CLIENT */ + + +#ifdef MYSQL_SERVER +Binlog_checkpoint_log_event::Binlog_checkpoint_log_event( + const char *binlog_file_name_arg, + uint binlog_file_len_arg) + :Log_event(), + binlog_file_name(my_strndup(binlog_file_name_arg, binlog_file_len_arg, + MYF(MY_WME))), + binlog_file_len(binlog_file_len_arg) +{ + cache_type= EVENT_NO_CACHE; +} +#endif /* MYSQL_SERVER */ + + +Binlog_checkpoint_log_event::Binlog_checkpoint_log_event( + const char *buf, uint event_len, + const Format_description_log_event *description_event) + :Log_event(buf, description_event), binlog_file_name(0) +{ + uint8 header_size= description_event->common_header_len; + uint8 post_header_len= + description_event->post_header_len[BINLOG_CHECKPOINT_EVENT-1]; + if (event_len < header_size + post_header_len || + post_header_len < BINLOG_CHECKPOINT_HEADER_LEN) + return; + buf+= header_size; + /* See uint4korr and int4store below */ + compile_time_assert(BINLOG_CHECKPOINT_HEADER_LEN == 4); + binlog_file_len= uint4korr(buf); + if (event_len - (header_size + post_header_len) < binlog_file_len) + return; + binlog_file_name= my_strndup(buf + post_header_len, binlog_file_len, + MYF(MY_WME)); + return; +} + + +#ifndef MYSQL_CLIENT +bool Binlog_checkpoint_log_event::write(IO_CACHE *file) +{ + uchar buf[BINLOG_CHECKPOINT_HEADER_LEN]; + int4store(buf, binlog_file_len); + return write_header(file, BINLOG_CHECKPOINT_HEADER_LEN + binlog_file_len) || + wrapper_my_b_safe_write(file, buf, BINLOG_CHECKPOINT_HEADER_LEN) || + wrapper_my_b_safe_write(file, (const uchar *)binlog_file_name, + binlog_file_len) || + write_footer(file); +} +#endif /* MYSQL_CLIENT */ + + +/************************************************************************** + Global transaction ID stuff +**************************************************************************/ + +Gtid_log_event::Gtid_log_event(const char *buf, uint event_len, + const Format_description_log_event *description_event) + : Log_event(buf, description_event), seq_no(0), commit_id(0) +{ + uint8 header_size= description_event->common_header_len; + uint8 post_header_len= description_event->post_header_len[GTID_EVENT-1]; + if (event_len < header_size + post_header_len || + post_header_len < GTID_HEADER_LEN) + return; + + buf+= header_size; + seq_no= uint8korr(buf); + buf+= 8; + domain_id= uint4korr(buf); + buf+= 4; + flags2= *buf; + if (flags2 & FL_GROUP_COMMIT_ID) + { + if (event_len < (uint)header_size + GTID_HEADER_LEN + 2) + { + seq_no= 0; // So is_valid() returns false + return; + } + ++buf; + commit_id= uint8korr(buf); + } +} + + +#ifdef MYSQL_SERVER + +Gtid_log_event::Gtid_log_event(THD *thd_arg, uint64 seq_no_arg, + uint32 domain_id_arg, bool standalone, + uint16 flags_arg, bool is_transactional, + uint64 commit_id_arg) + : Log_event(thd_arg, flags_arg, is_transactional), + seq_no(seq_no_arg), commit_id(commit_id_arg), domain_id(domain_id_arg), + flags2((standalone ? FL_STANDALONE : 0) | (commit_id_arg ? FL_GROUP_COMMIT_ID : 0)) +{ + cache_type= Log_event::EVENT_NO_CACHE; +} + + +/* + Used to record GTID while sending binlog to slave, without having to + fully contruct every Gtid_log_event() needlessly. +*/ +bool +Gtid_log_event::peek(const char *event_start, size_t event_len, + uint8 checksum_alg, + uint32 *domain_id, uint32 *server_id, uint64 *seq_no, + uchar *flags2, const Format_description_log_event *fdev) +{ + const char *p; + + if (checksum_alg == BINLOG_CHECKSUM_ALG_CRC32) + { + if (event_len > BINLOG_CHECKSUM_LEN) + event_len-= BINLOG_CHECKSUM_LEN; + else + event_len= 0; + } + else + DBUG_ASSERT(checksum_alg == BINLOG_CHECKSUM_ALG_UNDEF || + checksum_alg == BINLOG_CHECKSUM_ALG_OFF); + + if (event_len < (uint32)fdev->common_header_len + GTID_HEADER_LEN) + return true; + *server_id= uint4korr(event_start + SERVER_ID_OFFSET); + p= event_start + fdev->common_header_len; + *seq_no= uint8korr(p); + p+= 8; + *domain_id= uint4korr(p); + p+= 4; + *flags2= (uchar)*p; + return false; +} + + +bool +Gtid_log_event::write(IO_CACHE *file) +{ + uchar buf[GTID_HEADER_LEN+2]; + size_t write_len; + + int8store(buf, seq_no); + int4store(buf+8, domain_id); + buf[12]= flags2; + if (flags2 & FL_GROUP_COMMIT_ID) + { + int8store(buf+13, commit_id); + write_len= GTID_HEADER_LEN + 2; + } + else + { + bzero(buf+13, GTID_HEADER_LEN-13); + write_len= GTID_HEADER_LEN; + } + return write_header(file, write_len) || + wrapper_my_b_safe_write(file, buf, write_len) || + write_footer(file); +} + + +/* + Replace a GTID event with either a BEGIN event, dummy event, or nothing, as + appropriate to work with old slave that does not know global transaction id. + + The need_dummy_event argument is an IN/OUT argument. It is passed as TRUE + if slave has capability lower than MARIA_SLAVE_CAPABILITY_TOLERATE_HOLES. + It is returned TRUE if we return a BEGIN (or dummy) event to be sent to the + slave, FALSE if event should be skipped completely. +*/ +int +Gtid_log_event::make_compatible_event(String *packet, bool *need_dummy_event, + ulong ev_offset, uint8 checksum_alg) +{ + uchar flags2; + if (packet->length() - ev_offset < LOG_EVENT_HEADER_LEN + GTID_HEADER_LEN) + return 1; + flags2= (*packet)[ev_offset + LOG_EVENT_HEADER_LEN + 12]; + if (flags2 & FL_STANDALONE) + { + if (*need_dummy_event) + return Query_log_event::dummy_event(packet, ev_offset, checksum_alg); + else + return 0; + } + + *need_dummy_event= true; + return Query_log_event::begin_event(packet, ev_offset, checksum_alg); +} + + +#ifdef HAVE_REPLICATION +void +Gtid_log_event::pack_info(THD *thd, Protocol *protocol) +{ + char buf[6+5+10+1+10+1+20+1+4+20+1]; + char *p; + p = strmov(buf, (flags2 & FL_STANDALONE ? "GTID " : "BEGIN GTID ")); + p= longlong10_to_str(domain_id, p, 10); + *p++= '-'; + p= longlong10_to_str(server_id, p, 10); + *p++= '-'; + p= longlong10_to_str(seq_no, p, 10); + if (flags2 & FL_GROUP_COMMIT_ID) + { + p= strmov(p, " cid="); + p= longlong10_to_str(commit_id, p, 10); + } + + protocol->store(buf, p-buf, &my_charset_bin); +} + +static char gtid_begin_string[] = "BEGIN"; + +int +Gtid_log_event::do_apply_event(rpl_group_info *rgi) +{ + thd->variables.server_id= this->server_id; + thd->variables.gtid_domain_id= this->domain_id; + thd->variables.gtid_seq_no= this->seq_no; + + if (opt_gtid_strict_mode && opt_bin_log && opt_log_slave_updates) + { + /* Need to reset prior "ok" status to give an error. */ + thd->clear_error(); + thd->stmt_da->reset_diagnostics_area(); + if (mysql_bin_log.check_strict_gtid_sequence(this->domain_id, + this->server_id, this->seq_no)) + return 1; + } + if (flags2 & FL_STANDALONE) + return 0; + + /* Execute this like a BEGIN query event. */ + thd->set_query_and_id(gtid_begin_string, sizeof(gtid_begin_string)-1, + &my_charset_bin, next_query_id()); + Parser_state parser_state; + if (!parser_state.init(thd, thd->query(), thd->query_length())) + { + mysql_parse(thd, thd->query(), thd->query_length(), &parser_state); + /* Finalize server status flags after executing a statement. */ + thd->update_server_status(); + log_slow_statement(thd); + if (unlikely(thd->is_fatal_error)) + thd->is_slave_error= 1; + else if (likely(!thd->is_slave_error)) + general_log_write(thd, COM_QUERY, thd->query(), thd->query_length()); + } + + thd->reset_query(); + free_root(thd->mem_root,MYF(MY_KEEP_PREALLOC)); + return thd->is_slave_error; +} + + +int +Gtid_log_event::do_update_pos(rpl_group_info *rgi) +{ + rgi->inc_event_relay_log_pos(); + return 0; +} + + +Log_event::enum_skip_reason +Gtid_log_event::do_shall_skip(rpl_group_info *rgi) +{ + Relay_log_info *rli= rgi->rli; + /* + An event skipped due to @@skip_replication must not be counted towards the + number of events to be skipped due to @@sql_slave_skip_counter. + */ + if (flags & LOG_EVENT_SKIP_REPLICATION_F && + opt_replicate_events_marked_for_skip != RPL_SKIP_REPLICATE) + return Log_event::EVENT_SKIP_IGNORE; + + if (rli->slave_skip_counter > 0) + { + if (!(flags2 & FL_STANDALONE)) + { + thd->variables.option_bits|= OPTION_BEGIN; + DBUG_ASSERT(rgi->rli->get_flag(Relay_log_info::IN_TRANSACTION)); + } + return Log_event::continue_group(rgi); + } + return Log_event::do_shall_skip(rgi); +} + + +#endif /* HAVE_REPLICATION */ + +#else /* !MYSQL_SERVER */ + +void +Gtid_log_event::print(FILE *file, PRINT_EVENT_INFO *print_event_info) +{ + Write_on_release_cache cache(&print_event_info->head_cache, file, + Write_on_release_cache::FLUSH_F); + char buf[21]; + char buf2[21]; + + if (!print_event_info->short_form) + { + print_header(&cache, print_event_info, FALSE); + longlong10_to_str(seq_no, buf, 10); + if (flags2 & FL_GROUP_COMMIT_ID) + { + longlong10_to_str(commit_id, buf2, 10); + my_b_printf(&cache, "\tGTID %u-%u-%s cid=%s\n", + domain_id, server_id, buf, buf2); + } + else + my_b_printf(&cache, "\tGTID %u-%u-%s\n", domain_id, server_id, buf); + + if (!print_event_info->domain_id_printed || + print_event_info->domain_id != domain_id) + { + my_b_printf(&cache, "/*!100001 SET @@session.gtid_domain_id=%u*/%s\n", + domain_id, print_event_info->delimiter); + print_event_info->domain_id= domain_id; + print_event_info->domain_id_printed= true; + } + + if (!print_event_info->server_id_printed || + print_event_info->server_id != server_id) + { + my_b_printf(&cache, "/*!100001 SET @@session.server_id=%u*/%s\n", + server_id, print_event_info->delimiter); + print_event_info->server_id= server_id; + print_event_info->server_id_printed= true; + } + + my_b_printf(&cache, "/*!100001 SET @@session.gtid_seq_no=%s*/%s\n", + buf, print_event_info->delimiter); + } + if (!(flags2 & FL_STANDALONE)) + my_b_printf(&cache, "BEGIN\n%s\n", print_event_info->delimiter); +} + +#endif /* MYSQL_SERVER */ + + +/* GTID list. */ + +Gtid_list_log_event::Gtid_list_log_event(const char *buf, uint event_len, + const Format_description_log_event *description_event) + : Log_event(buf, description_event), count(0), list(0), sub_id_list(0) +{ + uint32 i; + uint32 val; + uint8 header_size= description_event->common_header_len; + uint8 post_header_len= description_event->post_header_len[GTID_LIST_EVENT-1]; + if (event_len < header_size + post_header_len || + post_header_len < GTID_LIST_HEADER_LEN) + return; + + buf+= header_size; + val= uint4korr(buf); + count= val & ((1<<28)-1); + gl_flags= val & ((uint32)0xf << 28); + buf+= 4; + if (event_len - (header_size + post_header_len) < count*element_size || + (!(list= (rpl_gtid *)my_malloc(count*sizeof(*list) + (count == 0), + MYF(MY_WME))))) + return; + + for (i= 0; i < count; ++i) + { + list[i].domain_id= uint4korr(buf); + buf+= 4; + list[i].server_id= uint4korr(buf); + buf+= 4; + list[i].seq_no= uint8korr(buf); + buf+= 8; + } + +#if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) + if ((gl_flags & FLAG_IGN_GTIDS)) + { + uint32 i; + if (!(sub_id_list= (uint64 *)my_malloc(count*sizeof(uint64), MYF(MY_WME)))) + { + my_free(list); + list= NULL; + return; + } + for (i= 0; i < count; ++i) + { + if (!(sub_id_list[i]= + rpl_global_gtid_slave_state.next_sub_id(list[i].domain_id))) + { + my_free(list); + my_free(sub_id_list); + list= NULL; + sub_id_list= NULL; + return; + } + } + } +#endif +} + + +#ifdef MYSQL_SERVER + +Gtid_list_log_event::Gtid_list_log_event(rpl_binlog_state *gtid_set, + uint32 gl_flags_) + : count(gtid_set->count()), gl_flags(gl_flags_), list(0), sub_id_list(0) +{ + cache_type= EVENT_NO_CACHE; + /* Failure to allocate memory will be caught by is_valid() returning false. */ + if (count < (1<<28) && + (list = (rpl_gtid *)my_malloc(count * sizeof(*list) + (count == 0), + MYF(MY_WME)))) + gtid_set->get_gtid_list(list, count); +} + + +Gtid_list_log_event::Gtid_list_log_event(slave_connection_state *gtid_set, + uint32 gl_flags_) + : count(gtid_set->count()), gl_flags(gl_flags_), list(0), sub_id_list(0) +{ + cache_type= EVENT_NO_CACHE; + /* Failure to allocate memory will be caught by is_valid() returning false. */ + if (count < (1<<28) && + (list = (rpl_gtid *)my_malloc(count * sizeof(*list) + (count == 0), + MYF(MY_WME)))) + { + gtid_set->get_gtid_list(list, count); +#if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) + if (gl_flags & FLAG_IGN_GTIDS) + { + uint32 i; + + if (!(sub_id_list= (uint64 *)my_malloc(count * sizeof(uint64), + MYF(MY_WME)))) + { + my_free(list); + list= NULL; + return; + } + for (i= 0; i < count; ++i) + { + if (!(sub_id_list[i]= + rpl_global_gtid_slave_state.next_sub_id(list[i].domain_id))) + { + my_free(list); + my_free(sub_id_list); + list= NULL; + sub_id_list= NULL; + return; + } + } + } +#endif + } +} + + +#if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) +bool +Gtid_list_log_event::to_packet(String *packet) +{ + uint32 i; + uchar *p; + uint32 needed_length; + + DBUG_ASSERT(count < 1<<28); + + needed_length= packet->length() + get_data_size(); + if (packet->reserve(needed_length)) + return true; + p= (uchar *)packet->ptr() + packet->length();; + packet->length(needed_length); + int4store(p, (count & ((1<<28)-1)) | gl_flags); + p += 4; + /* Initialise the padding for empty Gtid_list. */ + if (count == 0) + int2store(p, 0); + for (i= 0; i < count; ++i) + { + int4store(p, list[i].domain_id); + int4store(p+4, list[i].server_id); + int8store(p+8, list[i].seq_no); + p += 16; + } + + return false; +} + + +bool +Gtid_list_log_event::write(IO_CACHE *file) +{ + char buf[128]; + String packet(buf, sizeof(buf), system_charset_info); + + packet.length(0); + if (to_packet(&packet)) + return true; + return + write_header(file, get_data_size()) || + wrapper_my_b_safe_write(file, (uchar *)packet.ptr(), packet.length()) || + write_footer(file); +} + + +int +Gtid_list_log_event::do_apply_event(rpl_group_info *rgi) +{ + Relay_log_info const *rli= rgi->rli; + int ret; + if (gl_flags & FLAG_IGN_GTIDS) + { + uint32 i; + for (i= 0; i < count; ++i) + { + if ((ret= rpl_global_gtid_slave_state.record_gtid(thd, &list[i], + sub_id_list[i], + false, false))) + return ret; + rpl_global_gtid_slave_state.update_state_hash(sub_id_list[i], &list[i]); + } + } + ret= Log_event::do_apply_event(rgi); + if (rli->until_condition == Relay_log_info::UNTIL_GTID && + (gl_flags & FLAG_UNTIL_REACHED)) + { + char str_buf[128]; + String str(str_buf, sizeof(str_buf), system_charset_info); + const_cast<Relay_log_info*>(rli)->until_gtid_pos.to_string(&str); + sql_print_information("Slave SQL thread stops because it reached its" + " UNTIL master_gtid_pos %s", str.c_ptr_safe()); + const_cast<Relay_log_info*>(rli)->abort_slave= true; + } + return ret; +} + + +Log_event::enum_skip_reason +Gtid_list_log_event::do_shall_skip(rpl_group_info *rgi) +{ + enum_skip_reason reason= Log_event::do_shall_skip(rgi); + if (reason == EVENT_SKIP_COUNT) + reason= EVENT_SKIP_NOT; + return reason; +} + + +void +Gtid_list_log_event::pack_info(THD *thd, Protocol *protocol) +{ + char buf_mem[1024]; + String buf(buf_mem, sizeof(buf_mem), system_charset_info); + uint32 i; + bool first; + + buf.length(0); + buf.append(STRING_WITH_LEN("[")); + first= true; + for (i= 0; i < count; ++i) + rpl_slave_state_tostring_helper(&buf, &list[i], &first); + buf.append(STRING_WITH_LEN("]")); + + protocol->store(&buf); +} +#endif /* HAVE_REPLICATION */ + +#else /* !MYSQL_SERVER */ + +void +Gtid_list_log_event::print(FILE *file, PRINT_EVENT_INFO *print_event_info) +{ + if (!print_event_info->short_form) + { + Write_on_release_cache cache(&print_event_info->head_cache, file, + Write_on_release_cache::FLUSH_F); + char buf[21]; + uint32 i; + + print_header(&cache, print_event_info, FALSE); + my_b_printf(&cache, "\tGtid list ["); + for (i= 0; i < count; ++i) + { + longlong10_to_str(list[i].seq_no, buf, 10); + my_b_printf(&cache, "%u-%u-%s", list[i].domain_id, + list[i].server_id, buf); + if (i < count-1) + my_b_printf(&cache, ",\n# "); + } + my_b_printf(&cache, "]\n"); + } +} + +#endif /* MYSQL_SERVER */ + + +/* + Used to record gtid_list event while sending binlog to slave, without having to + fully contruct the event object. +*/ +bool +Gtid_list_log_event::peek(const char *event_start, uint32 event_len, + uint8 checksum_alg, + rpl_gtid **out_gtid_list, uint32 *out_list_len, + const Format_description_log_event *fdev) +{ + const char *p; + uint32 count_field, count; + rpl_gtid *gtid_list; + + if (checksum_alg == BINLOG_CHECKSUM_ALG_CRC32) + { + if (event_len > BINLOG_CHECKSUM_LEN) + event_len-= BINLOG_CHECKSUM_LEN; + else + event_len= 0; + } + else + DBUG_ASSERT(checksum_alg == BINLOG_CHECKSUM_ALG_UNDEF || + checksum_alg == BINLOG_CHECKSUM_ALG_OFF); + + if (event_len < (uint32)fdev->common_header_len + GTID_LIST_HEADER_LEN) + return true; + p= event_start + fdev->common_header_len; + count_field= uint4korr(p); + p+= 4; + count= count_field & ((1<<28)-1); + if (event_len < (uint32)fdev->common_header_len + GTID_LIST_HEADER_LEN + + 16 * count) + return true; + if (!(gtid_list= (rpl_gtid *)my_malloc(sizeof(rpl_gtid)*count + (count == 0), + MYF(MY_WME)))) + return true; + *out_gtid_list= gtid_list; + *out_list_len= count; + while (count--) + { + gtid_list->domain_id= uint4korr(p); + p+= 4; + gtid_list->server_id= uint4korr(p); + p+= 4; + gtid_list->seq_no= uint8korr(p); + p+= 8; + ++gtid_list; + } + + return false; +} + + +/************************************************************************** Intvar_log_event methods **************************************************************************/ @@ -5958,19 +6921,13 @@ void Intvar_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) Intvar_log_event::do_apply_event() */ -int Intvar_log_event::do_apply_event(Relay_log_info const *rli) +int Intvar_log_event::do_apply_event(rpl_group_info *rgi) { DBUG_ENTER("Intvar_log_event::do_apply_event"); - /* - We are now in a statement until the associated query log event has - been processed. - */ - const_cast<Relay_log_info*>(rli)->set_flag(Relay_log_info::IN_STMT); - - if (rli->deferred_events_collecting) + if (rgi->deferred_events_collecting) { DBUG_PRINT("info",("deferring event")); - DBUG_RETURN(rli->deferred_events->add(this)); + DBUG_RETURN(rgi->deferred_events->add(this)); } switch (type) { @@ -5985,15 +6942,15 @@ int Intvar_log_event::do_apply_event(Relay_log_info const *rli) DBUG_RETURN(0); } -int Intvar_log_event::do_update_pos(Relay_log_info *rli) +int Intvar_log_event::do_update_pos(rpl_group_info *rgi) { - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); return 0; } Log_event::enum_skip_reason -Intvar_log_event::do_shall_skip(Relay_log_info *rli) +Intvar_log_event::do_shall_skip(rpl_group_info *rgi) { /* It is a common error to set the slave skip counter to 1 instead of @@ -6003,7 +6960,7 @@ Intvar_log_event::do_shall_skip(Relay_log_info *rli) that we do not change the value of the slave skip counter since it will be decreased by the following insert event. */ - return continue_group(rli); + return continue_group(rgi); } #endif @@ -6071,31 +7028,25 @@ void Rand_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) -int Rand_log_event::do_apply_event(Relay_log_info const *rli) +int Rand_log_event::do_apply_event(rpl_group_info *rgi) { - /* - We are now in a statement until the associated query log event has - been processed. - */ - const_cast<Relay_log_info*>(rli)->set_flag(Relay_log_info::IN_STMT); - - if (rli->deferred_events_collecting) - return rli->deferred_events->add(this); + if (rgi->deferred_events_collecting) + return rgi->deferred_events->add(this); thd->rand.seed1= (ulong) seed1; thd->rand.seed2= (ulong) seed2; return 0; } -int Rand_log_event::do_update_pos(Relay_log_info *rli) +int Rand_log_event::do_update_pos(rpl_group_info *rgi) { - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); return 0; } Log_event::enum_skip_reason -Rand_log_event::do_shall_skip(Relay_log_info *rli) +Rand_log_event::do_shall_skip(rpl_group_info *rgi) { /* It is a common error to set the slave skip counter to 1 instead of @@ -6105,7 +7056,7 @@ Rand_log_event::do_shall_skip(Relay_log_info *rli) that we do not change the value of the slave skip counter since it will be decreased by the following insert event. */ - return continue_group(rli); + return continue_group(rgi); } /** @@ -6119,14 +7070,14 @@ Rand_log_event::do_shall_skip(Relay_log_info *rli) bool slave_execute_deferred_events(THD *thd) { bool res= false; - Relay_log_info *rli= thd->rli_slave; + rpl_group_info *rgi= thd->rgi_slave; - DBUG_ASSERT(rli && (!rli->deferred_events_collecting || rli->deferred_events)); + DBUG_ASSERT(rgi && (!rgi->deferred_events_collecting || rgi->deferred_events)); - if (!rli->deferred_events_collecting || rli->deferred_events->is_empty()) + if (!rgi->deferred_events_collecting || rgi->deferred_events->is_empty()) return res; - res= rli->deferred_events->execute(rli); + res= rgi->deferred_events->execute(rgi); return res; } @@ -6201,15 +7152,53 @@ void Xid_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) -int Xid_log_event::do_apply_event(Relay_log_info const *rli) +int Xid_log_event::do_apply_event(rpl_group_info *rgi) { bool res; + int err; + rpl_gtid gtid; + uint64 sub_id; + Relay_log_info const *rli= rgi->rli; + + /* + Record any GTID in the same transaction, so slave state is transactionally + consistent. + */ + if ((sub_id= rgi->gtid_sub_id)) + { + /* Clear the GTID from the RLI so we don't accidentally reuse it. */ + rgi->gtid_sub_id= 0; + + gtid= rgi->current_gtid; + err= rpl_global_gtid_slave_state.record_gtid(thd, >id, sub_id, true, false); + if (err) + { + rli->report(ERROR_LEVEL, ER_CANNOT_UPDATE_GTID_STATE, + "Error during XID COMMIT: failed to update GTID state in " + "%s.%s: %d: %s", + "mysql", rpl_gtid_slave_state_table_name.str, + thd->stmt_da->sql_errno(), thd->stmt_da->message()); + trans_rollback(thd); + thd->is_slave_error= 1; + return err; + } + + DBUG_EXECUTE_IF("gtid_fail_after_record_gtid", + { my_error(ER_ERROR_DURING_COMMIT, MYF(0), HA_ERR_WRONG_COMMAND); + thd->is_slave_error= 1; + return 1; + }); + } + /* For a slave Xid_log_event is COMMIT */ general_log_print(thd, COM_QUERY, "COMMIT /* implicit, from Xid_log_event */"); res= trans_commit(thd); /* Automatically rolls back on error. */ thd->mdl_context.release_transactional_locks(); + if (!res && sub_id) + rpl_global_gtid_slave_state.update_state_hash(sub_id, >id); + /* Increment the global status commit count variable */ @@ -6219,14 +7208,16 @@ int Xid_log_event::do_apply_event(Relay_log_info const *rli) } Log_event::enum_skip_reason -Xid_log_event::do_shall_skip(Relay_log_info *rli) +Xid_log_event::do_shall_skip(rpl_group_info *rgi) { DBUG_ENTER("Xid_log_event::do_shall_skip"); - if (rli->slave_skip_counter > 0) { + if (rgi->rli->slave_skip_counter > 0) + { + DBUG_ASSERT(!rgi->rli->get_flag(Relay_log_info::IN_TRANSACTION)); thd->variables.option_bits&= ~OPTION_BEGIN; DBUG_RETURN(Log_event::EVENT_SKIP_COUNT); } - DBUG_RETURN(Log_event::do_shall_skip(rli)); + DBUG_RETURN(Log_event::do_shall_skip(rgi)); } #endif /* !MYSQL_CLIENT */ @@ -6635,17 +7626,17 @@ void User_var_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) */ #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) -int User_var_log_event::do_apply_event(Relay_log_info const *rli) +int User_var_log_event::do_apply_event(rpl_group_info *rgi) { Item *it= 0; CHARSET_INFO *charset; DBUG_ENTER("User_var_log_event::do_apply_event"); query_id_t sav_query_id= 0; /* memorize orig id when deferred applying */ - if (rli->deferred_events_collecting) + if (rgi->deferred_events_collecting) { set_deferred(current_thd->query_id); - DBUG_RETURN(rli->deferred_events->add(this)); + DBUG_RETURN(rgi->deferred_events->add(this)); } else if (is_deferred()) { @@ -6661,12 +7652,6 @@ int User_var_log_event::do_apply_event(Relay_log_info const *rli) double real_val; longlong int_val; - /* - We are now in a statement until the associated query log event has - been processed. - */ - const_cast<Relay_log_info*>(rli)->set_flag(Relay_log_info::IN_STMT); - if (is_null) { it= new Item_null(); @@ -6731,14 +7716,14 @@ int User_var_log_event::do_apply_event(Relay_log_info const *rli) DBUG_RETURN(0); } -int User_var_log_event::do_update_pos(Relay_log_info *rli) +int User_var_log_event::do_update_pos(rpl_group_info *rgi) { - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); return 0; } Log_event::enum_skip_reason -User_var_log_event::do_shall_skip(Relay_log_info *rli) +User_var_log_event::do_shall_skip(rpl_group_info *rgi) { /* It is a common error to set the slave skip counter to 1 instead @@ -6748,7 +7733,7 @@ User_var_log_event::do_shall_skip(Relay_log_info *rli) that we do not change the value of the slave skip counter since it will be decreased by the following insert event. */ - return continue_group(rli); + return continue_group(rgi); } #endif /* !MYSQL_CLIENT */ @@ -6907,7 +7892,7 @@ Slave_log_event::Slave_log_event(const char* buf, #ifndef MYSQL_CLIENT -int Slave_log_event::do_apply_event(Relay_log_info const *rli) +int Slave_log_event::do_apply_event(rpl_group_info *rgi) { if (mysql_bin_log.is_open()) return mysql_bin_log.write(this); @@ -6951,8 +7936,11 @@ void Stop_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) Start_log_event_v3::do_apply_event(), not here. Because if we come here, the master was sane. */ -int Stop_log_event::do_update_pos(Relay_log_info *rli) + +int Stop_log_event::do_update_pos(rpl_group_info *rgi) { + Relay_log_info *rli= rgi->rli; + DBUG_ENTER("Stop_log_event::do_update_pos"); /* We do not want to update master_log pos because we get a rotate event before stop, so by now group_master_log_name is set to the next log. @@ -6960,14 +7948,15 @@ int Stop_log_event::do_update_pos(Relay_log_info *rli) could give false triggers in MASTER_POS_WAIT() that we have reached the target position when in fact we have not. */ - if (thd->variables.option_bits & OPTION_BEGIN) - rli->inc_event_relay_log_pos(); - else + if (rli->get_flag(Relay_log_info::IN_TRANSACTION)) + rgi->inc_event_relay_log_pos(); + else if (!rgi->is_parallel_exec) { - rli->inc_group_relay_log_pos(0); + rpl_global_gtid_slave_state.record_and_update_gtid(thd, rgi); + rli->inc_group_relay_log_pos(0, rgi); flush_relay_log_info(rli); } - return 0; + DBUG_RETURN(0); } #endif /* !MYSQL_CLIENT */ @@ -7181,17 +8170,19 @@ void Create_file_log_event::pack_info(THD *thd, Protocol *protocol) */ #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) -int Create_file_log_event::do_apply_event(Relay_log_info const *rli) +int Create_file_log_event::do_apply_event(rpl_group_info *rgi) { char proc_info[17+FN_REFLEN+10], *fname_buf; char *ext; int fd = -1; IO_CACHE file; int error = 1; + Relay_log_info const *rli= rgi->rli; bzero((char*)&file, sizeof(file)); fname_buf= strmov(proc_info, "Making temp file "); - ext= slave_load_file_stem(fname_buf, file_id, server_id, ".info"); + ext= slave_load_file_stem(fname_buf, file_id, server_id, ".info", + &rli->mi->connection_name); thd_proc_info(thd, proc_info); /* old copy may exist already */ mysql_file_delete(key_file_log_event_info, fname_buf, MYF(0)); @@ -7361,15 +8352,17 @@ int Append_block_log_event::get_create_or_append() const Append_block_log_event::do_apply_event() */ -int Append_block_log_event::do_apply_event(Relay_log_info const *rli) +int Append_block_log_event::do_apply_event(rpl_group_info *rgi) { char proc_info[17+FN_REFLEN+10], *fname= proc_info+17; int fd; int error = 1; + Relay_log_info const *rli= rgi->rli; DBUG_ENTER("Append_block_log_event::do_apply_event"); fname= strmov(proc_info, "Making temp file "); - slave_load_file_stem(fname, file_id, server_id, ".data"); + slave_load_file_stem(fname, file_id, server_id, ".data", + &rli->mi->cmp_connection_name); thd_proc_info(thd, proc_info); if (get_create_or_append()) { @@ -7510,10 +8503,12 @@ void Delete_file_log_event::pack_info(THD *thd, Protocol *protocol) */ #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) -int Delete_file_log_event::do_apply_event(Relay_log_info const *rli) +int Delete_file_log_event::do_apply_event(rpl_group_info *rgi) { char fname[FN_REFLEN+10]; - char *ext= slave_load_file_stem(fname, file_id, server_id, ".data"); + Relay_log_info const *rli= rgi->rli; + char *ext= slave_load_file_stem(fname, file_id, server_id, ".data", + &rli->mi->cmp_connection_name); mysql_file_delete(key_file_log_event_data, fname, MYF(MY_WME)); strmov(ext, ".info"); mysql_file_delete(key_file_log_event_info, fname, MYF(MY_WME)); @@ -7608,7 +8603,7 @@ void Execute_load_log_event::pack_info(THD *thd, Protocol *protocol) Execute_load_log_event::do_apply_event() */ -int Execute_load_log_event::do_apply_event(Relay_log_info const *rli) +int Execute_load_log_event::do_apply_event(rpl_group_info *rgi) { char fname[FN_REFLEN+10]; char *ext; @@ -7616,8 +8611,10 @@ int Execute_load_log_event::do_apply_event(Relay_log_info const *rli) int error= 1; IO_CACHE file; Load_log_event *lev= 0; + Relay_log_info const *rli= rgi->rli; - ext= slave_load_file_stem(fname, file_id, server_id, ".info"); + ext= slave_load_file_stem(fname, file_id, server_id, ".info", + &rli->mi->cmp_connection_name); if ((fd= mysql_file_open(key_file_log_event_info, fname, O_RDONLY | O_BINARY | O_NOFOLLOW, MYF(MY_WME))) < 0 || @@ -7649,8 +8646,7 @@ int Execute_load_log_event::do_apply_event(Relay_log_info const *rli) calls mysql_load()). */ - const_cast<Relay_log_info*>(rli)->future_group_master_log_pos= log_pos; - if (lev->do_apply_event(0,rli,1)) + if (lev->do_apply_event(0,rgi,1)) { /* We want to indicate the name of the file that could not be loaded @@ -7731,13 +8727,13 @@ int Begin_load_query_log_event::get_create_or_append() const #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) Log_event::enum_skip_reason -Begin_load_query_log_event::do_shall_skip(Relay_log_info *rli) +Begin_load_query_log_event::do_shall_skip(rpl_group_info *rgi) { /* If the slave skip counter is 1, then we should not start executing on the next event. */ - return continue_group(rli); + return continue_group(rgi); } #endif @@ -7879,13 +8875,14 @@ void Execute_load_query_log_event::pack_info(THD *thd, Protocol *protocol) int -Execute_load_query_log_event::do_apply_event(Relay_log_info const *rli) +Execute_load_query_log_event::do_apply_event(rpl_group_info *rgi) { char *p; char *buf; char *fname; char *fname_end; int error; + Relay_log_info const *rli= rgi->rli; buf= (char*) my_malloc(q_len + 1 - (fn_pos_end - fn_pos_start) + (FN_REFLEN + 10) + 10 + 8 + 5, MYF(MY_WME)); @@ -7904,7 +8901,8 @@ Execute_load_query_log_event::do_apply_event(Relay_log_info const *rli) memcpy(p, query, fn_pos_start); p+= fn_pos_start; fname= (p= strmake(p, STRING_WITH_LEN(" INFILE \'"))); - p= slave_load_file_stem(p, file_id, server_id, ".data"); + p= slave_load_file_stem(p, file_id, server_id, ".data", + &rli->mi->cmp_connection_name); fname_end= p= strend(p); // Safer than p=p+5 *(p++)='\''; switch (dup_handling) { @@ -7921,7 +8919,7 @@ Execute_load_query_log_event::do_apply_event(Relay_log_info const *rli) p= strmake(p, STRING_WITH_LEN(" INTO ")); p= strmake(p, query+fn_pos_end, q_len-fn_pos_end); - error= Query_log_event::do_apply_event(rli, buf, p-buf); + error= Query_log_event::do_apply_event(rgi, buf, p-buf); /* Forging file name for deletion in same buffer */ *fname_end= 0; @@ -8285,8 +9283,9 @@ int Rows_log_event::do_add_row_data(uchar *row_data, size_t length) #endif #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) -int Rows_log_event::do_apply_event(Relay_log_info const *rli) +int Rows_log_event::do_apply_event(rpl_group_info *rgi) { + Relay_log_info const *rli= rgi->rli; DBUG_ENTER("Rows_log_event::do_apply_event(Relay_log_info*)"); int error= 0; /* @@ -8303,7 +9302,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) */ DBUG_ASSERT(get_flags(STMT_END_F)); - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); thd->clear_error(); DBUG_RETURN(0); } @@ -8313,7 +9312,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) do_apply_event(). We still check here to prevent future coding errors. */ - DBUG_ASSERT(rli->sql_thd == thd); + DBUG_ASSERT(rgi->thd == thd); /* If there is no locks taken, this is the first binrow event seen @@ -8332,6 +9331,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) call might reset the value of current_stmt_binlog_format, so we need to do any changes to that value after this function. */ + delete_explain_query(thd->lex); lex_start(thd); mysql_reset_thd_for_next_command(thd); /* @@ -8365,7 +9365,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) /* A small test to verify that objects have consistent types */ DBUG_ASSERT(sizeof(thd->variables.option_bits) == sizeof(OPTION_RELAXED_UNIQUE_CHECKS)); - if (open_and_lock_tables(thd, rli->tables_to_lock, FALSE, 0)) + if (open_and_lock_tables(thd, rgi->tables_to_lock, FALSE, 0)) { uint actual_error= thd->stmt_da->sql_errno(); if (thd->is_slave_error || thd->is_fatal_error) @@ -8382,7 +9382,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) "unexpected success or fatal error")); thd->is_slave_error= 1; } - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); DBUG_RETURN(actual_error); } @@ -8396,7 +9396,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) { DBUG_PRINT("debug", ("Checking compability of tables to lock - tables_to_lock: %p", - rli->tables_to_lock)); + rgi->tables_to_lock)); /** When using RBR and MyISAM MERGE tables the base tables that make @@ -8410,8 +9410,8 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) NOTE: The base tables are added here are removed when close_thread_tables is called. */ - RPL_TABLE_LIST *ptr= rli->tables_to_lock; - for (uint i= 0 ; ptr && (i < rli->tables_to_lock_count); + RPL_TABLE_LIST *ptr= rgi->tables_to_lock; + for (uint i= 0 ; ptr && (i < rgi->tables_to_lock_count); ptr= static_cast<RPL_TABLE_LIST*>(ptr->next_global), i++) { DBUG_ASSERT(ptr->m_tabledef_valid); @@ -8427,7 +9427,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) having severe errors which should not be skiped. */ thd->is_slave_error= 1; - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); DBUG_RETURN(ERR_BAD_TABLE_DEF); } DBUG_PRINT("debug", ("Table: %s.%s is compatible with master" @@ -8452,18 +9452,18 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) Rows_log_event, we can invalidate the query cache for the associated table. */ - TABLE_LIST *ptr= rli->tables_to_lock; - for (uint i=0 ; ptr && (i < rli->tables_to_lock_count); ptr= ptr->next_global, i++) - const_cast<Relay_log_info*>(rli)->m_table_map.set_table(ptr->table_id, ptr->table); + TABLE_LIST *ptr= rgi->tables_to_lock; + for (uint i=0 ; ptr && (i < rgi->tables_to_lock_count); ptr= ptr->next_global, i++) + rgi->m_table_map.set_table(ptr->table_id, ptr->table); #ifdef HAVE_QUERY_CACHE - query_cache.invalidate_locked_for_write(thd, rli->tables_to_lock); + query_cache.invalidate_locked_for_write(thd, rgi->tables_to_lock); #endif } TABLE* table= - m_table= const_cast<Relay_log_info*>(rli)->m_table_map.get_table(m_table_id); + m_table= rgi->m_table_map.get_table(m_table_id); DBUG_PRINT("debug", ("m_table: 0x%lx, m_table_id: %lu", (ulong) m_table, m_table_id)); @@ -8486,17 +9486,6 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) */ thd->set_time(when, when_sec_part); - /* - Now we are in a statement and will stay in a statement until we - see a STMT_END_F. - - We set this flag here, before actually applying any rows, in - case the SQL thread is stopped and we need to detect that we're - inside a statement and halting abruptly might cause problems - when restarting. - */ - const_cast<Relay_log_info*>(rli)->set_flag(Relay_log_info::IN_STMT); - if ( m_width == table->s->fields && bitmap_is_set_all(&m_cols)) set_flags(COMPLETE_ROWS_F); @@ -8536,7 +9525,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) set the initial time of this ROWS statement if it was not done before in some other ROWS event. */ - const_cast<Relay_log_info*>(rli)->set_row_stmt_start_timestamp(); + rgi->set_row_stmt_start_timestamp(); while (error == 0 && m_curr_row < m_rows_end) { @@ -8545,7 +9534,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) if (!table->in_use) table->in_use= thd; - error= do_exec_row(rli); + error= do_exec_row(rgi); if (error) DBUG_PRINT("info", ("error: %s", HA_ERR(error))); @@ -8585,7 +9574,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) (ulong) m_curr_row, (ulong) m_curr_row_end, (ulong) m_rows_end)); if (!m_curr_row_end && !error) - error= unpack_current_row(rli); + error= unpack_current_row(rgi); // at this moment m_curr_row_end should be set DBUG_ASSERT(error || m_curr_row_end != NULL); @@ -8646,7 +9635,7 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) DBUG_RETURN(error); } - if (get_flags(STMT_END_F) && (error= rows_event_stmt_cleanup(rli, thd))) + if (get_flags(STMT_END_F) && (error= rows_event_stmt_cleanup(rgi, thd))) slave_rows_error_report(ERROR_LEVEL, thd->is_error() ? 0 : error, rli, thd, table, @@ -8656,17 +9645,17 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) } Log_event::enum_skip_reason -Rows_log_event::do_shall_skip(Relay_log_info *rli) +Rows_log_event::do_shall_skip(rpl_group_info *rgi) { /* If the slave skip counter is 1 and this event does not end a statement, then we should not start executing on the next event. Otherwise, we defer the decision to the normal skipping logic. */ - if (rli->slave_skip_counter == 1 && !get_flags(STMT_END_F)) + if (rgi->rli->slave_skip_counter == 1 && !get_flags(STMT_END_F)) return Log_event::EVENT_SKIP_IGNORE; else - return Log_event::do_shall_skip(rli); + return Log_event::do_shall_skip(rgi); } /** @@ -8680,9 +9669,11 @@ Rows_log_event::do_shall_skip(Relay_log_info *rli) @retval non-zero Error at the commit. */ -static int rows_event_stmt_cleanup(Relay_log_info const *rli, THD * thd) +static int rows_event_stmt_cleanup(rpl_group_info *rgi, THD * thd) { int error; + DBUG_ENTER("rows_event_stmt_cleanup"); + { /* This is the end of a statement or transaction, so close (and @@ -8737,9 +9728,16 @@ static int rows_event_stmt_cleanup(Relay_log_info const *rli, THD * thd) */ thd->reset_current_stmt_binlog_format_row(); - const_cast<Relay_log_info*>(rli)->cleanup_context(thd, 0); + /* + Reset modified_non_trans_table that we have set in + rows_log_event::do_apply_event() + */ + if (!thd->in_multi_stmt_transaction_mode()) + thd->transaction.all.modified_non_trans_table= 0; + + rgi->cleanup_context(thd, 0); } - return error; + DBUG_RETURN(error); } /** @@ -8753,8 +9751,9 @@ static int rows_event_stmt_cleanup(Relay_log_info const *rli, THD * thd) @retval non-zero Error in the statement commit */ int -Rows_log_event::do_update_pos(Relay_log_info *rli) +Rows_log_event::do_update_pos(rpl_group_info *rgi) { + Relay_log_info *rli= rgi->rli; DBUG_ENTER("Rows_log_event::do_update_pos"); int error= 0; @@ -8768,7 +9767,7 @@ Rows_log_event::do_update_pos(Relay_log_info *rli) Step the group log position if we are not in a transaction, otherwise increase the event log position. */ - rli->stmt_done(log_pos, when); + rli->stmt_done(log_pos, when, thd, rgi); /* Clear any errors in thd->net.last_err*. It is not known if this is needed or not. It is believed that any errors that may exist in @@ -8779,7 +9778,7 @@ Rows_log_event::do_update_pos(Relay_log_info *rli) } else { - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); } DBUG_RETURN(error); @@ -8991,7 +9990,7 @@ void Annotate_rows_log_event::print(FILE *file, PRINT_EVENT_INFO *pinfo) #endif #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) -int Annotate_rows_log_event::do_apply_event(Relay_log_info const *rli) +int Annotate_rows_log_event::do_apply_event(rpl_group_info *rgi) { m_save_thd_query_txt= thd->query(); m_save_thd_query_len= thd->query_length(); @@ -9001,18 +10000,18 @@ int Annotate_rows_log_event::do_apply_event(Relay_log_info const *rli) #endif #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) -int Annotate_rows_log_event::do_update_pos(Relay_log_info *rli) +int Annotate_rows_log_event::do_update_pos(rpl_group_info *rgi) { - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); return 0; } #endif #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) Log_event::enum_skip_reason -Annotate_rows_log_event::do_shall_skip(Relay_log_info *rli) +Annotate_rows_log_event::do_shall_skip(rpl_group_info *rgi) { - return continue_group(rli); + return continue_group(rgi); } #endif @@ -9474,19 +10473,20 @@ enum enum_tbl_map_status rli->tables_to_lock. */ static enum_tbl_map_status -check_table_map(Relay_log_info const *rli, RPL_TABLE_LIST *table_list) +check_table_map(rpl_group_info *rgi, RPL_TABLE_LIST *table_list) { DBUG_ENTER("check_table_map"); enum_tbl_map_status res= OK_TO_PROCESS; + Relay_log_info *rli= rgi->rli; - if (rli->sql_thd->slave_thread /* filtering is for slave only */ && - (!rpl_filter->db_ok(table_list->db) || - (rpl_filter->is_on() && !rpl_filter->tables_ok("", table_list)))) + if (rgi->thd->slave_thread /* filtering is for slave only */ && + (!rli->mi->rpl_filter->db_ok(table_list->db) || + (rli->mi->rpl_filter->is_on() && !rli->mi->rpl_filter->tables_ok("", table_list)))) res= FILTERED_OUT; else { - RPL_TABLE_LIST *ptr= static_cast<RPL_TABLE_LIST*>(rli->tables_to_lock); - for(uint i=0 ; ptr && (i< rli->tables_to_lock_count); + RPL_TABLE_LIST *ptr= static_cast<RPL_TABLE_LIST*>(rgi->tables_to_lock); + for(uint i=0 ; ptr && (i< rgi->tables_to_lock_count); ptr= static_cast<RPL_TABLE_LIST*>(ptr->next_local), i++) { if (ptr->table_id == table_list->table_id) @@ -9509,14 +10509,15 @@ check_table_map(Relay_log_info const *rli, RPL_TABLE_LIST *table_list) DBUG_RETURN(res); } -int Table_map_log_event::do_apply_event(Relay_log_info const *rli) +int Table_map_log_event::do_apply_event(rpl_group_info *rgi) { RPL_TABLE_LIST *table_list; char *db_mem, *tname_mem; size_t dummy_len; void *memory; + Rpl_filter *filter; + Relay_log_info const *rli= rgi->rli; DBUG_ENTER("Table_map_log_event::do_apply_event(Relay_log_info*)"); - DBUG_ASSERT(rli->sql_thd == thd); /* Step the query id to mark what columns that are actually used. */ thd->set_query_id(next_query_id()); @@ -9528,7 +10529,9 @@ int Table_map_log_event::do_apply_event(Relay_log_info const *rli) NullS))) DBUG_RETURN(HA_ERR_OUT_OF_MEM); - strmov(db_mem, rpl_filter->get_rewrite_db(m_dbnam, &dummy_len)); + /* call from mysql_client_binlog_statement() will not set rli->mi */ + filter= rgi->thd->slave_thread ? rli->mi->rpl_filter : global_rpl_filter; + strmov(db_mem, filter->get_rewrite_db(m_dbnam, &dummy_len)); strmov(tname_mem, m_tblnam); table_list->init_one_table(db_mem, strlen(db_mem), @@ -9539,7 +10542,7 @@ int Table_map_log_event::do_apply_event(Relay_log_info const *rli) table_list->updating= 1; table_list->required_type= FRMTYPE_TABLE; DBUG_PRINT("debug", ("table: %s is mapped to %u", table_list->table_name, table_list->table_id)); - enum_tbl_map_status tblmap_status= check_table_map(rli, table_list); + enum_tbl_map_status tblmap_status= check_table_map(rgi, table_list); if (tblmap_status == OK_TO_PROCESS) { DBUG_ASSERT(thd->lex->query_tables != table_list); @@ -9565,9 +10568,9 @@ int Table_map_log_event::do_apply_event(Relay_log_info const *rli) We record in the slave's information that the table should be locked by linking the table into the list of tables to lock. */ - table_list->next_global= table_list->next_local= rli->tables_to_lock; - const_cast<Relay_log_info*>(rli)->tables_to_lock= table_list; - const_cast<Relay_log_info*>(rli)->tables_to_lock_count++; + table_list->next_global= table_list->next_local= rgi->tables_to_lock; + rgi->tables_to_lock= table_list; + rgi->tables_to_lock_count++; /* 'memory' is freed in clear_tables_to_lock */ } else // FILTERED_OUT, SAME_ID_MAPPING_* @@ -9615,18 +10618,18 @@ int Table_map_log_event::do_apply_event(Relay_log_info const *rli) } Log_event::enum_skip_reason -Table_map_log_event::do_shall_skip(Relay_log_info *rli) +Table_map_log_event::do_shall_skip(rpl_group_info *rgi) { /* If the slave skip counter is 1, then we should not start executing on the next event. */ - return continue_group(rli); + return continue_group(rgi); } -int Table_map_log_event::do_update_pos(Relay_log_info *rli) +int Table_map_log_event::do_update_pos(rpl_group_info *rgi) { - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); return 0; } @@ -9802,23 +10805,6 @@ Write_rows_log_event::do_before_row_operations(const Slave_reporting_capability */ } - /* - We need TIMESTAMP_NO_AUTO_SET otherwise ha_write_row() will not use fill - any TIMESTAMP column with data from the row but instead will use - the event's current time. - As we replicate from TIMESTAMP to TIMESTAMP and slave has no extra - columns, we know that all TIMESTAMP columns on slave will receive explicit - data from the row, so TIMESTAMP_NO_AUTO_SET is ok. - When we allow a table without TIMESTAMP to be replicated to a table having - more columns including a TIMESTAMP column, or when we allow a TIMESTAMP - column to be replicated into a BIGINT column and the slave's table has a - TIMESTAMP column, then the slave's TIMESTAMP column will take its value - from set_time() which we called earlier (consistent with SBR). And then in - some cases we won't want TIMESTAMP_NO_AUTO_SET (will require some code to - analyze if explicit data is provided for slave's TIMESTAMP columns). - */ - m_table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - /* Honor next number column if present */ m_table->next_number_field= m_table->found_next_number_field; /* @@ -9936,7 +10922,7 @@ is_duplicate_key_error(int errcode) */ int -Rows_log_event::write_row(const Relay_log_info *const rli, +Rows_log_event::write_row(rpl_group_info *rgi, const bool overwrite) { DBUG_ENTER("write_row"); @@ -9951,7 +10937,7 @@ Rows_log_event::write_row(const Relay_log_info *const rli, table->file->ht->db_type != DB_TYPE_NDBCLUSTER); /* unpack row into table->record[0] */ - if ((error= unpack_current_row(rli))) + if ((error= unpack_current_row(rgi))) DBUG_RETURN(error); if (m_curr_row == m_rows_buf) @@ -10068,7 +11054,7 @@ Rows_log_event::write_row(const Relay_log_info *const rli, if (!get_flags(COMPLETE_ROWS_F)) { restore_record(table,record[1]); - error= unpack_current_row(rli); + error= unpack_current_row(rgi); } #ifndef DBUG_OFF @@ -10134,10 +11120,10 @@ Rows_log_event::write_row(const Relay_log_info *const rli, #endif int -Write_rows_log_event::do_exec_row(const Relay_log_info *const rli) +Write_rows_log_event::do_exec_row(rpl_group_info *rgi) { DBUG_ASSERT(m_table != NULL); - int error= write_row(rli, slave_exec_mode == SLAVE_EXEC_MODE_IDEMPOTENT); + int error= write_row(rgi, slave_exec_mode == SLAVE_EXEC_MODE_IDEMPOTENT); if (error && !thd->is_error()) { @@ -10382,13 +11368,13 @@ static inline void issue_long_find_row_warning(Log_event_type type, const char *table_name, bool is_index_scan, - const Relay_log_info *rli) + rpl_group_info *rgi) { if ((global_system_variables.log_warnings > 1 && - !const_cast<Relay_log_info*>(rli)->is_long_find_row_note_printed())) + !rgi->is_long_find_row_note_printed())) { time_t now= my_time(0); - time_t stmt_ts= const_cast<Relay_log_info*>(rli)->get_row_stmt_start_timestamp(); + time_t stmt_ts= rgi->get_row_stmt_start_timestamp(); DBUG_EXECUTE_IF("inject_long_find_row_note", stmt_ts-=(LONG_FIND_ROW_THRESHOLD*2);); @@ -10397,7 +11383,7 @@ void issue_long_find_row_warning(Log_event_type type, if (delta > LONG_FIND_ROW_THRESHOLD) { - const_cast<Relay_log_info*>(rli)->set_long_find_row_note_printed(); + rgi->set_long_find_row_note_printed(); const char* evt_type= type == DELETE_ROWS_EVENT ? " DELETE" : "n UPDATE"; const char* scan_type= is_index_scan ? "scanning an index" : "scanning the table"; @@ -10443,7 +11429,7 @@ void issue_long_find_row_warning(Log_event_type type, for any following update/delete command. */ -int Rows_log_event::find_row(const Relay_log_info *rli) +int Rows_log_event::find_row(rpl_group_info *rgi) { DBUG_ENTER("Rows_log_event::find_row"); @@ -10461,7 +11447,7 @@ int Rows_log_event::find_row(const Relay_log_info *rli) */ prepare_record(table, m_width, FALSE); - error= unpack_current_row(rli); + error= unpack_current_row(rgi); #ifndef DBUG_OFF DBUG_PRINT("info",("looking for the following record")); @@ -10726,7 +11712,7 @@ int Rows_log_event::find_row(const Relay_log_info *rli) end: if (is_table_scan || is_index_scan) issue_long_find_row_warning(get_type_code(), m_table->alias.c_ptr(), - is_index_scan, rli); + is_index_scan, rgi); table->default_column_bitmaps(); DBUG_RETURN(error); } @@ -10794,12 +11780,12 @@ Delete_rows_log_event::do_after_row_operations(const Slave_reporting_capability return error; } -int Delete_rows_log_event::do_exec_row(const Relay_log_info *const rli) +int Delete_rows_log_event::do_exec_row(rpl_group_info *rgi) { int error; DBUG_ASSERT(m_table != NULL); - if (!(error= find_row(rli))) + if (!(error= find_row(rgi))) { /* Delete the record found, located in record[0] @@ -10903,8 +11889,6 @@ Update_rows_log_event::do_before_row_operations(const Slave_reporting_capability if ((err= find_key())) return err; - m_table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - return 0; } @@ -10922,11 +11906,11 @@ Update_rows_log_event::do_after_row_operations(const Slave_reporting_capability } int -Update_rows_log_event::do_exec_row(const Relay_log_info *const rli) +Update_rows_log_event::do_exec_row(rpl_group_info *rgi) { DBUG_ASSERT(m_table != NULL); - int error= find_row(rli); + int error= find_row(rgi); if (error) { /* @@ -10934,7 +11918,7 @@ Update_rows_log_event::do_exec_row(const Relay_log_info *const rli) able to skip to the next pair of updates */ m_curr_row= m_curr_row_end; - unpack_current_row(rli); + unpack_current_row(rgi); return error; } @@ -10953,7 +11937,7 @@ Update_rows_log_event::do_exec_row(const Relay_log_info *const rli) m_curr_row= m_curr_row_end; /* this also updates m_curr_row_end */ - if ((error= unpack_current_row(rli))) + if ((error= unpack_current_row(rgi))) goto err; /* @@ -11076,8 +12060,9 @@ Incident_log_event::print(FILE *file, #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) int -Incident_log_event::do_apply_event(Relay_log_info const *rli) +Incident_log_event::do_apply_event(rpl_group_info *rgi) { + Relay_log_info const *rli= rgi->rli; DBUG_ENTER("Incident_log_event::do_apply_event"); rli->report(ERROR_LEVEL, ER_SLAVE_INCIDENT, ER(ER_SLAVE_INCIDENT), @@ -11128,7 +12113,9 @@ st_print_event_info::st_print_event_info() auto_increment_increment(0),auto_increment_offset(0), charset_inited(0), lc_time_names_number(~0), charset_database_number(ILLEGAL_CHARSET_INFO_NUMBER), - thread_id(0), thread_id_printed(false), skip_replication(0), + thread_id(0), thread_id_printed(false), server_id(0), + server_id_printed(false), domain_id(0), domain_id_printed(false), + skip_replication(0), base64_output_mode(BASE64_OUTPUT_UNSPEC), printed_fd_event(FALSE) { /* @@ -11166,6 +12153,9 @@ Heartbeat_log_event::Heartbeat_log_event(const char* buf, uint event_len, There is a dummy replacement for this in the embedded library that returns FALSE; this is used by XtraDB to allow it to access replication stuff while still being able to use the same plugin in both stand-alone and embedded. + + In this function it's ok to use active_mi, as this is only called for + the main replication server. */ bool rpl_get_position_info(const char **log_file_name, ulonglong *log_pos, const char **group_relay_log_name, @@ -11175,11 +12165,21 @@ bool rpl_get_position_info(const char **log_file_name, ulonglong *log_pos, return FALSE; #else const Relay_log_info *rli= &(active_mi->rli); - *log_file_name= rli->group_master_log_name; - *log_pos= rli->group_master_log_pos + - (rli->future_event_relay_log_pos - rli->group_relay_log_pos); - *group_relay_log_name= rli->group_relay_log_name; - *relay_log_pos= rli->future_event_relay_log_pos; + if (opt_slave_parallel_threads == 0) + { + *log_file_name= rli->group_master_log_name; + *log_pos= rli->group_master_log_pos + + (rli->future_event_relay_log_pos - rli->group_relay_log_pos); + *group_relay_log_name= rli->group_relay_log_name; + *relay_log_pos= rli->future_event_relay_log_pos; + } + else + { + *log_file_name= ""; + *log_pos= 0; + *group_relay_log_name= ""; + *relay_log_pos= 0; + } return TRUE; #endif } diff --git a/sql/log_event.h b/sql/log_event.h index 1d90111f1ca..4dc33f7a747 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -1,4 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. + Copyright (c) 2009, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -48,6 +49,8 @@ #include "sql_class.h" /* THD */ #endif +#include "rpl_gtid.h" + /* Forward declarations */ class String; @@ -258,6 +261,9 @@ struct sql_ex_info #define INCIDENT_HEADER_LEN 2 #define HEARTBEAT_HEADER_LEN 0 #define ANNOTATE_ROWS_HEADER_LEN 0 +#define BINLOG_CHECKPOINT_HEADER_LEN 4 +#define GTID_HEADER_LEN 19 +#define GTID_LIST_HEADER_LEN 4 /* Max number of possible extra bytes in a replication event compared to a @@ -549,7 +555,7 @@ struct sql_ex_info /* Shouldn't be defined before */ #define EXPECTED_OPTIONS \ - ((ULL(1) << 14) | (ULL(1) << 26) | (ULL(1) << 27) | (ULL(1) << 19)) + ((1ULL << 14) | (1ULL << 26) | (1ULL << 27) | (1ULL << 19)) #if OPTIONS_WRITTEN_TO_BIN_LOG != EXPECTED_OPTIONS #error OPTIONS_WRITTEN_TO_BIN_LOG must NOT change their values! @@ -572,6 +578,40 @@ enum enum_binlog_checksum_alg { #define BINLOG_CHECKSUM_LEN CHECKSUM_CRC32_SIGNATURE_LEN #define BINLOG_CHECKSUM_ALG_DESC_LEN 1 /* 1 byte checksum alg descriptor */ +/* + These are capability numbers for MariaDB slave servers. + + Newer MariaDB slaves set this to inform the master about their capabilities. + This allows the master to decide which events it can send to the slave + without breaking replication on old slaves that maybe do not understand + all events from newer masters. + + As new releases are backwards compatible, a given capability implies also + all capabilities with smaller number. + + Older MariaDB slaves and other MySQL slave servers do not set this, so they + are recorded with capability 0. +*/ + +/* MySQL or old MariaDB slave with no announced capability. */ +#define MARIA_SLAVE_CAPABILITY_UNKNOWN 0 +/* MariaDB >= 5.3, which understands ANNOTATE_ROWS_EVENT. */ +#define MARIA_SLAVE_CAPABILITY_ANNOTATE 1 +/* + MariaDB >= 5.5. This version has the capability to tolerate events omitted + from the binlog stream without breaking replication (MySQL slaves fail + because they mis-compute the offsets into the master's binlog). +*/ +#define MARIA_SLAVE_CAPABILITY_TOLERATE_HOLES 2 +/* MariaDB >= 10.0, which knows about binlog_checkpoint_log_event. */ +#define MARIA_SLAVE_CAPABILITY_BINLOG_CHECKPOINT 3 +/* MariaDB >= 10.0.1, which knows about global transaction id events. */ +#define MARIA_SLAVE_CAPABILITY_GTID 4 + +/* Our capability. */ +#define MARIA_SLAVE_CAPABILITY_MINE MARIA_SLAVE_CAPABILITY_GTID + + /** @enum Log_event_type @@ -647,6 +687,26 @@ enum Log_event_type MARIA_EVENTS_BEGIN= 160, /* New Maria event numbers start from here */ ANNOTATE_ROWS_EVENT= 160, + /* + Binlog checkpoint event. Used for XA crash recovery on the master, not used + in replication. + A binlog checkpoint event specifies a binlog file such that XA crash + recovery can start from that file - and it is guaranteed to find all XIDs + that are prepared in storage engines but not yet committed. + */ + BINLOG_CHECKPOINT_EVENT= 161, + /* + Gtid event. For global transaction ID, used to start a new event group, + instead of the old BEGIN query event, and also to mark stand-alone + events. + */ + GTID_EVENT= 162, + /* + Gtid list event. Logged at the start of every binlog, to record the + current replication state. This consists of the last GTID seen for + each replication domain. + */ + GTID_LIST_EVENT= 163, /* Add new MariaDB events here - right above this comment! */ @@ -719,6 +779,11 @@ typedef struct st_print_event_info uint charset_database_number; uint thread_id; bool thread_id_printed; + uint32 server_id; + bool server_id_printed; + uint32 domain_id; + bool domain_id_printed; + /* Track when @@skip_replication changes so we need to output a SET statement for it. @@ -1187,6 +1252,7 @@ public: #endif virtual Log_event_type get_type_code() = 0; virtual bool is_valid() const = 0; + virtual my_off_t get_header_len(my_off_t len) { return len; } void set_artificial_event() { flags |= LOG_EVENT_ARTIFICIAL_F; } void set_relay_log_event() { flags |= LOG_EVENT_RELAY_LOG_F; } bool is_artificial_event() const { return flags & LOG_EVENT_ARTIFICIAL_F; } @@ -1251,9 +1317,9 @@ public: @see do_apply_event */ - int apply_event(Relay_log_info const *rli) + int apply_event(rpl_group_info *rgi) { - return do_apply_event(rli); + return do_apply_event(rgi); } @@ -1265,9 +1331,9 @@ public: @see do_update_pos */ - int update_pos(Relay_log_info *rli) + int update_pos(rpl_group_info *rgi) { - return do_update_pos(rli); + return do_update_pos(rgi); } /** @@ -1276,11 +1342,66 @@ public: @see do_shall_skip */ - enum_skip_reason shall_skip(Relay_log_info *rli) + enum_skip_reason shall_skip(rpl_group_info *rgi) { - return do_shall_skip(rli); + return do_shall_skip(rgi); } + + /* + Check if an event is non-final part of a stand-alone event group, + such as Intvar_log_event (such events should be processed as part + of the following event group, not individually). + See also is_part_of_group() + */ + static bool is_part_of_group(enum Log_event_type ev_type) + { + switch (ev_type) + { + case GTID_EVENT: + case INTVAR_EVENT: + case RAND_EVENT: + case USER_VAR_EVENT: + case TABLE_MAP_EVENT: + case ANNOTATE_ROWS_EVENT: + return true; + case DELETE_ROWS_EVENT: + case UPDATE_ROWS_EVENT: + case WRITE_ROWS_EVENT: + /* + ToDo: also check for non-final Rows_log_event (though such events + are usually in a BEGIN-COMMIT group). + */ + default: + return false; + } + } + /* + Same as above, but works on the object. In addition this is true for all + rows event except the last one. + */ + virtual bool is_part_of_group() { return 0; } + + static bool is_group_event(enum Log_event_type ev_type) + { + switch (ev_type) + { + case START_EVENT_V3: + case STOP_EVENT: + case ROTATE_EVENT: + case SLAVE_EVENT: + case FORMAT_DESCRIPTION_EVENT: + case INCIDENT_EVENT: + case HEARTBEAT_LOG_EVENT: + case BINLOG_CHECKPOINT_EVENT: + case GTID_LIST_EVENT: + return false; + + default: + return true; + } + } + protected: /** @@ -1293,14 +1414,14 @@ protected: A typical usage is: @code - enum_skip_reason do_shall_skip(Relay_log_info *rli) { - return continue_group(rli); + enum_skip_reason do_shall_skip(rpl_group_info *rgi) { + return continue_group(rgi); } @endcode @return Skip reason */ - enum_skip_reason continue_group(Relay_log_info *rli); + enum_skip_reason continue_group(rpl_group_info *rgi); /** Primitive to apply an event to the database. @@ -1317,7 +1438,7 @@ protected: @retval 0 Event applied successfully @retval errno Error code if event application failed */ - virtual int do_apply_event(Relay_log_info const *rli) + virtual int do_apply_event(rpl_group_info *rgi) { return 0; /* Default implementation does nothing */ } @@ -1346,7 +1467,7 @@ protected: 1). Observe that handler errors are returned by the do_apply_event() function, and not by this one. */ - virtual int do_update_pos(Relay_log_info *rli); + virtual int do_update_pos(rpl_group_info *rgi); /** @@ -1378,7 +1499,7 @@ protected: The event shall be skipped because the slave skip counter was non-zero. The caller shall decrease the counter by one. */ - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); #endif }; @@ -1853,6 +1974,8 @@ public: my_free(data_buf); } Log_event_type get_type_code() { return QUERY_EVENT; } + static int dummy_event(String *packet, ulong ev_offset, uint8 checksum_alg); + static int begin_event(String *packet, ulong ev_offset, uint8 checksum_alg); #ifdef MYSQL_SERVER bool write(IO_CACHE* file); virtual bool write_post_header_for_derived(IO_CACHE* file) { return FALSE; } @@ -1868,13 +1991,15 @@ public: public: /* !!! Public in this patch to allow old usage */ #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); - virtual int do_apply_event(Relay_log_info const *rli); - virtual int do_update_pos(Relay_log_info *rli); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); - int do_apply_event(Relay_log_info const *rli, + int do_apply_event(rpl_group_info *rgi, const char *query_arg, uint32 q_len_arg); + static bool peek_is_commit_rollback(const char *event_start, + size_t event_len, uint8 checksum_alg); #endif /* HAVE_REPLICATION */ /* If true, the event always be applied by slave SQL thread or be printed by @@ -1898,6 +2023,9 @@ public: /* !!! Public in this patch to allow old usage */ !strncasecmp(query, "SAVEPOINT", 9) || !strncasecmp(query, "ROLLBACK", 8); } + bool is_begin() { return !strcmp(query, "BEGIN"); } + bool is_commit() { return !strcmp(query, "COMMIT"); } + bool is_rollback() { return !strcmp(query, "ROLLBACK"); } }; @@ -1984,7 +2112,7 @@ public: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const* rli); + virtual int do_apply_event(rpl_group_info *rgi); #endif }; @@ -2297,12 +2425,12 @@ public: public: /* !!! Public in this patch to allow old usage */ #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const* rli) + virtual int do_apply_event(rpl_group_info *rgi) { - return do_apply_event(thd->slave_net,rli,0); + return do_apply_event(thd->slave_net,rgi,0); } - int do_apply_event(NET *net, Relay_log_info const *rli, + int do_apply_event(NET *net, rpl_group_info *rgi, bool use_rli_only_for_errors); #endif }; @@ -2370,6 +2498,8 @@ public: const Format_description_log_event* description_event); ~Start_log_event_v3() {} Log_event_type get_type_code() { return START_EVENT_V3;} + my_off_t get_header_len(my_off_t l __attribute__((unused))) + { return LOG_EVENT_MINIMAL_HEADER_LEN; } #ifdef MYSQL_SERVER bool write(IO_CACHE* file); #endif @@ -2381,14 +2511,14 @@ public: protected: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info*) + virtual int do_apply_event(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info*) { /* Events from ourself should be skipped, but they should not decrease the slave skip counter. */ - if (this->server_id == ::server_id) + if (this->server_id == global_system_variables.server_id) return Log_event::EVENT_SKIP_IGNORE; else return Log_event::EVENT_SKIP_NOT; @@ -2477,9 +2607,9 @@ public: static bool is_version_before_checksum(const master_version_split *version_split); protected: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); - virtual int do_update_pos(Relay_log_info *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); #endif }; @@ -2553,12 +2683,13 @@ Intvar_log_event(THD* thd_arg,uchar type_arg, ulonglong val_arg, bool write(IO_CACHE* file); #endif bool is_valid() const { return 1; } + bool is_part_of_group() { return 1; } private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); - virtual int do_update_pos(Relay_log_info *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); #endif }; @@ -2632,12 +2763,13 @@ class Rand_log_event: public Log_event bool write(IO_CACHE* file); #endif bool is_valid() const { return 1; } + bool is_part_of_group() { return 1; } private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); - virtual int do_update_pos(Relay_log_info *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); #endif }; @@ -2684,8 +2816,8 @@ class Xid_log_event: public Log_event private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); - enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual int do_apply_event(rpl_group_info *rgi); + enum_skip_reason do_shall_skip(rpl_group_info *rgi); #endif }; @@ -2753,12 +2885,13 @@ public: void set_deferred(query_id_t qid) { deferred= true; query_id= qid; } #endif bool is_valid() const { return name != 0; } + bool is_part_of_group() { return 1; } private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); - virtual int do_update_pos(Relay_log_info *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); #endif }; @@ -2791,14 +2924,14 @@ public: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_update_pos(Relay_log_info *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli) + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi) { /* Events from ourself should be skipped, but they should not decrease the slave skip counter. */ - if (this->server_id == ::server_id) + if (this->server_id == global_system_variables.server_id) return Log_event::EVENT_SKIP_IGNORE; else return Log_event::EVENT_SKIP_NOT; @@ -2885,6 +3018,8 @@ public: my_free((void*) new_log_ident); } Log_event_type get_type_code() { return ROTATE_EVENT;} + my_off_t get_header_len(my_off_t l __attribute__((unused))) + { return LOG_EVENT_MINIMAL_HEADER_LEN; } int get_data_size() { return ident_len + ROTATE_HEADER_LEN;} bool is_valid() const { return new_log_ident != 0; } #ifdef MYSQL_SERVER @@ -2893,12 +3028,262 @@ public: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_update_pos(Relay_log_info *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); +#endif +}; + + +class Binlog_checkpoint_log_event: public Log_event +{ +public: + char *binlog_file_name; + uint binlog_file_len; + +#ifdef MYSQL_SERVER + Binlog_checkpoint_log_event(const char *binlog_file_name_arg, + uint binlog_file_len_arg); +#ifdef HAVE_REPLICATION + void pack_info(THD *thd, Protocol *protocol); +#endif +#else + void print(FILE *file, PRINT_EVENT_INFO *print_event_info); +#endif + Binlog_checkpoint_log_event(const char *buf, uint event_len, + const Format_description_log_event *description_event); + ~Binlog_checkpoint_log_event() { my_free(binlog_file_name); } + Log_event_type get_type_code() { return BINLOG_CHECKPOINT_EVENT;} + int get_data_size() { return binlog_file_len + BINLOG_CHECKPOINT_HEADER_LEN;} + bool is_valid() const { return binlog_file_name != 0; } +#ifdef MYSQL_SERVER + bool write(IO_CACHE* file); + enum_skip_reason do_shall_skip(rpl_group_info *rgi); #endif }; +/** + @class Gtid_log_event + + This event is logged as part of every event group to give the global + transaction id (GTID) of that group. + + It replaces the BEGIN query event used in earlier versions to begin most + event groups, but is also used for events that used to be stand-alone. + + @section Gtid_log_event_binary_format Binary Format + + The binary format for Gtid_log_event has 6 extra reserved bytes to make the + length a total of 19 byte (+ 19 bytes of header in common with all events). + This is just the minimal size for a BEGIN query event, which makes it easy + to replace this event with such BEGIN event to remain compatible with old + slave servers. + + <table> + <caption>Post-Header</caption> + + <tr> + <th>Name</th> + <th>Format</th> + <th>Description</th> + </tr> + + <tr> + <td>seq_no</td> + <td>8 byte unsigned integer</td> + <td>increasing id within one server_id. Starts at 1, holes in the sequence + may occur</td> + </tr> + + <tr> + <td>domain_id</td> + <td>4 byte unsigned integer</td> + <td>Replication domain id, identifying independent replication streams></td> + </tr> + + <tr> + <td>flags</td> + <td>1 byte bitfield</td> + <td>Bit 0 set indicates stand-alone event (no terminating COMMIT)</td> + </tr> + + <tr> + <td>Reserved</td> + <td>6 bytes</td> + <td>Reserved bytes, set to 0. Maybe be used for future expansion.</td> + </tr> + </table> + + The Body of Gtid_log_event is empty. The total event size is 19 bytes + + the normal 19 bytes common-header. +*/ + +class Gtid_log_event: public Log_event +{ +public: + uint64 seq_no; + uint64 commit_id; + uint32 domain_id; + uchar flags2; + + /* Flags2. */ + + /* FL_STANDALONE is set when there is no terminating COMMIT event. */ + static const uchar FL_STANDALONE= 1; + /* + FL_GROUP_COMMIT_ID is set when event group is part of a group commit on the + master. Groups with same commit_id are part of the same group commit. + */ + static const uchar FL_GROUP_COMMIT_ID= 2; + +#ifdef MYSQL_SERVER + Gtid_log_event(THD *thd_arg, uint64 seq_no, uint32 domain_id, bool standalone, + uint16 flags, bool is_transactional, uint64 commit_id); +#ifdef HAVE_REPLICATION + void pack_info(THD *thd, Protocol *protocol); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); +#endif +#else + void print(FILE *file, PRINT_EVENT_INFO *print_event_info); +#endif + Gtid_log_event(const char *buf, uint event_len, + const Format_description_log_event *description_event); + ~Gtid_log_event() { } + Log_event_type get_type_code() { return GTID_EVENT; } + int get_data_size() + { + return GTID_HEADER_LEN + ((flags2 & FL_GROUP_COMMIT_ID) ? 2 : 0); + } + bool is_valid() const { return seq_no != 0; } +#ifdef MYSQL_SERVER + bool write(IO_CACHE *file); + static int make_compatible_event(String *packet, bool *need_dummy_event, + ulong ev_offset, uint8 checksum_alg); + static bool peek(const char *event_start, size_t event_len, + uint8 checksum_alg, + uint32 *domain_id, uint32 *server_id, uint64 *seq_no, + uchar *flags2, const Format_description_log_event *fdev); +#endif +}; + + +/** + @class Gtid_list_log_event + + This event is logged at the start of every binlog file to record the + current replication state: the last global transaction id (GTID) applied + on the server within each replication domain. + + It consists of a list of GTIDs, one for each replication domain ever seen + on the server. + + @section Gtid_list_log_event_binary_format Binary Format + + <table> + <caption>Post-Header</caption> + + <tr> + <th>Name</th> + <th>Format</th> + <th>Description</th> + </tr> + + <tr> + <td>count</td> + <td>4 byte unsigned integer</td> + <td>The lower 28 bits are the number of GTIDs. The upper 4 bits are + flags bits.</td> + </tr> + </table> + + <table> + <caption>Body</caption> + + <tr> + <th>Name</th> + <th>Format</th> + <th>Description</th> + </tr> + + <tr> + <td>domain_id</td> + <td>4 byte unsigned integer</td> + <td>Replication domain id of one GTID</td> + </tr> + + <tr> + <td>server_id</td> + <td>4 byte unsigned integer</td> + <td>Server id of one GTID</td> + </tr> + + <tr> + <td>seq_no</td> + <td>8 byte unsigned integer</td> + <td>sequence number of one GTID</td> + </tr> + </table> + + The three elements in the body repeat COUNT times to form the GTID list. + + At the time of writing, only one flag bit is in use. + + Bit 28 of `count' is used for flag FLAG_UNTIL_REACHED, which is sent in a + Gtid_list event from the master to the slave to indicate that the START + SLAVE UNTIL master_gtid_pos=xxx condition has been reached. (This flag is + only sent in "fake" events generated on the fly, it is not written into + the binlog). +*/ + +class Gtid_list_log_event: public Log_event +{ +public: + uint32 count; + uint32 gl_flags; + struct rpl_gtid *list; + uint64 *sub_id_list; + + static const uint element_size= 4+4+8; + static const uint32 FLAG_UNTIL_REACHED= (1<<28); + static const uint32 FLAG_IGN_GTIDS= (1<<29); + +#ifdef MYSQL_SERVER + Gtid_list_log_event(rpl_binlog_state *gtid_set, uint32 gl_flags); + Gtid_list_log_event(slave_connection_state *gtid_set, uint32 gl_flags); +#ifdef HAVE_REPLICATION + void pack_info(THD *thd, Protocol *protocol); +#endif +#else + void print(FILE *file, PRINT_EVENT_INFO *print_event_info); +#endif + Gtid_list_log_event(const char *buf, uint event_len, + const Format_description_log_event *description_event); + ~Gtid_list_log_event() { my_free(list); my_free(sub_id_list); } + Log_event_type get_type_code() { return GTID_LIST_EVENT; } + int get_data_size() { + /* + Replacing with dummy event, needed for older slaves, requires a minimum + of 6 bytes in the body. + */ + return (count==0 ? + GTID_LIST_HEADER_LEN+2 : GTID_LIST_HEADER_LEN+count*element_size); + } + bool is_valid() const { return list != NULL; } +#if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) + bool to_packet(String *packet); + bool write(IO_CACHE *file); + virtual int do_apply_event(rpl_group_info *rgi); + enum_skip_reason do_shall_skip(rpl_group_info *rgi); +#endif + static bool peek(const char *event_start, uint32 event_len, + uint8 checksum_alg, + rpl_gtid **out_gtid_list, uint32 *out_list_len, + const Format_description_log_event *fdev); +}; + + /* the classes below are for the new LOAD DATA INFILE logging */ /** @@ -2970,7 +3355,7 @@ public: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); + virtual int do_apply_event(rpl_group_info *rgi); #endif }; @@ -3025,7 +3410,7 @@ public: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); + virtual int do_apply_event(rpl_group_info *rgi); #endif }; @@ -3066,7 +3451,7 @@ public: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); + virtual int do_apply_event(rpl_group_info *rgi); #endif }; @@ -3106,7 +3491,7 @@ public: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); + virtual int do_apply_event(rpl_group_info *rgi); #endif }; @@ -3139,7 +3524,7 @@ public: Log_event_type get_type_code() { return BEGIN_LOAD_QUERY_EVENT; } private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); #endif }; @@ -3205,7 +3590,7 @@ public: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); + virtual int do_apply_event(rpl_group_info *rgi); #endif }; @@ -3261,6 +3646,7 @@ public: virtual int get_data_size(); virtual Log_event_type get_type_code(); virtual bool is_valid() const; + virtual bool is_part_of_group() { return 1; } #ifndef MYSQL_CLIENT virtual bool write_data_header(IO_CACHE*); @@ -3277,9 +3663,9 @@ public: #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) private: - virtual int do_apply_event(Relay_log_info const*); - virtual int do_update_pos(Relay_log_info*); - virtual enum_skip_reason do_shall_skip(Relay_log_info*); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info*); #endif private: @@ -3672,6 +4058,7 @@ public: virtual Log_event_type get_type_code() { return TABLE_MAP_EVENT; } virtual bool is_valid() const { return m_memory != NULL; /* we check malloc */ } + virtual bool is_part_of_group() { return 1; } virtual int get_data_size() { return (uint) m_data_size; } #ifdef MYSQL_SERVER @@ -3692,9 +4079,9 @@ public: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); - virtual int do_update_pos(Relay_log_info *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); #endif #ifdef MYSQL_SERVER @@ -3837,6 +4224,7 @@ public: { return m_rows_buf && m_cols.bitmap; } + bool is_part_of_group() { return get_flags(STMT_END_F) != 0; } uint m_row_count; /* The number of rows added to the event */ @@ -3898,16 +4286,16 @@ protected: uint m_key_nr; /* Key number */ int find_key(); // Find a best key to use in find_row() - int find_row(const Relay_log_info *const); - int write_row(const Relay_log_info *const, const bool); + int find_row(rpl_group_info *); + int write_row(rpl_group_info *, const bool); // Unpack the current row into m_table->record[0] - int unpack_current_row(const Relay_log_info *const rli) + int unpack_current_row(rpl_group_info *rgi) { DBUG_ASSERT(m_table); ASSERT_OR_RETURN_ERROR(m_curr_row < m_rows_end, HA_ERR_CORRUPT_EVENT); - return ::unpack_row(rli, m_table, m_width, m_curr_row, &m_cols, + return ::unpack_row(rgi, m_table, m_width, m_curr_row, &m_cols, &m_curr_row_end, &m_master_reclength, m_rows_end); } #endif @@ -3915,9 +4303,9 @@ protected: private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); - virtual int do_update_pos(Relay_log_info *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); /* Primitive to prepare for a sequence of row executions. @@ -3968,7 +4356,7 @@ private: 0 if execution succeeded, 1 if execution failed. */ - virtual int do_exec_row(const Relay_log_info *const rli) = 0; + virtual int do_exec_row(rpl_group_info *rli) = 0; #endif /* defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) */ friend class Old_rows_log_event; @@ -4024,7 +4412,7 @@ private: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) virtual int do_before_row_operations(const Slave_reporting_capability *const); virtual int do_after_row_operations(const Slave_reporting_capability *const,int); - virtual int do_exec_row(const Relay_log_info *const); + virtual int do_exec_row(rpl_group_info *); #endif }; @@ -4098,7 +4486,7 @@ protected: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) virtual int do_before_row_operations(const Slave_reporting_capability *const); virtual int do_after_row_operations(const Slave_reporting_capability *const,int); - virtual int do_exec_row(const Relay_log_info *const); + virtual int do_exec_row(rpl_group_info *); #endif /* defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) */ }; @@ -4163,7 +4551,7 @@ protected: #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) virtual int do_before_row_operations(const Slave_reporting_capability *const); virtual int do_after_row_operations(const Slave_reporting_capability *const,int); - virtual int do_exec_row(const Relay_log_info *const); + virtual int do_exec_row(rpl_group_info *); #endif }; @@ -4249,7 +4637,7 @@ public: #endif #if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); + virtual int do_apply_event(rpl_group_info *rgi); #endif virtual bool write_data_header(IO_CACHE *file); @@ -4335,16 +4723,6 @@ bool event_checksum_test(uchar *buf, ulong event_len, uint8 alg); uint8 get_checksum_alg(const char* buf, ulong len); extern TYPELIB binlog_checksum_typelib; -#ifndef MYSQL_CLIENT -/** - The function is called by slave applier in case there are - active table filtering rules to force gathering events associated - with Query-log-event into an array to execute - them once the fate of the Query is determined for execution. -*/ -bool slave_execute_deferred_events(THD *thd); -#endif - /** @} (end of group Replication) */ diff --git a/sql/log_event_old.cc b/sql/log_event_old.cc index 70b3ec12356..87a31f53d87 100644 --- a/sql/log_event_old.cc +++ b/sql/log_event_old.cc @@ -36,12 +36,13 @@ // Old implementation of do_apply_event() int -Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info *rli) +Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, rpl_group_info *rgi) { DBUG_ENTER("Old_rows_log_event::do_apply_event(st_relay_log_info*)"); int error= 0; THD *ev_thd= ev->thd; uchar const *row_start= ev->m_rows_buf; + const Relay_log_info *rli= rgi->rli; /* If m_table_id == ~0UL, then we have a dummy event that does not @@ -57,7 +58,7 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info */ DBUG_ASSERT(ev->get_flags(Old_rows_log_event::STMT_END_F)); - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(ev_thd); + rgi->slave_close_thread_tables(ev_thd); ev_thd->clear_error(); DBUG_RETURN(0); } @@ -67,7 +68,7 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info do_apply_event(). We still check here to prevent future coding errors. */ - DBUG_ASSERT(rli->sql_thd == ev_thd); + DBUG_ASSERT(rgi->thd == ev_thd); /* If there is no locks taken, this is the first binrow event seen @@ -86,6 +87,7 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info call might reset the value of current_stmt_binlog_format, so we need to do any changes to that value after this function. */ + delete_explain_query(thd->lex); lex_start(ev_thd); mysql_reset_thd_for_next_command(ev_thd); @@ -97,7 +99,7 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info */ ev_thd->lex->set_stmt_row_injection(); - if (open_and_lock_tables(ev_thd, rli->tables_to_lock, FALSE, 0)) + if (open_and_lock_tables(ev_thd, rgi->tables_to_lock, FALSE, 0)) { uint actual_error= ev_thd->stmt_da->sql_errno(); if (ev_thd->is_slave_error || ev_thd->is_fatal_error) @@ -112,7 +114,7 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info "unexpected success or fatal error")); ev_thd->is_slave_error= 1; } - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); DBUG_RETURN(actual_error); } @@ -125,8 +127,8 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info */ { - RPL_TABLE_LIST *ptr= rli->tables_to_lock; - for (uint i= 0 ; ptr&& (i< rli->tables_to_lock_count); + RPL_TABLE_LIST *ptr= rgi->tables_to_lock; + for (uint i= 0 ; ptr&& (i< rgi->tables_to_lock_count); ptr= static_cast<RPL_TABLE_LIST*>(ptr->next_global), i++) { DBUG_ASSERT(ptr->m_tabledef_valid); @@ -135,7 +137,7 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info ptr->table, &conv_table)) { ev_thd->is_slave_error= 1; - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(ev_thd); + rgi->slave_close_thread_tables(ev_thd); DBUG_RETURN(Old_rows_log_event::ERR_BAD_TABLE_DEF); } DBUG_PRINT("debug", ("Table: %s.%s is compatible with master" @@ -160,15 +162,15 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info Old_rows_log_event, we can invalidate the query cache for the associated table. */ - TABLE_LIST *ptr= rli->tables_to_lock; - for (uint i=0; ptr && (i < rli->tables_to_lock_count); ptr= ptr->next_global, i++) - const_cast<Relay_log_info*>(rli)->m_table_map.set_table(ptr->table_id, ptr->table); + TABLE_LIST *ptr= rgi->tables_to_lock; + for (uint i=0; ptr && (i < rgi->tables_to_lock_count); ptr= ptr->next_global, i++) + rgi->m_table_map.set_table(ptr->table_id, ptr->table); #ifdef HAVE_QUERY_CACHE - query_cache.invalidate_locked_for_write(thd, rli->tables_to_lock); + query_cache.invalidate_locked_for_write(thd, rgi->tables_to_lock); #endif } - TABLE* table= const_cast<Relay_log_info*>(rli)->m_table_map.get_table(ev->m_table_id); + TABLE* table= rgi->m_table_map.get_table(ev->m_table_id); if (table) { @@ -204,22 +206,11 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info /* A small test to verify that objects have consistent types */ DBUG_ASSERT(sizeof(ev_thd->variables.option_bits) == sizeof(OPTION_RELAXED_UNIQUE_CHECKS)); - /* - Now we are in a statement and will stay in a statement until we - see a STMT_END_F. - - We set this flag here, before actually applying any rows, in - case the SQL thread is stopped and we need to detect that we're - inside a statement and halting abruptly might cause problems - when restarting. - */ - const_cast<Relay_log_info*>(rli)->set_flag(Relay_log_info::IN_STMT); - error= do_before_row_operations(table); while (error == 0 && row_start < ev->m_rows_end) { uchar const *row_end= NULL; - if ((error= do_prepare_row(ev_thd, rli, table, row_start, &row_end))) + if ((error= do_prepare_row(ev_thd, rgi, table, row_start, &row_end))) break; // We should perform the after-row operation even in // the case of error @@ -279,7 +270,7 @@ Old_rows_log_event::do_apply_event(Old_rows_log_event *ev, const Relay_log_info rollback at the caller along with sbr. */ ev_thd->reset_current_stmt_binlog_format_row(); - const_cast<Relay_log_info*>(rli)->cleanup_context(ev_thd, error); + rgi->cleanup_context(ev_thd, error); ev_thd->is_slave_error= 1; DBUG_RETURN(error); } @@ -927,22 +918,6 @@ int Write_rows_log_event_old::do_before_row_operations(TABLE *table) from the start. */ table->file->ha_start_bulk_insert(0); - /* - We need TIMESTAMP_NO_AUTO_SET otherwise ha_write_row() will not use fill - any TIMESTAMP column with data from the row but instead will use - the event's current time. - As we replicate from TIMESTAMP to TIMESTAMP and slave has no extra - columns, we know that all TIMESTAMP columns on slave will receive explicit - data from the row, so TIMESTAMP_NO_AUTO_SET is ok. - When we allow a table without TIMESTAMP to be replicated to a table having - more columns including a TIMESTAMP column, or when we allow a TIMESTAMP - column to be replicated into a BIGINT column and the slave's table has a - TIMESTAMP column, then the slave's TIMESTAMP column will take its value - from set_time() which we called earlier (consistent with SBR). And then in - some cases we won't want TIMESTAMP_NO_AUTO_SET (will require some code to - analyze if explicit data is provided for slave's TIMESTAMP columns). - */ - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; return error; } @@ -968,7 +943,7 @@ int Write_rows_log_event_old::do_after_row_operations(TABLE *table, int error) int Write_rows_log_event_old::do_prepare_row(THD *thd_arg, - Relay_log_info const *rli, + rpl_group_info *rgi, TABLE *table, uchar const *row_start, uchar const **row_end) @@ -977,7 +952,7 @@ Write_rows_log_event_old::do_prepare_row(THD *thd_arg, DBUG_ASSERT(row_start && row_end); int error; - error= unpack_row_old(const_cast<Relay_log_info*>(rli), + error= unpack_row_old(rgi, table, m_width, table->record[0], row_start, m_rows_end, &m_cols, row_end, &m_master_reclength, @@ -1052,7 +1027,7 @@ int Delete_rows_log_event_old::do_after_row_operations(TABLE *table, int error) int Delete_rows_log_event_old::do_prepare_row(THD *thd_arg, - Relay_log_info const *rli, + rpl_group_info *rgi, TABLE *table, uchar const *row_start, uchar const **row_end) @@ -1065,7 +1040,7 @@ Delete_rows_log_event_old::do_prepare_row(THD *thd_arg, */ DBUG_ASSERT(table->s->fields >= m_width); - error= unpack_row_old(const_cast<Relay_log_info*>(rli), + error= unpack_row_old(rgi, table, m_width, table->record[0], row_start, m_rows_end, &m_cols, row_end, &m_master_reclength, @@ -1131,8 +1106,6 @@ int Update_rows_log_event_old::do_before_row_operations(TABLE *table) if (!m_memory) return HA_ERR_OUT_OF_MEM; - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - return error; } @@ -1151,7 +1124,7 @@ int Update_rows_log_event_old::do_after_row_operations(TABLE *table, int error) int Update_rows_log_event_old::do_prepare_row(THD *thd_arg, - Relay_log_info const *rli, + rpl_group_info *rgi, TABLE *table, uchar const *row_start, uchar const **row_end) @@ -1165,14 +1138,14 @@ int Update_rows_log_event_old::do_prepare_row(THD *thd_arg, DBUG_ASSERT(table->s->fields >= m_width); /* record[0] is the before image for the update */ - error= unpack_row_old(const_cast<Relay_log_info*>(rli), + error= unpack_row_old(rgi, table, m_width, table->record[0], row_start, m_rows_end, &m_cols, row_end, &m_master_reclength, table->read_set, PRE_GA_UPDATE_ROWS_EVENT); row_start = *row_end; /* m_after_image is the after image for the update */ - error= unpack_row_old(const_cast<Relay_log_info*>(rli), + error= unpack_row_old(rgi, table, m_width, m_after_image, row_start, m_rows_end, &m_cols, row_end, &m_master_reclength, @@ -1468,10 +1441,11 @@ int Old_rows_log_event::do_add_row_data(uchar *row_data, size_t length) #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) -int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) +int Old_rows_log_event::do_apply_event(rpl_group_info *rgi) { DBUG_ENTER("Old_rows_log_event::do_apply_event(Relay_log_info*)"); int error= 0; + Relay_log_info const *rli= rgi->rli; /* If m_table_id == ~0UL, then we have a dummy event that does not @@ -1487,7 +1461,7 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) */ DBUG_ASSERT(get_flags(STMT_END_F)); - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); thd->clear_error(); DBUG_RETURN(0); } @@ -1497,7 +1471,7 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) do_apply_event(). We still check here to prevent future coding errors. */ - DBUG_ASSERT(rli->sql_thd == thd); + DBUG_ASSERT(rgi->thd == thd); /* If there is no locks taken, this is the first binrow event seen @@ -1515,8 +1489,8 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) */ lex_start(thd); - if ((error= lock_tables(thd, rli->tables_to_lock, - rli->tables_to_lock_count, 0))) + if ((error= lock_tables(thd, rgi->tables_to_lock, + rgi->tables_to_lock_count, 0))) { if (thd->is_slave_error || thd->is_fatal_error) { @@ -1538,7 +1512,7 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) "Error in %s event: when locking tables", get_type_str()); } - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); DBUG_RETURN(error); } @@ -1551,8 +1525,8 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) */ { - RPL_TABLE_LIST *ptr= rli->tables_to_lock; - for (uint i= 0 ; ptr&& (i< rli->tables_to_lock_count); + RPL_TABLE_LIST *ptr= rgi->tables_to_lock; + for (uint i= 0 ; ptr&& (i< rgi->tables_to_lock_count); ptr= static_cast<RPL_TABLE_LIST*>(ptr->next_global), i++) { TABLE *conv_table; @@ -1560,7 +1534,7 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) ptr->table, &conv_table)) { thd->is_slave_error= 1; - const_cast<Relay_log_info*>(rli)->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); DBUG_RETURN(ERR_BAD_TABLE_DEF); } ptr->m_conv_table= conv_table; @@ -1582,18 +1556,18 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) Old_rows_log_event, we can invalidate the query cache for the associated table. */ - for (TABLE_LIST *ptr= rli->tables_to_lock ; ptr ; ptr= ptr->next_global) + for (TABLE_LIST *ptr= rgi->tables_to_lock ; ptr ; ptr= ptr->next_global) { - const_cast<Relay_log_info*>(rli)->m_table_map.set_table(ptr->table_id, ptr->table); + rgi->m_table_map.set_table(ptr->table_id, ptr->table); } #ifdef HAVE_QUERY_CACHE - query_cache.invalidate_locked_for_write(thd, rli->tables_to_lock); + query_cache.invalidate_locked_for_write(thd, rgi->tables_to_lock); #endif } TABLE* table= - m_table= const_cast<Relay_log_info*>(rli)->m_table_map.get_table(m_table_id); + m_table= rgi->m_table_map.get_table(m_table_id); if (table) { @@ -1629,17 +1603,6 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) /* A small test to verify that objects have consistent types */ DBUG_ASSERT(sizeof(thd->variables.option_bits) == sizeof(OPTION_RELAXED_UNIQUE_CHECKS)); - /* - Now we are in a statement and will stay in a statement until we - see a STMT_END_F. - - We set this flag here, before actually applying any rows, in - case the SQL thread is stopped and we need to detect that we're - inside a statement and halting abruptly might cause problems - when restarting. - */ - const_cast<Relay_log_info*>(rli)->set_flag(Relay_log_info::IN_STMT); - if ( m_width == table->s->fields && bitmap_is_set_all(&m_cols)) set_flags(COMPLETE_ROWS_F); @@ -1673,7 +1636,7 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) if (!table->in_use) table->in_use= thd; - error= do_exec_row(rli); + error= do_exec_row(rgi); DBUG_PRINT("info", ("error: %d", error)); DBUG_ASSERT(error != HA_ERR_RECORD_DELETED); @@ -1712,7 +1675,7 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) (ulong) m_curr_row, (ulong) m_curr_row_end, (ulong) m_rows_end)); if (!m_curr_row_end && !error) - unpack_current_row(rli); + unpack_current_row(rgi); // at this moment m_curr_row_end should be set DBUG_ASSERT(error || m_curr_row_end != NULL); @@ -1749,7 +1712,7 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) rollback at the caller along with sbr. */ thd->reset_current_stmt_binlog_format_row(); - const_cast<Relay_log_info*>(rli)->cleanup_context(thd, error); + rgi->cleanup_context(thd, error); thd->is_slave_error= 1; DBUG_RETURN(error); } @@ -1778,7 +1741,7 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) problem. When WL#2975 is implemented, just remove the member Relay_log_info::last_event_start_time and all its occurrences. */ - const_cast<Relay_log_info*>(rli)->last_event_start_time= my_time(0); + rgi->last_event_start_time= my_time(0); } if (get_flags(STMT_END_F)) @@ -1831,7 +1794,7 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) */ thd->reset_current_stmt_binlog_format_row(); - const_cast<Relay_log_info*>(rli)->cleanup_context(thd, 0); + rgi->cleanup_context(thd, 0); } DBUG_RETURN(error); @@ -1839,22 +1802,23 @@ int Old_rows_log_event::do_apply_event(Relay_log_info const *rli) Log_event::enum_skip_reason -Old_rows_log_event::do_shall_skip(Relay_log_info *rli) +Old_rows_log_event::do_shall_skip(rpl_group_info *rgi) { /* If the slave skip counter is 1 and this event does not end a statement, then we should not start executing on the next event. Otherwise, we defer the decision to the normal skipping logic. */ - if (rli->slave_skip_counter == 1 && !get_flags(STMT_END_F)) + if (rgi->rli->slave_skip_counter == 1 && !get_flags(STMT_END_F)) return Log_event::EVENT_SKIP_IGNORE; else - return Log_event::do_shall_skip(rli); + return Log_event::do_shall_skip(rgi); } int -Old_rows_log_event::do_update_pos(Relay_log_info *rli) +Old_rows_log_event::do_update_pos(rpl_group_info *rgi) { + Relay_log_info *rli= rgi->rli; DBUG_ENTER("Old_rows_log_event::do_update_pos"); int error= 0; @@ -1868,7 +1832,7 @@ Old_rows_log_event::do_update_pos(Relay_log_info *rli) Step the group log position if we are not in a transaction, otherwise increase the event log position. */ - rli->stmt_done(log_pos, when); + rli->stmt_done(log_pos, when, thd, rgi); /* Clear any errors in thd->net.last_err*. It is not known if this is needed or not. It is believed that any errors that may exist in @@ -1879,7 +1843,7 @@ Old_rows_log_event::do_update_pos(Relay_log_info *rli) } else { - rli->inc_event_relay_log_pos(); + rgi->inc_event_relay_log_pos(); } DBUG_RETURN(error); @@ -2016,8 +1980,7 @@ void Old_rows_log_event::print_helper(FILE *file, */ int -Old_rows_log_event::write_row(const Relay_log_info *const rli, - const bool overwrite) +Old_rows_log_event::write_row(rpl_group_info *rgi, const bool overwrite) { DBUG_ENTER("write_row"); DBUG_ASSERT(m_table != NULL && thd != NULL); @@ -2034,7 +1997,7 @@ Old_rows_log_event::write_row(const Relay_log_info *const rli, DBUG_RETURN(error); /* unpack row into table->record[0] */ - error= unpack_current_row(rli); // TODO: how to handle errors? + error= unpack_current_row(rgi); // TODO: how to handle errors? #ifndef DBUG_OFF DBUG_DUMP("record[0]", table->record[0], table->s->reclength); @@ -2141,7 +2104,7 @@ Old_rows_log_event::write_row(const Relay_log_info *const rli, if (!get_flags(COMPLETE_ROWS_F)) { restore_record(table,record[1]); - error= unpack_current_row(rli); + error= unpack_current_row(rgi); } #ifndef DBUG_OFF @@ -2236,7 +2199,7 @@ Old_rows_log_event::write_row(const Relay_log_info *const rli, for any following update/delete command. */ -int Old_rows_log_event::find_row(const Relay_log_info *rli) +int Old_rows_log_event::find_row(rpl_group_info *rgi) { DBUG_ENTER("find_row"); @@ -2249,7 +2212,7 @@ int Old_rows_log_event::find_row(const Relay_log_info *rli) // TODO: shall we check and report errors here? prepare_record(table, m_width, FALSE /* don't check errors */); - error= unpack_current_row(rli); + error= unpack_current_row(rgi); #ifndef DBUG_OFF DBUG_PRINT("info",("looking for the following record")); @@ -2595,22 +2558,6 @@ Write_rows_log_event_old::do_before_row_operations(const Slave_reporting_capabil from the start. */ m_table->file->ha_start_bulk_insert(0); - /* - We need TIMESTAMP_NO_AUTO_SET otherwise ha_write_row() will not use fill - any TIMESTAMP column with data from the row but instead will use - the event's current time. - As we replicate from TIMESTAMP to TIMESTAMP and slave has no extra - columns, we know that all TIMESTAMP columns on slave will receive explicit - data from the row, so TIMESTAMP_NO_AUTO_SET is ok. - When we allow a table without TIMESTAMP to be replicated to a table having - more columns including a TIMESTAMP column, or when we allow a TIMESTAMP - column to be replicated into a BIGINT column and the slave's table has a - TIMESTAMP column, then the slave's TIMESTAMP column will take its value - from set_time() which we called earlier (consistent with SBR). And then in - some cases we won't want TIMESTAMP_NO_AUTO_SET (will require some code to - analyze if explicit data is provided for slave's TIMESTAMP columns). - */ - m_table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; return error; } @@ -2637,10 +2584,10 @@ Write_rows_log_event_old::do_after_row_operations(const Slave_reporting_capabili int -Write_rows_log_event_old::do_exec_row(const Relay_log_info *const rli) +Write_rows_log_event_old::do_exec_row(rpl_group_info *rgi) { DBUG_ASSERT(m_table != NULL); - int error= write_row(rli, TRUE /* overwrite */); + int error= write_row(rgi, TRUE /* overwrite */); if (error && !thd->net.last_errno) thd->net.last_errno= error; @@ -2739,12 +2686,12 @@ Delete_rows_log_event_old::do_after_row_operations(const Slave_reporting_capabil } -int Delete_rows_log_event_old::do_exec_row(const Relay_log_info *const rli) +int Delete_rows_log_event_old::do_exec_row(rpl_group_info *rgi) { int error; DBUG_ASSERT(m_table != NULL); - if (!(error= find_row(rli))) + if (!(error= find_row(rgi))) { /* Delete the record found, located in record[0] @@ -2820,8 +2767,6 @@ Update_rows_log_event_old::do_before_row_operations(const Slave_reporting_capabi return HA_ERR_OUT_OF_MEM; } - m_table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - return 0; } @@ -2840,11 +2785,11 @@ Update_rows_log_event_old::do_after_row_operations(const Slave_reporting_capabil int -Update_rows_log_event_old::do_exec_row(const Relay_log_info *const rli) +Update_rows_log_event_old::do_exec_row(rpl_group_info *rgi) { DBUG_ASSERT(m_table != NULL); - int error= find_row(rli); + int error= find_row(rgi); if (error) { /* @@ -2852,7 +2797,7 @@ Update_rows_log_event_old::do_exec_row(const Relay_log_info *const rli) able to skip to the next pair of updates */ m_curr_row= m_curr_row_end; - unpack_current_row(rli); + unpack_current_row(rgi); return error; } @@ -2870,7 +2815,7 @@ Update_rows_log_event_old::do_exec_row(const Relay_log_info *const rli) store_record(m_table,record[1]); m_curr_row= m_curr_row_end; - error= unpack_current_row(rli); // this also updates m_curr_row_end + error= unpack_current_row(rgi); // this also updates m_curr_row_end /* Now we have the right row to update. The old row (the one we're diff --git a/sql/log_event_old.h b/sql/log_event_old.h index d9d9f25737b..192123e603f 100644 --- a/sql/log_event_old.h +++ b/sql/log_event_old.h @@ -145,6 +145,7 @@ public: { return m_rows_buf && m_cols.bitmap; } + bool is_part_of_group() { return 1; } uint m_row_count; /* The number of rows added to the event */ @@ -195,15 +196,15 @@ protected: const uchar *m_curr_row_end; /* One-after the end of the current row */ uchar *m_key; /* Buffer to keep key value during searches */ - int find_row(const Relay_log_info *const); - int write_row(const Relay_log_info *const, const bool); + int find_row(rpl_group_info *); + int write_row(rpl_group_info *, const bool); // Unpack the current row into m_table->record[0] - int unpack_current_row(const Relay_log_info *const rli) + int unpack_current_row(rpl_group_info *rgi) { DBUG_ASSERT(m_table); ASSERT_OR_RETURN_ERROR(m_curr_row < m_rows_end, HA_ERR_CORRUPT_EVENT); - return ::unpack_row(rli, m_table, m_width, m_curr_row, &m_cols, + return ::unpack_row(rgi, m_table, m_width, m_curr_row, &m_cols, &m_curr_row_end, &m_master_reclength, m_rows_end); } #endif @@ -211,9 +212,9 @@ protected: private: #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) - virtual int do_apply_event(Relay_log_info const *rli); - virtual int do_update_pos(Relay_log_info *rli); - virtual enum_skip_reason do_shall_skip(Relay_log_info *rli); + virtual int do_apply_event(rpl_group_info *rgi); + virtual int do_update_pos(rpl_group_info *rgi); + virtual enum_skip_reason do_shall_skip(rpl_group_info *rgi); /* Primitive to prepare for a sequence of row executions. @@ -264,7 +265,7 @@ private: 0 if execution succeeded, 1 if execution failed. */ - virtual int do_exec_row(const Relay_log_info *const rli) = 0; + virtual int do_exec_row(rpl_group_info *rgi) = 0; #endif /* !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) */ /********** END OF CUT & PASTE FROM Rows_log_event **********/ @@ -272,7 +273,7 @@ private: #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) - int do_apply_event(Old_rows_log_event*,const Relay_log_info*); + int do_apply_event(Old_rows_log_event*, rpl_group_info *rgi); /* Primitive to prepare for a sequence of row executions. @@ -321,7 +322,7 @@ private: RETURN VALUE Error code, if something went wrong, 0 otherwise. */ - virtual int do_prepare_row(THD*, Relay_log_info const*, TABLE*, + virtual int do_prepare_row(THD*, rpl_group_info*, TABLE*, uchar const *row_start, uchar const **row_end) = 0; @@ -384,7 +385,7 @@ private: #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) virtual int do_before_row_operations(const Slave_reporting_capability *const); virtual int do_after_row_operations(const Slave_reporting_capability *const,int); - virtual int do_exec_row(const Relay_log_info *const); + virtual int do_exec_row(rpl_group_info *); #endif /********** END OF CUT & PASTE FROM Write_rows_log_event **********/ @@ -400,13 +401,13 @@ private: #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) // use old definition of do_apply_event() - virtual int do_apply_event(const Relay_log_info *rli) - { return Old_rows_log_event::do_apply_event(this,rli); } + virtual int do_apply_event(rpl_group_info *rgi) + { return Old_rows_log_event::do_apply_event(this, rgi); } // primitives for old version of do_apply_event() virtual int do_before_row_operations(TABLE *table); virtual int do_after_row_operations(TABLE *table, int error); - virtual int do_prepare_row(THD*, Relay_log_info const*, TABLE*, + virtual int do_prepare_row(THD*, rpl_group_info*, TABLE*, uchar const *row_start, uchar const **row_end); virtual int do_exec_row(TABLE *table); @@ -460,7 +461,7 @@ protected: #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) virtual int do_before_row_operations(const Slave_reporting_capability *const); virtual int do_after_row_operations(const Slave_reporting_capability *const,int); - virtual int do_exec_row(const Relay_log_info *const); + virtual int do_exec_row(rpl_group_info *); #endif /* !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) */ /********** END OF CUT & PASTE FROM Update_rows_log_event **********/ @@ -478,13 +479,13 @@ private: #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) // use old definition of do_apply_event() - virtual int do_apply_event(const Relay_log_info *rli) - { return Old_rows_log_event::do_apply_event(this,rli); } + virtual int do_apply_event(rpl_group_info *rgi) + { return Old_rows_log_event::do_apply_event(this, rgi); } // primitives for old version of do_apply_event() virtual int do_before_row_operations(TABLE *table); virtual int do_after_row_operations(TABLE *table, int error); - virtual int do_prepare_row(THD*, Relay_log_info const*, TABLE*, + virtual int do_prepare_row(THD*, rpl_group_info*, TABLE*, uchar const *row_start, uchar const **row_end); virtual int do_exec_row(TABLE *table); #endif /* !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) */ @@ -535,7 +536,7 @@ protected: #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) virtual int do_before_row_operations(const Slave_reporting_capability *const); virtual int do_after_row_operations(const Slave_reporting_capability *const,int); - virtual int do_exec_row(const Relay_log_info *const); + virtual int do_exec_row(rpl_group_info *); #endif /********** END CUT & PASTE FROM Delete_rows_log_event **********/ @@ -553,13 +554,13 @@ private: #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) // use old definition of do_apply_event() - virtual int do_apply_event(const Relay_log_info *rli) - { return Old_rows_log_event::do_apply_event(this,rli); } + virtual int do_apply_event(rpl_group_info *rgi) + { return Old_rows_log_event::do_apply_event(this, rgi); } // primitives for old version of do_apply_event() virtual int do_before_row_operations(TABLE *table); virtual int do_after_row_operations(TABLE *table, int error); - virtual int do_prepare_row(THD*, Relay_log_info const*, TABLE*, + virtual int do_prepare_row(THD*, rpl_group_info*, TABLE*, uchar const *row_start, uchar const **row_end); virtual int do_exec_row(TABLE *table); #endif diff --git a/sql/log_slow.h b/sql/log_slow.h index 92a2d1bf4f6..e8faf79a047 100644 --- a/sql/log_slow.h +++ b/sql/log_slow.h @@ -18,6 +18,7 @@ #define LOG_SLOW_VERBOSITY_INIT 0 #define LOG_SLOW_VERBOSITY_INNODB 1 << 0 #define LOG_SLOW_VERBOSITY_QUERY_PLAN 1 << 1 +#define LOG_SLOW_VERBOSITY_EXPLAIN 1 << 2 #define QPLAN_INIT QPLAN_QC_NO diff --git a/sql/mdl.cc b/sql/mdl.cc index 415d56887d5..e739a9aff78 100644 --- a/sql/mdl.cc +++ b/sql/mdl.cc @@ -85,7 +85,8 @@ const char *MDL_key::m_namespace_to_wait_state_name[NAMESPACE_END]= "Waiting for stored procedure metadata lock", "Waiting for trigger metadata lock", "Waiting for event metadata lock", - "Waiting for commit lock" + "Waiting for commit lock", + "User lock" /* Be compatible with old status. */ }; static bool mdl_initialized= 0; @@ -107,6 +108,7 @@ public: void init(); void destroy(); MDL_lock *find_or_insert(const MDL_key *key); + unsigned long get_lock_owner(const MDL_key *key); void remove(MDL_lock *lock); private: bool move_from_hash_to_lock_mutex(MDL_lock *lock); @@ -382,6 +384,7 @@ public: bool ignore_lock_priority) const; inline static MDL_lock *create(const MDL_key *key); + inline unsigned long get_lock_owner() const; void reschedule_waiters(); @@ -857,6 +860,43 @@ bool MDL_map::move_from_hash_to_lock_mutex(MDL_lock *lock) /** + * Return thread id of the owner of the lock, if it is owned. + */ + +unsigned long +MDL_map::get_lock_owner(const MDL_key *mdl_key) +{ + MDL_lock *lock; + unsigned long res= 0; + + if (mdl_key->mdl_namespace() == MDL_key::GLOBAL || + mdl_key->mdl_namespace() == MDL_key::COMMIT) + { + lock= (mdl_key->mdl_namespace() == MDL_key::GLOBAL) ? m_global_lock : + m_commit_lock; + mysql_prlock_rdlock(&lock->m_rwlock); + res= lock->get_lock_owner(); + mysql_prlock_unlock(&lock->m_rwlock); + } + else + { + my_hash_value_type hash_value= my_calc_hash(&m_locks, + mdl_key->ptr(), + mdl_key->length()); + mysql_mutex_lock(&m_mutex); + lock= (MDL_lock*) my_hash_search_using_hash_value(&m_locks, + hash_value, + mdl_key->ptr(), + mdl_key->length()); + if (lock) + res= lock->get_lock_owner(); + mysql_mutex_unlock(&m_mutex); + } + return res; +} + + +/** Destroy MDL_lock object or delegate this responsibility to whatever thread that holds the last outstanding reference to it. @@ -1621,6 +1661,23 @@ MDL_lock::can_grant_lock(enum_mdl_type type_arg, } +/** + Return thread id of the thread to which the first ticket was + granted. +*/ + +inline unsigned long +MDL_lock::get_lock_owner() const +{ + Ticket_iterator it(m_granted); + MDL_ticket *ticket; + + if ((ticket= it++)) + return thd_get_thread_id(ticket->get_ctx()->get_thd()); + return 0; +} + + /** Remove a ticket from waiting or pending queue and wakeup up waiters. */ void MDL_lock::remove_ticket(Ticket_list MDL_lock::*list, MDL_ticket *ticket) @@ -2094,31 +2151,37 @@ MDL_context::acquire_lock(MDL_request *mdl_request, ulong lock_wait_timeout) find_deadlock(); - if (lock->needs_notification(ticket)) + struct timespec abs_shortwait; + set_timespec(abs_shortwait, 1); + wait_status= MDL_wait::EMPTY; + + while (cmp_timespec(abs_shortwait, abs_timeout) <= 0) { - struct timespec abs_shortwait; - set_timespec(abs_shortwait, 1); - wait_status= MDL_wait::EMPTY; + /* abs_timeout is far away. Wait a short while and notify locks. */ + wait_status= m_wait.timed_wait(m_thd, &abs_shortwait, FALSE, + mdl_request->key.get_wait_state_name()); - while (cmp_timespec(abs_shortwait, abs_timeout) <= 0) + if (wait_status != MDL_wait::EMPTY) + break; + /* Check if the client is gone while we were waiting. */ + if (! thd_is_connected(m_thd)) { - /* abs_timeout is far away. Wait a short while and notify locks. */ - wait_status= m_wait.timed_wait(m_thd, &abs_shortwait, FALSE, - mdl_request->key.get_wait_state_name()); - - if (wait_status != MDL_wait::EMPTY) - break; + /* + * The client is disconnected. Don't wait forever: + * assume it's the same as a wait timeout, this + * ensures all error handling is correct. + */ + wait_status= MDL_wait::TIMEOUT; + break; + } - mysql_prlock_wrlock(&lock->m_rwlock); + mysql_prlock_wrlock(&lock->m_rwlock); + if (lock->needs_notification(ticket)) lock->notify_conflicting_locks(this); - mysql_prlock_unlock(&lock->m_rwlock); - set_timespec(abs_shortwait, 1); - } - if (wait_status == MDL_wait::EMPTY) - wait_status= m_wait.timed_wait(m_thd, &abs_timeout, TRUE, - mdl_request->key.get_wait_state_name()); + mysql_prlock_unlock(&lock->m_rwlock); + set_timespec(abs_shortwait, 1); } - else + if (wait_status == MDL_wait::EMPTY) wait_status= m_wait.timed_wait(m_thd, &abs_timeout, TRUE, mdl_request->key.get_wait_state_name()); @@ -2204,7 +2267,8 @@ bool MDL_context::acquire_locks(MDL_request_list *mdl_requests, /* Sort requests according to MDL_key. */ if (! (sort_buf= (MDL_request **)my_malloc(req_count * sizeof(MDL_request*), - MYF(MY_WME)))) + MYF(MY_WME | + MY_THREAD_SPECIFIC)))) DBUG_RETURN(TRUE); for (p_req= sort_buf; p_req < sort_buf + req_count; p_req++) @@ -2612,7 +2676,7 @@ void MDL_context::release_lock(MDL_ticket *ticket) the corresponding lists, i.e. stored in reverse temporal order. This allows to employ this function to: - back off in case of a lock conflict. - - release all locks in the end of a statment or transaction + - release all locks in the end of a statement or transaction - rollback to a savepoint. */ @@ -2724,6 +2788,22 @@ MDL_context::is_lock_owner(MDL_key::enum_mdl_namespace mdl_namespace, /** + Return thread id of the owner of the lock or 0 if + there is no owner. + @note: Lock type is not considered at all, the function + simply checks that there is some lock for the given key. + + @return thread id of the owner of the lock or 0 +*/ + +unsigned long +MDL_context::get_lock_owner(MDL_key *key) +{ + return mdl_locks.get_lock_owner(key); +} + + +/** Check if we have any pending locks which conflict with existing shared lock. @pre The ticket must match an acquired lock. @@ -2736,6 +2816,11 @@ bool MDL_ticket::has_pending_conflicting_lock() const return m_lock->has_pending_conflicting_lock(m_type); } +/** Return a key identifying this lock. */ +MDL_key *MDL_ticket::get_key() const +{ + return &m_lock->key; +} /** Releases metadata locks that were acquired after a specific savepoint. diff --git a/sql/mdl.h b/sql/mdl.h index 68f24a7a0e8..a86b45e180f 100644 --- a/sql/mdl.h +++ b/sql/mdl.h @@ -212,6 +212,7 @@ public: TRIGGER, EVENT, COMMIT, + USER_LOCK, /* user level locks. */ /* This should be the last ! */ NAMESPACE_END }; @@ -492,6 +493,7 @@ public: } enum_mdl_type get_type() const { return m_type; } MDL_lock *get_lock() const { return m_lock; } + MDL_key *get_key() const; void downgrade_exclusive_lock(enum_mdl_type type); bool has_stronger_or_equal_type(enum_mdl_type type) const; @@ -653,6 +655,7 @@ public: bool is_lock_owner(MDL_key::enum_mdl_namespace mdl_namespace, const char *db, const char *name, enum_mdl_type mdl_type); + unsigned long get_lock_owner(MDL_key *mdl_key); bool has_lock(const MDL_savepoint &mdl_savepoint, MDL_ticket *mdl_ticket); @@ -721,9 +724,9 @@ private: Lists of MDL tickets: --------------------- The entire set of locks acquired by a connection can be separated - in three subsets according to their: locks released at the end of - statement, at the end of transaction and locks are released - explicitly. + in three subsets according to their duration: locks released at + the end of statement, at the end of transaction and locks are + released explicitly. Statement and transactional locks are locks with automatic scope. They are accumulated in the course of a transaction, and released @@ -732,11 +735,12 @@ private: locks). They must not be (and never are) released manually, i.e. with release_lock() call. - Locks with explicit duration are taken for locks that span + Tickets with explicit duration are taken for locks that span multiple transactions or savepoints. These are: HANDLER SQL locks (HANDLER SQL is transaction-agnostic), LOCK TABLES locks (you can COMMIT/etc - under LOCK TABLES, and the locked tables stay locked), and + under LOCK TABLES, and the locked tables stay locked), user level + locks (GET_LOCK()/RELEASE_LOCK() functions) and locks implementing "global read lock". Statement/transactional locks are always prepended to the @@ -745,20 +749,19 @@ private: a savepoint, we start popping and releasing tickets from the front until we reach the last ticket acquired after the savepoint. - Locks with explicit duration stored are not stored in any + Locks with explicit duration are not stored in any particular order, and among each other can be split into - three sets: + four sets: - [LOCK TABLES locks] [HANDLER locks] [GLOBAL READ LOCK locks] + [LOCK TABLES locks] [USER locks] [HANDLER locks] [GLOBAL READ LOCK locks] The following is known about these sets: - * GLOBAL READ LOCK locks are always stored after LOCK TABLES - locks and after HANDLER locks. This is because one can't say - SET GLOBAL read_only=1 or FLUSH TABLES WITH READ LOCK - if one has locked tables. One can, however, LOCK TABLES - after having entered the read only mode. Note, that - subsequent LOCK TABLES statement will unlock the previous + * GLOBAL READ LOCK locks are always stored last. + This is because one can't say SET GLOBAL read_only=1 or + FLUSH TABLES WITH READ LOCK if one has locked tables. One can, + however, LOCK TABLES after having entered the read only mode. + Note, that subsequent LOCK TABLES statement will unlock the previous set of tables, but not the GRL! There are no HANDLER locks after GRL locks because SET GLOBAL read_only performs a FLUSH TABLES WITH @@ -853,6 +856,18 @@ extern bool mysql_notify_thread_having_shared_lock(THD *thd, THD *in_use, extern "C" const char* thd_enter_cond(MYSQL_THD thd, mysql_cond_t *cond, mysql_mutex_t *mutex, const char *msg); extern "C" void thd_exit_cond(MYSQL_THD thd, const char *old_msg); +extern "C" unsigned long thd_get_thread_id(const MYSQL_THD thd); + +/** + Check if a connection in question is no longer connected. + + @details + Replication apply thread is always connected. Otherwise, + does a poll on the associated socket to check if the client + is gone. +*/ +extern "C" int thd_is_connected(MYSQL_THD thd); + #ifndef DBUG_OFF extern mysql_mutex_t LOCK_open; diff --git a/sql/multi_range_read.cc b/sql/multi_range_read.cc index d9aeff21400..651ee87933e 100644 --- a/sql/multi_range_read.cc +++ b/sql/multi_range_read.cc @@ -1200,9 +1200,9 @@ bool DsMrr_impl::setup_buffer_sharing(uint key_size_in_keybuf, statistics? */ uint parts= my_count_bits(key_tuple_map); - ulong rpc; + ha_rows rpc; ulonglong rowids_size= rowid_buf_elem_size; - if ((rpc= key_info->rec_per_key[parts - 1])) + if ((rpc= (ha_rows) key_info->actual_rec_per_key(parts - 1))) rowids_size= rowid_buf_elem_size * rpc; double fraction_for_rowids= diff --git a/sql/my_apc.cc b/sql/my_apc.cc new file mode 100644 index 00000000000..dcb1e3d99b1 --- /dev/null +++ b/sql/my_apc.cc @@ -0,0 +1,270 @@ +/* + Copyright (c) 2011, 2013 Monty Program Ab. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + + +#ifndef MY_APC_STANDALONE + +#include "sql_class.h" + +#endif + +/* For standalone testing of APC system, see unittest/sql/my_apc-t.cc */ + +/* + Initialize the target. + + @note + Initialization must be done prior to enabling/disabling the target, or making + any call requests to it. + Initial state after initialization is 'disabled'. +*/ +void Apc_target::init(mysql_mutex_t *target_mutex) +{ + DBUG_ASSERT(!enabled); + LOCK_thd_data_ptr= target_mutex; +#ifndef DBUG_OFF + n_calls_processed= 0; +#endif +} + + +/* + Destroy the target. The target must be disabled when this call is made. +*/ +void Apc_target::destroy() +{ + DBUG_ASSERT(!enabled); +} + + +/* + Enter ther state where the target is available for serving APC requests +*/ +void Apc_target::enable() +{ + /* Ok to do without getting/releasing the mutex: */ + enabled++; +} + + +/* + Make the target unavailable for serving APC requests. + + @note + This call will serve all requests that were already enqueued +*/ + +void Apc_target::disable() +{ + bool process= FALSE; + DBUG_ASSERT(enabled); + mysql_mutex_lock(LOCK_thd_data_ptr); + if (!(--enabled)) + process= TRUE; + mysql_mutex_unlock(LOCK_thd_data_ptr); + if (process) + process_apc_requests(); +} + + +/* [internal] Put request qe into the request list */ + +void Apc_target::enqueue_request(Call_request *qe) +{ + mysql_mutex_assert_owner(LOCK_thd_data_ptr); + if (apc_calls) + { + Call_request *after= apc_calls->prev; + qe->next= apc_calls; + apc_calls->prev= qe; + + qe->prev= after; + after->next= qe; + } + else + { + apc_calls= qe; + qe->next= qe->prev= qe; + } +} + + +/* + [internal] Remove request qe from the request queue. + + The request is not necessarily first in the queue. +*/ + +void Apc_target::dequeue_request(Call_request *qe) +{ + mysql_mutex_assert_owner(LOCK_thd_data_ptr); + if (apc_calls == qe) + { + if ((apc_calls= apc_calls->next) == qe) + { + apc_calls= NULL; + } + } + + qe->prev->next= qe->next; + qe->next->prev= qe->prev; +} + +#ifdef HAVE_PSI_INTERFACE + +/* One key for all conds */ +PSI_cond_key key_show_explain_request_COND; + +static PSI_cond_info show_explain_psi_conds[]= +{ + { &key_show_explain_request_COND, "show_explain", 0 /* not using PSI_FLAG_GLOBAL*/ } +}; + +void init_show_explain_psi_keys(void) +{ + if (PSI_server == NULL) + return; + + PSI_server->register_cond("sql", show_explain_psi_conds, + array_elements(show_explain_psi_conds)); +} +#endif + + +/* + Make an APC (Async Procedure Call) to another thread. + + @detail + Make an APC call: schedule it for execution and wait until the target + thread has executed it. + + - The caller is responsible for making sure he's not posting request + to the thread he's calling this function from. + + - The caller must have locked target_mutex. The function will release it. + + @retval FALSE - Ok, the call has been made + @retval TRUE - Call wasnt made (either the target is in disabled state or + timeout occured) +*/ + +bool Apc_target::make_apc_call(THD *caller_thd, Apc_call *call, + int timeout_sec, bool *timed_out) +{ + bool res= TRUE; + *timed_out= FALSE; + + if (enabled) + { + /* Create and post the request */ + Call_request apc_request; + apc_request.call= call; + apc_request.processed= FALSE; + mysql_cond_init(key_show_explain_request_COND, &apc_request.COND_request, + NULL); + enqueue_request(&apc_request); + apc_request.what="enqueued by make_apc_call"; + + struct timespec abstime; + const int timeout= timeout_sec; + set_timespec(abstime, timeout); + + int wait_res= 0; + const char *old_msg; + old_msg= caller_thd->enter_cond(&apc_request.COND_request, + LOCK_thd_data_ptr, "show_explain"); + /* todo: how about processing other errors here? */ + while (!apc_request.processed && (wait_res != ETIMEDOUT)) + { + /* We own LOCK_thd_data_ptr */ + wait_res= mysql_cond_timedwait(&apc_request.COND_request, + LOCK_thd_data_ptr, &abstime); + // &apc_request.LOCK_request, &abstime); + if (caller_thd->killed) + break; + } + + if (!apc_request.processed) + { + /* + The wait has timed out, or this thread was KILLed. + Remove the request from the queue (ok to do because we own + LOCK_thd_data_ptr) + */ + apc_request.processed= TRUE; + dequeue_request(&apc_request); + *timed_out= TRUE; + res= TRUE; + } + else + { + /* Request was successfully executed and dequeued by the target thread */ + res= FALSE; + } + /* + exit_cond() will call mysql_mutex_unlock(LOCK_thd_data_ptr) for us: + */ + caller_thd->exit_cond(old_msg); + + /* Destroy all APC request data */ + mysql_cond_destroy(&apc_request.COND_request); + } + else + { + mysql_mutex_unlock(LOCK_thd_data_ptr); + } + return res; +} + + +/* + Process all APC requests. + This should be called periodically by the APC target thread. +*/ + +void Apc_target::process_apc_requests() +{ + while (1) + { + Call_request *request; + + mysql_mutex_lock(LOCK_thd_data_ptr); + if (!(request= get_first_in_queue())) + { + /* No requests in the queue */ + mysql_mutex_unlock(LOCK_thd_data_ptr); + break; + } + + /* + Remove the request from the queue (we're holding queue lock so we can be + sure that request owner won't try to remove it) + */ + request->what="dequeued by process_apc_requests"; + dequeue_request(request); + request->processed= TRUE; + + request->call->call_in_target_thread(); + request->what="func called by process_apc_requests"; + +#ifndef DBUG_OFF + n_calls_processed++; +#endif + mysql_cond_signal(&request->COND_request); + mysql_mutex_unlock(LOCK_thd_data_ptr); + } +} + diff --git a/sql/my_apc.h b/sql/my_apc.h new file mode 100644 index 00000000000..a12db5093a4 --- /dev/null +++ b/sql/my_apc.h @@ -0,0 +1,140 @@ +#ifndef SQL_MY_APC_INCLUDED +#define SQL_MY_APC_INCLUDED +/* + Copyright (c) 2011, 2013 Monty Program Ab. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* + Interface + ~~~~~~~~~ + ( + - This is an APC request queue + - We assume there is a particular owner thread which periodically calls + process_apc_requests() to serve the call requests. + - Other threads can post call requests, and block until they are exectued. + ) + + Implementation + ~~~~~~~~~~~~~~ + - The target has a mutex-guarded request queue. + + - After the request has been put into queue, the requestor waits for request + to be satisfied. The worker satisifes the request and signals the + requestor. +*/ + +class THD; + +/* + Target for asynchronous procedure calls (APCs). + - A target is running in some particular thread, + - One can make calls to it from other threads. +*/ +class Apc_target +{ + mysql_mutex_t *LOCK_thd_data_ptr; +public: + Apc_target() : enabled(0), apc_calls(NULL) {} + ~Apc_target() { DBUG_ASSERT(!enabled && !apc_calls);} + + void init(mysql_mutex_t *target_mutex); + void destroy(); + void enable(); + void disable(); + + void process_apc_requests(); + /* + A lightweight function, intended to be used in frequent checks like this: + + if (apc_target.have_requests()) apc_target.process_apc_requests() + */ + inline bool have_apc_requests() + { + return test(apc_calls); + } + + inline bool is_enabled() { return enabled; } + + /* Functor class for calls you can schedule */ + class Apc_call + { + public: + /* This function will be called in the target thread */ + virtual void call_in_target_thread()= 0; + virtual ~Apc_call() {} + }; + + /* Make a call in the target thread (see function definition for details) */ + bool make_apc_call(THD *caller_thd, Apc_call *call, int timeout_sec, bool *timed_out); + +#ifndef DBUG_OFF + int n_calls_processed; /* Number of calls served by this target */ +#endif +private: + class Call_request; + + /* + Non-zero value means we're enabled. It's an int, not bool, because one can + call enable() N times (and then needs to call disable() N times before the + target is really disabled) + */ + int enabled; + + /* + Circular, double-linked list of all enqueued call requests. + We use this structure, because we + - process requests sequentially: requests are added at the end of the + list and removed from the front. With circular list, we can keep one + pointer, and access both front an back of the list with it. + - a thread that has posted a request may time out (or be KILLed) and + cancel the request, which means we need a fast request-removal + operation. + */ + Call_request *apc_calls; + + class Call_request + { + public: + Apc_call *call; /* Functor to be called */ + + /* The caller will actually wait for "processed==TRUE" */ + bool processed; + + /* Condition that will be signalled when the request has been served */ + mysql_cond_t COND_request; + + /* Double linked-list linkage */ + Call_request *next; + Call_request *prev; + + const char *what; /* (debug) state of the request */ + }; + + void enqueue_request(Call_request *qe); + void dequeue_request(Call_request *qe); + + /* return the first call request in queue, or NULL if there are none enqueued */ + Call_request *get_first_in_queue() + { + return apc_calls; + } +}; + +#ifdef HAVE_PSI_INTERFACE +void init_show_explain_psi_keys(void); +#endif + +#endif //SQL_MY_APC_INCLUDED + diff --git a/sql/mysqld.cc b/sql/mysqld.cc index dcc9aec97a6..7066eba00ac 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -52,6 +52,7 @@ #include "des_key_file.h" // load_des_key_file #include "sql_manager.h" // stop_handle_manager, start_handle_manager #include "sql_expression_cache.h" // subquery_cache_miss, subquery_cache_hit +#include "sys_vars_shared.h" #include <m_ctype.h> #include <my_dir.h> @@ -464,14 +465,15 @@ ulong delay_key_write_options; uint protocol_version; uint lower_case_table_names; ulong tc_heuristic_recover= 0; -uint volatile thread_count; +int32 thread_count; int32 thread_running; +int32 slave_open_temp_tables; ulong thread_created; ulong back_log, connect_timeout, concurrency, server_id; ulong table_cache_size, table_def_size; ulong what_to_log; -ulong slow_launch_time, slave_open_temp_tables; -ulong open_files_limit, max_binlog_size, max_relay_log_size; +ulong slow_launch_time; +ulong open_files_limit, max_binlog_size; ulong slave_trans_retries; uint slave_net_timeout; ulong slave_exec_mode_options; @@ -488,6 +490,9 @@ ulong executed_events=0; query_id_t global_query_id; my_atomic_rwlock_t global_query_id_lock; my_atomic_rwlock_t thread_running_lock; +my_atomic_rwlock_t thread_count_lock; +my_atomic_rwlock_t statistics_lock; +my_atomic_rwlock_t slave_executed_entries_lock; ulong aborted_threads, aborted_connects; ulong delayed_insert_timeout, delayed_insert_limit, delayed_queue_size; ulong delayed_insert_threads, delayed_insert_writes, delayed_rows_in_use; @@ -497,6 +502,7 @@ ulong binlog_cache_use= 0, binlog_cache_disk_use= 0; ulong binlog_stmt_cache_use= 0, binlog_stmt_cache_disk_use= 0; ulong max_connections, max_connect_errors; ulong extra_max_connections; +ulong slave_retried_transactions; ulonglong denied_connections; my_decimal decimal_zero; @@ -539,6 +545,11 @@ ulong rpl_recovery_rank=0; */ ulong stored_program_cache_size= 0; +ulong opt_slave_parallel_threads= 0; +ulong opt_binlog_commit_wait_count= 0; +ulong opt_binlog_commit_wait_usec= 0; +ulong opt_slave_parallel_max_queued= 131072; + const double log_10[] = { 1e000, 1e001, 1e002, 1e003, 1e004, 1e005, 1e006, 1e007, 1e008, 1e009, 1e010, 1e011, 1e012, 1e013, 1e014, 1e015, 1e016, 1e017, 1e018, 1e019, @@ -616,7 +627,8 @@ MYSQL_FILE *bootstrap_file; int bootstrap_error; I_List<THD> threads; -Rpl_filter* rpl_filter; +Rpl_filter* cur_rpl_filter; +Rpl_filter* global_rpl_filter; Rpl_filter* binlog_filter; THD *first_global_thread() @@ -653,12 +665,13 @@ SHOW_COMP_OPTION have_ssl, have_symlink, have_dlopen, have_query_cache; SHOW_COMP_OPTION have_geometry, have_rtree_keys; SHOW_COMP_OPTION have_crypt, have_compress; SHOW_COMP_OPTION have_profiling; +SHOW_COMP_OPTION have_openssl; /* Thread specific variables */ pthread_key(MEM_ROOT**,THR_MALLOC); pthread_key(THD*, THR_THD); -mysql_mutex_t LOCK_thread_count; +mysql_mutex_t LOCK_thread_count, LOCK_thread_cache; mysql_mutex_t LOCK_status, LOCK_error_log, LOCK_short_uuid_generator, LOCK_delayed_insert, LOCK_delayed_status, LOCK_delayed_create, @@ -689,8 +702,7 @@ pthread_attr_t connection_attrib; mysql_mutex_t LOCK_server_started; mysql_cond_t COND_server_started; -int mysqld_server_started= 0; - +int mysqld_server_started=0, mysqld_server_initialized= 0; File_parser_dummy_hook file_parser_dummy_hook; /* replication parameters, if master_host is not NULL, we are a slave */ @@ -732,14 +744,16 @@ char **orig_argv; #ifdef HAVE_PSI_INTERFACE #ifdef HAVE_MMAP -PSI_mutex_key key_PAGE_lock, key_LOCK_sync, key_LOCK_active, key_LOCK_pool; +PSI_mutex_key key_PAGE_lock, key_LOCK_sync, key_LOCK_active, key_LOCK_pool, + key_LOCK_pending_checkpoint; #endif /* HAVE_MMAP */ #ifdef HAVE_OPENSSL PSI_mutex_key key_LOCK_des_key_file; #endif /* HAVE_OPENSSL */ -PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_prep_xids, +PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, + key_BINLOG_LOCK_binlog_background_thread, key_delayed_insert_mutex, key_hash_filo_lock, key_LOCK_active_mi, key_LOCK_connection_count, key_LOCK_crypt, key_LOCK_delayed_create, key_LOCK_delayed_insert, key_LOCK_delayed_status, key_LOCK_error_log, @@ -753,16 +767,20 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_prep_xids, key_master_info_sleep_lock, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, - key_relay_log_info_sleep_lock, + key_rpl_group_info_sleep_lock, key_structure_guard_mutex, key_TABLE_SHARE_LOCK_ha_data, - key_LOCK_error_messages, key_LOG_INFO_lock, key_LOCK_thread_count, + key_LOCK_error_messages, key_LOG_INFO_lock, + key_LOCK_thread_count, key_LOCK_thread_cache, key_PARTITION_LOCK_auto_inc; PSI_mutex_key key_RELAYLOG_LOCK_index; +PSI_mutex_key key_LOCK_slave_state, key_LOCK_binlog_state, + key_LOCK_rpl_thread, key_LOCK_rpl_thread_pool, key_LOCK_parallel_entry; PSI_mutex_key key_LOCK_stats, key_LOCK_global_user_client_stats, key_LOCK_global_table_stats, key_LOCK_global_index_stats, - key_LOCK_wakeup_ready; + key_LOCK_wakeup_ready, key_LOCK_wait_commit; +PSI_mutex_key key_LOCK_gtid_waiting; PSI_mutex_key key_LOCK_prepare_ordered, key_LOCK_commit_ordered; @@ -773,6 +791,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_LOCK_sync, "TC_LOG_MMAP::LOCK_sync", 0}, { &key_LOCK_active, "TC_LOG_MMAP::LOCK_active", 0}, { &key_LOCK_pool, "TC_LOG_MMAP::LOCK_pool", 0}, + { &key_LOCK_pool, "TC_LOG_MMAP::LOCK_pending_checkpoint", 0}, #endif /* HAVE_MMAP */ #ifdef HAVE_OPENSSL @@ -780,7 +799,8 @@ static PSI_mutex_info all_server_mutexes[]= #endif /* HAVE_OPENSSL */ { &key_BINLOG_LOCK_index, "MYSQL_BIN_LOG::LOCK_index", 0}, - { &key_BINLOG_LOCK_prep_xids, "MYSQL_BIN_LOG::LOCK_prep_xids", 0}, + { &key_BINLOG_LOCK_xid_list, "MYSQL_BIN_LOG::LOCK_xid_list", 0}, + { &key_BINLOG_LOCK_binlog_background_thread, "MYSQL_BIN_LOG::LOCK_binlog_background_thread", 0}, { &key_RELAYLOG_LOCK_index, "MYSQL_RELAY_LOG::LOCK_index", 0}, { &key_delayed_insert_mutex, "Delayed_insert::mutex", 0}, { &key_hash_filo_lock, "hash_filo::lock", 0}, @@ -805,6 +825,8 @@ static PSI_mutex_info all_server_mutexes[]= { &key_LOCK_global_table_stats, "LOCK_global_table_stats", PSI_FLAG_GLOBAL}, { &key_LOCK_global_index_stats, "LOCK_global_index_stats", PSI_FLAG_GLOBAL}, { &key_LOCK_wakeup_ready, "THD::LOCK_wakeup_ready", 0}, + { &key_LOCK_wait_commit, "wait_for_commit::LOCK_wait_commit", 0}, + { &key_LOCK_gtid_waiting, "gtid_waiting::LOCK_gtid_waiting", 0}, { &key_LOCK_thd_data, "THD::LOCK_thd_data", 0}, { &key_LOCK_user_conn, "LOCK_user_conn", PSI_FLAG_GLOBAL}, { &key_LOCK_uuid_short_generator, "LOCK_uuid_short_generator", PSI_FLAG_GLOBAL}, @@ -816,7 +838,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_relay_log_info_data_lock, "Relay_log_info::data_lock", 0}, { &key_relay_log_info_log_space_lock, "Relay_log_info::log_space_lock", 0}, { &key_relay_log_info_run_lock, "Relay_log_info::run_lock", 0}, - { &key_relay_log_info_sleep_lock, "Relay_log_info::sleep_lock", 0}, + { &key_rpl_group_info_sleep_lock, "Rpl_group_info::sleep_lock", 0}, { &key_structure_guard_mutex, "Query_cache::structure_guard_mutex", 0}, { &key_TABLE_SHARE_LOCK_ha_data, "TABLE_SHARE::LOCK_ha_data", 0}, { &key_LOCK_error_messages, "LOCK_error_messages", PSI_FLAG_GLOBAL}, @@ -824,7 +846,13 @@ static PSI_mutex_info all_server_mutexes[]= { &key_LOCK_commit_ordered, "LOCK_commit_ordered", PSI_FLAG_GLOBAL}, { &key_LOG_INFO_lock, "LOG_INFO::lock", 0}, { &key_LOCK_thread_count, "LOCK_thread_count", PSI_FLAG_GLOBAL}, - { &key_PARTITION_LOCK_auto_inc, "HA_DATA_PARTITION::LOCK_auto_inc", 0} + { &key_LOCK_thread_cache, "LOCK_thread_cache", PSI_FLAG_GLOBAL}, + { &key_PARTITION_LOCK_auto_inc, "HA_DATA_PARTITION::LOCK_auto_inc", 0}, + { &key_LOCK_slave_state, "LOCK_slave_state", 0}, + { &key_LOCK_binlog_state, "LOCK_binlog_state", 0}, + { &key_LOCK_rpl_thread, "LOCK_rpl_thread", 0}, + { &key_LOCK_rpl_thread_pool, "LOCK_rpl_thread_pool", 0}, + { &key_LOCK_parallel_entry, "LOCK_parallel_entry", 0} }; PSI_rwlock_key key_rwlock_LOCK_grant, key_rwlock_LOCK_logger, @@ -848,7 +876,9 @@ static PSI_rwlock_info all_server_rwlocks[]= PSI_cond_key key_PAGE_cond, key_COND_active, key_COND_pool; #endif /* HAVE_MMAP */ -PSI_cond_key key_BINLOG_COND_prep_xids, key_BINLOG_update_cond, +PSI_cond_key key_BINLOG_COND_xid_list, key_BINLOG_update_cond, + key_BINLOG_COND_binlog_background_thread, + key_BINLOG_COND_binlog_background_thread_end, key_COND_cache_status_changed, key_COND_manager, key_COND_rpl_status, key_COND_server_started, key_delayed_insert_cond, key_delayed_insert_cond_client, @@ -857,13 +887,17 @@ PSI_cond_key key_BINLOG_COND_prep_xids, key_BINLOG_update_cond, key_master_info_sleep_cond, key_relay_log_info_data_cond, key_relay_log_info_log_space_cond, key_relay_log_info_start_cond, key_relay_log_info_stop_cond, - key_relay_log_info_sleep_cond, + key_rpl_group_info_sleep_cond, key_TABLE_SHARE_cond, key_user_level_lock_cond, key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache, key_BINLOG_COND_queue_busy; -PSI_cond_key key_RELAYLOG_update_cond, key_COND_wakeup_ready; +PSI_cond_key key_RELAYLOG_update_cond, key_COND_wakeup_ready, + key_COND_wait_commit; PSI_cond_key key_RELAYLOG_COND_queue_busy; PSI_cond_key key_TC_LOG_MMAP_COND_queue_busy; +PSI_cond_key key_COND_rpl_thread, key_COND_rpl_thread_pool, + key_COND_parallel_entry, key_COND_prepare_ordered; +PSI_cond_key key_COND_wait_gtid; static PSI_cond_info all_server_conds[]= { @@ -876,12 +910,15 @@ static PSI_cond_info all_server_conds[]= { &key_COND_pool, "TC_LOG_MMAP::COND_pool", 0}, { &key_TC_LOG_MMAP_COND_queue_busy, "TC_LOG_MMAP::COND_queue_busy", 0}, #endif /* HAVE_MMAP */ - { &key_BINLOG_COND_prep_xids, "MYSQL_BIN_LOG::COND_prep_xids", 0}, + { &key_BINLOG_COND_xid_list, "MYSQL_BIN_LOG::COND_xid_list", 0}, { &key_BINLOG_update_cond, "MYSQL_BIN_LOG::update_cond", 0}, + { &key_BINLOG_COND_binlog_background_thread, "MYSQL_BIN_LOG::COND_binlog_background_thread", 0}, + { &key_BINLOG_COND_binlog_background_thread_end, "MYSQL_BIN_LOG::COND_binlog_background_thread_end", 0}, { &key_BINLOG_COND_queue_busy, "MYSQL_BIN_LOG::COND_queue_busy", 0}, { &key_RELAYLOG_update_cond, "MYSQL_RELAY_LOG::update_cond", 0}, { &key_RELAYLOG_COND_queue_busy, "MYSQL_RELAY_LOG::COND_queue_busy", 0}, { &key_COND_wakeup_ready, "THD::COND_wakeup_ready", 0}, + { &key_COND_wait_commit, "wait_for_commit::COND_wait_commit", 0}, { &key_COND_cache_status_changed, "Query_cache::COND_cache_status_changed", 0}, { &key_COND_manager, "COND_manager", PSI_FLAG_GLOBAL}, { &key_COND_rpl_status, "COND_rpl_status", PSI_FLAG_GLOBAL}, @@ -897,17 +934,23 @@ static PSI_cond_info all_server_conds[]= { &key_relay_log_info_log_space_cond, "Relay_log_info::log_space_cond", 0}, { &key_relay_log_info_start_cond, "Relay_log_info::start_cond", 0}, { &key_relay_log_info_stop_cond, "Relay_log_info::stop_cond", 0}, - { &key_relay_log_info_sleep_cond, "Relay_log_info::sleep_cond", 0}, + { &key_rpl_group_info_sleep_cond, "Rpl_group_info::sleep_cond", 0}, { &key_TABLE_SHARE_cond, "TABLE_SHARE::cond", 0}, { &key_user_level_lock_cond, "User_level_lock::cond", 0}, { &key_COND_thread_count, "COND_thread_count", PSI_FLAG_GLOBAL}, { &key_COND_thread_cache, "COND_thread_cache", PSI_FLAG_GLOBAL}, - { &key_COND_flush_thread_cache, "COND_flush_thread_cache", PSI_FLAG_GLOBAL} + { &key_COND_flush_thread_cache, "COND_flush_thread_cache", PSI_FLAG_GLOBAL}, + { &key_COND_rpl_thread, "COND_rpl_thread", 0}, + { &key_COND_rpl_thread_pool, "COND_rpl_thread_pool", 0}, + { &key_COND_parallel_entry, "COND_parallel_entry", 0}, + { &key_COND_prepare_ordered, "COND_prepare_ordered", 0}, + { &key_COND_wait_gtid, "COND_wait_gtid", 0} }; PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert, key_thread_handle_manager, key_thread_main, - key_thread_one_connection, key_thread_signal_hand; + key_thread_one_connection, key_thread_signal_hand, + key_thread_slave_init, key_rpl_parallel_thread; static PSI_thread_info all_server_threads[]= { @@ -932,7 +975,9 @@ static PSI_thread_info all_server_threads[]= { &key_thread_handle_manager, "manager", PSI_FLAG_GLOBAL}, { &key_thread_main, "main", PSI_FLAG_GLOBAL}, { &key_thread_one_connection, "one_connection", 0}, - { &key_thread_signal_hand, "signal_handler", PSI_FLAG_GLOBAL} + { &key_thread_signal_hand, "signal_handler", PSI_FLAG_GLOBAL}, + { &key_thread_slave_init, "slave_init", PSI_FLAG_GLOBAL}, + { &key_rpl_parallel_thread, "rpl_parallel_thread", 0} }; PSI_file_key key_file_binlog, key_file_binlog_index, key_file_casetest, @@ -944,6 +989,7 @@ PSI_file_key key_file_binlog, key_file_binlog_index, key_file_casetest, key_file_trg, key_file_trn, key_file_init; PSI_file_key key_file_query_log, key_file_slow_log; PSI_file_key key_file_relaylog, key_file_relaylog_index; +PSI_file_key key_file_binlog_state; static PSI_file_info all_server_files[]= { @@ -974,7 +1020,8 @@ static PSI_file_info all_server_files[]= { &key_file_tclog, "tclog", 0}, { &key_file_trg, "trigger_name", 0}, { &key_file_trn, "trigger", 0}, - { &key_file_init, "init", 0} + { &key_file_init, "init", 0}, + { &key_file_binlog_state, "binlog_state", 0} }; /** @@ -1109,7 +1156,7 @@ private: void Buffered_logs::init() { - init_alloc_root(&m_root, 1024, 0); + init_alloc_root(&m_root, 1024, 0, MYF(0)); } void Buffered_logs::cleanup() @@ -1267,6 +1314,9 @@ struct st_VioSSLFd *ssl_acceptor_fd; */ uint connection_count= 0, extra_connection_count= 0; +my_bool opt_gtid_strict_mode= FALSE; + + /* Function declarations */ pthread_handler_t signal_hand(void *arg); @@ -1461,8 +1511,21 @@ static void close_connections(void) Events::deinit(); end_slave(); - /* Give threads time to die. */ - for (int i= 0; thread_count && i < 100; i++) + /* + Give threads time to die. + + In 5.5, this was waiting 100 rounds @ 20 milliseconds/round, so as little + as 2 seconds, depending on thread scheduling. + + From 10.0, we increase this to 1000 rounds / 20 seconds. The rationale is + that on a server with heavy I/O load, it is quite possible for eg. an + fsync() of the binlog or whatever to cause something like LOCK_log to be + held for more than 2 seconds. We do not want to force kill threads in + such cases, if it can be avoided. Note that normally, the wait will be + much smaller than even 2 seconds, this is only a safety fallback against + stuck threads so server shutdown is not held up forever. + */ + for (int i= 0; *(volatile int32*) &thread_count && i < 1000; i++) my_sleep(20000); /* @@ -1756,11 +1819,14 @@ extern "C" void unireg_abort(int exit_code) static void mysqld_exit(int exit_code) { + DBUG_ENTER("mysqld_exit"); /* Important note: we wait for the signal thread to end, but if a kill -15 signal was sent, the signal thread did spawn the kill_server_thread thread, which is running concurrently. */ + rpl_deinit_gtid_waiting(); + rpl_deinit_gtid_slave_state(); wait_for_signal_thread_to_end(); mysql_audit_finalize(); clean_up_mutexes(); @@ -1768,6 +1834,7 @@ static void mysqld_exit(int exit_code) my_end((opt_endinfo ? MY_CHECK_ERROR | MY_GIVE_INFO : 0)); #ifdef WITH_PERFSCHEMA_STORAGE_ENGINE shutdown_performance_schema(); // we do it as late as possible + //DBUG_LEAVE; #endif exit(exit_code); /* purecov: inspected */ } @@ -1780,7 +1847,12 @@ void clean_up(bool print_message) if (cleanup_done++) return; /* purecov: inspected */ - close_active_mi(); +#ifdef HAVE_REPLICATION + // We must call end_slave() as clean_up may have been called during startup + end_slave(); + if (use_slave_mask) + bitmap_free(&slave_error_mask); +#endif stop_handle_manager(); release_ddl_log(); @@ -1795,21 +1867,17 @@ void clean_up(bool print_message) injector::free_instance(); mysql_bin_log.cleanup(); -#ifdef HAVE_REPLICATION - if (use_slave_mask) - bitmap_free(&slave_error_mask); -#endif my_tz_free(); my_dboptions_cache_free(); ignore_db_dirs_free(); -#ifndef NO_EMBEDDED_ACCESS_CHECKS servers_free(1); +#ifndef NO_EMBEDDED_ACCESS_CHECKS acl_free(1); grant_free(); #endif query_cache_destroy(); hostname_cache_free(); - item_user_lock_free(); + item_func_sleep_free(); lex_free(); /* Free some memory */ item_create_cleanup(); if (!opt_noacl) @@ -1849,12 +1917,11 @@ void clean_up(bool print_message) #endif my_uuid_end(); delete binlog_filter; - delete rpl_filter; + delete global_rpl_filter; end_ssl(); #ifndef EMBEDDED_LIBRARY vio_end(); #endif /*!EMBEDDED_LIBRARY*/ - my_regex_end(); #if defined(ENABLED_DEBUG_SYNC) /* End the debug sync facility. See debug_sync.cc. */ debug_sync_end(); @@ -1875,6 +1942,9 @@ void clean_up(bool print_message) sys_var_end(); my_atomic_rwlock_destroy(&global_query_id_lock); my_atomic_rwlock_destroy(&thread_running_lock); + my_atomic_rwlock_destroy(&thread_count_lock); + my_atomic_rwlock_destroy(&statistics_lock); + my_atomic_rwlock_destroy(&slave_executed_entries_lock); free_charsets(); mysql_mutex_lock(&LOCK_thread_count); DBUG_PRINT("quit", ("got thread count lock")); @@ -1918,6 +1988,7 @@ static void clean_up_mutexes() DBUG_ENTER("clean_up_mutexes"); mysql_rwlock_destroy(&LOCK_grant); mysql_mutex_destroy(&LOCK_thread_count); + mysql_mutex_destroy(&LOCK_thread_cache); mysql_mutex_destroy(&LOCK_status); mysql_mutex_destroy(&LOCK_delayed_insert); mysql_mutex_destroy(&LOCK_delayed_status); @@ -1955,6 +2026,7 @@ static void clean_up_mutexes() mysql_mutex_destroy(&LOCK_server_started); mysql_cond_destroy(&COND_server_started); mysql_mutex_destroy(&LOCK_prepare_ordered); + mysql_cond_destroy(&COND_prepare_ordered); mysql_mutex_destroy(&LOCK_commit_ordered); DBUG_VOID_RETURN; } @@ -2418,7 +2490,7 @@ void close_connection(THD *thd, uint sql_errno) { sleep(0); /* Workaround to avoid tailcall optimisation */ } - MYSQL_AUDIT_NOTIFY_CONNECTION_DISCONNECT(thd, sql_errno); + mysql_audit_notify_connection_disconnect(thd, sql_errno); DBUG_VOID_RETURN; } #endif /* EMBEDDED_LIBRARY */ @@ -2465,6 +2537,28 @@ void dec_connection_count(THD *thd) /* + Delete THD and decrement thread counters, including thread_running +*/ + +void delete_running_thd(THD *thd) +{ + mysql_mutex_lock(&LOCK_thread_count); + thd->unlink(); + mysql_mutex_unlock(&LOCK_thread_count); + + delete thd; + dec_thread_running(); + thread_safe_decrement32(&thread_count, &thread_count_lock); + if (!thread_count) + { + mysql_mutex_lock(&LOCK_thread_count); + mysql_cond_broadcast(&COND_thread_count); + mysql_mutex_unlock(&LOCK_thread_count); + } +} + + +/* Unlink thd from global list of available connections and free thd SYNOPSIS @@ -2488,7 +2582,6 @@ void unlink_thd(THD *thd) mysql_mutex_unlock(&LOCK_status); mysql_mutex_lock(&LOCK_thread_count); - thread_count--; thd->unlink(); /* Used by binlog_reset_master. It would be cleaner to use @@ -2499,6 +2592,8 @@ void unlink_thd(THD *thd) mysql_mutex_unlock(&LOCK_thread_count); delete thd; + thread_safe_decrement32(&thread_count, &thread_count_lock); + DBUG_VOID_RETURN; } @@ -2510,7 +2605,7 @@ void unlink_thd(THD *thd) cache_thread() NOTES - LOCK_thread_count has to be locked + LOCK_thread_cache is used to protect the cache variables RETURN 0 Thread was not put in cache @@ -2521,7 +2616,9 @@ void unlink_thd(THD *thd) static bool cache_thread() { - mysql_mutex_assert_owner(&LOCK_thread_count); + DBUG_ENTER("cache_thread"); + + mysql_mutex_lock(&LOCK_thread_cache); if (cached_thread_count < thread_cache_size && ! abort_loop && !kill_cached_threads) { @@ -2539,7 +2636,7 @@ static bool cache_thread() #endif while (!abort_loop && ! wake_thread && ! kill_cached_threads) - mysql_cond_wait(&COND_thread_cache, &LOCK_thread_count); + mysql_cond_wait(&COND_thread_cache, &LOCK_thread_cache); cached_thread_count--; if (kill_cached_threads) mysql_cond_signal(&COND_flush_thread_cache); @@ -2548,6 +2645,8 @@ static bool cache_thread() THD *thd; wake_thread--; thd= thread_cache.get(); + mysql_mutex_unlock(&LOCK_thread_cache); + thd->thread_stack= (char*) &thd; // For store_globals (void) thd->store_globals(); @@ -2573,11 +2672,16 @@ static bool cache_thread() thd->mysys_var->abort= 0; thd->thr_create_utime= microsecond_interval_timer(); thd->start_utime= thd->thr_create_utime; + + /* Link thd into list of all active threads (THD's) */ + mysql_mutex_lock(&LOCK_thread_count); threads.append(thd); - return(1); + mysql_mutex_unlock(&LOCK_thread_count); + DBUG_RETURN(1); } } - return(0); + mysql_mutex_unlock(&LOCK_thread_cache); + DBUG_RETURN(0); } @@ -2605,24 +2709,28 @@ bool one_thread_per_connection_end(THD *thd, bool put_in_cache) DBUG_ENTER("one_thread_per_connection_end"); unlink_thd(thd); /* Mark that current_thd is not valid anymore */ - my_pthread_setspecific_ptr(THR_THD, 0); - if (put_in_cache) + set_current_thd(0); + if (put_in_cache && cache_thread()) + DBUG_RETURN(0); // Thread is reused + + /* + It's safe to check for thread_count outside of the mutex + as we are only interested to see if it was counted to 0 by the + above unlink_thd() call. We should only signal COND_thread_count if + thread_count is likely to be 0. (false positives are ok) + */ + if (!thread_count) { mysql_mutex_lock(&LOCK_thread_count); - put_in_cache= cache_thread(); + DBUG_PRINT("signal", ("Broadcasting COND_thread_count")); + mysql_cond_broadcast(&COND_thread_count); mysql_mutex_unlock(&LOCK_thread_count); - if (put_in_cache) - DBUG_RETURN(0); // Thread is reused } - - /* It's safe to broadcast outside a lock (COND... is not deleted here) */ - DBUG_PRINT("signal", ("Broadcasting COND_thread_count")); DBUG_LEAVE; // Must match DBUG_ENTER() #if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY) ERR_remove_state(0); #endif my_thread_end(); - mysql_cond_broadcast(&COND_thread_count); pthread_exit(0); return 0; // Avoid compiler warnings @@ -2631,15 +2739,17 @@ bool one_thread_per_connection_end(THD *thd, bool put_in_cache) void flush_thread_cache() { - mysql_mutex_lock(&LOCK_thread_count); + DBUG_ENTER("flush_thread_cache"); + mysql_mutex_lock(&LOCK_thread_cache); kill_cached_threads++; while (cached_thread_count) { mysql_cond_broadcast(&COND_thread_cache); - mysql_cond_wait(&COND_flush_thread_cache, &LOCK_thread_count); + mysql_cond_wait(&COND_flush_thread_cache, &LOCK_thread_cache); } kill_cached_threads--; - mysql_mutex_unlock(&LOCK_thread_count); + mysql_mutex_unlock(&LOCK_thread_cache); + DBUG_VOID_RETURN; } @@ -3158,9 +3268,9 @@ void my_message_sql(uint error, const char *str, myf MyFlags) THD *thd= current_thd; MYSQL_ERROR::enum_warning_level level; sql_print_message_func func; - DBUG_ENTER("my_message_sql"); - DBUG_PRINT("error", ("error: %u message: '%s' Flag: %d", error, str, MyFlags)); + DBUG_PRINT("error", ("error: %u message: '%s' Flag: %lu", error, str, + MyFlags)); DBUG_ASSERT(str != NULL); DBUG_ASSERT(error != 0); @@ -3199,7 +3309,6 @@ void my_message_sql(uint error, const char *str, myf MyFlags) } -#ifndef EMBEDDED_LIBRARY extern "C" void *my_str_malloc_mysqld(size_t size); extern "C" void my_str_free_mysqld(void *ptr); @@ -3213,7 +3322,6 @@ void my_str_free_mysqld(void *ptr) { my_free(ptr); } -#endif /* EMBEDDED_LIBRARY */ #ifdef __WIN__ @@ -3245,24 +3353,56 @@ sizeof(load_default_groups)/sizeof(load_default_groups[0]); /** This function is used to check for stack overrun for pathological cases of regular expressions and 'like' expressions. - The call to current_thd is quite expensive, so we try to avoid it - for the normal cases. +*/ +extern "C" int +check_enough_stack_size_slow() +{ + uchar stack_top; + THD *my_thd= current_thd; + if (my_thd != NULL) + return check_stack_overrun(my_thd, STACK_MIN_SIZE * 2, &stack_top); + return 0; +} + + +/* + The call to current_thd in check_enough_stack_size_slow is quite expensive, + so we try to avoid it for the normal cases. The size of each stack frame for the wildcmp() routines is ~128 bytes, so checking *every* recursive call is not necessary. */ extern "C" int check_enough_stack_size(int recurse_level) { - uchar stack_top; if (recurse_level % 16 != 0) return 0; + return check_enough_stack_size_slow(); +} +#endif - THD *my_thd= current_thd; - if (my_thd != NULL) - return check_stack_overrun(my_thd, STACK_MIN_SIZE * 2, &stack_top); - return 0; + + +/* + Initialize my_str_malloc() and my_str_free() +*/ +static void init_libstrings() +{ + my_str_malloc= &my_str_malloc_mysqld; + my_str_free= &my_str_free_mysqld; +#ifndef EMBEDDED_LIBRARY + my_string_stack_guard= check_enough_stack_size; +#endif } + + +static void init_pcre() +{ + pcre_malloc= pcre_stack_malloc= my_str_malloc_mysqld; + pcre_free= pcre_stack_free= my_str_free_mysqld; +#ifndef EMBEDDED_LIBRARY + pcre_stack_guard= check_enough_stack_size_slow; #endif +} /** @@ -3320,6 +3460,7 @@ SHOW_VAR com_status_vars[]= { {"create_function", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_CREATE_SPFUNCTION]), SHOW_LONG_STATUS}, {"create_index", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_CREATE_INDEX]), SHOW_LONG_STATUS}, {"create_procedure", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_CREATE_PROCEDURE]), SHOW_LONG_STATUS}, + {"create_role", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_CREATE_ROLE]), SHOW_LONG_STATUS}, {"create_server", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_CREATE_SERVER]), SHOW_LONG_STATUS}, {"create_table", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_CREATE_TABLE]), SHOW_LONG_STATUS}, {"create_trigger", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_CREATE_TRIGGER]), SHOW_LONG_STATUS}, @@ -3335,6 +3476,7 @@ SHOW_VAR com_status_vars[]= { {"drop_function", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_DROP_FUNCTION]), SHOW_LONG_STATUS}, {"drop_index", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_DROP_INDEX]), SHOW_LONG_STATUS}, {"drop_procedure", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_DROP_PROCEDURE]), SHOW_LONG_STATUS}, + {"drop_role", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_DROP_ROLE]), SHOW_LONG_STATUS}, {"drop_server", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_DROP_SERVER]), SHOW_LONG_STATUS}, {"drop_table", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_DROP_TABLE]), SHOW_LONG_STATUS}, {"drop_trigger", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_DROP_TRIGGER]), SHOW_LONG_STATUS}, @@ -3344,6 +3486,7 @@ SHOW_VAR com_status_vars[]= { {"execute_sql", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_EXECUTE]), SHOW_LONG_STATUS}, {"flush", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_FLUSH]), SHOW_LONG_STATUS}, {"grant", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_GRANT]), SHOW_LONG_STATUS}, + {"grant_role", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_GRANT_ROLE]), SHOW_LONG_STATUS}, {"ha_close", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_HA_CLOSE]), SHOW_LONG_STATUS}, {"ha_open", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_HA_OPEN]), SHOW_LONG_STATUS}, {"ha_read", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_HA_READ]), SHOW_LONG_STATUS}, @@ -3369,6 +3512,7 @@ SHOW_VAR com_status_vars[]= { {"resignal", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_RESIGNAL]), SHOW_LONG_STATUS}, {"revoke", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_REVOKE]), SHOW_LONG_STATUS}, {"revoke_all", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_REVOKE_ALL]), SHOW_LONG_STATUS}, + {"revoke_role", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_REVOKE_ROLE]), SHOW_LONG_STATUS}, {"rollback", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_ROLLBACK]), SHOW_LONG_STATUS}, {"rollback_to_savepoint",(char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_ROLLBACK_TO_SAVEPOINT]), SHOW_LONG_STATUS}, {"savepoint", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SAVEPOINT]), SHOW_LONG_STATUS}, @@ -3393,6 +3537,7 @@ SHOW_VAR com_status_vars[]= { {"show_engine_status", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHOW_ENGINE_STATUS]), SHOW_LONG_STATUS}, {"show_errors", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHOW_ERRORS]), SHOW_LONG_STATUS}, {"show_events", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHOW_EVENTS]), SHOW_LONG_STATUS}, + {"show_explain", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHOW_EXPLAIN]), SHOW_LONG_STATUS}, {"show_fields", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHOW_FIELDS]), SHOW_LONG_STATUS}, #ifndef DBUG_OFF {"show_function_code", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHOW_FUNC_CODE]), SHOW_LONG_STATUS}, @@ -3424,9 +3569,10 @@ SHOW_VAR com_status_vars[]= { {"show_user_statistics", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHOW_USER_STATS]), SHOW_LONG_STATUS}, {"show_variables", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHOW_VARIABLES]), SHOW_LONG_STATUS}, {"show_warnings", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHOW_WARNS]), SHOW_LONG_STATUS}, + {"shutdown", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SHUTDOWN]), SHOW_LONG_STATUS}, {"signal", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SIGNAL]), SHOW_LONG_STATUS}, - {"slave_start", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SLAVE_START]), SHOW_LONG_STATUS}, - {"slave_stop", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SLAVE_STOP]), SHOW_LONG_STATUS}, + {"start_all_slaves", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SLAVE_ALL_START]), SHOW_LONG_STATUS}, + {"start_slave", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SLAVE_START]), SHOW_LONG_STATUS}, {"stmt_close", (char*) offsetof(STATUS_VAR, com_stmt_close), SHOW_LONG_STATUS}, {"stmt_execute", (char*) offsetof(STATUS_VAR, com_stmt_execute), SHOW_LONG_STATUS}, {"stmt_fetch", (char*) offsetof(STATUS_VAR, com_stmt_fetch), SHOW_LONG_STATUS}, @@ -3434,6 +3580,8 @@ SHOW_VAR com_status_vars[]= { {"stmt_reprepare", (char*) offsetof(STATUS_VAR, com_stmt_reprepare), SHOW_LONG_STATUS}, {"stmt_reset", (char*) offsetof(STATUS_VAR, com_stmt_reset), SHOW_LONG_STATUS}, {"stmt_send_long_data", (char*) offsetof(STATUS_VAR, com_stmt_send_long_data), SHOW_LONG_STATUS}, + {"stop_all_slaves", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SLAVE_ALL_STOP]), SHOW_LONG_STATUS}, + {"stop_slave", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_SLAVE_STOP]), SHOW_LONG_STATUS}, {"truncate", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_TRUNCATE]), SHOW_LONG_STATUS}, {"uninstall_plugin", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_UNINSTALL_PLUGIN]), SHOW_LONG_STATUS}, {"unlock_tables", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_UNLOCK_TABLES]), SHOW_LONG_STATUS}, @@ -3448,21 +3596,89 @@ SHOW_VAR com_status_vars[]= { {NullS, NullS, SHOW_LONG} }; +#ifdef SAFEMALLOC +/* + Return the id for the current THD, to allow safemalloc to associate + the memory with the right id. +*/ + +extern "C" my_thread_id mariadb_dbug_id() +{ + THD *thd; + if ((thd= current_thd)) + { + return thd->thread_id; + } + return my_thread_dbug_id(); +} +#endif /* SAFEMALLOC */ + +/* Thread Mem Usage By P.Linux */ +extern "C" { +static void my_malloc_size_cb_func(long long size, my_bool is_thread_specific) +{ + /* If thread specific memory */ + if (is_thread_specific) + { + THD *thd= current_thd; + if (mysqld_server_initialized || thd) + { + /* + THD may not be set if we are called from my_net_init() before THD + thread has started. + However, this should never happen, so better to assert and + fix this. + */ + DBUG_ASSERT(thd); + if (thd) + { + DBUG_PRINT("info", ("memory_used: %lld size: %lld", + (longlong) thd->status_var.memory_used, size)); + thd->status_var.memory_used+= size; + DBUG_ASSERT((longlong) thd->status_var.memory_used >= 0); + } + } + } + // workaround for gcc 4.2.4-1ubuntu4 -fPIE (from DEB_BUILD_HARDENING=1) + int64 volatile * volatile ptr=&global_status_var.memory_used; + my_atomic_add64(ptr, size); +} +} + + +/* + Init common variables +*/ + static int init_common_variables() { umask(((~my_umask) & 0666)); my_decimal_set_zero(&decimal_zero); // set decimal_zero constant; + if (pthread_key_create(&THR_THD,NULL) || + pthread_key_create(&THR_MALLOC,NULL)) + { + sql_print_error("Can't create thread-keys"); + return 1; + } + + set_current_thd(0); + set_malloc_size_cb(my_malloc_size_cb_func); + + init_libstrings(); tzset(); // Set tzname sf_leaking_memory= 0; // no memory leaks from now on +#ifdef SAFEMALLOC + sf_malloc_dbug_id= mariadb_dbug_id; +#endif max_system_variables.pseudo_thread_id= (ulong)~0; server_start_time= flush_status_time= my_time(0); - rpl_filter= new Rpl_filter; + global_rpl_filter= new Rpl_filter; binlog_filter= new Rpl_filter; - if (!rpl_filter || !binlog_filter) + if (!global_rpl_filter || !binlog_filter) { sql_perror("Could not allocate replication and binlog filters"); return 1; @@ -3752,12 +3968,7 @@ static int init_common_variables() if (item_create_init()) return 1; item_init(); -#ifndef EMBEDDED_LIBRARY - my_regex_init(&my_charset_latin1, check_enough_stack_size); - my_string_stack_guard= check_enough_stack_size; -#else - my_regex_init(&my_charset_latin1, NULL); -#endif + init_pcre(); /* Process a comma-separated character set list and choose the first available character set. This is mostly for @@ -3916,7 +4127,9 @@ You should consider changing lower_case_table_names to 1 or 2", static int init_thread_environment() { + DBUG_ENTER("init_thread_environment"); mysql_mutex_init(key_LOCK_thread_count, &LOCK_thread_count, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_LOCK_thread_cache, &LOCK_thread_cache, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_status, &LOCK_status, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_delayed_insert, &LOCK_delayed_insert, MY_MUTEX_INIT_FAST); @@ -3929,6 +4142,7 @@ static int init_thread_environment() mysql_mutex_init(key_LOCK_active_mi, &LOCK_active_mi, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_global_system_variables, &LOCK_global_system_variables, MY_MUTEX_INIT_FAST); + mysql_mutex_record_order(&LOCK_active_mi, &LOCK_global_system_variables); mysql_rwlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash); mysql_mutex_init(key_LOCK_prepared_stmt_count, @@ -3948,6 +4162,7 @@ static int init_thread_environment() &LOCK_global_index_stats, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_prepare_ordered, &LOCK_prepare_ordered, MY_MUTEX_INIT_SLOW); + mysql_cond_init(key_COND_prepare_ordered, &COND_prepare_ordered, NULL); mysql_mutex_init(key_LOCK_commit_ordered, &LOCK_commit_ordered, MY_MUTEX_INIT_SLOW); @@ -3983,19 +4198,19 @@ static int init_thread_environment() #ifdef HAVE_EVENT_SCHEDULER Events::init_mutexes(); #endif + init_show_explain_psi_keys(); /* Parameter for threads created for connections */ (void) pthread_attr_init(&connection_attrib); (void) pthread_attr_setdetachstate(&connection_attrib, PTHREAD_CREATE_DETACHED); pthread_attr_setscope(&connection_attrib, PTHREAD_SCOPE_SYSTEM); - if (pthread_key_create(&THR_THD,NULL) || - pthread_key_create(&THR_MALLOC,NULL)) - { - sql_print_error("Can't create thread-keys"); - return 1; - } - return 0; +#ifdef HAVE_REPLICATION + rpl_init_gtid_slave_state(); + rpl_init_gtid_waiting(); +#endif + + DBUG_RETURN(0); } @@ -4430,7 +4645,7 @@ a file name for --log-bin-index option", opt_binlog_index_name); plugin_ref plugin; handlerton *hton; if ((plugin= ha_resolve_by_name(0, &name))) - hton= plugin_data(plugin, handlerton*); + hton= plugin_hton(plugin); else { sql_print_error("Unknown/unsupported storage engine: %s", @@ -4488,7 +4703,7 @@ a file name for --log-bin-index option", opt_binlog_index_name); } if (opt_bin_log && mysql_bin_log.open(opt_bin_logname, LOG_BIN, 0, - WRITE_CACHE, 0, max_binlog_size, 0, TRUE)) + WRITE_CACHE, max_binlog_size, 0, TRUE)) unireg_abort(1); #ifdef HAVE_REPLICATION @@ -4530,6 +4745,8 @@ a file name for --log-bin-index option", opt_binlog_index_name); init_update_queries(); init_global_user_stats(); init_global_client_stats(); + if (!opt_bootstrap) + servers_init(0); DBUG_RETURN(0); } @@ -4705,6 +4922,7 @@ static void test_lc_time_sz() } #endif//DBUG_OFF + #ifdef __WIN__ int win_main(int argc, char **argv) #else @@ -4717,6 +4935,8 @@ int mysqld_main(int argc, char **argv) */ my_progname= argv[0]; sf_leaking_memory= 1; // no safemalloc memory leak reports if we exit early + mysqld_server_started= mysqld_server_initialized= 0; + #ifdef HAVE_NPTL ld_assume_kernel_is_set= (getenv("LD_ASSUME_KERNEL") != 0); #endif @@ -4759,7 +4979,7 @@ int mysqld_main(int argc, char **argv) my_getopt_skip_unknown= TRUE; /* prepare all_early_options array */ - my_init_dynamic_array(&all_early_options, sizeof(my_option), 100, 25); + my_init_dynamic_array(&all_early_options, sizeof(my_option), 100, 25, MYF(0)); sys_var_add_options(&all_early_options, sys_var::PARSE_EARLY); add_terminator(&all_early_options); @@ -4912,9 +5132,9 @@ int mysqld_main(int argc, char **argv) set_user(mysqld_user, user_info); } - if (opt_bin_log && !server_id) + if (opt_bin_log && !global_system_variables.server_id) { - server_id= 1; + global_system_variables.server_id= ::server_id= 1; #ifdef EXTRA_DEBUG sql_print_warning("You have enabled the binary log, but you haven't set " "server-id to a non-zero value: we force server id to 1; " @@ -4948,12 +5168,6 @@ int mysqld_main(int argc, char **argv) #endif /* - Initialize my_str_malloc() and my_str_free() - */ - my_str_malloc= &my_str_malloc_mysqld; - my_str_free= &my_str_free_mysqld; - - /* init signals & alarm After this we can't quit by a simple unireg_abort */ @@ -4977,9 +5191,6 @@ int mysqld_main(int argc, char **argv) if (!opt_noacl) (void) grant_init(); - if (!opt_bootstrap) - servers_init(0); - if (!opt_noacl) { #ifdef HAVE_DLOPEN @@ -4992,16 +5203,6 @@ int mysqld_main(int argc, char **argv) opt_skip_slave_start= 1; binlog_unsafe_map_init(); - /* - init_slave() must be called after the thread keys are created. - Some parts of the code (e.g. SHOW STATUS LIKE 'slave_running' and other - places) assume that active_mi != 0, so let's fail if it's 0 (out of - memory); a message has already been printed. - */ - if (init_slave() && !active_mi) - { - unireg_abort(1); - } #ifdef WITH_PERFSCHEMA_STORAGE_ENGINE initialize_performance_schema_acl(opt_bootstrap); @@ -5019,6 +5220,12 @@ int mysqld_main(int argc, char **argv) execute_ddl_log_recovery(); + /* + We must have LOCK_open before LOCK_global_system_variables because + LOCK_open is held while sql_plugin.c::intern_sys_var_ptr() is called. + */ + mysql_mutex_record_order(&LOCK_open, &LOCK_global_system_variables); + if (Events::init(opt_noacl || opt_bootstrap)) unireg_abort(1); @@ -5034,21 +5241,33 @@ int mysqld_main(int argc, char **argv) exit(0); } } + + /* It's now safe to use thread specific memory */ + mysqld_server_initialized= 1; + + create_shutdown_thread(); + start_handle_manager(); + + /* Copy default global rpl_filter to global_rpl_filter */ + copy_filter_setting(global_rpl_filter, get_or_create_rpl_filter("", 0)); + + /* + init_slave() must be called after the thread keys are created. + Some parts of the code (e.g. SHOW STATUS LIKE 'slave_running' and other + places) assume that active_mi != 0, so let's fail if it's 0 (out of + memory); a message has already been printed. + */ + if (init_slave() && !active_mi) + { + unireg_abort(1); + } + if (opt_init_file && *opt_init_file) { if (read_init_file(opt_init_file)) unireg_abort(1); } - /* - We must have LOCK_open before LOCK_global_system_variables because - LOCK_open is hold while sql_plugin.c::intern_sys_var_ptr() is called. - */ - mysql_mutex_record_order(&LOCK_open, &LOCK_global_system_variables); - - create_shutdown_thread(); - start_handle_manager(); - sql_print_information(ER_DEFAULT(ER_STARTUP),my_progname,server_version, ((unix_sock == INVALID_SOCKET) ? (char*) "" : mysqld_unix_port), @@ -5058,7 +5277,6 @@ int mysqld_main(int argc, char **argv) Service.SetRunning(); #endif - /* Signal threads waiting for server to be started */ mysql_mutex_lock(&LOCK_server_started); mysqld_server_started= 1; @@ -5346,11 +5564,11 @@ static void bootstrap(MYSQL_FILE *file) THD *thd= new THD; thd->bootstrap=1; - my_net_init(&thd->net,(st_vio*) 0); + my_net_init(&thd->net,(st_vio*) 0, MYF(0)); thd->max_client_packet_length= thd->net.max_packet; thd->security_ctx->master_access= ~(ulong)0; thd->thread_id= thd->variables.pseudo_thread_id= thread_id++; - thread_count++; + thread_count++; // Safe as only one thread running in_bootstrap= TRUE; bootstrap_file=file; @@ -5434,55 +5652,69 @@ void handle_connection_in_main_thread(THD *thd) void create_thread_to_handle_connection(THD *thd) { + DBUG_ENTER("create_thread_to_handle_connection"); + mysql_mutex_assert_owner(&LOCK_thread_count); + + /* Check if we can get thread from the cache */ if (cached_thread_count > wake_thread) { - /* Get thread from cache */ - thread_cache.push_back(thd); - wake_thread++; - mysql_cond_signal(&COND_thread_cache); + mysql_mutex_lock(&LOCK_thread_cache); + /* Recheck condition when we have the lock */ + if (cached_thread_count > wake_thread) + { + mysql_mutex_unlock(&LOCK_thread_count); + /* Get thread from cache */ + thread_cache.push_back(thd); + wake_thread++; + mysql_cond_signal(&COND_thread_cache); + mysql_mutex_unlock(&LOCK_thread_cache); + DBUG_PRINT("info",("Thread created")); + DBUG_VOID_RETURN; + } + mysql_mutex_unlock(&LOCK_thread_cache); } - else + + char error_message_buff[MYSQL_ERRMSG_SIZE]; + /* Create new thread to handle connection */ + int error; + thread_created++; + threads.append(thd); + DBUG_PRINT("info",(("creating thread %lu"), thd->thread_id)); + thd->prior_thr_create_utime= microsecond_interval_timer(); + if ((error= mysql_thread_create(key_thread_one_connection, + &thd->real_id, &connection_attrib, + handle_one_connection, + (void*) thd))) { - char error_message_buff[MYSQL_ERRMSG_SIZE]; - /* Create new thread to handle connection */ - int error; - thread_created++; - threads.append(thd); - DBUG_PRINT("info",(("creating thread %lu"), thd->thread_id)); - thd->prior_thr_create_utime= microsecond_interval_timer(); - if ((error= mysql_thread_create(key_thread_one_connection, - &thd->real_id, &connection_attrib, - handle_one_connection, - (void*) thd))) - { - /* purecov: begin inspected */ - DBUG_PRINT("error", - ("Can't create thread to handle request (error %d)", - error)); + /* purecov: begin inspected */ + DBUG_PRINT("error", + ("Can't create thread to handle request (error %d)", + error)); + thd->killed= KILL_CONNECTION; // Safety + mysql_mutex_unlock(&LOCK_thread_count); - thread_count--; - thd->killed= KILL_CONNECTION; // Safety - mysql_mutex_unlock(&LOCK_thread_count); + mysql_mutex_lock(&LOCK_connection_count); + (*thd->scheduler->connection_count)--; + mysql_mutex_unlock(&LOCK_connection_count); - mysql_mutex_lock(&LOCK_connection_count); - (*thd->scheduler->connection_count)--; - mysql_mutex_unlock(&LOCK_connection_count); + statistic_increment(aborted_connects,&LOCK_status); + /* Can't use my_error() since store_globals has not been called. */ + my_snprintf(error_message_buff, sizeof(error_message_buff), + ER_THD(thd, ER_CANT_CREATE_THREAD), error); + net_send_error(thd, ER_CANT_CREATE_THREAD, error_message_buff, NULL); + close_connection(thd, ER_OUT_OF_RESOURCES); - statistic_increment(aborted_connects,&LOCK_status); - /* Can't use my_error() since store_globals has not been called. */ - my_snprintf(error_message_buff, sizeof(error_message_buff), - ER_THD(thd, ER_CANT_CREATE_THREAD), error); - net_send_error(thd, ER_CANT_CREATE_THREAD, error_message_buff, NULL); - close_connection(thd, ER_OUT_OF_RESOURCES); - mysql_mutex_lock(&LOCK_thread_count); - delete thd; - mysql_mutex_unlock(&LOCK_thread_count); - return; - /* purecov: end */ - } + mysql_mutex_lock(&LOCK_thread_count); + thd->unlink(); + mysql_mutex_unlock(&LOCK_thread_count); + delete thd; + thread_safe_decrement32(&thread_count, &thread_count_lock); + return; + /* purecov: end */ } mysql_mutex_unlock(&LOCK_thread_count); DBUG_PRINT("info",("Thread created")); + DBUG_VOID_RETURN; } @@ -5529,10 +5761,10 @@ static void create_new_thread(THD *thd) mysql_mutex_unlock(&LOCK_connection_count); - /* Start a new thread to handle connection. */ + thread_safe_increment32(&thread_count, &thread_count_lock); + /* Start a new thread to handle connection. */ mysql_mutex_lock(&LOCK_thread_count); - /* The initialization of thread_id is done in create_embedded_thd() for the embedded library. @@ -5540,8 +5772,6 @@ static void create_new_thread(THD *thd) */ thd->thread_id= thd->variables.pseudo_thread_id= thread_id++; - thread_count++; - MYSQL_CALLBACK(thd->scheduler, add_connection, (thd)); DBUG_VOID_RETURN; @@ -5749,35 +5979,24 @@ void handle_connections_sockets() } #endif /* HAVE_LIBWRAP */ - { - size_socket dummyLen; - struct sockaddr_storage dummy; - dummyLen = sizeof(dummy); - if ( getsockname(new_sock,(struct sockaddr *)&dummy, - (SOCKET_SIZE_TYPE *)&dummyLen) < 0 ) - { - sql_perror("Error on new connection socket"); - (void) mysql_socket_shutdown(new_sock, SHUT_RDWR); - (void) closesocket(new_sock); - continue; - } - } - /* ** Don't allow too many connections */ + DBUG_PRINT("info", ("Creating THD for new connection")); if (!(thd= new THD)) { (void) mysql_socket_shutdown(new_sock, SHUT_RDWR); (void) closesocket(new_sock); continue; } + /* Set to get io buffers to be part of THD */ + set_current_thd(thd); if (!(vio_tmp=vio_new(new_sock, sock == unix_sock ? VIO_TYPE_SOCKET : VIO_TYPE_TCPIP, sock == unix_sock ? VIO_LOCALHOST: 0)) || - my_net_init(&thd->net,vio_tmp)) + my_net_init(&thd->net, vio_tmp, MYF(MY_THREAD_SPECIFIC))) { /* Only delete the temporary vio if we didn't already attach it to the @@ -5792,6 +6011,7 @@ void handle_connections_sockets() (void) closesocket(new_sock); } delete thd; + set_current_thd(0); continue; } if (sock == unix_sock) @@ -5803,6 +6023,7 @@ void handle_connections_sockets() thd->scheduler= extra_thread_scheduler; } create_new_thread(thd); + set_current_thd(0); } DBUG_VOID_RETURN; } @@ -5896,16 +6117,19 @@ pthread_handler_t handle_connections_namedpipes(void *arg) CloseHandle(hConnectedPipe); continue; } + set_current_thd(thd); if (!(thd->net.vio= vio_new_win32pipe(hConnectedPipe)) || - my_net_init(&thd->net, thd->net.vio)) + my_net_init(&thd->net, thd->net.vio, MYF(MY_THREAD_SPECIFIC))) { close_connection(thd, ER_OUT_OF_RESOURCES); delete thd; + set_current_thd(0); continue; } /* Host is unknown */ thd->security_ctx->host= my_strdup(my_localhost, MYF(0)); create_new_thread(thd); + set_current_thd(0); } CloseHandle(connectOverlapped.hEvent); DBUG_LEAVE; @@ -6085,6 +6309,7 @@ pthread_handler_t handle_connections_shared_memory(void *arg) errmsg= "Could not set client to read mode"; goto errorconn; } + set_current_thd(thd); if (!(thd->net.vio= vio_new_win32shared_memory(handle_client_file_map, handle_client_map, event_client_wrote, @@ -6092,7 +6317,7 @@ pthread_handler_t handle_connections_shared_memory(void *arg) event_server_wrote, event_server_read, event_conn_closed)) || - my_net_init(&thd->net, thd->net.vio)) + my_net_init(&thd->net, thd->net.vio, MYF(MY_THREAD_SPECIFIC))) { close_connection(thd, ER_OUT_OF_RESOURCES); errmsg= 0; @@ -6101,6 +6326,7 @@ pthread_handler_t handle_connections_shared_memory(void *arg) thd->security_ctx->host= my_strdup(my_localhost, MYF(0)); /* Host is unknown */ create_new_thread(thd); connect_number++; + set_current_thd(thd); continue; errorconn: @@ -6128,6 +6354,7 @@ errorconn: CloseHandle(event_conn_closed); delete thd; } + set_current_thd(0); /* End shared memory handling */ error: @@ -6165,6 +6392,7 @@ error: */ struct my_option my_long_options[]= + { {"help", '?', "Display this help and exit.", &opt_help, &opt_help, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, @@ -6442,28 +6670,28 @@ struct my_option my_long_options[]= "while having selected a different or no database. If you need cross " "database updates to work, make sure you have 3.23.28 or later, and use " "replicate-wild-do-table=db_name.%.", - 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + 0, 0, 0, GET_STR | GET_ASK_ADDR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"replicate-do-table", OPT_REPLICATE_DO_TABLE, "Tells the slave thread to restrict replication to the specified table. " "To specify more than one table, use the directive multiple times, once " "for each table. This will work for cross-database updates, in contrast " - "to replicate-do-db.", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + "to replicate-do-db.", 0, 0, 0, GET_STR | GET_ASK_ADDR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"replicate-ignore-db", OPT_REPLICATE_IGNORE_DB, "Tells the slave thread to not replicate to the specified database. To " "specify more than one database to ignore, use the directive multiple " "times, once for each database. This option will not work if you use " "cross database updates. If you need cross database updates to work, " "make sure you have 3.23.28 or later, and use replicate-wild-ignore-" - "table=db_name.%. ", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + "table=db_name.%. ", 0, 0, 0, GET_STR | GET_ASK_ADDR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"replicate-ignore-table", OPT_REPLICATE_IGNORE_TABLE, "Tells the slave thread to not replicate to the specified table. To specify " "more than one table to ignore, use the directive multiple times, once for " "each table. This will work for cross-database updates, in contrast to " - "replicate-ignore-db.", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + "replicate-ignore-db.", 0, 0, 0, GET_STR | GET_ASK_ADDR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"replicate-rewrite-db", OPT_REPLICATE_REWRITE_DB, "Updates to a database with a different name than the original. Example: " "replicate-rewrite-db=master_db_name->slave_db_name.", - 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + 0, 0, 0, GET_STR | GET_ASK_ADDR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, #ifdef HAVE_REPLICATION {"replicate-same-server-id", 0, "In replication, if set to 1, do not skip events having our server id. " @@ -6479,7 +6707,7 @@ struct my_option my_long_options[]= "database updates. Example: replicate-wild-do-table=foo%.bar% will " "replicate only updates to tables in all databases that start with foo " "and whose table names start with bar.", - 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + 0, 0, 0, GET_STR | GET_ASK_ADDR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"replicate-wild-ignore-table", OPT_REPLICATE_WILD_IGNORE_TABLE, "Tells the slave thread to not replicate to the tables that match the " "given wildcard pattern. To specify more than one table to ignore, use " @@ -6487,7 +6715,7 @@ struct my_option my_long_options[]= "cross-database updates. Example: replicate-wild-ignore-table=foo%.bar% " "will not do updates to tables in databases that start with foo and whose " "table names start with bar.", - 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + 0, 0, 0, GET_STR | GET_ASK_ADDR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"safe-mode", OPT_SAFE, "Skip some optimize stages (for testing). Deprecated.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"safe-user-create", 0, @@ -6630,66 +6858,76 @@ static int show_rpl_status(THD *thd, SHOW_VAR *var, char *buff) static int show_slave_running(THD *thd, SHOW_VAR *var, char *buff) { + Master_info *mi; + bool tmp; + LINT_INIT(tmp); + var->type= SHOW_MY_BOOL; - mysql_mutex_lock(&LOCK_active_mi); var->value= buff; - *((my_bool *)buff)= (my_bool) (active_mi && - active_mi->slave_running == MYSQL_SLAVE_RUN_CONNECT && - active_mi->rli.slave_running); - mysql_mutex_unlock(&LOCK_active_mi); - return 0; -} - -static int show_slave_retried_trans(THD *thd, SHOW_VAR *var, char *buff) -{ - /* - TODO: with multimaster, have one such counter per line in - SHOW SLAVE STATUS, and have the sum over all lines here. - */ + mysql_mutex_unlock(&LOCK_status); mysql_mutex_lock(&LOCK_active_mi); - if (active_mi) - { - var->type= SHOW_LONG; - var->value= buff; - mysql_mutex_lock(&active_mi->rli.data_lock); - *((long *)buff)= (long)active_mi->rli.retried_trans; - mysql_mutex_unlock(&active_mi->rli.data_lock); - } + mi= master_info_index-> + get_master_info(&thd->variables.default_master_connection, + MYSQL_ERROR::WARN_LEVEL_NOTE); + if (mi) + tmp= (my_bool) (mi->slave_running == MYSQL_SLAVE_RUN_CONNECT && + mi->rli.slave_running); + mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_status); + if (mi) + *((my_bool *)buff)= tmp; else var->type= SHOW_UNDEF; - mysql_mutex_unlock(&LOCK_active_mi); return 0; } + static int show_slave_received_heartbeats(THD *thd, SHOW_VAR *var, char *buff) { + Master_info *mi; + longlong tmp; + LINT_INIT(tmp); + + var->type= SHOW_LONGLONG; + var->value= buff; + mysql_mutex_unlock(&LOCK_status); mysql_mutex_lock(&LOCK_active_mi); - if (active_mi) - { - var->type= SHOW_LONGLONG; - var->value= buff; - mysql_mutex_lock(&active_mi->rli.data_lock); - *((longlong *)buff)= active_mi->received_heartbeats; - mysql_mutex_unlock(&active_mi->rli.data_lock); - } + mi= master_info_index-> + get_master_info(&thd->variables.default_master_connection, + MYSQL_ERROR::WARN_LEVEL_NOTE); + if (mi) + tmp= mi->received_heartbeats; + mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_status); + if (mi) + *((longlong *)buff)= tmp; else var->type= SHOW_UNDEF; - mysql_mutex_unlock(&LOCK_active_mi); return 0; } + static int show_heartbeat_period(THD *thd, SHOW_VAR *var, char *buff) { + Master_info *mi; + float tmp; + LINT_INIT(tmp); + + var->type= SHOW_CHAR; + var->value= buff; + mysql_mutex_unlock(&LOCK_status); mysql_mutex_lock(&LOCK_active_mi); - if (active_mi) - { - var->type= SHOW_CHAR; - var->value= buff; - sprintf(buff, "%.3f", active_mi->heartbeat_period); - } + mi= master_info_index-> + get_master_info(&thd->variables.default_master_connection, + MYSQL_ERROR::WARN_LEVEL_NOTE); + if (mi) + tmp= mi->heartbeat_period; + mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_status); + if (mi) + sprintf(buff, "%.3f", tmp); else var->type= SHOW_UNDEF; - mysql_mutex_unlock(&LOCK_active_mi); return 0; } @@ -7020,6 +7258,42 @@ static int show_default_keycache(THD *thd, SHOW_VAR *var, char *buff) return 0; } +#ifndef DBUG_OFF +static int debug_status_func(THD *thd, SHOW_VAR *var, char *buff) +{ +#define add_var(X,Y,Z) \ + v->name= X; \ + v->value= (char*)Y; \ + v->type= Z; \ + v++; + + var->type= SHOW_ARRAY; + var->value= buff; + + SHOW_VAR *v= (SHOW_VAR *)buff; + + if (_db_keyword_(0, "role_merge_stats", 1)) + { + static SHOW_VAR roles[]= { + {"global", (char*) &role_global_merges, SHOW_ULONG}, + {"db", (char*) &role_db_merges, SHOW_ULONG}, + {"table", (char*) &role_table_merges, SHOW_ULONG}, + {"column", (char*) &role_column_merges, SHOW_ULONG}, + {"routine", (char*) &role_routine_merges, SHOW_ULONG}, + {NullS, NullS, SHOW_LONG} + }; + + add_var("role_merges", roles, SHOW_ARRAY); + } + + v->name= 0; + +#undef add_var + + return 0; +} +#endif + #ifdef HAVE_POOL_OF_THREADS int show_threadpool_idle_threads(THD *thd, SHOW_VAR *var, char *buff) { @@ -7047,12 +7321,15 @@ SHOW_VAR status_vars[]= { {"Bytes_received", (char*) offsetof(STATUS_VAR, bytes_received), SHOW_LONGLONG_STATUS}, {"Bytes_sent", (char*) offsetof(STATUS_VAR, bytes_sent), SHOW_LONGLONG_STATUS}, {"Com", (char*) com_status_vars, SHOW_ARRAY}, - {"Compression", (char*) &show_net_compression, SHOW_FUNC}, + {"Compression", (char*) &show_net_compression, SHOW_SIMPLE_FUNC}, {"Connections", (char*) &thread_id, SHOW_LONG_NOFLUSH}, {"Cpu_time", (char*) offsetof(STATUS_VAR, cpu_time), SHOW_DOUBLE_STATUS}, {"Created_tmp_disk_tables", (char*) offsetof(STATUS_VAR, created_tmp_disk_tables), SHOW_LONG_STATUS}, {"Created_tmp_files", (char*) &my_tmp_file_created, SHOW_LONG}, {"Created_tmp_tables", (char*) offsetof(STATUS_VAR, created_tmp_tables), SHOW_LONG_STATUS}, +#ifndef DBUG_OFF + {"Debug", (char*) &debug_status_func, SHOW_FUNC}, +#endif {"Delayed_errors", (char*) &delayed_insert_errors, SHOW_LONG}, {"Delayed_insert_threads", (char*) &delayed_insert_threads, SHOW_LONG_NOFLUSH}, {"Delayed_writes", (char*) &delayed_insert_writes, SHOW_LONG}, @@ -7065,8 +7342,8 @@ SHOW_VAR status_vars[]= { {"Feature_locale", (char*) offsetof(STATUS_VAR, feature_locale), SHOW_LONG_STATUS}, {"Feature_subquery", (char*) offsetof(STATUS_VAR, feature_subquery), SHOW_LONG_STATUS}, {"Feature_timezone", (char*) offsetof(STATUS_VAR, feature_timezone), SHOW_LONG_STATUS}, - {"Feature_trigger", (char*) offsetof(STATUS_VAR, feature_trigger), SHOW_LONG_STATUS}, - {"Feature_xml", (char*) offsetof(STATUS_VAR, feature_xml), SHOW_LONG_STATUS}, + {"Feature_trigger", (char*) offsetof(STATUS_VAR, feature_trigger), SHOW_LONG_STATUS}, + {"Feature_xml", (char*) offsetof(STATUS_VAR, feature_xml), SHOW_LONG_STATUS}, {"Flush_commands", (char*) &refresh_version, SHOW_LONG_NOFLUSH}, {"Handler_commit", (char*) offsetof(STATUS_VAR, ha_commit_count), SHOW_LONG_STATUS}, {"Handler_delete", (char*) offsetof(STATUS_VAR, ha_delete_count), SHOW_LONG_STATUS}, @@ -7074,8 +7351,8 @@ SHOW_VAR status_vars[]= { {"Handler_icp_attempts", (char*) offsetof(STATUS_VAR, ha_icp_attempts), SHOW_LONG_STATUS}, {"Handler_icp_match", (char*) offsetof(STATUS_VAR, ha_icp_match), SHOW_LONG_STATUS}, {"Handler_mrr_init", (char*) offsetof(STATUS_VAR, ha_mrr_init_count), SHOW_LONG_STATUS}, - {"Handler_mrr_key_refills", (char*) offsetof(STATUS_VAR, ha_mrr_key_refills_count), SHOW_LONG_STATUS}, - {"Handler_mrr_rowid_refills", (char*) offsetof(STATUS_VAR, ha_mrr_rowid_refills_count), SHOW_LONG_STATUS}, + {"Handler_mrr_key_refills", (char*) offsetof(STATUS_VAR, ha_mrr_key_refills_count), SHOW_LONG_STATUS}, + {"Handler_mrr_rowid_refills",(char*) offsetof(STATUS_VAR, ha_mrr_rowid_refills_count), SHOW_LONG_STATUS}, {"Handler_prepare", (char*) offsetof(STATUS_VAR, ha_prepare_count), SHOW_LONG_STATUS}, {"Handler_read_first", (char*) offsetof(STATUS_VAR, ha_read_first_count), SHOW_LONG_STATUS}, {"Handler_read_key", (char*) offsetof(STATUS_VAR, ha_read_key_count), SHOW_LONG_STATUS}, @@ -7095,18 +7372,20 @@ SHOW_VAR status_vars[]= { {"Key", (char*) &show_default_keycache, SHOW_FUNC}, {"Last_query_cost", (char*) offsetof(STATUS_VAR, last_query_cost), SHOW_DOUBLE_STATUS}, {"Max_used_connections", (char*) &max_used_connections, SHOW_LONG}, + {"Memory_used", (char*) offsetof(STATUS_VAR, memory_used), SHOW_LONGLONG_STATUS}, {"Not_flushed_delayed_rows", (char*) &delayed_rows_in_use, SHOW_LONG_NOFLUSH}, {"Open_files", (char*) &my_file_opened, SHOW_LONG_NOFLUSH}, {"Open_streams", (char*) &my_stream_opened, SHOW_LONG_NOFLUSH}, - {"Open_table_definitions", (char*) &show_table_definitions, SHOW_FUNC}, - {"Open_tables", (char*) &show_open_tables, SHOW_FUNC}, + {"Open_table_definitions", (char*) &show_table_definitions, SHOW_SIMPLE_FUNC}, + {"Open_tables", (char*) &show_open_tables, SHOW_SIMPLE_FUNC}, {"Opened_files", (char*) &my_file_total_opened, SHOW_LONG_NOFLUSH}, + {"Opened_plugin_libraries", (char*) &dlopen_count, SHOW_LONG}, {"Opened_table_definitions", (char*) offsetof(STATUS_VAR, opened_shares), SHOW_LONG_STATUS}, {"Opened_tables", (char*) offsetof(STATUS_VAR, opened_tables), SHOW_LONG_STATUS}, - {"Opened_views", (char*) offsetof(STATUS_VAR, opened_views), SHOW_LONG_STATUS}, - {"Prepared_stmt_count", (char*) &show_prepared_stmt_count, SHOW_FUNC}, - {"Rows_read", (char*) offsetof(STATUS_VAR, rows_read), SHOW_LONGLONG_STATUS}, + {"Opened_views", (char*) offsetof(STATUS_VAR, opened_views), SHOW_LONG_STATUS}, + {"Prepared_stmt_count", (char*) &show_prepared_stmt_count, SHOW_SIMPLE_FUNC}, {"Rows_sent", (char*) offsetof(STATUS_VAR, rows_sent), SHOW_LONGLONG_STATUS}, + {"Rows_read", (char*) offsetof(STATUS_VAR, rows_read), SHOW_LONGLONG_STATUS}, {"Rows_tmp_read", (char*) offsetof(STATUS_VAR, rows_tmp_read), SHOW_LONGLONG_STATUS}, #ifdef HAVE_QUERY_CACHE {"Qcache_free_blocks", (char*) &query_cache.free_memory_blocks, SHOW_LONG_NOFLUSH}, @@ -7118,22 +7397,22 @@ SHOW_VAR status_vars[]= { {"Qcache_queries_in_cache", (char*) &query_cache.queries_in_cache, SHOW_LONG_NOFLUSH}, {"Qcache_total_blocks", (char*) &query_cache.total_blocks, SHOW_LONG_NOFLUSH}, #endif /*HAVE_QUERY_CACHE*/ - {"Queries", (char*) &show_queries, SHOW_FUNC}, + {"Queries", (char*) &show_queries, SHOW_SIMPLE_FUNC}, {"Questions", (char*) offsetof(STATUS_VAR, questions), SHOW_LONG_STATUS}, #ifdef HAVE_REPLICATION - {"Rpl_status", (char*) &show_rpl_status, SHOW_FUNC}, + {"Rpl_status", (char*) &show_rpl_status, SHOW_SIMPLE_FUNC}, #endif {"Select_full_join", (char*) offsetof(STATUS_VAR, select_full_join_count), SHOW_LONG_STATUS}, {"Select_full_range_join", (char*) offsetof(STATUS_VAR, select_full_range_join_count), SHOW_LONG_STATUS}, {"Select_range", (char*) offsetof(STATUS_VAR, select_range_count), SHOW_LONG_STATUS}, {"Select_range_check", (char*) offsetof(STATUS_VAR, select_range_check_count), SHOW_LONG_STATUS}, {"Select_scan", (char*) offsetof(STATUS_VAR, select_scan_count), SHOW_LONG_STATUS}, - {"Slave_open_temp_tables", (char*) &slave_open_temp_tables, SHOW_LONG}, + {"Slave_open_temp_tables", (char*) &slave_open_temp_tables, SHOW_INT}, #ifdef HAVE_REPLICATION - {"Slave_heartbeat_period", (char*) &show_heartbeat_period, SHOW_FUNC}, - {"Slave_received_heartbeats",(char*) &show_slave_received_heartbeats, SHOW_FUNC}, - {"Slave_retried_transactions",(char*) &show_slave_retried_trans, SHOW_FUNC}, - {"Slave_running", (char*) &show_slave_running, SHOW_FUNC}, + {"Slave_retried_transactions",(char*)&slave_retried_transactions, SHOW_LONG}, + {"Slave_heartbeat_period", (char*) &show_heartbeat_period, SHOW_SIMPLE_FUNC}, + {"Slave_received_heartbeats",(char*) &show_slave_received_heartbeats, SHOW_SIMPLE_FUNC}, + {"Slave_running", (char*) &show_slave_running, SHOW_SIMPLE_FUNC}, #endif {"Slow_launch_threads", (char*) &slow_launch_threads, SHOW_LONG}, {"Slow_queries", (char*) offsetof(STATUS_VAR, long_query_count), SHOW_LONG_STATUS}, @@ -7143,29 +7422,29 @@ SHOW_VAR status_vars[]= { {"Sort_scan", (char*) offsetof(STATUS_VAR, filesort_scan_count), SHOW_LONG_STATUS}, #ifdef HAVE_OPENSSL #ifndef EMBEDDED_LIBRARY - {"Ssl_accept_renegotiates", (char*) &show_ssl_ctx_sess_accept_renegotiate, SHOW_FUNC}, - {"Ssl_accepts", (char*) &show_ssl_ctx_sess_accept, SHOW_FUNC}, - {"Ssl_callback_cache_hits", (char*) &show_ssl_ctx_sess_cb_hits, SHOW_FUNC}, - {"Ssl_cipher", (char*) &show_ssl_get_cipher, SHOW_FUNC}, - {"Ssl_cipher_list", (char*) &show_ssl_get_cipher_list, SHOW_FUNC}, - {"Ssl_client_connects", (char*) &show_ssl_ctx_sess_connect, SHOW_FUNC}, - {"Ssl_connect_renegotiates", (char*) &show_ssl_ctx_sess_connect_renegotiate, SHOW_FUNC}, - {"Ssl_ctx_verify_depth", (char*) &show_ssl_ctx_get_verify_depth, SHOW_FUNC}, - {"Ssl_ctx_verify_mode", (char*) &show_ssl_ctx_get_verify_mode, SHOW_FUNC}, - {"Ssl_default_timeout", (char*) &show_ssl_get_default_timeout, SHOW_FUNC}, - {"Ssl_finished_accepts", (char*) &show_ssl_ctx_sess_accept_good, SHOW_FUNC}, - {"Ssl_finished_connects", (char*) &show_ssl_ctx_sess_connect_good, SHOW_FUNC}, - {"Ssl_session_cache_hits", (char*) &show_ssl_ctx_sess_hits, SHOW_FUNC}, - {"Ssl_session_cache_misses", (char*) &show_ssl_ctx_sess_misses, SHOW_FUNC}, - {"Ssl_session_cache_mode", (char*) &show_ssl_ctx_get_session_cache_mode, SHOW_FUNC}, - {"Ssl_session_cache_overflows", (char*) &show_ssl_ctx_sess_cache_full, SHOW_FUNC}, - {"Ssl_session_cache_size", (char*) &show_ssl_ctx_sess_get_cache_size, SHOW_FUNC}, - {"Ssl_session_cache_timeouts", (char*) &show_ssl_ctx_sess_timeouts, SHOW_FUNC}, - {"Ssl_sessions_reused", (char*) &show_ssl_session_reused, SHOW_FUNC}, - {"Ssl_used_session_cache_entries",(char*) &show_ssl_ctx_sess_number, SHOW_FUNC}, - {"Ssl_verify_depth", (char*) &show_ssl_get_verify_depth, SHOW_FUNC}, - {"Ssl_verify_mode", (char*) &show_ssl_get_verify_mode, SHOW_FUNC}, - {"Ssl_version", (char*) &show_ssl_get_version, SHOW_FUNC}, + {"Ssl_accept_renegotiates", (char*) &show_ssl_ctx_sess_accept_renegotiate, SHOW_SIMPLE_FUNC}, + {"Ssl_accepts", (char*) &show_ssl_ctx_sess_accept, SHOW_SIMPLE_FUNC}, + {"Ssl_callback_cache_hits", (char*) &show_ssl_ctx_sess_cb_hits, SHOW_SIMPLE_FUNC}, + {"Ssl_cipher", (char*) &show_ssl_get_cipher, SHOW_SIMPLE_FUNC}, + {"Ssl_cipher_list", (char*) &show_ssl_get_cipher_list, SHOW_SIMPLE_FUNC}, + {"Ssl_client_connects", (char*) &show_ssl_ctx_sess_connect, SHOW_SIMPLE_FUNC}, + {"Ssl_connect_renegotiates", (char*) &show_ssl_ctx_sess_connect_renegotiate, SHOW_SIMPLE_FUNC}, + {"Ssl_ctx_verify_depth", (char*) &show_ssl_ctx_get_verify_depth, SHOW_SIMPLE_FUNC}, + {"Ssl_ctx_verify_mode", (char*) &show_ssl_ctx_get_verify_mode, SHOW_SIMPLE_FUNC}, + {"Ssl_default_timeout", (char*) &show_ssl_get_default_timeout, SHOW_SIMPLE_FUNC}, + {"Ssl_finished_accepts", (char*) &show_ssl_ctx_sess_accept_good, SHOW_SIMPLE_FUNC}, + {"Ssl_finished_connects", (char*) &show_ssl_ctx_sess_connect_good, SHOW_SIMPLE_FUNC}, + {"Ssl_session_cache_hits", (char*) &show_ssl_ctx_sess_hits, SHOW_SIMPLE_FUNC}, + {"Ssl_session_cache_misses", (char*) &show_ssl_ctx_sess_misses, SHOW_SIMPLE_FUNC}, + {"Ssl_session_cache_mode", (char*) &show_ssl_ctx_get_session_cache_mode, SHOW_SIMPLE_FUNC}, + {"Ssl_session_cache_overflows", (char*) &show_ssl_ctx_sess_cache_full, SHOW_SIMPLE_FUNC}, + {"Ssl_session_cache_size", (char*) &show_ssl_ctx_sess_get_cache_size, SHOW_SIMPLE_FUNC}, + {"Ssl_session_cache_timeouts", (char*) &show_ssl_ctx_sess_timeouts, SHOW_SIMPLE_FUNC}, + {"Ssl_sessions_reused", (char*) &show_ssl_session_reused, SHOW_SIMPLE_FUNC}, + {"Ssl_used_session_cache_entries",(char*) &show_ssl_ctx_sess_number, SHOW_SIMPLE_FUNC}, + {"Ssl_verify_depth", (char*) &show_ssl_get_verify_depth, SHOW_SIMPLE_FUNC}, + {"Ssl_verify_mode", (char*) &show_ssl_get_verify_mode, SHOW_SIMPLE_FUNC}, + {"Ssl_version", (char*) &show_ssl_get_version, SHOW_SIMPLE_FUNC}, #endif #endif /* HAVE_OPENSSL */ {"Syncs", (char*) &my_sync_count, SHOW_LONG_NOFLUSH}, @@ -7183,16 +7462,16 @@ SHOW_VAR status_vars[]= { {"Tc_log_page_waits", (char*) &tc_log_page_waits, SHOW_LONG}, #endif #ifdef HAVE_POOL_OF_THREADS - {"Threadpool_idle_threads", (char *) &show_threadpool_idle_threads, SHOW_FUNC}, + {"Threadpool_idle_threads", (char *) &show_threadpool_idle_threads, SHOW_SIMPLE_FUNC}, {"Threadpool_threads", (char *) &tp_stats.num_worker_threads, SHOW_INT}, #endif {"Threads_cached", (char*) &cached_thread_count, SHOW_LONG_NOFLUSH}, {"Threads_connected", (char*) &connection_count, SHOW_INT}, {"Threads_created", (char*) &thread_created, SHOW_LONG_NOFLUSH}, {"Threads_running", (char*) &thread_running, SHOW_INT}, - {"Uptime", (char*) &show_starttime, SHOW_FUNC}, + {"Uptime", (char*) &show_starttime, SHOW_SIMPLE_FUNC}, #ifdef ENABLED_PROFILING - {"Uptime_since_flush_status",(char*) &show_flushstatustime, SHOW_FUNC}, + {"Uptime_since_flush_status",(char*) &show_flushstatustime, SHOW_SIMPLE_FUNC}, #endif {NullS, NullS, SHOW_LONG} }; @@ -7241,7 +7520,7 @@ static int option_cmp(my_option *a, my_option *b) static void print_help() { MEM_ROOT mem_root; - init_alloc_root(&mem_root, 4096, 4096); + init_alloc_root(&mem_root, 4096, 4096, MYF(0)); pop_dynamic(&all_options); sys_var_add_options(&all_options, sys_var::PARSE_EARLY); @@ -7385,10 +7664,14 @@ static int mysql_init_variables(void) protocol_version= PROTOCOL_VERSION; what_to_log= ~ (1L << (uint) COM_TIME); refresh_version= 2L; /* Increments on each reload. 0 and 1 are reserved */ + denied_connections= 0; executed_events= 0; global_query_id= thread_id= 1L; my_atomic_rwlock_init(&global_query_id_lock); my_atomic_rwlock_init(&thread_running_lock); + my_atomic_rwlock_init(&thread_count_lock); + my_atomic_rwlock_init(&statistics_lock); + my_atomic_rwlock_init(slave_executed_entries_lock); strmov(server_version, MYSQL_SERVER_VERSION); threads.empty(); thread_cache.empty(); @@ -7412,6 +7695,7 @@ static int mysql_init_variables(void) relay_log_info_file= (char*) "relay-log.info"; report_user= report_password = report_host= 0; /* TO BE DELETED */ opt_relay_logname= opt_relaylog_index_name= 0; + slave_retried_transactions= 0; /* Variables in libraries */ charsets_dir= 0; @@ -7436,8 +7720,13 @@ static int mysql_init_variables(void) #if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY) have_ssl=SHOW_OPTION_YES; +#if HAVE_YASSL + have_openssl= SHOW_OPTION_NO; +#else + have_openssl= SHOW_OPTION_YES; +#endif #else - have_ssl=SHOW_OPTION_NO; + have_openssl= have_ssl= SHOW_OPTION_NO; #endif #ifdef HAVE_BROKEN_REALPATH have_symlink=SHOW_OPTION_NO; @@ -7555,7 +7844,7 @@ mysqld_get_one_option(int optid, default_collation_name= 0; break; case 'l': - WARN_DEPRECATED(NULL, 7, 0, "--log", "'--general-log'/'--general-log-file'"); + WARN_DEPRECATED(NULL, 10, 1, "--log", "'--general-log'/'--general-log-file'"); opt_log=1; break; case 'h': @@ -7632,12 +7921,12 @@ mysqld_get_one_option(int optid, #ifdef HAVE_REPLICATION case (int)OPT_REPLICATE_IGNORE_DB: { - rpl_filter->add_ignore_db(argument); + cur_rpl_filter->add_ignore_db(argument); break; } case (int)OPT_REPLICATE_DO_DB: { - rpl_filter->add_do_db(argument); + cur_rpl_filter->add_do_db(argument); break; } case (int)OPT_REPLICATE_REWRITE_DB: @@ -7668,7 +7957,7 @@ mysqld_get_one_option(int optid, return 1; } - rpl_filter->add_db_rewrite(key, val); + cur_rpl_filter->add_db_rewrite(key, val); break; } @@ -7684,7 +7973,7 @@ mysqld_get_one_option(int optid, } case (int)OPT_REPLICATE_DO_TABLE: { - if (rpl_filter->add_do_table(argument)) + if (cur_rpl_filter->add_do_table(argument)) { sql_print_error("Could not add do table rule '%s'!\n", argument); return 1; @@ -7693,7 +7982,7 @@ mysqld_get_one_option(int optid, } case (int)OPT_REPLICATE_WILD_DO_TABLE: { - if (rpl_filter->add_wild_do_table(argument)) + if (cur_rpl_filter->add_wild_do_table(argument)) { sql_print_error("Could not add do table rule '%s'!\n", argument); return 1; @@ -7702,7 +7991,7 @@ mysqld_get_one_option(int optid, } case (int)OPT_REPLICATE_WILD_IGNORE_TABLE: { - if (rpl_filter->add_wild_ignore_table(argument)) + if (cur_rpl_filter->add_wild_ignore_table(argument)) { sql_print_error("Could not add ignore table rule '%s'!\n", argument); return 1; @@ -7711,7 +8000,7 @@ mysqld_get_one_option(int optid, } case (int)OPT_REPLICATE_IGNORE_TABLE: { - if (rpl_filter->add_ignore_table(argument)) + if (cur_rpl_filter->add_ignore_table(argument)) { sql_print_error("Could not add ignore table rule '%s'!\n", argument); return 1; @@ -7720,7 +8009,7 @@ mysqld_get_one_option(int optid, } #endif /* HAVE_REPLICATION */ case (int) OPT_SLOW_QUERY_LOG: - WARN_DEPRECATED(NULL, 7, 0, "--log-slow-queries", "'--slow-query-log'/'--slow-query-log-file'"); + WARN_DEPRECATED(NULL, 10, 1, "--log-slow-queries", "'--slow-query-log'/'--slow-query-log-file'"); opt_slow_log= 1; break; case (int) OPT_SAFE: @@ -7737,7 +8026,7 @@ mysqld_get_one_option(int optid, case (int) OPT_SKIP_PRIOR: opt_specialflag|= SPECIAL_NO_PRIOR; sql_print_warning("The --skip-thread-priority startup option is deprecated " - "and will be removed in MySQL 7.0. This option has no effect " + "and will be removed in MySQL 11.0. This option has no effect " "as the implied behavior is already the default."); break; case (int) OPT_SKIP_HOST_CACHE: @@ -7759,6 +8048,7 @@ mysqld_get_one_option(int optid, break; case OPT_SERVER_ID: server_id_supplied = 1; + ::server_id= global_system_variables.server_id; break; case OPT_ONE_THREAD: thread_handling= SCHEDULER_NO_THREADS; @@ -7831,7 +8121,7 @@ mysqld_get_one_option(int optid, C_MODE_START static void* -mysql_getopt_value(const char *keyname, uint key_length, +mysql_getopt_value(const char *name, uint length, const struct my_option *option, int *error) { if (error) @@ -7844,7 +8134,7 @@ mysql_getopt_value(const char *keyname, uint key_length, case OPT_KEY_CACHE_PARTITIONS: { KEY_CACHE *key_cache; - if (!(key_cache= get_or_create_key_cache(keyname, key_length))) + if (!(key_cache= get_or_create_key_cache(name, length))) { if (error) *error= EXIT_OUT_OF_MEMORY; @@ -7863,6 +8153,22 @@ mysql_getopt_value(const char *keyname, uint key_length, return (uchar**) &key_cache->param_partitions; } } + case OPT_REPLICATE_DO_DB: + case OPT_REPLICATE_DO_TABLE: + case OPT_REPLICATE_IGNORE_DB: + case OPT_REPLICATE_IGNORE_TABLE: + case OPT_REPLICATE_WILD_DO_TABLE: + case OPT_REPLICATE_WILD_IGNORE_TABLE: + case OPT_REPLICATE_REWRITE_DB: + { + /* Store current filter for mysqld_get_one_option() */ + if (!(cur_rpl_filter= get_or_create_rpl_filter(name, length))) + { + if (error) + *error= EXIT_OUT_OF_MEMORY; + } + return 0; + } } return option->value; } @@ -7903,7 +8209,7 @@ static int get_options(int *argc_ptr, char ***argv_ptr) /* prepare all_options array */ my_init_dynamic_array(&all_options, sizeof(my_option), array_elements(my_long_options), - array_elements(my_long_options)/4); + array_elements(my_long_options)/4, MYF(0)); for (my_option *opt= my_long_options; opt < my_long_options + array_elements(my_long_options) - 1; opt++) @@ -7993,7 +8299,7 @@ static int get_options(int *argc_ptr, char ***argv_ptr) global_system_variables.sql_mode= expand_sql_mode(global_system_variables.sql_mode); -#if defined(HAVE_BROKEN_REALPATH) +#if !defined(HAVE_REALPATH) || defined(HAVE_BROKEN_REALPATH) my_use_symdir=0; my_disable_symlinks=1; have_symlink=SHOW_OPTION_NO; @@ -8098,8 +8404,30 @@ static int get_options(int *argc_ptr, char ***argv_ptr) if (!max_long_data_size_used) max_long_data_size= global_system_variables.max_allowed_packet; - /* Rember if max_user_connections was 0 at startup */ + /* Remember if max_user_connections was 0 at startup */ max_user_connections_checking= global_system_variables.max_user_connections != 0; + + { + sys_var *max_relay_log_size_var, *max_binlog_size_var; + /* If max_relay_log_size is 0, then set it to max_binlog_size */ + if (!global_system_variables.max_relay_log_size) + global_system_variables.max_relay_log_size= max_binlog_size; + + /* + Fix so that DEFAULT and limit checking works with max_relay_log_size + (Yes, this is a hack, but it's required as the definition of + max_relay_log_size allows it to be set to 0). + */ + max_relay_log_size_var= intern_find_sys_var("max_relay_log_size", 0); + max_binlog_size_var= intern_find_sys_var("max_binlog_size", 0); + if (max_binlog_size_var && max_relay_log_size_var) + { + max_relay_log_size_var->option.min_value= + max_binlog_size_var->option.min_value; + max_relay_log_size_var->option.def_value= + max_binlog_size_var->option.def_value; + } + } return 0; } @@ -8404,7 +8732,7 @@ void refresh_status(THD *thd) add_to_status(&global_status_var, &thd->status_var); /* Reset thread's status variables */ - bzero((uchar*) &thd->status_var, sizeof(thd->status_var)); + thd->set_status_var_init(); bzero((uchar*) &thd->org_status_var, sizeof(thd->org_status_var)); thd->start_bytes_received= 0; @@ -8418,32 +8746,8 @@ void refresh_status(THD *thd) /* Set max_used_connections to the number of currently open - connections. Lock LOCK_thread_count out of LOCK_status to avoid - deadlocks. Status reset becomes not atomic, but status data is - not exact anyway. + connections. This is not perfect, but status data is not exact anyway. */ - mysql_mutex_lock(&LOCK_thread_count); max_used_connections= thread_count-delayed_insert_threads; - mysql_mutex_unlock(&LOCK_thread_count); } - -/***************************************************************************** - Instantiate variables for missing storage engines - This section should go away soon -*****************************************************************************/ - -/***************************************************************************** - Instantiate templates -*****************************************************************************/ - -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -/* Used templates */ -template class I_List<THD>; -template class I_List_iterator<THD>; -template class I_List<i_string>; -template class I_List<i_string_pair>; -template class I_List<Statement>; -template class I_List_iterator<Statement>; -#endif - diff --git a/sql/mysqld.h b/sql/mysqld.h index 56c48d3d23f..71fdd3ee5bd 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -56,6 +56,7 @@ void kill_mysql(void); void close_connection(THD *thd, uint sql_errno= 0); void handle_connection_in_main_thread(THD *thd); void create_thread_to_handle_connection(THD *thd); +void delete_running_thd(THD *thd); void unlink_thd(THD *thd); bool one_thread_per_connection_end(THD *thd, bool put_in_cache); void flush_thread_cache(); @@ -89,12 +90,12 @@ extern bool opt_ignore_builtin_innodb; extern my_bool opt_character_set_client_handshake; extern bool volatile abort_loop; extern bool in_bootstrap; -extern uint volatile thread_count; extern uint connection_count; extern my_bool opt_safe_user_create; extern my_bool opt_safe_show_db, opt_local_infile, opt_myisam_use_mmap; extern my_bool opt_slave_compressed_protocol, use_temp_pool; extern ulong slave_exec_mode_options; +extern ulong slave_retried_transactions; extern ulonglong slave_type_conversions_options; extern my_bool read_only, opt_readonly; extern my_bool lower_case_file_system; @@ -105,6 +106,7 @@ extern char* opt_secure_backup_file_priv; extern size_t opt_secure_backup_file_priv_len; extern my_bool opt_log_slow_admin_statements, opt_log_slow_slave_statements; extern my_bool sp_automatic_privileges, opt_noacl; +extern ulong use_stat_tables; extern my_bool opt_old_style_user_limits, trust_function_creators; extern uint opt_crash_binlog_innodb; extern char *shared_memory_base_name, *mysqld_unix_port; @@ -151,7 +153,7 @@ extern ulong delayed_insert_timeout; extern ulong delayed_insert_limit, delayed_queue_size; extern ulong delayed_insert_threads, delayed_insert_writes; extern ulong delayed_rows_in_use,delayed_insert_errors; -extern ulong slave_open_temp_tables; +extern int32 slave_open_temp_tables; extern ulonglong query_cache_size; extern ulong query_cache_min_res_unit; extern ulong slow_launch_threads, slow_launch_time; @@ -169,11 +171,15 @@ extern ulong max_prepared_stmt_count, prepared_stmt_count; extern ulong open_files_limit; extern ulonglong binlog_cache_size, binlog_stmt_cache_size; extern ulonglong max_binlog_cache_size, max_binlog_stmt_cache_size; -extern ulong max_binlog_size, max_relay_log_size; +extern ulong max_binlog_size; extern ulong slave_max_allowed_packet; extern ulong opt_binlog_rows_event_max_size; extern ulong rpl_recovery_rank, thread_cache_size; extern ulong stored_program_cache_size; +extern ulong opt_slave_parallel_threads; +extern ulong opt_slave_parallel_max_queued; +extern ulong opt_binlog_commit_wait_count; +extern ulong opt_binlog_commit_wait_usec; extern ulong back_log; extern ulong executed_events; extern char language[FN_REFLEN]; @@ -198,7 +204,7 @@ extern handlerton *myisam_hton; extern handlerton *heap_hton; extern const char *load_default_groups[]; extern struct my_option my_long_options[]; -extern int mysqld_server_started; +extern int mysqld_server_started, mysqld_server_initialized; extern "C" MYSQL_PLUGIN_IMPORT int orig_argc; extern "C" MYSQL_PLUGIN_IMPORT char **orig_argv; extern pthread_attr_t connection_attrib; @@ -218,14 +224,15 @@ extern pthread_key(MEM_ROOT**,THR_MALLOC); #ifdef HAVE_PSI_INTERFACE #ifdef HAVE_MMAP extern PSI_mutex_key key_PAGE_lock, key_LOCK_sync, key_LOCK_active, - key_LOCK_pool; + key_LOCK_pool, key_LOCK_pending_checkpoint; #endif /* HAVE_MMAP */ #ifdef HAVE_OPENSSL extern PSI_mutex_key key_LOCK_des_key_file; #endif -extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_prep_xids, +extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, + key_BINLOG_LOCK_binlog_background_thread, key_delayed_insert_mutex, key_hash_filo_lock, key_LOCK_active_mi, key_LOCK_connection_count, key_LOCK_crypt, key_LOCK_delayed_create, key_LOCK_delayed_insert, key_LOCK_delayed_status, key_LOCK_error_log, @@ -239,14 +246,17 @@ extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_prep_xids, key_master_info_sleep_lock, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, - key_relay_log_info_sleep_lock, + key_rpl_group_info_sleep_lock, key_structure_guard_mutex, key_TABLE_SHARE_LOCK_ha_data, key_LOCK_error_messages, key_LOCK_thread_count, key_PARTITION_LOCK_auto_inc; extern PSI_mutex_key key_RELAYLOG_LOCK_index; +extern PSI_mutex_key key_LOCK_slave_state, key_LOCK_binlog_state, + key_LOCK_rpl_thread, key_LOCK_rpl_thread_pool, key_LOCK_parallel_entry; extern PSI_mutex_key key_LOCK_stats, key_LOCK_global_user_client_stats, key_LOCK_global_table_stats, - key_LOCK_global_index_stats, key_LOCK_wakeup_ready; + key_LOCK_global_index_stats, key_LOCK_wakeup_ready, key_LOCK_wait_commit; +extern PSI_mutex_key key_LOCK_gtid_waiting; extern PSI_rwlock_key key_rwlock_LOCK_grant, key_rwlock_LOCK_logger, key_rwlock_LOCK_sys_init_connect, key_rwlock_LOCK_sys_init_slave, @@ -256,7 +266,9 @@ extern PSI_rwlock_key key_rwlock_LOCK_grant, key_rwlock_LOCK_logger, extern PSI_cond_key key_PAGE_cond, key_COND_active, key_COND_pool; #endif /* HAVE_MMAP */ -extern PSI_cond_key key_BINLOG_COND_prep_xids, key_BINLOG_update_cond, +extern PSI_cond_key key_BINLOG_COND_xid_list, key_BINLOG_update_cond, + key_BINLOG_COND_binlog_background_thread, + key_BINLOG_COND_binlog_background_thread_end, key_COND_cache_status_changed, key_COND_manager, key_COND_rpl_status, key_COND_server_started, key_delayed_insert_cond, key_delayed_insert_cond_client, @@ -265,16 +277,21 @@ extern PSI_cond_key key_BINLOG_COND_prep_xids, key_BINLOG_update_cond, key_master_info_sleep_cond, key_relay_log_info_data_cond, key_relay_log_info_log_space_cond, key_relay_log_info_start_cond, key_relay_log_info_stop_cond, - key_relay_log_info_sleep_cond, + key_rpl_group_info_sleep_cond, key_TABLE_SHARE_cond, key_user_level_lock_cond, key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache; -extern PSI_cond_key key_RELAYLOG_update_cond, key_COND_wakeup_ready; +extern PSI_cond_key key_RELAYLOG_update_cond, key_COND_wakeup_ready, + key_COND_wait_commit; extern PSI_cond_key key_RELAYLOG_COND_queue_busy; extern PSI_cond_key key_TC_LOG_MMAP_COND_queue_busy; +extern PSI_cond_key key_COND_rpl_thread, key_COND_rpl_thread_pool, + key_COND_parallel_entry; +extern PSI_cond_key key_COND_wait_gtid; extern PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert, key_thread_handle_manager, key_thread_kill_server, key_thread_main, - key_thread_one_connection, key_thread_signal_hand; + key_thread_one_connection, key_thread_signal_hand, key_thread_slave_init, + key_rpl_parallel_thread; extern PSI_file_key key_file_binlog, key_file_binlog_index, key_file_casetest, key_file_dbopt, key_file_des_key_file, key_file_ERRMSG, key_select_to_file, @@ -285,6 +302,7 @@ extern PSI_file_key key_file_binlog, key_file_binlog_index, key_file_casetest, key_file_trg, key_file_trn, key_file_init; extern PSI_file_key key_file_query_log, key_file_slow_log; extern PSI_file_key key_file_relaylog, key_file_relaylog_index; +extern PSI_file_key key_file_binlog_state; void init_server_psi_keys(); #endif /* HAVE_PSI_INTERFACE */ @@ -328,7 +346,7 @@ extern MYSQL_PLUGIN_IMPORT key_map key_map_full; /* Should be threaded Server mutex locks and condition variables. */ extern mysql_mutex_t - LOCK_user_locks, LOCK_status, + LOCK_item_func_sleep, LOCK_status, LOCK_error_log, LOCK_delayed_insert, LOCK_short_uuid_generator, LOCK_delayed_status, LOCK_delayed_create, LOCK_crypt, LOCK_timezone, LOCK_slave_list, LOCK_active_mi, LOCK_manager, @@ -345,7 +363,9 @@ extern mysql_rwlock_t LOCK_system_variables_hash; extern mysql_cond_t COND_thread_count; extern mysql_cond_t COND_manager; extern int32 thread_running; -extern my_atomic_rwlock_t thread_running_lock; +extern int32 thread_count; +extern my_atomic_rwlock_t thread_running_lock, thread_count_lock; +extern my_atomic_rwlock_t slave_executed_entries_lock; extern char *opt_ssl_ca, *opt_ssl_capath, *opt_ssl_cert, *opt_ssl_cipher, *opt_ssl_key; @@ -431,6 +451,7 @@ enum enum_query_type typedef int64 query_id_t; extern query_id_t global_query_id; extern my_atomic_rwlock_t global_query_id_lock; +extern my_atomic_rwlock_t statistics_lock; void unireg_end(void) __attribute__((noreturn)); @@ -441,7 +462,7 @@ inline query_id_t next_query_id() my_atomic_rwlock_wrlock(&global_query_id_lock); id= my_atomic_add64(&global_query_id, 1); my_atomic_rwlock_wrunlock(&global_query_id_lock); - return (id+1); + return (id); } inline query_id_t get_query_id() @@ -471,42 +492,44 @@ inline void table_case_convert(char * name, uint length) name, length, name, length); } -inline ulong sql_rnd_with_mutex() +inline void thread_safe_increment32(int32 *value, my_atomic_rwlock_t *lock) { - mysql_mutex_lock(&LOCK_thread_count); - ulong tmp=(ulong) (my_rnd(&sql_rand) * 0xffffffff); /* make all bits random */ - mysql_mutex_unlock(&LOCK_thread_count); - return tmp; + my_atomic_rwlock_wrlock(lock); + (void) my_atomic_add32(value, 1); + my_atomic_rwlock_wrunlock(lock); } -inline int32 -inc_thread_running() +inline void thread_safe_decrement32(int32 *value, my_atomic_rwlock_t *lock) { - int32 num_thread_running; - my_atomic_rwlock_wrlock(&thread_running_lock); - num_thread_running= my_atomic_add32(&thread_running, 1); - my_atomic_rwlock_wrunlock(&thread_running_lock); - return (num_thread_running+1); + my_atomic_rwlock_wrlock(lock); + (void) my_atomic_add32(value, -1); + my_atomic_rwlock_wrunlock(lock); } -inline int32 -dec_thread_running() +inline void thread_safe_increment64(int64 *value, my_atomic_rwlock_t *lock) +{ + my_atomic_rwlock_wrlock(lock); + (void) my_atomic_add64(value, 1); + my_atomic_rwlock_wrunlock(lock); +} + +inline void thread_safe_decrement64(int64 *value, my_atomic_rwlock_t *lock) +{ + my_atomic_rwlock_wrlock(lock); + (void) my_atomic_add64(value, -1); + my_atomic_rwlock_wrunlock(lock); +} + +inline void +inc_thread_running() { - int32 num_thread_running; - my_atomic_rwlock_wrlock(&thread_running_lock); - num_thread_running= my_atomic_add32(&thread_running, -1); - my_atomic_rwlock_wrunlock(&thread_running_lock); - return (num_thread_running-1); + thread_safe_increment32(&thread_running, &thread_running_lock); } -inline int32 -get_thread_running() +inline void +dec_thread_running() { - int32 num_thread_running; - my_atomic_rwlock_wrlock(&thread_running_lock); - num_thread_running= my_atomic_load32(&thread_running); - my_atomic_rwlock_wrunlock(&thread_running_lock); - return num_thread_running; + thread_safe_decrement32(&thread_running, &thread_running_lock); } void set_server_version(void); @@ -526,6 +549,10 @@ inline THD *_current_thd(void) } #endif #define current_thd _current_thd() +inline int set_current_thd(THD *thd) +{ + return my_pthread_setspecific_ptr(THR_THD, thd); +} /* @todo remove, make it static in ha_maria.cc @@ -534,6 +561,8 @@ inline THD *_current_thd(void) extern handlerton *maria_hton; extern uint extra_connection_count; +extern uint64 global_gtid_counter; +extern my_bool opt_gtid_strict_mode; extern my_bool opt_userstat_running, debug_assert_if_crashed_table; extern uint mysqld_extra_port; extern ulong opt_progress_report_time; diff --git a/sql/net_serv.cc b/sql/net_serv.cc index 4b78492c857..64a6cd7826c 100644 --- a/sql/net_serv.cc +++ b/sql/net_serv.cc @@ -120,14 +120,15 @@ static my_bool net_write_buff(NET *net,const uchar *packet,ulong len); /** Init with packet info. */ -my_bool my_net_init(NET *net, Vio* vio) +my_bool my_net_init(NET *net, Vio* vio, uint my_flags) { DBUG_ENTER("my_net_init"); + DBUG_PRINT("enter", ("my_flags: %u", my_flags)); net->vio = vio; my_net_local_init(net); /* Set some limits */ if (!(net->buff=(uchar*) my_malloc((size_t) net->max_packet+ NET_HEADER_SIZE + COMP_HEADER_SIZE +1, - MYF(MY_WME)))) + MYF(MY_WME | my_flags)))) DBUG_RETURN(1); net->buff_end=net->buff+net->max_packet; net->error=0; net->return_status=0; @@ -139,6 +140,7 @@ my_bool my_net_init(NET *net, Vio* vio) net->net_skip_rest_factor= 0; net->last_errno=0; net->unused= 0; + net->thread_specific_malloc= test(my_flags & MY_THREAD_SPECIFIC); if (vio != 0) /* If real connection */ { @@ -193,7 +195,9 @@ my_bool net_realloc(NET *net, size_t length) */ if (!(buff= (uchar*) my_realloc((char*) net->buff, pkt_length + NET_HEADER_SIZE + COMP_HEADER_SIZE + 1, - MYF(MY_WME)))) + MYF(MY_WME | + (net->thread_specific_malloc ? + MY_THREAD_SPECIFIC : 0))))) { /* @todo: 1 and 2 codes are identical. */ net->error= 1; @@ -603,7 +607,10 @@ net_real_write(NET *net,const uchar *packet, size_t len) uchar *b; uint header_length=NET_HEADER_SIZE+COMP_HEADER_SIZE; if (!(b= (uchar*) my_malloc(len + NET_HEADER_SIZE + - COMP_HEADER_SIZE + 1, MYF(MY_WME)))) + COMP_HEADER_SIZE + 1, + MYF(MY_WME | + (net->thread_specific_malloc ? + MY_THREAD_SPECIFIC : 0))))) { net->error= 2; net->last_errno= ER_OUT_OF_RESOURCES; diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 66bd287d86a..6c1ddab9710 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -117,6 +117,7 @@ #include "records.h" // init_read_record, end_read_record #include <m_ctype.h> #include "sql_select.h" +#include "sql_statistics.h" #include "filesort.h" // filesort_free_buffers #ifndef EXTRA_DEBUG @@ -860,6 +861,14 @@ class PARAM : public RANGE_OPT_PARAM { public: ha_rows quick_rows[MAX_KEY]; + + /* + This will collect 'possible keys' based on the range optimization. + + Queries with a JOIN object actually use ref optimizer (see add_key_field) + to collect possible_keys. This is used by single table UPDATE/DELETE. + */ + key_map possible_keys; longlong baseflag; uint max_key_part, range_count; @@ -1787,7 +1796,8 @@ QUICK_RANGE_SELECT::QUICK_RANGE_SELECT(THD *thd, TABLE *table, uint key_nr, index= key_nr; head= table; key_part_info= head->key_info[index].key_part; - my_init_dynamic_array(&ranges, sizeof(QUICK_RANGE*), 16, 16); + my_init_dynamic_array(&ranges, sizeof(QUICK_RANGE*), 16, 16, + MYF(MY_THREAD_SPECIFIC)); /* 'thd' is not accessible in QUICK_RANGE_SELECT::reset(). */ mrr_buf_size= thd->variables.mrr_buff_size; @@ -1796,7 +1806,8 @@ QUICK_RANGE_SELECT::QUICK_RANGE_SELECT(THD *thd, TABLE *table, uint key_nr, if (!no_alloc && !parent_alloc) { // Allocates everything through the internal memroot - init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0); + init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0, + MYF(MY_THREAD_SPECIFIC)); thd->mem_root= &alloc; } else @@ -1806,7 +1817,7 @@ QUICK_RANGE_SELECT::QUICK_RANGE_SELECT(THD *thd, TABLE *table, uint key_nr, /* Allocate a bitmap for used columns (Q: why not on MEM_ROOT?) */ if (!(bitmap= (my_bitmap_map*) my_malloc(head->s->column_bitmap_size, - MYF(MY_WME)))) + MYF(MY_WME | MY_THREAD_SPECIFIC)))) { column_bitmap.bitmap= 0; *create_error= 1; @@ -1891,7 +1902,8 @@ QUICK_INDEX_SORT_SELECT::QUICK_INDEX_SORT_SELECT(THD *thd_param, index= MAX_KEY; head= table; bzero(&read_record, sizeof(read_record)); - init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0); + init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0, + MYF(MY_THREAD_SPECIFIC)); DBUG_VOID_RETURN; } @@ -1962,7 +1974,8 @@ QUICK_ROR_INTERSECT_SELECT::QUICK_ROR_INTERSECT_SELECT(THD *thd_param, head= table; record= head->record[0]; if (!parent_alloc) - init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0); + init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0, + MYF(MY_THREAD_SPECIFIC)); else bzero(&alloc, sizeof(MEM_ROOT)); last_rowid= (uchar*) alloc_root(parent_alloc? parent_alloc : &alloc, @@ -2248,7 +2261,8 @@ QUICK_ROR_UNION_SELECT::QUICK_ROR_UNION_SELECT(THD *thd_param, head= table; rowid_length= table->file->ref_length; record= head->record[0]; - init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0); + init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0, + MYF(MY_THREAD_SPECIFIC)); thd_param->mem_root= &alloc; } @@ -2930,7 +2944,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, DBUG_PRINT("enter",("keys_to_use: %lu prev_tables: %lu const_tables: %lu", (ulong) keys_to_use.to_ulonglong(), (ulong) prev_tables, (ulong) const_tables)); - DBUG_PRINT("info", ("records: %lu", (ulong) head->file->stats.records)); + DBUG_PRINT("info", ("records: %lu", (ulong) head->stat_records())); delete quick; quick=0; needed_reg.clear_all(); @@ -2938,7 +2952,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, DBUG_ASSERT(!head->is_filled_at_execution()); if (keys_to_use.is_clear_all() || head->is_filled_at_execution()) DBUG_RETURN(0); - records= head->file->stats.records; + records= head->stat_records(); if (!records) records++; /* purecov: inspected */ scan_time= (double) records / TIME_FOR_COMPARE + 1; @@ -2949,6 +2963,8 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, read_time= (double) records + scan_time + 1; // Force to use index else if (read_time <= 2.0 && !force_quick_range) DBUG_RETURN(0); /* No need for quick select */ + + possible_keys.clear_all(); DBUG_PRINT("info",("Time to scan table: %g", read_time)); @@ -2980,9 +2996,11 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, param.using_real_indexes= TRUE; param.remove_jump_scans= TRUE; param.force_default_mrr= ordered_output; + param.possible_keys.clear_all(); thd->no_errors=1; // Don't warn about NULL - init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0); + init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0, + MYF(MY_THREAD_SPECIFIC)); if (!(param.key_parts= (KEY_PART*) alloc_root(&alloc, sizeof(KEY_PART) * @@ -3075,7 +3093,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, if (group_trp) { param.table->quick_condition_rows= min(group_trp->records, - head->file->stats.records); + head->stat_records()); if (group_trp->read_cost < best_read_time) { best_trp= group_trp; @@ -3190,6 +3208,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, quick= NULL; } } + possible_keys= param.possible_keys; free_mem: free_root(&alloc,MYF(0)); // Return memory & allocator @@ -3197,6 +3216,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, thd->no_errors=0; } + DBUG_EXECUTE("info", print_quick(quick, &needed_reg);); /* @@ -3207,8 +3227,442 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, } /**************************************************************************** + * Condition selectivity module + ****************************************************************************/ + + +/* + Build descriptors of pseudo-indexes over columns to perform range analysis + + SYNOPSIS + create_key_parts_for_pseudo_indexes() + param IN/OUT data structure for the descriptors to be built + used_fields bitmap of columns for which the descriptors are to be built + + DESCRIPTION + For each column marked in the bitmap used_fields the function builds + a descriptor of a single-component pseudo-index over this column that + can be used for the range analysis of the predicates over this columns. + The descriptors are created in the memory of param->mem_root. + + RETURN + FALSE in the case of success + TRUE otherwise +*/ + +static +bool create_key_parts_for_pseudo_indexes(RANGE_OPT_PARAM *param, + MY_BITMAP *used_fields) +{ + Field **field_ptr; + TABLE *table= param->table; + uint parts= 0; + + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + if (bitmap_is_set(used_fields, (*field_ptr)->field_index)) + parts++; + } + + KEY_PART *key_part; + uint keys= 0; + + if (!(key_part= (KEY_PART *) alloc_root(param->mem_root, + sizeof(KEY_PART) * parts))) + return TRUE; + + param->key_parts= key_part; + + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + if (bitmap_is_set(used_fields, (*field_ptr)->field_index)) + { + Field *field= *field_ptr; + uint16 store_length; + key_part->key= keys; + key_part->part= 0; + key_part->length= (uint16) field->key_length(); + store_length= key_part->length; + if (field->real_maybe_null()) + store_length+= HA_KEY_NULL_LENGTH; + if (field->real_type() == MYSQL_TYPE_VARCHAR) + store_length+= HA_KEY_BLOB_LENGTH; + key_part->store_length= store_length; + key_part->field= field; + key_part->image_type= Field::itRAW; + key_part->flag= 0; + param->key[keys]= key_part; + keys++; + key_part++; + } + } + param->keys= keys; + param->key_parts_end= key_part; + + return FALSE; +} + + +/* + Estimate the number of rows in all ranges built for a column + by the range optimizer + + SYNOPSIS + records_in_column_ranges() + param the data structure to access descriptors of pseudo indexes + built over columns used in the condition of the processed query + idx the index of the descriptor of interest in param + tree the tree representing ranges built for the interesting column + + DESCRIPTION + This function retrieves the ranges represented by the SEL_ARG 'tree' and + for each of them r it calls the function get_column_range_cardinality() + that estimates the number of expected rows in r. It is assumed that param + is the data structure containing the descriptors of pseudo-indexes that + has been built to perform range analysis of the range conditions imposed + on the columns used in the processed query, while idx is the index of the + descriptor created in 'param' exactly for the column for which 'tree' + has been built by the range optimizer. + + RETURN + the number of rows in the retrieved ranges +*/ + +static +double records_in_column_ranges(PARAM *param, uint idx, + SEL_ARG *tree) +{ + SEL_ARG_RANGE_SEQ seq; + KEY_MULTI_RANGE range; + range_seq_t seq_it; + double rows; + Field *field; + uint flags= 0; + double total_rows= 0; + RANGE_SEQ_IF seq_if = {NULL, sel_arg_range_seq_init, + sel_arg_range_seq_next, 0, 0}; + + /* Handle cases when we don't have a valid non-empty list of range */ + if (!tree) + return HA_POS_ERROR; + if (tree->type == SEL_ARG::IMPOSSIBLE) + return (0L); + + field= tree->field; + + seq.keyno= idx; + seq.real_keyno= MAX_KEY; + seq.param= param; + seq.start= tree; + + seq_it= seq_if.init((void *) &seq, 0, flags); + + while (!seq_if.next(seq_it, &range)) + { + key_range *min_endp, *max_endp; + min_endp= range.start_key.length? &range.start_key : NULL; + max_endp= range.end_key.length? &range.end_key : NULL; + rows= get_column_range_cardinality(field, min_endp, max_endp, + range.range_flag); + if (HA_POS_ERROR == rows) + { + total_rows= HA_POS_ERROR; + break; + } + total_rows += rows; + } + return total_rows; +} + + +/* + Calculate the selectivity of the condition imposed on the rows of a table + + SYNOPSIS + calculate_cond_selectivity_for_table() + thd the context handle + table the table of interest + cond conditions imposed on the rows of the table + + DESCRIPTION + This function calculates the selectivity of range conditions cond imposed + on the rows of 'table' in the processed query. + The calculated selectivity is assigned to the field table->cond_selectivity. + + NOTE + Currently the selectivities of range conditions over different columns are + considered independent. + + RETURN + FALSE on success + TRUE otherwise +*/ + +bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item *cond) +{ + uint keynr; + uint max_quick_key_parts= 0; + MY_BITMAP *used_fields= &table->cond_set; + double table_records= table->stat_records(); + DBUG_ENTER("calculate_cond_selectivity_for_table"); + + table->cond_selectivity= 1.0; + + if (table_records == 0) + DBUG_RETURN(FALSE); + + if (table->pos_in_table_list->schema_table) + DBUG_RETURN(FALSE); + + if (thd->variables.optimizer_use_condition_selectivity > 2 && + !bitmap_is_clear_all(used_fields)) + { + /* + Calculate the selectivity of the range conditions not supported + by any index + */ + + PARAM param; + MEM_ROOT alloc; + SEL_TREE *tree; + SEL_ARG **key, **end; + double rows; + uint idx= 0; + + init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0, + MYF(MY_THREAD_SPECIFIC)); + param.thd= thd; + param.mem_root= &alloc; + param.old_root= thd->mem_root; + param.table= table; + param.is_ror_scan= FALSE; + + if (create_key_parts_for_pseudo_indexes(¶m, used_fields)) + goto free_alloc; + + param.prev_tables= param.read_tables= 0; + param.current_table= table->map; + param.using_real_indexes= FALSE; + param.real_keynr[0]= 0; + param.alloced_sel_args= 0; + + thd->no_errors=1; + + tree= get_mm_tree(¶m, cond); + + if (!tree) + goto free_alloc; + + table->reginfo.impossible_range= 0; + if (tree->type == SEL_TREE::IMPOSSIBLE) + { + rows= 0; + table->reginfo.impossible_range= 1; + goto free_alloc; + } + else if (tree->type == SEL_TREE::ALWAYS) + { + rows= table_records; + goto free_alloc; + } + else if (tree->type == SEL_TREE::MAYBE) + { + rows= table_records; + goto free_alloc; + } + + for (key= tree->keys, end= key + param.keys; key != end; key++, idx++) + { + if (*key) + { + if ((*key)->type == SEL_ARG::IMPOSSIBLE) + { + rows= 0; + table->reginfo.impossible_range= 1; + goto free_alloc; + } + else + { + rows= records_in_column_ranges(¶m, idx, *key); + if (rows != HA_POS_ERROR) + (*key)->field->cond_selectivity= rows/table_records; + } + } + } + + for (Field **field_ptr= table->field; *field_ptr; field_ptr++) + { + Field *table_field= *field_ptr; + if (bitmap_is_set(table->read_set, table_field->field_index) && + table_field->cond_selectivity < 1.0) + table->cond_selectivity*= table_field->cond_selectivity; + } + + free_alloc: + thd->mem_root= param.old_root; + free_root(&alloc, MYF(0)); + + } + + /* Calculate the selectivity of the range conditions supported by indexes */ + + bitmap_clear_all(used_fields); + + for (keynr= 0; keynr < table->s->keys; keynr++) + { + if (table->quick_keys.is_set(keynr)) + set_if_bigger(max_quick_key_parts, table->quick_key_parts[keynr]); + } + + for (uint quick_key_parts= max_quick_key_parts; + quick_key_parts; quick_key_parts--) + { + for (keynr= 0; keynr < table->s->keys; keynr++) + { + if (table->quick_keys.is_set(keynr) && + table->quick_key_parts[keynr] == quick_key_parts) + { + uint i; + uint used_key_parts= table->quick_key_parts[keynr]; + double quick_cond_selectivity= table->quick_rows[keynr] / + table_records; + KEY *key_info= table->key_info + keynr; + KEY_PART_INFO* key_part= key_info->key_part; + for (i= 0; i < used_key_parts; i++, key_part++) + { + if (bitmap_is_set(used_fields, key_part->fieldnr-1)) + break; + bitmap_set_bit(used_fields, key_part->fieldnr-1); + } + if (i) + { + table->cond_selectivity*= quick_cond_selectivity; + if (i != used_key_parts) + { + double f1= key_info->actual_rec_per_key(i-1); + double f2= key_info->actual_rec_per_key(i); + table->cond_selectivity*= f1 / f2; + } + } + } + } + } + + /* Calculate selectivity of probably highly selective predicates */ + ulong check_rows= + min(thd->variables.optimizer_selectivity_sampling_limit, + (ulong) (table_records * SELECTIVITY_SAMPLING_SHARE)); + if (cond && check_rows > SELECTIVITY_SAMPLING_THRESHOLD && + thd->variables.optimizer_use_condition_selectivity > 4) + { + find_selective_predicates_list_processor_data *dt= + (find_selective_predicates_list_processor_data *) + alloc_root(thd->mem_root, + sizeof(find_selective_predicates_list_processor_data)); + if (!dt) + DBUG_RETURN(TRUE); + dt->list.empty(); + dt->table= table; + if (cond->walk(&Item::find_selective_predicates_list_processor, 0, + (uchar*) dt)) + DBUG_RETURN(TRUE); + if (dt->list.elements > 0) + { + check_rows= check_selectivity(thd, check_rows, table, &dt->list); + if (check_rows > SELECTIVITY_SAMPLING_THRESHOLD) + { + COND_STATISTIC *stat; + List_iterator_fast<COND_STATISTIC> it(dt->list); + double examined_rows= check_rows; + while ((stat= it++)) + { + if (!stat->positive) + { + DBUG_PRINT("info", ("To avoid 0 assigned 1 to the counter")); + stat->positive= 1; // avoid 0 + } + DBUG_PRINT("info", ("The predicate selectivity : %g", + (double)stat->positive / examined_rows)); + double selectivity= ((double)stat->positive) / examined_rows; + table->cond_selectivity*= selectivity; + /* + If a field is involved then we register its selectivity in case + there in an equality with the field. + For example in case + t1.a LIKE "%bla%" and t1.a = t2.b + the selectivity we have found could be used also for t2. + */ + if (stat->field_arg) + { + stat->field_arg->cond_selectivity*= selectivity; + + if (stat->field_arg->next_equal_field) + { + for (Field *next_field= stat->field_arg->next_equal_field; + next_field != stat->field_arg; + next_field= next_field->next_equal_field) + { + next_field->cond_selectivity*= selectivity; + next_field->table->cond_selectivity*= selectivity; + } + } + } + } + + } + /* This list and its elements put to mem_root so should not be freed */ + table->cond_selectivity_sampling_explain= &dt->list; + } + } + + DBUG_RETURN(FALSE); +} + +/**************************************************************************** + * Condition selectivity code ends + ****************************************************************************/ + +/**************************************************************************** * Partition pruning module ****************************************************************************/ + +/* + Store field key image to table record + + SYNOPSIS + store_key_image_to_rec() + field Field which key image should be stored + ptr Field value in key format + len Length of the value, in bytes + + DESCRIPTION + Copy the field value from its key image to the table record. The source + is the value in key image format, occupying len bytes in buffer pointed + by ptr. The destination is table record, in "field value in table record" + format. +*/ + +void store_key_image_to_rec(Field *field, uchar *ptr, uint len) +{ + /* Do the same as print_key() does */ + my_bitmap_map *old_map; + + if (field->real_maybe_null()) + { + if (*ptr) + { + field->set_null(); + return; + } + field->set_notnull(); + ptr++; + } + old_map= dbug_tmp_use_all_columns(field->table, + field->table->write_set); + field->set_key_image(ptr, len); + dbug_tmp_restore_column_map(field->table->write_set, old_map); +} + #ifdef WITH_PARTITION_STORAGE_ENGINE /* @@ -3427,7 +3881,8 @@ bool prune_partitions(THD *thd, TABLE *table, Item *pprune_cond) my_bitmap_map *old_sets[2]; prune_param.part_info= part_info; - init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0); + init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0, + MYF(MY_THREAD_SPECIFIC)); range_par->mem_root= &alloc; range_par->old_root= thd->mem_root; @@ -3543,44 +3998,6 @@ end: /* - Store field key image to table record - - SYNOPSIS - store_key_image_to_rec() - field Field which key image should be stored - ptr Field value in key format - len Length of the value, in bytes - - DESCRIPTION - Copy the field value from its key image to the table record. The source - is the value in key image format, occupying len bytes in buffer pointed - by ptr. The destination is table record, in "field value in table record" - format. -*/ - -void store_key_image_to_rec(Field *field, uchar *ptr, uint len) -{ - /* Do the same as print_key() does */ - my_bitmap_map *old_map; - - if (field->real_maybe_null()) - { - if (*ptr) - { - field->set_null(); - return; - } - field->set_notnull(); - ptr++; - } - old_map= dbug_tmp_use_all_columns(field->table, - field->table->write_set); - field->set_key_image(ptr, len); - dbug_tmp_restore_column_map(field->table->write_set, old_map); -} - - -/* For SEL_ARG* array, store sel_arg->min values into table record buffer SYNOPSIS @@ -4681,7 +5098,7 @@ TABLE_READ_PLAN *get_best_disjunct_quick(PARAM *param, SEL_IMERGE *imerge, DBUG_PRINT("info", ("index_merge scans cost %g", imerge_cost)); if (imerge_too_expensive || (imerge_cost > read_time) || ((non_cpk_scan_records+cpk_scan_records >= - param->table->file->stats.records) && + param->table->stat_records()) && read_time != DBL_MAX)) { /* @@ -4752,7 +5169,7 @@ TABLE_READ_PLAN *get_best_disjunct_quick(PARAM *param, SEL_IMERGE *imerge, imerge_trp->read_cost= imerge_cost; imerge_trp->records= non_cpk_scan_records + cpk_scan_records; imerge_trp->records= min(imerge_trp->records, - param->table->file->stats.records); + param->table->stat_records()); imerge_trp->range_scans= range_scans; imerge_trp->range_scans_end= range_scans + n_child_scans; read_time= imerge_cost; @@ -4823,7 +5240,7 @@ skip_to_ror_scan: ((TRP_ROR_INTERSECT*)(*cur_roru_plan))->index_scan_costs; roru_total_records += (*cur_roru_plan)->records; roru_intersect_part *= (*cur_roru_plan)->records / - param->table->file->stats.records; + param->table->stat_records(); } /* @@ -4833,7 +5250,7 @@ skip_to_ror_scan: in disjunction do not share key parts. */ roru_total_records -= (ha_rows)(roru_intersect_part* - param->table->file->stats.records); + param->table->stat_records()); /* ok, got a ROR read plan for each of the disjuncts Calculate cost: cost(index_union_scan(scan_1, ... scan_n)) = @@ -5114,12 +5531,12 @@ static inline ha_rows get_table_cardinality_for_index_intersect(TABLE *table) { if (table->file->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT) - return table->file->stats.records; + return table->stat_records(); else { ha_rows d; double q; - for (q= (double)table->file->stats.records, d= 1 ; q >= 10; q/= 10, d*= 10 ) ; + for (q= (double)table->stat_records(), d= 1 ; q >= 10; q/= 10, d*= 10 ) ; return (ha_rows) (floor(q+0.5) * d); } } @@ -5522,9 +5939,8 @@ ha_rows records_in_index_intersect_extension(PARTIAL_INDEX_INTERSECT_INFO *curr, ha_rows ext_records= ext_index_scan->records; if (i < used_key_parts) { - ulong *rec_per_key= key_info->rec_per_key+i-1; - ulong f1= rec_per_key[0] ? rec_per_key[0] : 1; - ulong f2= rec_per_key[1] ? rec_per_key[1] : 1; + double f1= key_info->actual_rec_per_key(i-1); + double f2= key_info->actual_rec_per_key(i); ext_records= (ha_rows) ((double) ext_records / f2 * f1); } if (ext_records < table_cardinality) @@ -6016,7 +6432,7 @@ ROR_INTERSECT_INFO* ror_intersect_init(const PARAM *param) info->is_covering= FALSE; info->index_scan_costs= 0.0; info->index_records= 0; - info->out_rows= (double) param->table->file->stats.records; + info->out_rows= (double) param->table->stat_records(); bitmap_clear_all(&info->covered_fields); return info; } @@ -6142,7 +6558,7 @@ static double ror_scan_selectivity(const ROR_INTERSECT_INFO *info, min_range.flag= HA_READ_KEY_EXACT; max_range.key= key_val; max_range.flag= HA_READ_AFTER_KEY; - ha_rows prev_records= info->param->table->file->stats.records; + ha_rows prev_records= info->param->table->stat_records(); DBUG_ENTER("ror_scan_selectivity"); for (sel_arg= scan->sel_arg; sel_arg; @@ -6369,7 +6785,7 @@ TRP_ROR_INTERSECT *get_best_ror_intersect(const PARAM *param, SEL_TREE *tree, double min_cost= DBL_MAX; DBUG_ENTER("get_best_ror_intersect"); - if ((tree->n_ror_scans < 2) || !param->table->file->stats.records || + if ((tree->n_ror_scans < 2) || !param->table->stat_records() || !optimizer_flag(param->thd, OPTIMIZER_SWITCH_INDEX_MERGE_INTERSECT)) DBUG_RETURN(NULL); @@ -7247,7 +7663,8 @@ static SEL_TREE *get_func_mm_tree(RANGE_OPT_PARAM *param, Item_func *cond_func, param PARAM from SQL_SELECT::test_quick_select cond_func item for the predicate field_item field in the predicate - value constant in the predicate + value constant in the predicate (or a field already read from + a table in the case of dynamic range access) (for BETWEEN it contains the number of the field argument, for IN it's always 0) inv TRUE <> NOT cond_func is considered @@ -7516,24 +7933,41 @@ static SEL_TREE *get_mm_tree(RANGE_OPT_PARAM *param,COND *cond) DBUG_RETURN(ftree); } default: + + DBUG_ASSERT (!ftree); if (cond_func->arguments()[0]->real_item()->type() == Item::FIELD_ITEM) { field_item= (Item_field*) (cond_func->arguments()[0]->real_item()); - value= cond_func->arg_count > 1 ? cond_func->arguments()[1] : 0; + value= cond_func->arg_count > 1 ? cond_func->arguments()[1] : NULL; + if (value && value->is_expensive()) + DBUG_RETURN(0); + ftree= get_full_func_mm_tree(param, cond_func, field_item, value, inv); } - else if (cond_func->have_rev_func() && - cond_func->arguments()[1]->real_item()->type() == - Item::FIELD_ITEM) + /* + Even if get_full_func_mm_tree() was executed above and did not + return a range predicate it may still be possible to create one + by reversing the order of the operands. Note that this only + applies to predicates where both operands are fields. Example: A + query of the form + + WHERE t1.a OP t2.b + + In this case, arguments()[0] == t1.a and arguments()[1] == t2.b. + When creating range predicates for t2, get_full_func_mm_tree() + above will return NULL because 'field' belongs to t1 and only + predicates that applies to t2 are of interest. In this case a + call to get_full_func_mm_tree() with reversed operands (see + below) may succeed. + */ + if (!ftree && cond_func->have_rev_func() && + cond_func->arguments()[1]->real_item()->type() == Item::FIELD_ITEM) { field_item= (Item_field*) (cond_func->arguments()[1]->real_item()); value= cond_func->arguments()[0]; + if (value && value->is_expensive()) + DBUG_RETURN(0); + ftree= get_full_func_mm_tree(param, cond_func, field_item, value, inv); } - else - DBUG_RETURN(0); - if (value && value->is_expensive()) - DBUG_RETURN(0); - - ftree= get_full_func_mm_tree(param, cond_func, field_item, value, inv); } DBUG_RETURN(ftree); @@ -10074,6 +10508,7 @@ ha_rows check_quick_select(PARAM *param, uint idx, bool index_only, if (rows != HA_POS_ERROR) { param->quick_rows[keynr]= rows; + param->possible_keys.set_bit(keynr); if (update_tbl_stats) { param->table->quick_keys.set_bit(keynr); @@ -11568,78 +12003,134 @@ void QUICK_SELECT_I::add_key_name(String *str, bool *first) } -void QUICK_RANGE_SELECT::add_info_string(String *str) +Explain_quick_select* QUICK_RANGE_SELECT::get_explain(MEM_ROOT *alloc) { - bool first= TRUE; - - add_key_name(str, &first); + Explain_quick_select *res; + if ((res= new (alloc) Explain_quick_select(QS_TYPE_RANGE))) + res->range.set(alloc, head->key_info[index].name, max_used_key_length); + return res; +} + + +Explain_quick_select* QUICK_GROUP_MIN_MAX_SELECT::get_explain(MEM_ROOT *alloc) +{ + Explain_quick_select *res; + if ((res= new (alloc) Explain_quick_select(QS_TYPE_GROUP_MIN_MAX))) + res->range.set(alloc, head->key_info[index].name, max_used_key_length); + return res; } -void QUICK_INDEX_MERGE_SELECT::add_info_string(String *str) + +Explain_quick_select* QUICK_INDEX_SORT_SELECT::get_explain(MEM_ROOT *alloc) { + Explain_quick_select *res; + if (!(res= new (alloc) Explain_quick_select(get_type()))) + return NULL; + QUICK_RANGE_SELECT *quick; - bool first= TRUE; + Explain_quick_select *child_explain; List_iterator_fast<QUICK_RANGE_SELECT> it(quick_selects); - - str->append(STRING_WITH_LEN("sort_union(")); while ((quick= it++)) { - quick->add_key_name(str, &first); + if ((child_explain= quick->get_explain(alloc))) + res->children.push_back(child_explain); + else + return NULL; } + if (pk_quick_select) - pk_quick_select->add_key_name(str, &first); - str->append(')'); + { + if ((child_explain= pk_quick_select->get_explain(alloc))) + res->children.push_back(child_explain); + else + return NULL; + } + return res; } -void QUICK_INDEX_INTERSECT_SELECT::add_info_string(String *str) + +/* + Same as QUICK_INDEX_SORT_SELECT::get_explain(), but primary key is printed + first +*/ + +Explain_quick_select* QUICK_INDEX_INTERSECT_SELECT::get_explain(MEM_ROOT *alloc) { - QUICK_RANGE_SELECT *quick; - bool first= TRUE; - List_iterator_fast<QUICK_RANGE_SELECT> it(quick_selects); + Explain_quick_select *res; + Explain_quick_select *child_explain; + + if (!(res= new (alloc) Explain_quick_select(get_type()))) + return NULL; - str->append(STRING_WITH_LEN("sort_intersect(")); if (pk_quick_select) - pk_quick_select->add_key_name(str, &first); + { + if ((child_explain= pk_quick_select->get_explain(alloc))) + res->children.push_back(child_explain); + else + return NULL; + } + + QUICK_RANGE_SELECT *quick; + List_iterator_fast<QUICK_RANGE_SELECT> it(quick_selects); while ((quick= it++)) { - quick->add_key_name(str, &first); + if ((child_explain= quick->get_explain(alloc))) + res->children.push_back(child_explain); + else + return NULL; } - str->append(')'); + return res; } -void QUICK_ROR_INTERSECT_SELECT::add_info_string(String *str) + +Explain_quick_select* QUICK_ROR_INTERSECT_SELECT::get_explain(MEM_ROOT *alloc) { - bool first= TRUE; + Explain_quick_select *res; + Explain_quick_select *child_explain; + + if (!(res= new (alloc) Explain_quick_select(get_type()))) + return NULL; + QUICK_SELECT_WITH_RECORD *qr; List_iterator_fast<QUICK_SELECT_WITH_RECORD> it(quick_selects); - - str->append(STRING_WITH_LEN("intersect(")); while ((qr= it++)) { - qr->quick->add_key_name(str, &first); + if ((child_explain= qr->quick->get_explain(alloc))) + res->children.push_back(child_explain); + else + return NULL; } + if (cpk_quick) - cpk_quick->add_key_name(str, &first); - str->append(')'); + { + if ((child_explain= cpk_quick->get_explain(alloc))) + res->children.push_back(child_explain); + else + return NULL; + } + return res; } -void QUICK_ROR_UNION_SELECT::add_info_string(String *str) +Explain_quick_select* QUICK_ROR_UNION_SELECT::get_explain(MEM_ROOT *alloc) { + Explain_quick_select *res; + Explain_quick_select *child_explain; + + if (!(res= new (alloc) Explain_quick_select(get_type()))) + return NULL; + QUICK_SELECT_I *quick; - bool first= TRUE; List_iterator_fast<QUICK_SELECT_I> it(quick_selects); - - str->append(STRING_WITH_LEN("union(")); while ((quick= it++)) { - if (first) - first= FALSE; + if ((child_explain= quick->get_explain(alloc))) + res->children.push_back(child_explain); else - str->append(','); - quick->add_info_string(str); + return NULL; } - str->append(')'); + + return res; } @@ -12039,8 +12530,12 @@ get_best_group_min_max(PARAM *param, SEL_TREE *tree, double read_time) uchar cur_key_infix[MAX_KEY_LENGTH]; uint cur_used_key_parts; - /* Check (B1) - if current index is covering. */ - if (!table->covering_keys.is_set(cur_index)) + /* + Check (B1) - if current index is covering. Exclude UNIQUE indexes, because + loose scan may still be chosen for them due to imperfect cost calculations. + */ + if (!table->covering_keys.is_set(cur_index) || + cur_index_info->flags & HA_NOSAME) goto next_index; /* @@ -12894,11 +13389,11 @@ void cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts, double *read_cost, ha_rows *records) { ha_rows table_records; - uint num_groups; - uint num_blocks; - uint keys_per_block; - uint keys_per_group; - uint keys_per_subgroup; /* Average number of keys in sub-groups */ + ha_rows num_groups; + ha_rows num_blocks; + uint keys_per_block; + ha_rows keys_per_group; + ha_rows keys_per_subgroup; /* Average number of keys in sub-groups */ /* formed by a key infix. */ double p_overlap; /* Probability that a sub-group overlaps two blocks. */ double quick_prefix_selectivity; @@ -12906,25 +13401,25 @@ void cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts, double cpu_cost= 0; /* TODO: CPU cost of index_read calls? */ DBUG_ENTER("cost_group_min_max"); - table_records= table->file->stats.records; - keys_per_block= (table->file->stats.block_size / 2 / - (index_info->key_length + table->file->ref_length) - + 1); - num_blocks= (uint)(table_records / keys_per_block) + 1; + table_records= table->stat_records(); + keys_per_block= (uint) (table->file->stats.block_size / 2 / + (index_info->key_length + table->file->ref_length) + + 1); + num_blocks= (ha_rows)(table_records / keys_per_block) + 1; /* Compute the number of keys in a group. */ - keys_per_group= index_info->rec_per_key[group_key_parts - 1]; + keys_per_group= (ha_rows) index_info->actual_rec_per_key(group_key_parts - 1); if (keys_per_group == 0) /* If there is no statistics try to guess */ /* each group contains 10% of all records */ - keys_per_group= (uint)(table_records / 10) + 1; - num_groups= (uint)(table_records / keys_per_group) + 1; + keys_per_group= (table_records / 10) + 1; + num_groups= (table_records / keys_per_group) + 1; /* Apply the selectivity of the quick select for group prefixes. */ if (range_tree && (quick_prefix_records != HA_POS_ERROR)) { quick_prefix_selectivity= (double) quick_prefix_records / (double) table_records; - num_groups= (uint) rint(num_groups * quick_prefix_selectivity); + num_groups= (ha_rows) rint(num_groups * quick_prefix_selectivity); set_if_bigger(num_groups, 1); } @@ -12933,7 +13428,7 @@ void cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts, Compute the probability that two ends of a subgroup are inside different blocks. */ - keys_per_subgroup= index_info->rec_per_key[used_key_parts - 1]; + keys_per_subgroup= (ha_rows) index_info->actual_rec_per_key(used_key_parts - 1); if (keys_per_subgroup >= keys_per_block) /* If a subgroup is bigger than */ p_overlap= 1.0; /* a block, it will overlap at least two blocks. */ else @@ -12961,9 +13456,9 @@ void cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts, *records= num_groups; DBUG_PRINT("info", - ("table rows: %lu keys/block: %u keys/group: %u result rows: %lu blocks: %u", - (ulong)table_records, keys_per_block, keys_per_group, - (ulong) *records, num_blocks)); + ("table rows: %lu keys/block: %u keys/group: %lu result rows: %lu blocks: %lu", + (ulong)table_records, keys_per_block, (ulong) keys_per_group, + (ulong) *records, (ulong) num_blocks)); DBUG_VOID_RETURN; } @@ -13130,7 +13625,8 @@ QUICK_GROUP_MIN_MAX_SELECT(TABLE *table, JOIN *join_arg, bool have_min_arg, DBUG_ASSERT(!parent_alloc); if (!parent_alloc) { - init_sql_alloc(&alloc, join->thd->variables.range_alloc_block_size, 0); + init_sql_alloc(&alloc, join->thd->variables.range_alloc_block_size, 0, + MYF(MY_THREAD_SPECIFIC)); join->thd->mem_root= &alloc; } else @@ -13185,7 +13681,8 @@ int QUICK_GROUP_MIN_MAX_SELECT::init() if (min_max_arg_part) { - if (my_init_dynamic_array(&min_max_ranges, sizeof(QUICK_RANGE*), 16, 16)) + if (my_init_dynamic_array(&min_max_ranges, sizeof(QUICK_RANGE*), 16, 16, + MYF(MY_THREAD_SPECIFIC))) return 1; if (have_min) @@ -14393,11 +14890,3 @@ void QUICK_GROUP_MIN_MAX_SELECT::dbug_dump(int indent, bool verbose) #endif /* !DBUG_OFF */ -/***************************************************************************** -** Instantiate templates -*****************************************************************************/ - -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class List<QUICK_RANGE>; -template class List_iterator<QUICK_RANGE>; -#endif diff --git a/sql/opt_range.h b/sql/opt_range.h index fff6e414ad9..d61219b7dd0 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -52,7 +52,7 @@ typedef struct st_key_part { Field::imagetype image_type; } KEY_PART; - +class Explain_quick_select; /* A "MIN_TUPLE < tbl.key_tuple < MAX_TUPLE" interval. @@ -345,13 +345,8 @@ public: void add_key_name(String *str, bool *first); - /* - Append text representation of quick select structure (what and how is - merged) to str. The result is added to "Extra" field in EXPLAIN output. - This function is implemented only by quick selects that merge other quick - selects output and/or can produce output suitable for merging. - */ - virtual void add_info_string(String *str) {} + /* Save information about quick select's query plan */ + virtual Explain_quick_select* get_explain(MEM_ROOT *alloc)= 0; /* Return 1 if any index used by this quick select @@ -478,7 +473,7 @@ public: { file->position(record); } int get_type() { return QS_TYPE_RANGE; } void add_keys_and_lengths(String *key_names, String *used_lengths); - void add_info_string(String *str); + Explain_quick_select *get_explain(MEM_ROOT *alloc); #ifndef DBUG_OFF void dbug_dump(int indent, bool verbose); #endif @@ -615,6 +610,7 @@ public: #ifndef DBUG_OFF void dbug_dump(int indent, bool verbose); #endif + Explain_quick_select *get_explain(MEM_ROOT *alloc); bool push_quick_back(QUICK_RANGE_SELECT *quick_sel_range); @@ -663,7 +659,6 @@ public: int get_next(); int get_type() { return QS_TYPE_INDEX_MERGE; } void add_keys_and_lengths(String *key_names, String *used_lengths); - void add_info_string(String *str); }; class QUICK_INDEX_INTERSECT_SELECT : public QUICK_INDEX_SORT_SELECT @@ -679,7 +674,7 @@ public: int get_next(); int get_type() { return QS_TYPE_INDEX_INTERSECT; } void add_keys_and_lengths(String *key_names, String *used_lengths); - void add_info_string(String *str); + Explain_quick_select *get_explain(MEM_ROOT *alloc); }; @@ -717,7 +712,7 @@ public: bool unique_key_range() { return false; } int get_type() { return QS_TYPE_ROR_INTERSECT; } void add_keys_and_lengths(String *key_names, String *used_lengths); - void add_info_string(String *str); + Explain_quick_select *get_explain(MEM_ROOT *alloc); bool is_keys_used(const MY_BITMAP *fields); #ifndef DBUG_OFF void dbug_dump(int indent, bool verbose); @@ -796,7 +791,7 @@ public: bool unique_key_range() { return false; } int get_type() { return QS_TYPE_ROR_UNION; } void add_keys_and_lengths(String *key_names, String *used_lengths); - void add_info_string(String *str); + Explain_quick_select *get_explain(MEM_ROOT *alloc); bool is_keys_used(const MY_BITMAP *fields); #ifndef DBUG_OFF void dbug_dump(int indent, bool verbose); @@ -944,11 +939,8 @@ public: void dbug_dump(int indent, bool verbose); #endif bool is_agg_distinct() { return have_agg_distinct; } - virtual void append_loose_scan_type(String *str) - { - if (is_index_scan) - str->append(STRING_WITH_LEN(" (scanning)")); - } + bool loose_scan_is_scanning() { return is_index_scan; } + Explain_quick_select *get_explain(MEM_ROOT *alloc); }; @@ -990,6 +982,8 @@ class SQL_SELECT :public Sql_alloc { key_map quick_keys; // Possible quick keys key_map needed_reg; // Possible quick keys after prev tables. table_map const_tables,read_tables; + /* See PARAM::possible_keys */ + key_map possible_keys; bool free_cond; /* Currently not used and always FALSE */ SQL_SELECT(); @@ -1042,11 +1036,20 @@ SQL_SELECT *make_select(TABLE *head, table_map const_tables, table_map read_tables, COND *conds, bool allow_null_cond, int *error); +bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item *cond); + #ifdef WITH_PARTITION_STORAGE_ENGINE bool prune_partitions(THD *thd, TABLE *table, Item *pprune_cond); -void store_key_image_to_rec(Field *field, uchar *ptr, uint len); #endif +void store_key_image_to_rec(Field *field, uchar *ptr, uint len); extern String null_string; +/* check this number of rows (default value) */ +#define SELECTIVITY_SAMPLING_LIMIT 100 +/* but no more then this part of table (10%) */ +#define SELECTIVITY_SAMPLING_SHARE 0.10 +/* do not check if we are going check less then this number of records */ +#define SELECTIVITY_SAMPLING_THRESHOLD 10 + #endif diff --git a/sql/opt_range_mrr.cc b/sql/opt_range_mrr.cc index 1f4e36178db..8029dbf000f 100644 --- a/sql/opt_range_mrr.cc +++ b/sql/opt_range_mrr.cc @@ -268,8 +268,10 @@ walk_up_n_right: range->end_key.keypart_map= make_prev_keypart_map(cur->max_key_parts); if (!(cur->min_key_flag & ~NULL_RANGE) && !cur->max_key_flag && - (uint)key_tree->part+1 == seq->param->table->key_info[seq->real_keyno].key_parts && - (seq->param->table->key_info[seq->real_keyno].flags & HA_NOSAME) && + (seq->real_keyno == MAX_KEY || + ((uint)key_tree->part+1 == + seq->param->table->key_info[seq->real_keyno].key_parts && + (seq->param->table->key_info[seq->real_keyno].flags & HA_NOSAME))) && range->start_key.length == range->end_key.length && !memcmp(seq->param->min_key,seq->param->max_key,range->start_key.length)) range->range_flag= UNIQUE_RANGE | (cur->min_key_flag & NULL_RANGE); diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index 794b4f9b750..2cc7447202c 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -664,6 +664,9 @@ int check_and_do_in_subquery_rewrites(JOIN *join) 8. No execution method was already chosen (by a prepared statement) 9. Parent select is not a table-less select 10. Neither parent nor child select have STRAIGHT_JOIN option. + 11. It is first optimisation (the subquery could be moved from ON + clause during first optimisation and then be considered for SJ + on the second when it is too late) */ if (optimizer_flag(thd, OPTIMIZER_SWITCH_SEMIJOIN) && in_subs && // 1 @@ -677,7 +680,8 @@ int check_and_do_in_subquery_rewrites(JOIN *join) select_lex->outer_select()->leaf_tables.elements && // 9 !((join->select_options | // 10 select_lex->outer_select()->join->select_options) // 10 - & SELECT_STRAIGHT_JOIN)) // 10 + & SELECT_STRAIGHT_JOIN) && // 10 + select_lex->first_cond_optimization) // 11 { DBUG_PRINT("info", ("Subquery is semi-join conversion candidate")); @@ -1663,6 +1667,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred) parent_lex->ftfunc_list->push_front(ifm); } + parent_lex->have_merged_subqueries= TRUE; DBUG_RETURN(FALSE); } @@ -1776,6 +1781,8 @@ static bool convert_subq_to_jtbm(JOIN *parent_join, create_subquery_temptable_name(tbl_alias, hash_sj_engine->materialize_join-> select_lex->select_number); jtbm->alias= tbl_alias; + + parent_lex->have_merged_subqueries= TRUE; #if 0 /* Inject sj_on_expr into the parent's WHERE or ON */ if (emb_tbl_nest) @@ -3907,7 +3914,7 @@ SJ_TMP_TABLE::create_sj_weedout_tmp_table(THD *thd) using_unique_constraint= TRUE; /* STEP 3: Allocate memory for temptable description */ - init_sql_alloc(&own_root, TABLE_ALLOC_BLOCK_SIZE, 0); + init_sql_alloc(&own_root, TABLE_ALLOC_BLOCK_SIZE, 0, MYF(MY_THREAD_SPECIFIC)); if (!multi_alloc_root(&own_root, &table, sizeof(*table), &share, sizeof(*share), @@ -3920,7 +3927,7 @@ SJ_TMP_TABLE::create_sj_weedout_tmp_table(THD *thd) &tmpname, (uint) strlen(path)+1, &group_buff, (!using_unique_constraint ? uniq_tuple_length_arg : 0), - &bitmaps, bitmap_buffer_size(1)*3, + &bitmaps, bitmap_buffer_size(1)*5, NullS)) { if (temp_pool_slot != MY_BIT_NONE) diff --git a/sql/opt_subselect.h b/sql/opt_subselect.h index 0a74abec68b..07aa3c64633 100644 --- a/sql/opt_subselect.h +++ b/sql/opt_subselect.h @@ -284,6 +284,7 @@ public: { pos->records_read= best_loose_scan_records; pos->key= best_loose_scan_start_key; + pos->cond_selectivity= 1.0; pos->loosescan_picker.loosescan_key= best_loose_scan_key; pos->loosescan_picker.loosescan_parts= best_max_loose_keypart + 1; pos->use_join_buffer= FALSE; diff --git a/sql/partition_info.cc b/sql/partition_info.cc index 1bfa8864cb9..f94e5af119e 100644 --- a/sql/partition_info.cc +++ b/sql/partition_info.cc @@ -1581,29 +1581,21 @@ bool check_partition_dirs(partition_info *part_info) partition_element *subpart_elem; while ((subpart_elem= sub_it++)) { - if (test_if_data_home_dir(subpart_elem->data_file_name)) - goto dd_err; - if (test_if_data_home_dir(subpart_elem->index_file_name)) - goto id_err; + if (error_if_data_home_dir(subpart_elem->data_file_name, + "DATA DIRECTORY") || + error_if_data_home_dir(subpart_elem->index_file_name, + "INDEX DIRECTORY")) + return 1; } } else { - if (test_if_data_home_dir(part_elem->data_file_name)) - goto dd_err; - if (test_if_data_home_dir(part_elem->index_file_name)) - goto id_err; + if (error_if_data_home_dir(part_elem->data_file_name, "DATA DIRECTORY") || + error_if_data_home_dir(part_elem->index_file_name, "INDEX DIRECTORY")) + return 1; } } return 0; - -dd_err: - my_error(ER_WRONG_ARGUMENTS,MYF(0),"DATA DIRECTORY"); - return 1; - -id_err: - my_error(ER_WRONG_ARGUMENTS,MYF(0),"INDEX DIRECTORY"); - return 1; } @@ -2568,4 +2560,9 @@ void partition_info::print_debug(const char *str, uint *value) { } +bool check_partition_dirs(partition_info *part_info) +{ + return 0; +} + #endif /* WITH_PARTITION_STORAGE_ENGINE */ diff --git a/sql/partition_info.h b/sql/partition_info.h index cff941a858a..2a7c1bede0e 100644 --- a/sql/partition_info.h +++ b/sql/partition_info.h @@ -321,6 +321,7 @@ private: char *create_default_partition_names(uint part_no, uint num_parts, uint start_no); char *create_subpartition_name(uint subpart_no, const char *part_name); +public: bool has_unique_name(partition_element *element); }; diff --git a/sql/protocol.cc b/sql/protocol.cc index 2f19843e3e2..6b2cf82a447 100644 --- a/sql/protocol.cc +++ b/sql/protocol.cc @@ -776,7 +776,7 @@ bool Protocol::send_result_set_metadata(List<Item> *list, uint flags) pos= (char*) local_packet->ptr()+local_packet->length(); *pos++= 12; // Length of packed fields /* inject a NULL to test the client */ - DBUG_EXECUTE_IF("poison_rs_fields", pos[-1]= 0xfb;); + DBUG_EXECUTE_IF("poison_rs_fields", pos[-1]= (char) 0xfb;); if (item->charset_for_protocol() == &my_charset_bin || thd_charset == NULL) { /* No conversion */ diff --git a/sql/protocol.h b/sql/protocol.h index d9bc48ce45d..1c0a28560bd 100644 --- a/sql/protocol.h +++ b/sql/protocol.h @@ -35,6 +35,7 @@ class Protocol protected: THD *thd; String *packet; + /* Used by net_store_data() for charset conversions */ String *convert; uint field_pos; #ifndef DBUG_OFF @@ -49,6 +50,10 @@ protected: MYSQL_FIELD *next_mysql_field; MEM_ROOT *alloc; #endif + /* + The following two are low-level functions that are invoked from + higher-level store_xxx() funcs. The data is stored into this->packet. + */ bool net_store_data(const uchar *from, size_t length, CHARSET_INFO *fromcs, CHARSET_INFO *tocs); bool store_string_aux(const char *from, size_t length, diff --git a/sql/records.cc b/sql/records.cc index aca950d7435..1b230c41156 100644 --- a/sql/records.cc +++ b/sql/records.cc @@ -603,7 +603,7 @@ static int init_rr_cache(THD *thd, READ_RECORD *info) if (info->cache_records <= 2 || !(info->cache=(uchar*) my_malloc_lock(rec_cache_size+info->cache_records* info->struct_length+1, - MYF(0)))) + MYF(MY_THREAD_SPECIFIC)))) DBUG_RETURN(1); #ifdef HAVE_valgrind // Avoid warnings in qsort diff --git a/sql/repl_failsafe.cc b/sql/repl_failsafe.cc index 89fb1bb27de..35b43fd4305 100644 --- a/sql/repl_failsafe.cc +++ b/sql/repl_failsafe.cc @@ -89,14 +89,15 @@ void change_rpl_status(ulong from_status, ulong to_status) void unregister_slave(THD* thd, bool only_mine, bool need_mutex) { - if (thd->server_id) + uint32 thd_server_id= thd->variables.server_id; + if (thd_server_id) { if (need_mutex) mysql_mutex_lock(&LOCK_slave_list); SLAVE_INFO* old_si; if ((old_si = (SLAVE_INFO*)my_hash_search(&slave_list, - (uchar*)&thd->server_id, 4)) && + (uchar*)&thd_server_id, 4)) && (!only_mine || old_si->thd == thd)) my_hash_delete(&slave_list, (uchar*)old_si); @@ -127,7 +128,7 @@ int register_slave(THD* thd, uchar* packet, uint packet_length) if (!(si = (SLAVE_INFO*)my_malloc(sizeof(SLAVE_INFO), MYF(MY_WME)))) goto err2; - thd->server_id= si->server_id= uint4korr(p); + thd->variables.server_id= si->server_id= uint4korr(p); p+= 4; get_object(p,si->host, "Failed to register slave: too long 'report-host'"); get_object(p,si->user, "Failed to register slave: too long 'report-user'"); @@ -145,7 +146,7 @@ int register_slave(THD* thd, uchar* packet, uint packet_length) // si->rpl_recovery_rank= uint4korr(p); p += 4; if (!(si->master_id= uint4korr(p))) - si->master_id= server_id; + si->master_id= global_system_variables.server_id; si->thd= thd; mysql_mutex_lock(&LOCK_slave_list); diff --git a/sql/rpl_filter.cc b/sql/rpl_filter.cc index 9103cad337d..2b4a3093e0f 100644 --- a/sql/rpl_filter.cc +++ b/sql/rpl_filter.cc @@ -574,7 +574,7 @@ void Rpl_filter::init_table_rule_array(DYNAMIC_ARRAY* a, bool* a_inited) { my_init_dynamic_array(a, sizeof(TABLE_RULE_ENT*), TABLE_RULE_ARR_SIZE, - TABLE_RULE_ARR_SIZE); + TABLE_RULE_ARR_SIZE, MYF(0)); *a_inited = 1; } @@ -735,6 +735,18 @@ Rpl_filter::get_rewrite_db(const char* db, size_t *new_len) } +void +Rpl_filter::copy_rewrite_db(Rpl_filter *from) +{ + I_List_iterator<i_string_pair> it(from->rewrite_db); + i_string_pair* tmp; + DBUG_ASSERT(rewrite_db.is_empty()); + + /* TODO: Add memory checking here and in all add_xxxx functions ! */ + while ((tmp=it++)) + add_db_rewrite(tmp->key, tmp->val); +} + I_List<i_string>* Rpl_filter::get_do_db() { diff --git a/sql/rpl_filter.h b/sql/rpl_filter.h index 2eb0340b714..65d11cfb6e6 100644 --- a/sql/rpl_filter.h +++ b/sql/rpl_filter.h @@ -88,6 +88,7 @@ public: bool rewrite_db_is_empty(); const char* get_rewrite_db(const char* db, size_t *new_len); + void copy_rewrite_db(Rpl_filter *from); I_List<i_string>* get_do_db(); I_List<i_string>* get_ignore_db(); @@ -139,7 +140,7 @@ private: I_List<i_string_pair> rewrite_db; }; -extern Rpl_filter *rpl_filter; +extern Rpl_filter *global_rpl_filter; extern Rpl_filter *binlog_filter; #endif // RPL_FILTER_H diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc new file mode 100644 index 00000000000..d7923ed9130 --- /dev/null +++ b/sql/rpl_gtid.cc @@ -0,0 +1,2118 @@ +/* Copyright (c) 2013, Kristian Nielsen and MariaDB Services Ab. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + + +/* Definitions for MariaDB global transaction ID (GTID). */ + + +#include "sql_priv.h" +#include "my_sys.h" +#include "unireg.h" +#include "my_global.h" +#include "sql_base.h" +#include "sql_parse.h" +#include "key.h" +#include "rpl_gtid.h" +#include "rpl_rli.h" + + +const LEX_STRING rpl_gtid_slave_state_table_name= + { C_STRING_WITH_LEN("gtid_slave_pos") }; + + +void +rpl_slave_state::update_state_hash(uint64 sub_id, rpl_gtid *gtid) +{ + int err; + /* + Add the gtid to the HASH in the replication slave state. + + We must do this only _after_ commit, so that for parallel replication, + there will not be an attempt to delete the corresponding table row before + it is even committed. + */ + mysql_mutex_lock(&LOCK_slave_state); + err= update(gtid->domain_id, gtid->server_id, sub_id, gtid->seq_no); + mysql_mutex_unlock(&LOCK_slave_state); + if (err) + { + sql_print_warning("Slave: Out of memory during slave state maintenance. " + "Some no longer necessary rows in table " + "mysql.%s may be left undeleted.", + rpl_gtid_slave_state_table_name.str); + /* + Such failure is not fatal. We will fail to delete the row for this + GTID, but it will do no harm and will be removed automatically on next + server restart. + */ + } +} + + +int +rpl_slave_state::record_and_update_gtid(THD *thd, rpl_group_info *rgi) +{ + uint64 sub_id; + DBUG_ENTER("rpl_slave_state::record_and_update_gtid"); + + /* + Update the GTID position, if we have it and did not already update + it in a GTID transaction. + */ + if ((sub_id= rgi->gtid_sub_id)) + { + rgi->gtid_sub_id= 0; + if (record_gtid(thd, &rgi->current_gtid, sub_id, false, false)) + DBUG_RETURN(1); + update_state_hash(sub_id, &rgi->current_gtid); + } + DBUG_RETURN(0); +} + + +static void +rpl_slave_state_free_element(void *arg) +{ + struct rpl_slave_state::element *elem= (struct rpl_slave_state::element *)arg; + mysql_cond_destroy(&elem->COND_wait_gtid); + my_free(elem); +} + + +rpl_slave_state::rpl_slave_state() + : last_sub_id(0), inited(false), loaded(false) +{ + my_hash_init(&hash, &my_charset_bin, 32, offsetof(element, domain_id), + sizeof(uint32), NULL, rpl_slave_state_free_element, HASH_UNIQUE); +} + + +rpl_slave_state::~rpl_slave_state() +{ +} + + +void +rpl_slave_state::init() +{ + DBUG_ASSERT(!inited); + mysql_mutex_init(key_LOCK_slave_state, &LOCK_slave_state, MY_MUTEX_INIT_SLOW); + inited= true; +} + + +void +rpl_slave_state::truncate_hash() +{ + uint32 i; + + for (i= 0; i < hash.records; ++i) + { + element *e= (element *)my_hash_element(&hash, i); + list_element *l= e->list; + list_element *next; + while (l) + { + next= l->next; + my_free(l); + l= next; + } + /* The element itself is freed by the hash element free function. */ + } + my_hash_reset(&hash); +} + +void +rpl_slave_state::deinit() +{ + if (!inited) + return; + truncate_hash(); + my_hash_free(&hash); + mysql_mutex_destroy(&LOCK_slave_state); +} + + +int +rpl_slave_state::update(uint32 domain_id, uint32 server_id, uint64 sub_id, + uint64 seq_no) +{ + element *elem= NULL; + list_element *list_elem= NULL; + + if (!(elem= get_element(domain_id))) + return 1; + + if (seq_no > elem->highest_seq_no) + elem->highest_seq_no= seq_no; + if (elem->gtid_waiter && elem->min_wait_seq_no <= seq_no) + { + /* + Someone was waiting in MASTER_GTID_WAIT() for this GTID to appear. + Signal (and remove) them. The waiter will handle all the processing + of all pending MASTER_GTID_WAIT(), so we do not slow down the + replication SQL thread. + */ + mysql_mutex_assert_owner(&LOCK_slave_state); + elem->gtid_waiter= NULL; + mysql_cond_broadcast(&elem->COND_wait_gtid); + } + + if (!(list_elem= (list_element *)my_malloc(sizeof(*list_elem), MYF(MY_WME)))) + return 1; + list_elem->server_id= server_id; + list_elem->sub_id= sub_id; + list_elem->seq_no= seq_no; + + elem->add(list_elem); + if (last_sub_id < sub_id) + last_sub_id= sub_id; + + return 0; +} + + +struct rpl_slave_state::element * +rpl_slave_state::get_element(uint32 domain_id) +{ + struct element *elem; + + elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, 0); + if (elem) + return elem; + + if (!(elem= (element *)my_malloc(sizeof(*elem), MYF(MY_WME)))) + return NULL; + elem->list= NULL; + elem->domain_id= domain_id; + elem->highest_seq_no= 0; + elem->gtid_waiter= NULL; + mysql_cond_init(key_COND_wait_gtid, &elem->COND_wait_gtid, 0); + if (my_hash_insert(&hash, (uchar *)elem)) + { + my_free(elem); + return NULL; + } + return elem; +} + + +int +rpl_slave_state::put_back_list(uint32 domain_id, list_element *list) +{ + element *e; + if (!(e= (element *)my_hash_search(&hash, (const uchar *)&domain_id, 0))) + return 1; + while (list) + { + list_element *next= list->next; + e->add(list); + list= next; + } + return 0; +} + + +int +rpl_slave_state::truncate_state_table(THD *thd) +{ + TABLE_LIST tlist; + int err= 0; + + tmp_disable_binlog(thd); + tlist.init_one_table(STRING_WITH_LEN("mysql"), + rpl_gtid_slave_state_table_name.str, + rpl_gtid_slave_state_table_name.length, + NULL, TL_WRITE); + if (!(err= open_and_lock_tables(thd, &tlist, FALSE, 0))) + { + err= tlist.table->file->ha_truncate(); + + if (err) + { + ha_rollback_trans(thd, FALSE); + close_thread_tables(thd); + ha_rollback_trans(thd, TRUE); + } + else + { + ha_commit_trans(thd, FALSE); + close_thread_tables(thd); + ha_commit_trans(thd, TRUE); + } + thd->mdl_context.release_transactional_locks(); + } + + reenable_binlog(thd); + return err; +} + + +static const TABLE_FIELD_TYPE mysql_rpl_slave_state_coltypes[4]= { + { { C_STRING_WITH_LEN("domain_id") }, + { C_STRING_WITH_LEN("int(10) unsigned") }, + {NULL, 0} }, + { { C_STRING_WITH_LEN("sub_id") }, + { C_STRING_WITH_LEN("bigint(20) unsigned") }, + {NULL, 0} }, + { { C_STRING_WITH_LEN("server_id") }, + { C_STRING_WITH_LEN("int(10) unsigned") }, + {NULL, 0} }, + { { C_STRING_WITH_LEN("seq_no") }, + { C_STRING_WITH_LEN("bigint(20) unsigned") }, + {NULL, 0} }, +}; + +static const uint mysql_rpl_slave_state_pk_parts[]= {0, 1}; + +static const TABLE_FIELD_DEF mysql_gtid_slave_pos_tabledef= { + array_elements(mysql_rpl_slave_state_coltypes), + mysql_rpl_slave_state_coltypes, + array_elements(mysql_rpl_slave_state_pk_parts), + mysql_rpl_slave_state_pk_parts +}; + +class Gtid_db_intact : public Table_check_intact +{ +protected: + void report_error(uint, const char *fmt, ...) + { + va_list args; + va_start(args, fmt); + error_log_print(ERROR_LEVEL, fmt, args); + va_end(args); + } +}; + +static Gtid_db_intact gtid_table_intact; + +/* + Check that the mysql.gtid_slave_pos table has the correct definition. +*/ +int +gtid_check_rpl_slave_state_table(TABLE *table) +{ + int err; + + if ((err= gtid_table_intact.check(table, &mysql_gtid_slave_pos_tabledef))) + my_error(ER_GTID_OPEN_TABLE_FAILED, MYF(0), "mysql", + rpl_gtid_slave_state_table_name.str); + return err; +} + + +/* + Write a gtid to the replication slave state table. + + Do it as part of the transaction, to get slave crash safety, or as a separate + transaction if !in_transaction (eg. MyISAM or DDL). + + gtid The global transaction id for this event group. + sub_id Value allocated within the sub_id when the event group was + read (sub_id must be consistent with commit order in master binlog). + + Note that caller must later ensure that the new gtid and sub_id is inserted + into the appropriate HASH element with rpl_slave_state.add(), so that it can + be deleted later. But this must only be done after COMMIT if in transaction. +*/ +int +rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, + bool in_transaction, bool in_statement) +{ + TABLE_LIST tlist; + int err= 0; + bool table_opened= false; + TABLE *table; + list_element *elist= 0, *next; + element *elem; + ulonglong thd_saved_option= thd->variables.option_bits; + Query_tables_list lex_backup; + DBUG_ENTER("record_gtid"); + + if (unlikely(!loaded)) + { + /* + Probably the mysql.gtid_slave_pos table is missing (eg. upgrade) or + corrupt. + + We already complained loudly about this, but we can try to continue + until the DBA fixes it. + */ + DBUG_RETURN(0); + } + + if (!in_statement) + mysql_reset_thd_for_next_command(thd); + + DBUG_EXECUTE_IF("gtid_inject_record_gtid", + { + my_error(ER_CANNOT_UPDATE_GTID_STATE, MYF(0)); + DBUG_RETURN(1); + } ); + + thd->lex->reset_n_backup_query_tables_list(&lex_backup); + tlist.init_one_table(STRING_WITH_LEN("mysql"), + rpl_gtid_slave_state_table_name.str, + rpl_gtid_slave_state_table_name.length, + NULL, TL_WRITE); + if ((err= open_and_lock_tables(thd, &tlist, FALSE, 0))) + goto end; + table_opened= true; + table= tlist.table; + + if ((err= gtid_check_rpl_slave_state_table(table))) + goto end; + + if (!in_transaction) + { + DBUG_PRINT("info", ("resetting OPTION_BEGIN")); + thd->variables.option_bits&= + ~(ulonglong)(OPTION_NOT_AUTOCOMMIT|OPTION_BEGIN|OPTION_BIN_LOG); + } + else + thd->variables.option_bits&= ~(ulonglong)OPTION_BIN_LOG; + + bitmap_set_all(table->write_set); + + table->field[0]->store((ulonglong)gtid->domain_id, true); + table->field[1]->store(sub_id, true); + table->field[2]->store((ulonglong)gtid->server_id, true); + table->field[3]->store(gtid->seq_no, true); + DBUG_EXECUTE_IF("inject_crash_before_write_rpl_slave_state", DBUG_SUICIDE();); + if ((err= table->file->ha_write_row(table->record[0]))) + { + table->file->print_error(err, MYF(0)); + goto end; + } + + if(opt_bin_log && + (err= mysql_bin_log.bump_seq_no_counter_if_needed(gtid->domain_id, + gtid->seq_no))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + goto end; + } + + mysql_mutex_lock(&LOCK_slave_state); + if ((elem= get_element(gtid->domain_id)) == NULL) + { + mysql_mutex_unlock(&LOCK_slave_state); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + err= 1; + goto end; + } + if ((elist= elem->grab_list()) != NULL) + { + /* Delete any old stuff, but keep around the most recent one. */ + list_element *cur= elist; + uint64 best_sub_id= cur->sub_id; + list_element **best_ptr_ptr= &elist; + while ((next= cur->next)) + { + if (next->sub_id > best_sub_id) + { + best_sub_id= next->sub_id; + best_ptr_ptr= &cur->next; + } + cur= next; + } + /* + Delete the highest sub_id element from the old list, and put it back as + the single-element new list. + */ + cur= *best_ptr_ptr; + *best_ptr_ptr= cur->next; + cur->next= NULL; + elem->list= cur; + } + mysql_mutex_unlock(&LOCK_slave_state); + + if (!elist) + goto end; + + /* Now delete any already committed rows. */ + bitmap_set_bit(table->read_set, table->field[0]->field_index); + bitmap_set_bit(table->read_set, table->field[1]->field_index); + + if ((err= table->file->ha_index_init(0, 0))) + { + table->file->print_error(err, MYF(0)); + goto end; + } + while (elist) + { + uchar key_buffer[4+8]; + + DBUG_EXECUTE_IF("gtid_slave_pos_simulate_failed_delete", + { err= ENOENT; + table->file->print_error(err, MYF(0)); + /* `break' does not work inside DBUG_EXECUTE_IF */ + goto dbug_break; }); + + next= elist->next; + + table->field[1]->store(elist->sub_id, true); + /* domain_id is already set in table->record[0] from write_row() above. */ + key_copy(key_buffer, table->record[0], &table->key_info[0], 0, false); + if (table->file->ha_index_read_map(table->record[1], key_buffer, + HA_WHOLE_KEY, HA_READ_KEY_EXACT)) + /* We cannot find the row, assume it is already deleted. */ + ; + else if ((err= table->file->ha_delete_row(table->record[1]))) + table->file->print_error(err, MYF(0)); + /* + In case of error, we still discard the element from the list. We do + not want to endlessly error on the same element in case of table + corruption or such. + */ + my_free(elist); + elist= next; + if (err) + break; + } +IF_DBUG(dbug_break:, ) + table->file->ha_index_end(); + +end: + + if (table_opened) + { + if (err) + { + /* + If error, we need to put any remaining elist back into the HASH so we + can do another delete attempt later. + */ + if (elist) + { + mysql_mutex_lock(&LOCK_slave_state); + put_back_list(gtid->domain_id, elist); + mysql_mutex_unlock(&LOCK_slave_state); + } + + ha_rollback_trans(thd, FALSE); + close_thread_tables(thd); + } + else + { + ha_commit_trans(thd, FALSE); + close_thread_tables(thd); + } + if (in_transaction) + thd->mdl_context.release_statement_locks(); + else + thd->mdl_context.release_transactional_locks(); + } + thd->lex->restore_backup_query_tables_list(&lex_backup); + thd->variables.option_bits= thd_saved_option; + DBUG_RETURN(err); +} + + +uint64 +rpl_slave_state::next_sub_id(uint32 domain_id) +{ + uint64 sub_id= 0; + + mysql_mutex_lock(&LOCK_slave_state); + sub_id= ++last_sub_id; + mysql_mutex_unlock(&LOCK_slave_state); + + return sub_id; +} + + +bool +rpl_slave_state_tostring_helper(String *dest, const rpl_gtid *gtid, bool *first) +{ + if (*first) + *first= false; + else + if (dest->append(",",1)) + return true; + return + dest->append_ulonglong(gtid->domain_id) || + dest->append("-",1) || + dest->append_ulonglong(gtid->server_id) || + dest->append("-",1) || + dest->append_ulonglong(gtid->seq_no); +} + + +int +rpl_slave_state::iterate(int (*cb)(rpl_gtid *, void *), void *data, + rpl_gtid *extra_gtids, uint32 num_extra) +{ + uint32 i; + HASH gtid_hash; + uchar *rec; + rpl_gtid *gtid; + int res= 1; + + my_hash_init(>id_hash, &my_charset_bin, 32, offsetof(rpl_gtid, domain_id), + sizeof(uint32), NULL, NULL, HASH_UNIQUE); + for (i= 0; i < num_extra; ++i) + if (extra_gtids[i].server_id == global_system_variables.server_id && + my_hash_insert(>id_hash, (uchar *)(&extra_gtids[i]))) + goto err; + + mysql_mutex_lock(&LOCK_slave_state); + + for (i= 0; i < hash.records; ++i) + { + uint64 best_sub_id; + rpl_gtid best_gtid; + element *e= (element *)my_hash_element(&hash, i); + list_element *l= e->list; + + if (!l) + continue; /* Nothing here */ + + best_gtid.domain_id= e->domain_id; + best_gtid.server_id= l->server_id; + best_gtid.seq_no= l->seq_no; + best_sub_id= l->sub_id; + while ((l= l->next)) + { + if (l->sub_id > best_sub_id) + { + best_sub_id= l->sub_id; + best_gtid.server_id= l->server_id; + best_gtid.seq_no= l->seq_no; + } + } + + /* Check if we have something newer in the extra list. */ + rec= my_hash_search(>id_hash, (const uchar *)&best_gtid.domain_id, 0); + if (rec) + { + gtid= (rpl_gtid *)rec; + if (gtid->seq_no > best_gtid.seq_no) + memcpy(&best_gtid, gtid, sizeof(best_gtid)); + if (my_hash_delete(>id_hash, rec)) + { + mysql_mutex_unlock(&LOCK_slave_state); + goto err; + } + } + + if ((res= (*cb)(&best_gtid, data))) + { + mysql_mutex_unlock(&LOCK_slave_state); + goto err; + } + } + + mysql_mutex_unlock(&LOCK_slave_state); + + /* Also add any remaining extra domain_ids. */ + for (i= 0; i < gtid_hash.records; ++i) + { + gtid= (rpl_gtid *)my_hash_element(>id_hash, i); + if ((res= (*cb)(gtid, data))) + goto err; + } + + res= 0; + +err: + my_hash_free(>id_hash); + + return res; +} + + +struct rpl_slave_state_tostring_data { + String *dest; + bool first; +}; +static int +rpl_slave_state_tostring_cb(rpl_gtid *gtid, void *data) +{ + rpl_slave_state_tostring_data *p= (rpl_slave_state_tostring_data *)data; + return rpl_slave_state_tostring_helper(p->dest, gtid, &p->first); +} + + +/* + Prepare the current slave state as a string, suitable for sending to the + master to request to receive binlog events starting from that GTID state. + + The state consists of the most recently applied GTID for each domain_id, + ie. the one with the highest sub_id within each domain_id. + + Optinally, extra_gtids is a list of GTIDs from the binlog. This is used when + a server was previously a master and now needs to connect to a new master as + a slave. For each domain_id, if the GTID in the binlog was logged with our + own server_id _and_ has a higher seq_no than what is in the slave state, + then this should be used as the position to start replicating at. This + allows to promote a slave as new master, and connect the old master as a + slave with MASTER_GTID_POS=AUTO. +*/ +int +rpl_slave_state::tostring(String *dest, rpl_gtid *extra_gtids, uint32 num_extra) +{ + struct rpl_slave_state_tostring_data data; + data.first= true; + data.dest= dest; + + return iterate(rpl_slave_state_tostring_cb, &data, extra_gtids, num_extra); +} + + +/* + Lookup a domain_id in the current replication slave state. + + Returns false if the domain_id has no entries in the slave state. + Otherwise returns true, and fills in out_gtid with the corresponding + GTID. +*/ +bool +rpl_slave_state::domain_to_gtid(uint32 domain_id, rpl_gtid *out_gtid) +{ + element *elem; + list_element *list; + uint64 best_sub_id; + + mysql_mutex_lock(&LOCK_slave_state); + elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, 0); + if (!elem || !(list= elem->list)) + { + mysql_mutex_unlock(&LOCK_slave_state); + return false; + } + + out_gtid->domain_id= domain_id; + out_gtid->server_id= list->server_id; + out_gtid->seq_no= list->seq_no; + best_sub_id= list->sub_id; + + while ((list= list->next)) + { + if (best_sub_id > list->sub_id) + continue; + best_sub_id= list->sub_id; + out_gtid->server_id= list->server_id; + out_gtid->seq_no= list->seq_no; + } + + mysql_mutex_unlock(&LOCK_slave_state); + return true; +} + + +/* + Parse a GTID at the start of a string, and update the pointer to point + at the first character after the parsed GTID. + + Returns 0 on ok, non-zero on parse error. +*/ +static int +gtid_parser_helper(char **ptr, char *end, rpl_gtid *out_gtid) +{ + char *q; + char *p= *ptr; + uint64 v1, v2, v3; + int err= 0; + + q= end; + v1= (uint64)my_strtoll10(p, &q, &err); + if (err != 0 || v1 > (uint32)0xffffffff || q == end || *q != '-') + return 1; + p= q+1; + q= end; + v2= (uint64)my_strtoll10(p, &q, &err); + if (err != 0 || v2 > (uint32)0xffffffff || q == end || *q != '-') + return 1; + p= q+1; + q= end; + v3= (uint64)my_strtoll10(p, &q, &err); + if (err != 0) + return 1; + + out_gtid->domain_id= v1; + out_gtid->server_id= v2; + out_gtid->seq_no= v3; + *ptr= q; + return 0; +} + + +rpl_gtid * +gtid_parse_string_to_list(const char *str, size_t str_len, uint32 *out_len) +{ + char *p= const_cast<char *>(str); + char *end= p + str_len; + uint32 len= 0, alloc_len= 5; + rpl_gtid *list= NULL; + + for (;;) + { + rpl_gtid gtid; + + if (len >= (((uint32)1 << 28)-1) || gtid_parser_helper(&p, end, >id)) + { + my_free(list); + return NULL; + } + if ((!list || len >= alloc_len) && + !(list= + (rpl_gtid *)my_realloc(list, + (alloc_len= alloc_len*2) * sizeof(rpl_gtid), + MYF(MY_FREE_ON_ERROR|MY_ALLOW_ZERO_PTR)))) + return NULL; + list[len++]= gtid; + + if (p == end) + break; + if (*p != ',') + { + my_free(list); + return NULL; + } + ++p; + } + *out_len= len; + return list; +} + + +/* + Update the slave replication state with the GTID position obtained from + master when connecting with old-style (filename,offset) position. + + If RESET is true then all existing entries are removed. Otherwise only + domain_ids mentioned in the STATE_FROM_MASTER are changed. + + Returns 0 if ok, non-zero if error. +*/ +int +rpl_slave_state::load(THD *thd, char *state_from_master, size_t len, + bool reset, bool in_statement) +{ + char *end= state_from_master + len; + + if (reset) + { + if (truncate_state_table(thd)) + return 1; + truncate_hash(); + } + if (state_from_master == end) + return 0; + for (;;) + { + rpl_gtid gtid; + uint64 sub_id; + + if (gtid_parser_helper(&state_from_master, end, >id) || + !(sub_id= next_sub_id(gtid.domain_id)) || + record_gtid(thd, >id, sub_id, false, in_statement) || + update(gtid.domain_id, gtid.server_id, sub_id, gtid.seq_no)) + return 1; + if (state_from_master == end) + break; + if (*state_from_master != ',') + return 1; + ++state_from_master; + } + return 0; +} + + +bool +rpl_slave_state::is_empty() +{ + uint32 i; + bool result= true; + + mysql_mutex_lock(&LOCK_slave_state); + for (i= 0; i < hash.records; ++i) + { + element *e= (element *)my_hash_element(&hash, i); + if (e->list) + { + result= false; + break; + } + } + mysql_mutex_unlock(&LOCK_slave_state); + + return result; +} + + +rpl_binlog_state::rpl_binlog_state() +{ + my_hash_init(&hash, &my_charset_bin, 32, offsetof(element, domain_id), + sizeof(uint32), NULL, my_free, HASH_UNIQUE); + mysql_mutex_init(key_LOCK_binlog_state, &LOCK_binlog_state, + MY_MUTEX_INIT_SLOW); + initialized= 1; +} + + +void +rpl_binlog_state::reset_nolock() +{ + uint32 i; + + for (i= 0; i < hash.records; ++i) + my_hash_free(&((element *)my_hash_element(&hash, i))->hash); + my_hash_reset(&hash); +} + + +void +rpl_binlog_state::reset() +{ + mysql_mutex_lock(&LOCK_binlog_state); + reset_nolock(); + mysql_mutex_unlock(&LOCK_binlog_state); +} + + +void rpl_binlog_state::free() +{ + if (initialized) + { + initialized= 0; + reset_nolock(); + my_hash_free(&hash); + mysql_mutex_destroy(&LOCK_binlog_state); + } +} + + +bool +rpl_binlog_state::load(struct rpl_gtid *list, uint32 count) +{ + uint32 i; + bool res= false; + + mysql_mutex_lock(&LOCK_binlog_state); + reset_nolock(); + for (i= 0; i < count; ++i) + { + if (update_nolock(&(list[i]), false)) + { + res= true; + break; + } + } + mysql_mutex_unlock(&LOCK_binlog_state); + return res; +} + + +rpl_binlog_state::~rpl_binlog_state() +{ + free(); +} + + +/* + Update replication state with a new GTID. + + If the (domain_id, server_id) pair already exists, then the new GTID replaces + the old one for that domain id. Else a new entry is inserted. + + Returns 0 for ok, 1 for error. +*/ +int +rpl_binlog_state::update_nolock(const struct rpl_gtid *gtid, bool strict) +{ + element *elem; + + if ((elem= (element *)my_hash_search(&hash, + (const uchar *)(>id->domain_id), 0))) + { + if (strict && elem->last_gtid && elem->last_gtid->seq_no >= gtid->seq_no) + { + my_error(ER_GTID_STRICT_OUT_OF_ORDER, MYF(0), gtid->domain_id, + gtid->server_id, gtid->seq_no, elem->last_gtid->domain_id, + elem->last_gtid->server_id, elem->last_gtid->seq_no); + return 1; + } + if (elem->seq_no_counter < gtid->seq_no) + elem->seq_no_counter= gtid->seq_no; + if (!elem->update_element(gtid)) + return 0; + } + else if (!alloc_element_nolock(gtid)) + return 0; + + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return 1; +} + + +int +rpl_binlog_state::update(const struct rpl_gtid *gtid, bool strict) +{ + int res; + mysql_mutex_lock(&LOCK_binlog_state); + res= update_nolock(gtid, strict); + mysql_mutex_unlock(&LOCK_binlog_state); + return res; +} + + +/* + Fill in a new GTID, allocating next sequence number, and update state + accordingly. +*/ +int +rpl_binlog_state::update_with_next_gtid(uint32 domain_id, uint32 server_id, + rpl_gtid *gtid) +{ + element *elem; + int res= 0; + + gtid->domain_id= domain_id; + gtid->server_id= server_id; + + mysql_mutex_lock(&LOCK_binlog_state); + if ((elem= (element *)my_hash_search(&hash, (const uchar *)(&domain_id), 0))) + { + gtid->seq_no= ++elem->seq_no_counter; + if (!elem->update_element(gtid)) + goto end; + } + else + { + gtid->seq_no= 1; + if (!alloc_element_nolock(gtid)) + goto end; + } + + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + res= 1; +end: + mysql_mutex_unlock(&LOCK_binlog_state); + return res; +} + + +/* Helper functions for update. */ +int +rpl_binlog_state::element::update_element(const rpl_gtid *gtid) +{ + rpl_gtid *lookup_gtid; + + /* + By far the most common case is that successive events within same + replication domain have the same server id (it changes only when + switching to a new master). So save a hash lookup in this case. + */ + if (likely(last_gtid && last_gtid->server_id == gtid->server_id)) + { + last_gtid->seq_no= gtid->seq_no; + return 0; + } + + lookup_gtid= (rpl_gtid *) + my_hash_search(&hash, (const uchar *)>id->server_id, 0); + if (lookup_gtid) + { + lookup_gtid->seq_no= gtid->seq_no; + last_gtid= lookup_gtid; + return 0; + } + + /* Allocate a new GTID and insert it. */ + lookup_gtid= (rpl_gtid *)my_malloc(sizeof(*lookup_gtid), MYF(MY_WME)); + if (!lookup_gtid) + return 1; + memcpy(lookup_gtid, gtid, sizeof(*lookup_gtid)); + if (my_hash_insert(&hash, (const uchar *)lookup_gtid)) + { + my_free(lookup_gtid); + return 1; + } + last_gtid= lookup_gtid; + return 0; +} + + +int +rpl_binlog_state::alloc_element_nolock(const rpl_gtid *gtid) +{ + element *elem; + rpl_gtid *lookup_gtid; + + /* First time we see this domain_id; allocate a new element. */ + elem= (element *)my_malloc(sizeof(*elem), MYF(MY_WME)); + lookup_gtid= (rpl_gtid *)my_malloc(sizeof(*lookup_gtid), MYF(MY_WME)); + if (elem && lookup_gtid) + { + elem->domain_id= gtid->domain_id; + my_hash_init(&elem->hash, &my_charset_bin, 32, + offsetof(rpl_gtid, server_id), sizeof(uint32), NULL, my_free, + HASH_UNIQUE); + elem->last_gtid= lookup_gtid; + elem->seq_no_counter= gtid->seq_no; + memcpy(lookup_gtid, gtid, sizeof(*lookup_gtid)); + if (0 == my_hash_insert(&elem->hash, (const uchar *)lookup_gtid)) + { + lookup_gtid= NULL; /* Do not free. */ + if (0 == my_hash_insert(&hash, (const uchar *)elem)) + return 0; + } + my_hash_free(&elem->hash); + } + + /* An error. */ + if (elem) + my_free(elem); + if (lookup_gtid) + my_free(lookup_gtid); + return 1; +} + + +/* + Check that a new GTID can be logged without creating an out-of-order + sequence number with existing GTIDs. +*/ +bool +rpl_binlog_state::check_strict_sequence(uint32 domain_id, uint32 server_id, + uint64 seq_no) +{ + element *elem; + bool res= 0; + + mysql_mutex_lock(&LOCK_binlog_state); + if ((elem= (element *)my_hash_search(&hash, + (const uchar *)(&domain_id), 0)) && + elem->last_gtid && elem->last_gtid->seq_no >= seq_no) + { + my_error(ER_GTID_STRICT_OUT_OF_ORDER, MYF(0), domain_id, server_id, seq_no, + elem->last_gtid->domain_id, elem->last_gtid->server_id, + elem->last_gtid->seq_no); + res= 1; + } + mysql_mutex_unlock(&LOCK_binlog_state); + return res; +} + + +/* + When we see a new GTID that will not be binlogged (eg. slave thread + with --log-slave-updates=0), then we need to remember to allocate any + GTID seq_no of our own within that domain starting from there. + + Returns 0 if ok, non-zero if out-of-memory. +*/ +int +rpl_binlog_state::bump_seq_no_if_needed(uint32 domain_id, uint64 seq_no) +{ + element *elem; + int res; + + mysql_mutex_lock(&LOCK_binlog_state); + if ((elem= (element *)my_hash_search(&hash, (const uchar *)(&domain_id), 0))) + { + if (elem->seq_no_counter < seq_no) + elem->seq_no_counter= seq_no; + res= 0; + goto end; + } + + /* We need to allocate a new, empty element to remember the next seq_no. */ + if (!(elem= (element *)my_malloc(sizeof(*elem), MYF(MY_WME)))) + { + res= 1; + goto end; + } + + elem->domain_id= domain_id; + my_hash_init(&elem->hash, &my_charset_bin, 32, + offsetof(rpl_gtid, server_id), sizeof(uint32), NULL, my_free, + HASH_UNIQUE); + elem->last_gtid= NULL; + elem->seq_no_counter= seq_no; + if (0 == my_hash_insert(&hash, (const uchar *)elem)) + { + res= 0; + goto end; + } + + my_hash_free(&elem->hash); + my_free(elem); + res= 1; + +end: + mysql_mutex_unlock(&LOCK_binlog_state); + return res; +} + + +/* + Write binlog state to text file, so we can read it in again without having + to scan last binlog file (normal shutdown/startup, not crash recovery). + + The most recent GTID within each domain_id is written after any other GTID + within this domain. +*/ +int +rpl_binlog_state::write_to_iocache(IO_CACHE *dest) +{ + ulong i, j; + char buf[21]; + int res= 0; + + mysql_mutex_lock(&LOCK_binlog_state); + for (i= 0; i < hash.records; ++i) + { + size_t res; + element *e= (element *)my_hash_element(&hash, i); + if (!e->last_gtid) + { + DBUG_ASSERT(e->hash.records == 0); + continue; + } + for (j= 0; j <= e->hash.records; ++j) + { + const rpl_gtid *gtid; + if (j < e->hash.records) + { + gtid= (const rpl_gtid *)my_hash_element(&e->hash, j); + if (gtid == e->last_gtid) + continue; + } + else + gtid= e->last_gtid; + + longlong10_to_str(gtid->seq_no, buf, 10); + res= my_b_printf(dest, "%u-%u-%s\n", gtid->domain_id, gtid->server_id, buf); + if (res == (size_t) -1) + { + res= 1; + goto end; + } + } + } + +end: + mysql_mutex_unlock(&LOCK_binlog_state); + return res; +} + + +int +rpl_binlog_state::read_from_iocache(IO_CACHE *src) +{ + /* 10-digit - 10-digit - 20-digit \n \0 */ + char buf[10+1+10+1+20+1+1]; + char *p, *end; + rpl_gtid gtid; + int res= 0; + + mysql_mutex_lock(&LOCK_binlog_state); + reset_nolock(); + for (;;) + { + size_t len= my_b_gets(src, buf, sizeof(buf)); + if (!len) + break; + p= buf; + end= buf + len; + if (gtid_parser_helper(&p, end, >id) || + update_nolock(>id, false)) + { + res= 1; + break; + } + } + mysql_mutex_unlock(&LOCK_binlog_state); + return res; +} + + +rpl_gtid * +rpl_binlog_state::find_nolock(uint32 domain_id, uint32 server_id) +{ + element *elem; + if (!(elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, 0))) + return NULL; + return (rpl_gtid *)my_hash_search(&elem->hash, (const uchar *)&server_id, 0); +} + +rpl_gtid * +rpl_binlog_state::find(uint32 domain_id, uint32 server_id) +{ + rpl_gtid *p; + mysql_mutex_lock(&LOCK_binlog_state); + p= find_nolock(domain_id, server_id); + mysql_mutex_unlock(&LOCK_binlog_state); + return p; +} + +rpl_gtid * +rpl_binlog_state::find_most_recent(uint32 domain_id) +{ + element *elem; + rpl_gtid *gtid= NULL; + + mysql_mutex_lock(&LOCK_binlog_state); + elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, 0); + if (elem && elem->last_gtid) + gtid= elem->last_gtid; + mysql_mutex_unlock(&LOCK_binlog_state); + + return gtid; +} + + +uint32 +rpl_binlog_state::count() +{ + uint32 c= 0; + uint32 i; + + mysql_mutex_lock(&LOCK_binlog_state); + for (i= 0; i < hash.records; ++i) + c+= ((element *)my_hash_element(&hash, i))->hash.records; + mysql_mutex_unlock(&LOCK_binlog_state); + + return c; +} + + +int +rpl_binlog_state::get_gtid_list(rpl_gtid *gtid_list, uint32 list_size) +{ + uint32 i, j, pos; + int res= 0; + + mysql_mutex_lock(&LOCK_binlog_state); + pos= 0; + for (i= 0; i < hash.records; ++i) + { + element *e= (element *)my_hash_element(&hash, i); + if (!e->last_gtid) + { + DBUG_ASSERT(e->hash.records==0); + continue; + } + for (j= 0; j <= e->hash.records; ++j) + { + const rpl_gtid *gtid; + if (j < e->hash.records) + { + gtid= (rpl_gtid *)my_hash_element(&e->hash, j); + if (gtid == e->last_gtid) + continue; + } + else + gtid= e->last_gtid; + + if (pos >= list_size) + { + res= 1; + goto end; + } + memcpy(>id_list[pos++], gtid, sizeof(*gtid)); + } + } + +end: + mysql_mutex_unlock(&LOCK_binlog_state); + return res; +} + + +/* + Get a list of the most recently binlogged GTID, for each domain_id. + + This can be used when switching from being a master to being a slave, + to know where to start replicating from the new master. + + The returned list must be de-allocated with my_free(). + + Returns 0 for ok, non-zero for out-of-memory. +*/ +int +rpl_binlog_state::get_most_recent_gtid_list(rpl_gtid **list, uint32 *size) +{ + uint32 i; + uint32 alloc_size, out_size; + int res= 0; + + out_size= 0; + mysql_mutex_lock(&LOCK_binlog_state); + alloc_size= hash.records; + if (!(*list= (rpl_gtid *)my_malloc(alloc_size * sizeof(rpl_gtid), + MYF(MY_WME)))) + { + res= 1; + goto end; + } + for (i= 0; i < alloc_size; ++i) + { + element *e= (element *)my_hash_element(&hash, i); + if (!e->last_gtid) + continue; + memcpy(&((*list)[out_size++]), e->last_gtid, sizeof(rpl_gtid)); + } + +end: + mysql_mutex_unlock(&LOCK_binlog_state); + *size= out_size; + return res; +} + + +bool +rpl_binlog_state::append_pos(String *str) +{ + uint32 i; + bool first= true; + + mysql_mutex_lock(&LOCK_binlog_state); + for (i= 0; i < hash.records; ++i) + { + element *e= (element *)my_hash_element(&hash, i); + if (e->last_gtid && + rpl_slave_state_tostring_helper(str, e->last_gtid, &first)) + return true; + } + mysql_mutex_unlock(&LOCK_binlog_state); + + return false; +} + + +bool +rpl_binlog_state::append_state(String *str) +{ + uint32 i, j; + bool first= true; + bool res= false; + + mysql_mutex_lock(&LOCK_binlog_state); + for (i= 0; i < hash.records; ++i) + { + element *e= (element *)my_hash_element(&hash, i); + if (!e->last_gtid) + { + DBUG_ASSERT(e->hash.records==0); + continue; + } + for (j= 0; j <= e->hash.records; ++j) + { + const rpl_gtid *gtid; + if (j < e->hash.records) + { + gtid= (rpl_gtid *)my_hash_element(&e->hash, j); + if (gtid == e->last_gtid) + continue; + } + else + gtid= e->last_gtid; + + if (rpl_slave_state_tostring_helper(str, gtid, &first)) + { + res= true; + goto end; + } + } + } + +end: + mysql_mutex_unlock(&LOCK_binlog_state); + return res; +} + + +slave_connection_state::slave_connection_state() +{ + my_hash_init(&hash, &my_charset_bin, 32, + offsetof(entry, gtid) + offsetof(rpl_gtid, domain_id), + sizeof(uint32), NULL, my_free, HASH_UNIQUE); +} + + +slave_connection_state::~slave_connection_state() +{ + my_hash_free(&hash); +} + + +/* + Create a hash from the slave GTID state that is sent to master when slave + connects to start replication. + + The state is sent as <GTID>,<GTID>,...,<GTID>, for example: + + 0-2-112,1-4-1022 + + The state gives for each domain_id the GTID to start replication from for + the corresponding replication stream. So domain_id must be unique. + + Returns 0 if ok, non-zero if error due to malformed input. + + Note that input string is built by slave server, so it will not be incorrect + unless bug/corruption/malicious server. So we just need basic sanity check, + not fancy user-friendly error message. +*/ + +int +slave_connection_state::load(char *slave_request, size_t len) +{ + char *p, *end; + uchar *rec; + rpl_gtid *gtid; + const entry *e; + + reset(); + p= slave_request; + end= slave_request + len; + if (p == end) + return 0; + for (;;) + { + if (!(rec= (uchar *)my_malloc(sizeof(entry), MYF(MY_WME)))) + { + my_error(ER_OUTOFMEMORY, MYF(0), sizeof(*gtid)); + return 1; + } + gtid= &((entry *)rec)->gtid; + if (gtid_parser_helper(&p, end, gtid)) + { + my_free(rec); + my_error(ER_INCORRECT_GTID_STATE, MYF(0)); + return 1; + } + if ((e= (const entry *) + my_hash_search(&hash, (const uchar *)(>id->domain_id), 0))) + { + my_error(ER_DUPLICATE_GTID_DOMAIN, MYF(0), gtid->domain_id, + gtid->server_id, (ulonglong)gtid->seq_no, e->gtid.domain_id, + e->gtid.server_id, (ulonglong)e->gtid.seq_no, gtid->domain_id); + my_free(rec); + return 1; + } + ((entry *)rec)->flags= 0; + if (my_hash_insert(&hash, rec)) + { + my_free(rec); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return 1; + } + if (p == end) + break; /* Finished. */ + if (*p != ',') + { + my_error(ER_INCORRECT_GTID_STATE, MYF(0)); + return 1; + } + ++p; + } + + return 0; +} + + +int +slave_connection_state::load(const rpl_gtid *gtid_list, uint32 count) +{ + uint32 i; + + reset(); + for (i= 0; i < count; ++i) + if (update(>id_list[i])) + return 1; + return 0; +} + + +static int +slave_connection_state_load_cb(rpl_gtid *gtid, void *data) +{ + slave_connection_state *state= (slave_connection_state *)data; + return state->update(gtid); +} + + +/* + Same as rpl_slave_state::tostring(), but populates a slave_connection_state + instead. +*/ +int +slave_connection_state::load(rpl_slave_state *state, + rpl_gtid *extra_gtids, uint32 num_extra) +{ + reset(); + return state->iterate(slave_connection_state_load_cb, this, + extra_gtids, num_extra); +} + + +slave_connection_state::entry * +slave_connection_state::find_entry(uint32 domain_id) +{ + return (entry *) my_hash_search(&hash, (const uchar *)(&domain_id), 0); +} + + +rpl_gtid * +slave_connection_state::find(uint32 domain_id) +{ + entry *e= find_entry(domain_id); + if (!e) + return NULL; + return &e->gtid; +} + + +int +slave_connection_state::update(const rpl_gtid *in_gtid) +{ + entry *e; + uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), 0); + if (rec) + { + e= (entry *)rec; + e->gtid= *in_gtid; + return 0; + } + + if (!(e= (entry *)my_malloc(sizeof(*e), MYF(MY_WME)))) + return 1; + e->gtid= *in_gtid; + e->flags= 0; + if (my_hash_insert(&hash, (uchar *)e)) + { + my_free(e); + return 1; + } + + return 0; +} + + +void +slave_connection_state::remove(const rpl_gtid *in_gtid) +{ + uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), 0); +#ifndef DBUG_OFF + bool err; + rpl_gtid *slave_gtid= &((entry *)rec)->gtid; + DBUG_ASSERT(rec /* We should never try to remove not present domain_id. */); + DBUG_ASSERT(slave_gtid->server_id == in_gtid->server_id); + DBUG_ASSERT(slave_gtid->seq_no == in_gtid->seq_no); +#endif + + IF_DBUG(err=, ) + my_hash_delete(&hash, rec); + DBUG_ASSERT(!err); +} + + +void +slave_connection_state::remove_if_present(const rpl_gtid *in_gtid) +{ + uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), 0); + if (rec) + my_hash_delete(&hash, rec); +} + + +int +slave_connection_state::to_string(String *out_str) +{ + out_str->length(0); + return append_to_string(out_str); +} + + +int +slave_connection_state::append_to_string(String *out_str) +{ + uint32 i; + bool first; + + first= true; + for (i= 0; i < hash.records; ++i) + { + const entry *e= (const entry *)my_hash_element(&hash, i); + if (rpl_slave_state_tostring_helper(out_str, &e->gtid, &first)) + return 1; + } + return 0; +} + + +int +slave_connection_state::get_gtid_list(rpl_gtid *gtid_list, uint32 list_size) +{ + uint32 i, pos; + + pos= 0; + for (i= 0; i < hash.records; ++i) + { + entry *e; + if (pos >= list_size) + return 1; + e= (entry *)my_hash_element(&hash, i); + memcpy(>id_list[pos++], &e->gtid, sizeof(e->gtid)); + } + + return 0; +} + + +/* + Execute a MASTER_GTID_WAIT(). + The position to wait for is in gtid_str in string form. + The timeout in microseconds is in timeout_us, zero means no timeout. + + Returns: + 1 for error. + 0 for wait completed. + -1 for wait timed out. +*/ +int +gtid_waiting::wait_for_pos(THD *thd, String *gtid_str, longlong timeout_us) +{ + int err; + rpl_gtid *wait_pos; + uint32 count, i; + struct timespec wait_until, *wait_until_ptr; + + /* Wait for the empty position returns immediately. */ + if (gtid_str->length() == 0) + return 0; + + if (!(wait_pos= gtid_parse_string_to_list(gtid_str->ptr(), gtid_str->length(), + &count))) + { + my_error(ER_INCORRECT_GTID_STATE, MYF(0)); + return 1; + } + + if (timeout_us >= 0) + { + set_timespec_nsec(wait_until, (ulonglong)1000*timeout_us); + wait_until_ptr= &wait_until; + } + else + wait_until_ptr= NULL; + err= 0; + for (i= 0; i < count; ++i) + { + if ((err= wait_for_gtid(thd, &wait_pos[i], wait_until_ptr))) + break; + } + my_free(wait_pos); + return err; +} + + +void +gtid_waiting::promote_new_waiter(gtid_waiting::hash_element *he) +{ + queue_element *qe; + + mysql_mutex_assert_owner(&LOCK_gtid_waiting); + if (queue_empty(&he->queue)) + return; + qe= (queue_element *)queue_top(&he->queue); + qe->do_small_wait= true; + mysql_cond_signal(&qe->thd->COND_wakeup_ready); +} + +void +gtid_waiting::process_wait_hash(uint64 wakeup_seq_no, + gtid_waiting::hash_element *he) +{ + mysql_mutex_assert_owner(&LOCK_gtid_waiting); + + for (;;) + { + queue_element *qe; + + if (queue_empty(&he->queue)) + break; + qe= (queue_element *)queue_top(&he->queue); + if (qe->wait_seq_no > wakeup_seq_no) + break; + DBUG_ASSERT(!qe->done); + queue_remove_top(&he->queue); + qe->done= true;; + mysql_cond_signal(&qe->thd->COND_wakeup_ready); + } +} + + +/* + Execute a MASTER_GTID_WAIT() for one specific domain. + + The implementation is optimised primarily for (1) minimal performance impact + on the slave replication threads, and secondarily for (2) quick performance + of MASTER_GTID_WAIT() on a single GTID, which can be useful for consistent + read to clients in an async replication read-scaleout scenario. + + To achieve (1), we have a "small" wait and a "large" wait. The small wait + contends with the replication threads on the lock on the gtid_slave_pos, so + only minimal processing is done under that lock, and only a single waiter at + a time does the small wait. + + If there is already a small waiter, a new thread will either replace the + small waiter (if it needs to wait for an earlier sequence number), or + instead do a "large" wait. + + Once awoken on the small wait, the waiting thread releases the lock shared + with the SQL threads quickly, and then processes all waiters currently doing + the large wait using a different lock that does not impact replication. + + This way, the SQL threads only need to do a single check + possibly a + pthread_cond_signal() when updating the gtid_slave_state, and the time that + non-SQL threads contend for the lock on gtid_slave_state is minimized. + + There is always at least one thread that has the responsibility to ensure + that there is a small waiter; this thread has queue_element::do_small_wait + set to true. This thread will do the small wait until it is done, at which + point it will make sure to pass on the responsibility to another thread. + Normally only one thread has do_small_wait==true, but it can occasionally + happen that there is more than one, when threads race one another for the + lock on the small wait (this results in slightly increased activity on the + small lock but is otherwise harmless). + + Returns: + 0 Wait completed normally + -1 Wait completed due to timeout + 1 An error (my_error() will have been called to set the error in the da) +*/ +int +gtid_waiting::wait_for_gtid(THD *thd, rpl_gtid *wait_gtid, + struct timespec *wait_until) +{ + bool timed_out= false; +#ifdef HAVE_REPLICATION + queue_element elem; + uint32 domain_id= wait_gtid->domain_id; + uint64 seq_no= wait_gtid->seq_no; + hash_element *he; + rpl_slave_state::element *slave_state_elem= NULL; + const char *old_msg= NULL; + bool did_enter_cond= false; + + elem.wait_seq_no= seq_no; + elem.thd= thd; + elem.done= false; + + mysql_mutex_lock(&LOCK_gtid_waiting); + if (!(he= get_entry(wait_gtid->domain_id))) + { + mysql_mutex_unlock(&LOCK_gtid_waiting); + return 1; + } + /* + If there is already another waiter with seq_no no larger than our own, + we are sure that there is already a small waiter that will wake us up + (or later pass the small wait responsibility to us). So in this case, we + do not need to touch the small wait lock at all. + */ + elem.do_small_wait= + (queue_empty(&he->queue) || + ((queue_element *)queue_top(&he->queue))->wait_seq_no > seq_no); + + if (register_in_wait_queue(thd, wait_gtid, he, &elem)) + { + mysql_mutex_unlock(&LOCK_gtid_waiting); + return 1; + } + /* + Loop, doing either the small or large wait as appropriate, until either + the position waited for is reached, or we get a kill or timeout. + */ + for (;;) + { + mysql_mutex_assert_owner(&LOCK_gtid_waiting); + + if (elem.do_small_wait) + { + uint64 wakeup_seq_no; + queue_element *cur_waiter; + + mysql_mutex_lock(&rpl_global_gtid_slave_state.LOCK_slave_state); + /* + The elements in the gtid_slave_state_hash are never re-allocated once + they enter the hash, so we do not need to re-do the lookup after releasing + and re-aquiring the lock. + */ + if (!slave_state_elem && + !(slave_state_elem= rpl_global_gtid_slave_state.get_element(domain_id))) + { + mysql_mutex_unlock(&rpl_global_gtid_slave_state.LOCK_slave_state); + remove_from_wait_queue(he, &elem); + promote_new_waiter(he); + if (did_enter_cond) + thd->exit_cond(old_msg); + else + mysql_mutex_unlock(&LOCK_gtid_waiting); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return 1; + } + + if ((wakeup_seq_no= slave_state_elem->highest_seq_no) >= seq_no) + { + /* + We do not have to wait. (We will be removed from the wait queue when + we call process_wait_hash() below. + */ + mysql_mutex_unlock(&rpl_global_gtid_slave_state.LOCK_slave_state); + } + else if ((cur_waiter= slave_state_elem->gtid_waiter) && + slave_state_elem->min_wait_seq_no <= seq_no) + { + /* + There is already a suitable small waiter, go do the large wait. + (Normally we would not have needed to check the small wait in this + case, but it can happen if we race with another thread for the small + lock). + */ + elem.do_small_wait= false; + mysql_mutex_unlock(&rpl_global_gtid_slave_state.LOCK_slave_state); + } + else + { + /* + We have to do the small wait ourselves (stealing it from any thread + that might already be waiting for a later seq_no). + */ + slave_state_elem->gtid_waiter= &elem; + slave_state_elem->min_wait_seq_no= seq_no; + if (cur_waiter) + { + /* We stole the wait, so wake up the old waiting thread. */ + mysql_cond_signal(&slave_state_elem->COND_wait_gtid); + } + + /* Release the large lock, and do the small wait. */ + if (did_enter_cond) + { + thd->exit_cond(old_msg); + did_enter_cond= false; + } + else + mysql_mutex_unlock(&LOCK_gtid_waiting); + old_msg= + thd->enter_cond(&slave_state_elem->COND_wait_gtid, + &rpl_global_gtid_slave_state.LOCK_slave_state, + "Waiting in MASTER_GTID_WAIT() (primary waiter)"); + do + { + if (thd->check_killed()) + break; + else if (wait_until) + { + int err= + mysql_cond_timedwait(&slave_state_elem->COND_wait_gtid, + &rpl_global_gtid_slave_state.LOCK_slave_state, + wait_until); + if (err == ETIMEDOUT || err == ETIME) + { + timed_out= true; + break; + } + } + else + mysql_cond_wait(&slave_state_elem->COND_wait_gtid, + &rpl_global_gtid_slave_state.LOCK_slave_state); + } while (slave_state_elem->gtid_waiter == &elem); + wakeup_seq_no= slave_state_elem->highest_seq_no; + /* + If we aborted due to timeout or kill, remove us as waiter. + + If we were replaced by another waiter with a smaller seq_no, then we + no longer have responsibility for the small wait. + */ + if ((cur_waiter= slave_state_elem->gtid_waiter)) + { + if (cur_waiter == &elem) + slave_state_elem->gtid_waiter= NULL; + else if (slave_state_elem->min_wait_seq_no <= seq_no) + elem.do_small_wait= false; + } + thd->exit_cond(old_msg); + + mysql_mutex_lock(&LOCK_gtid_waiting); + } + + /* + Note that hash_entry pointers do not change once allocated, so we do + not need to lookup `he' again after re-aquiring LOCK_gtid_waiting. + */ + process_wait_hash(wakeup_seq_no, he); + } + else + { + /* Do the large wait. */ + if (!did_enter_cond) + { + old_msg= thd->enter_cond(&thd->COND_wakeup_ready, &LOCK_gtid_waiting, + "Waiting in MASTER_GTID_WAIT()"); + did_enter_cond= true; + } + while (!elem.done && !thd->check_killed()) + { + thd_wait_begin(thd, THD_WAIT_BINLOG); + if (wait_until) + { + int err= mysql_cond_timedwait(&thd->COND_wakeup_ready, + &LOCK_gtid_waiting, wait_until); + if (err == ETIMEDOUT || err == ETIME) + timed_out= true; + } + else + mysql_cond_wait(&thd->COND_wakeup_ready, &LOCK_gtid_waiting); + thd_wait_end(thd); + if (elem.do_small_wait || timed_out) + break; + } + } + + if ((thd->killed || timed_out) && !elem.done) + { + /* Aborted, so remove ourselves from the hash. */ + remove_from_wait_queue(he, &elem); + elem.done= true; + } + if (elem.done) + { + /* + If our wait is done, but we have (or were passed) responsibility for + the small wait, then we need to pass on that task to someone else. + */ + if (elem.do_small_wait) + promote_new_waiter(he); + break; + } + } + + if (did_enter_cond) + thd->exit_cond(old_msg); + else + mysql_mutex_unlock(&LOCK_gtid_waiting); + if (thd->killed) + thd->send_kill_message(); +#endif /* HAVE_REPLICATION */ + return timed_out ? -1 : 0; +} + + +static void +free_hash_element(void *p) +{ + gtid_waiting::hash_element *e= (gtid_waiting::hash_element *)p; + delete_queue(&e->queue); + my_free(e); +} + + +void +gtid_waiting::init() +{ + my_hash_init(&hash, &my_charset_bin, 32, + offsetof(hash_element, domain_id), sizeof(uint32), NULL, + free_hash_element, HASH_UNIQUE); + mysql_mutex_init(key_LOCK_gtid_waiting, &LOCK_gtid_waiting, 0); +} + + +void +gtid_waiting::destroy() +{ + mysql_mutex_destroy(&LOCK_gtid_waiting); + my_hash_free(&hash); +} + + +static int +cmp_queue_elem(void *, uchar *a, uchar *b) +{ + uint64 seq_no_a= *(uint64 *)a; + uint64 seq_no_b= *(uint64 *)b; + if (seq_no_a < seq_no_b) + return -1; + else if (seq_no_a == seq_no_b) + return 0; + else + return 1; +} + + +gtid_waiting::hash_element * +gtid_waiting::get_entry(uint32 domain_id) +{ + hash_element *e; + + if ((e= (hash_element *)my_hash_search(&hash, (const uchar *)&domain_id, 0))) + return e; + + if (!(e= (hash_element *)my_malloc(sizeof(*e), MYF(MY_WME)))) + { + my_error(ER_OUTOFMEMORY, MYF(0), sizeof(*e)); + return NULL; + } + + if (init_queue(&e->queue, 8, offsetof(queue_element, wait_seq_no), 0, + cmp_queue_elem, NULL, 1+offsetof(queue_element, queue_idx), 1)) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + my_free(e); + return NULL; + } + e->domain_id= domain_id; + if (my_hash_insert(&hash, (uchar *)e)) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + delete_queue(&e->queue); + my_free(e); + return NULL; + } + return e; +} + + +int +gtid_waiting::register_in_wait_queue(THD *thd, rpl_gtid *wait_gtid, + gtid_waiting::hash_element *he, + gtid_waiting::queue_element *elem) +{ + mysql_mutex_assert_owner(&LOCK_gtid_waiting); + + if (queue_insert_safe(&he->queue, (uchar *)elem)) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return 1; + } + + return 0; +} + + +void +gtid_waiting::remove_from_wait_queue(gtid_waiting::hash_element *he, + gtid_waiting::queue_element *elem) +{ + mysql_mutex_assert_owner(&LOCK_gtid_waiting); + + queue_remove(&he->queue, elem->queue_idx); +} diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h new file mode 100644 index 00000000000..54f352661a7 --- /dev/null +++ b/sql/rpl_gtid.h @@ -0,0 +1,280 @@ +/* Copyright (c) 2013, Kristian Nielsen and MariaDB Services Ab. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef RPL_GTID_H +#define RPL_GTID_H + +#include "hash.h" +#include "queues.h" + + +/* Definitions for MariaDB global transaction ID (GTID). */ + + +extern const LEX_STRING rpl_gtid_slave_state_table_name; + +class String; + +struct rpl_gtid +{ + uint32 domain_id; + uint32 server_id; + uint64 seq_no; +}; + + +enum enum_gtid_skip_type { + GTID_SKIP_NOT, GTID_SKIP_STANDALONE, GTID_SKIP_TRANSACTION +}; + + +/* + Structure to keep track of threads waiting in MASTER_GTID_WAIT(). + + Since replication is (mostly) single-threaded, we want to minimise the + performance impact on that from MASTER_GTID_WAIT(). To achieve this, we + are careful to keep the common lock between replication threads and + MASTER_GTID_WAIT threads held for as short as possible. We keep only + a single thread waiting to be notified by the replication threads; this + thread then handles all the (potentially heavy) lifting of dealing with + all current waiting threads. +*/ +struct gtid_waiting { + /* Elements in the hash, basically a priority queue for each domain. */ + struct hash_element { + QUEUE queue; + uint32 domain_id; + }; + /* A priority queue to handle waiters in one domain in seq_no order. */ + struct queue_element { + uint64 wait_seq_no; + THD *thd; + int queue_idx; + /* + do_small_wait is true if we have responsibility for ensuring that there + is a small waiter. + */ + bool do_small_wait; + /* + The flag `done' is set when the wait is completed (either due to reaching + the position waited for, or due to timeout or kill). The queue_element + is in the queue if and only if `done' is true. + */ + bool done; + }; + + mysql_mutex_t LOCK_gtid_waiting; + HASH hash; + + void init(); + void destroy(); + hash_element *get_entry(uint32 domain_id); + int wait_for_pos(THD *thd, String *gtid_str, longlong timeout_us); + void promote_new_waiter(gtid_waiting::hash_element *he); + int wait_for_gtid(THD *thd, rpl_gtid *wait_gtid, struct timespec *wait_until); + void process_wait_hash(uint64 wakeup_seq_no, gtid_waiting::hash_element *he); + int register_in_wait_queue(THD *thd, rpl_gtid *wait_gtid, hash_element *he, + queue_element *elem); + void remove_from_wait_queue(hash_element *he, queue_element *elem); +}; + + +/* + Replication slave state. + + For every independent replication stream (identified by domain_id), this + remembers the last gtid applied on the slave within this domain. + + Since events are always committed in-order within a single domain, this is + sufficient to maintain the state of the replication slave. +*/ +struct rpl_slave_state +{ + /* Elements in the list of GTIDs kept for each domain_id. */ + struct list_element + { + struct list_element *next; + uint64 sub_id; + uint64 seq_no; + uint32 server_id; + }; + + /* Elements in the HASH that hold the state for one domain_id. */ + struct element + { + struct list_element *list; + uint32 domain_id; + /* Highest seq_no seen so far in this domain. */ + uint64 highest_seq_no; + /* + If this is non-NULL, then it is the waiter responsible for the small + wait in MASTER_GTID_WAIT(). + */ + gtid_waiting::queue_element *gtid_waiter; + /* + If gtid_waiter is non-NULL, then this is the seq_no that its + MASTER_GTID_WAIT() is waiting on. When we reach this seq_no, we need to + signal the waiter on COND_wait_gtid. + */ + uint64 min_wait_seq_no; + mysql_cond_t COND_wait_gtid; + + list_element *grab_list() { list_element *l= list; list= NULL; return l; } + void add(list_element *l) + { + l->next= list; + list= l; + } + }; + + /* Mapping from domain_id to its element. */ + HASH hash; + /* Mutex protecting access to the state. */ + mysql_mutex_t LOCK_slave_state; + + uint64 last_sub_id; + bool inited; + bool loaded; + + rpl_slave_state(); + ~rpl_slave_state(); + + void init(); + void deinit(); + void truncate_hash(); + ulong count() const { return hash.records; } + int update(uint32 domain_id, uint32 server_id, uint64 sub_id, uint64 seq_no); + int truncate_state_table(THD *thd); + int record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, + bool in_transaction, bool in_statement); + uint64 next_sub_id(uint32 domain_id); + int iterate(int (*cb)(rpl_gtid *, void *), void *data, + rpl_gtid *extra_gtids, uint32 num_extra); + int tostring(String *dest, rpl_gtid *extra_gtids, uint32 num_extra); + bool domain_to_gtid(uint32 domain_id, rpl_gtid *out_gtid); + int load(THD *thd, char *state_from_master, size_t len, bool reset, + bool in_statement); + bool is_empty(); + + element *get_element(uint32 domain_id); + int put_back_list(uint32 domain_id, list_element *list); + + void update_state_hash(uint64 sub_id, rpl_gtid *gtid); + int record_and_update_gtid(THD *thd, struct rpl_group_info *rgi); +}; + + +/* + Binlog state. + This keeps the last GTID written to the binlog for every distinct + (domain_id, server_id) pair. + This will be logged at the start of the next binlog file as a + Gtid_list_log_event; this way, it is easy to find the binlog file + containing a gigen GTID, by simply scanning backwards from the newest + one until a lower seq_no is found in the Gtid_list_log_event at the + start of a binlog for the given domain_id and server_id. + + We also remember the last logged GTID for every domain_id. This is used + to know where to start when a master is changed to a slave. As a side + effect, it also allows to skip a hash lookup in the very common case of + logging a new GTID with same server id as last GTID. +*/ +struct rpl_binlog_state +{ + struct element { + uint32 domain_id; + HASH hash; /* Containing all server_id for one domain_id */ + /* The most recent entry in the hash. */ + rpl_gtid *last_gtid; + /* Counter to allocate next seq_no for this domain. */ + uint64 seq_no_counter; + + int update_element(const rpl_gtid *gtid); + }; + /* Mapping from domain_id to collection of elements. */ + HASH hash; + /* Mutex protecting access to the state. */ + mysql_mutex_t LOCK_binlog_state; + my_bool initialized; + + rpl_binlog_state(); + ~rpl_binlog_state(); + + void reset_nolock(); + void reset(); + void free(); + bool load(struct rpl_gtid *list, uint32 count); + int update_nolock(const struct rpl_gtid *gtid, bool strict); + int update(const struct rpl_gtid *gtid, bool strict); + int update_with_next_gtid(uint32 domain_id, uint32 server_id, + rpl_gtid *gtid); + int alloc_element_nolock(const rpl_gtid *gtid); + bool check_strict_sequence(uint32 domain_id, uint32 server_id, uint64 seq_no); + int bump_seq_no_if_needed(uint32 domain_id, uint64 seq_no); + int write_to_iocache(IO_CACHE *dest); + int read_from_iocache(IO_CACHE *src); + uint32 count(); + int get_gtid_list(rpl_gtid *gtid_list, uint32 list_size); + int get_most_recent_gtid_list(rpl_gtid **list, uint32 *size); + bool append_pos(String *str); + bool append_state(String *str); + rpl_gtid *find_nolock(uint32 domain_id, uint32 server_id); + rpl_gtid *find(uint32 domain_id, uint32 server_id); + rpl_gtid *find_most_recent(uint32 domain_id); +}; + + +/* + Represent the GTID state that a slave connection to a master requests + the master to start sending binlog events from. +*/ +struct slave_connection_state +{ + struct entry { + rpl_gtid gtid; + uint32 flags; + }; + static const uint32 START_OWN_SLAVE_POS= 0x1; + static const uint32 START_ON_EMPTY_DOMAIN= 0x2; + + /* Mapping from domain_id to the entry with GTID requested for that domain. */ + HASH hash; + + slave_connection_state(); + ~slave_connection_state(); + + void reset() { my_hash_reset(&hash); } + int load(char *slave_request, size_t len); + int load(const rpl_gtid *gtid_list, uint32 count); + int load(rpl_slave_state *state, rpl_gtid *extra_gtids, uint32 num_extra); + rpl_gtid *find(uint32 domain_id); + entry *find_entry(uint32 domain_id); + int update(const rpl_gtid *in_gtid); + void remove(const rpl_gtid *gtid); + void remove_if_present(const rpl_gtid *in_gtid); + ulong count() const { return hash.records; } + int to_string(String *out_str); + int append_to_string(String *out_str); + int get_gtid_list(rpl_gtid *gtid_list, uint32 list_size); +}; + + +extern bool rpl_slave_state_tostring_helper(String *dest, const rpl_gtid *gtid, + bool *first); +extern int gtid_check_rpl_slave_state_table(TABLE *table); +extern rpl_gtid *gtid_parse_string_to_list(const char *p, size_t len, + uint32 *out_len); + +#endif /* RPL_GTID_H */ diff --git a/sql/rpl_handler.cc b/sql/rpl_handler.cc index 9267190605c..2777dabf451 100644 --- a/sql/rpl_handler.cc +++ b/sql/rpl_handler.cc @@ -176,7 +176,7 @@ void delegates_destroy() plugins add to thd->lex will be automatically unlocked. */ #define FOREACH_OBSERVER(r, f, thd, args) \ - param.server_id= thd->server_id; \ + param.server_id= thd->variables.server_id; \ /* Use a struct to make sure that they are allocated adjacent, check delete_dynamic(). @@ -189,7 +189,7 @@ void delegates_destroy() DYNAMIC_ARRAY *plugins= &s.plugins; \ plugin_ref *plugins_buffer= s.plugins_buffer; \ my_init_dynamic_array2(plugins, sizeof(plugin_ref), \ - plugins_buffer, 8, 8); \ + plugins_buffer, 8, 8, MYF(0)); \ read_lock(); \ Observer_info_iterator iter= observer_info_iter(); \ Observer_info *info= iter++; \ @@ -348,7 +348,7 @@ int Binlog_transmit_delegate::reserve_header(THD *thd, ushort flags, ulong hlen; Binlog_transmit_param param; param.flags= flags; - param.server_id= thd->server_id; + param.server_id= thd->variables.server_id; int ret= 0; read_lock(); @@ -555,4 +555,24 @@ int unregister_binlog_relay_io_observer(Binlog_relay_IO_observer *observer, void { return binlog_relay_io_delegate->remove_observer(observer, (st_plugin_int *)p); } +#else +int register_binlog_transmit_observer(Binlog_transmit_observer *observer, void *p) +{ + return 0; +} + +int unregister_binlog_transmit_observer(Binlog_transmit_observer *observer, void *p) +{ + return 0; +} + +int register_binlog_relay_io_observer(Binlog_relay_IO_observer *observer, void *p) +{ + return 0; +} + +int unregister_binlog_relay_io_observer(Binlog_relay_IO_observer *observer, void *p) +{ + return 0; +} #endif /* HAVE_REPLICATION */ diff --git a/sql/rpl_handler.h b/sql/rpl_handler.h index 4743fffb9a0..e028fb49808 100644 --- a/sql/rpl_handler.h +++ b/sql/rpl_handler.h @@ -124,7 +124,7 @@ public: inited= FALSE; if (my_rwlock_init(&lock, NULL)) return; - init_sql_alloc(&memroot, 1024, 0); + init_sql_alloc(&memroot, 1024, 0, MYF(0)); inited= TRUE; } ~Delegate() diff --git a/sql/rpl_injector.cc b/sql/rpl_injector.cc index ec1a96e8a2b..a4b04d2e047 100644 --- a/sql/rpl_injector.cc +++ b/sql/rpl_injector.cc @@ -108,7 +108,7 @@ int injector::transaction::use_table(server_id_type sid, table tbl) if ((error= check_state(TABLE_STATE))) DBUG_RETURN(error); - server_id_type save_id= m_thd->server_id; + server_id_type save_id= m_thd->variables.server_id; m_thd->set_server_id(sid); error= m_thd->binlog_write_table_map(tbl.get_table(), tbl.is_transactional()); @@ -127,7 +127,7 @@ int injector::transaction::write_row (server_id_type sid, table tbl, if (error) DBUG_RETURN(error); - server_id_type save_id= m_thd->server_id; + server_id_type save_id= m_thd->variables.server_id; m_thd->set_server_id(sid); error= m_thd->binlog_write_row(tbl.get_table(), tbl.is_transactional(), cols, colcnt, record); @@ -146,7 +146,7 @@ int injector::transaction::delete_row(server_id_type sid, table tbl, if (error) DBUG_RETURN(error); - server_id_type save_id= m_thd->server_id; + server_id_type save_id= m_thd->variables.server_id; m_thd->set_server_id(sid); error= m_thd->binlog_delete_row(tbl.get_table(), tbl.is_transactional(), cols, colcnt, record); @@ -165,7 +165,7 @@ int injector::transaction::update_row(server_id_type sid, table tbl, if (error) DBUG_RETURN(error); - server_id_type save_id= m_thd->server_id; + server_id_type save_id= m_thd->variables.server_id; m_thd->set_server_id(sid); error= m_thd->binlog_update_row(tbl.get_table(), tbl.is_transactional(), cols, colcnt, before, after); diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index 3c5a99121fa..4a69eb4a6ee 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -20,6 +20,8 @@ #include "unireg.h" // REQUIRED by other includes #include "rpl_mi.h" #include "slave.h" // SLAVE_MAX_HEARTBEAT_PERIOD +#include "strfunc.h" +#include "sql_repl.h" #ifdef HAVE_REPLICATION @@ -27,20 +29,50 @@ static void init_master_log_pos(Master_info* mi); -Master_info::Master_info(bool is_slave_recovery) +Master_info::Master_info(LEX_STRING *connection_name_arg, + bool is_slave_recovery) :Slave_reporting_capability("I/O"), ssl(0), ssl_verify_server_cert(1), fd(-1), io_thd(0), rli(is_slave_recovery), port(MYSQL_PORT), checksum_alg_before_fd(BINLOG_CHECKSUM_ALG_UNDEF), connect_retry(DEFAULT_CONNECT_RETRY), inited(0), abort_slave(0), slave_running(0), slave_run_id(0), sync_counter(0), - heartbeat_period(0), received_heartbeats(0), master_id(0) + heartbeat_period(0), received_heartbeats(0), master_id(0), + using_gtid(USE_GTID_NO), events_queued_since_last_gtid(0), + gtid_reconnect_event_skip_count(0), gtid_event_seen(false) { host[0] = 0; user[0] = 0; password[0] = 0; ssl_ca[0]= 0; ssl_capath[0]= 0; ssl_cert[0]= 0; ssl_cipher[0]= 0; ssl_key[0]= 0; - my_init_dynamic_array(&ignore_server_ids, sizeof(::server_id), 16, 16); + /* + Store connection name and lower case connection name + It's safe to ignore any OMM errors as this is checked by error() + */ + connection_name.length= cmp_connection_name.length= + connection_name_arg->length; + if ((connection_name.str= (char*) my_malloc(connection_name_arg->length*2+2, + MYF(MY_WME)))) + { + cmp_connection_name.str= (connection_name.str + + connection_name_arg->length+1); + strmake(connection_name.str, connection_name_arg->str, + connection_name.length); + memcpy(cmp_connection_name.str, connection_name_arg->str, + connection_name.length+1); + my_casedn_str(system_charset_info, cmp_connection_name.str); + } + /* When MySQL restarted, all Rpl_filter settings which aren't in the my.cnf + * will lose. So if you want a setting will not lose after restarting, you + * should add them into my.cnf + * */ + rpl_filter= get_or_create_rpl_filter(connection_name.str, + connection_name.length); + copy_filter_setting(rpl_filter, global_rpl_filter); + + my_init_dynamic_array(&ignore_server_ids, + sizeof(global_system_variables.server_id), 16, 16, + MYF(0)); bzero((char*) &file, sizeof(file)); mysql_mutex_init(key_master_info_run_lock, &run_lock, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_master_info_data_lock, &data_lock, MY_MUTEX_INIT_FAST); @@ -55,6 +87,9 @@ Master_info::Master_info(bool is_slave_recovery) Master_info::~Master_info() { + rpl_filters.delete_element(connection_name.str, connection_name.length, + (void (*)(const char*, uchar*)) free_rpl_filter); + my_free(connection_name.str); delete_dynamic(&ignore_server_ids); mysql_mutex_destroy(&run_lock); mysql_mutex_destroy(&data_lock); @@ -113,12 +148,34 @@ void Master_info::clear_in_memory_info(bool all) } } + +const char * +Master_info::using_gtid_astext(enum enum_using_gtid arg) +{ + switch (arg) + { + case USE_GTID_NO: + return "No"; + case USE_GTID_SLAVE_POS: + return "Slave_Pos"; + default: + DBUG_ASSERT(arg == USE_GTID_CURRENT_POS); + return "Current_Pos"; + } +} + + void init_master_log_pos(Master_info* mi) { DBUG_ENTER("init_master_log_pos"); mi->master_log_name[0] = 0; mi->master_log_pos = BIN_LOG_HEADER_SIZE; // skip magic number + mi->using_gtid= Master_info::USE_GTID_NO; + mi->gtid_current_pos.reset(); + mi->events_queued_since_last_gtid= 0; + mi->gtid_reconnect_event_skip_count= 0; + mi->gtid_event_seen= false; /* Intentionally init ssl_verify_server_cert to 0, no option available */ mi->ssl_verify_server_cert= 0; @@ -148,8 +205,13 @@ enum { LINE_FOR_MASTER_BIND = 17, /* 6.0 added value of master_ignore_server_id */ LINE_FOR_REPLICATE_IGNORE_SERVER_IDS= 18, - /* Number of lines currently used when saving master info file */ - LINES_IN_MASTER_INFO= LINE_FOR_REPLICATE_IGNORE_SERVER_IDS + /* MySQL 5.6 fixed-position lines. */ + LINE_FOR_FIRST_MYSQL_5_6=19, + LINE_FOR_LAST_MYSQL_5_6=23, + /* Reserved lines for MySQL future versions. */ + LINE_FOR_LAST_MYSQL_FUTURE=33, + /* Number of (fixed-position) lines used when saving master info file */ + LINES_IN_MASTER_INFO= LINE_FOR_LAST_MYSQL_FUTURE }; int init_master_info(Master_info* mi, const char* master_info_fname, @@ -277,7 +339,7 @@ file '%s')", fname); int ssl= 0, ssl_verify_server_cert= 0; float master_heartbeat_period= 0.0; char *first_non_digit; - char dummy_buf[HOSTNAME_LENGTH+1]; + char buf[HOSTNAME_LENGTH+1]; /* Starting from 4.1.x master.info has new format. Now its @@ -371,7 +433,7 @@ file '%s')", fname); (this is just a reservation to avoid future upgrade problems) */ if (lines >= LINE_FOR_MASTER_BIND && - init_strvar_from_file(dummy_buf, sizeof(dummy_buf), &mi->file, "")) + init_strvar_from_file(buf, sizeof(buf), &mi->file, "")) goto errwithmsg; /* Starting from 6.0 list of server_id of ignorable servers might be @@ -383,6 +445,42 @@ file '%s')", fname); sql_print_error("Failed to initialize master info ignore_server_ids"); goto errwithmsg; } + + /* + Starting with MariaDB 10.0, we use a key=value syntax, which is nicer + in several ways. But we leave a bunch of empty lines to accomodate + any future old-style additions in MySQL (this will make it easier for + users moving from MariaDB to MySQL, to not have MySQL try to + interpret a MariaDB key=value line.) + */ + if (lines >= LINE_FOR_LAST_MYSQL_FUTURE) + { + uint i; + /* Skip lines used by / reserved for MySQL >= 5.6. */ + for (i= LINE_FOR_FIRST_MYSQL_5_6; i <= LINE_FOR_LAST_MYSQL_FUTURE; ++i) + { + if (init_strvar_from_file(buf, sizeof(buf), &mi->file, "")) + goto errwithmsg; + } + + /* + Parse any extra key=value lines. + Ignore unknown lines, to facilitate downgrades. + */ + while (!init_strvar_from_file(buf, sizeof(buf), &mi->file, 0)) + { + if (0 == strncmp(buf, STRING_WITH_LEN("using_gtid="))) + { + int val= atoi(buf + sizeof("using_gtid")); + if (val == Master_info::USE_GTID_CURRENT_POS) + mi->using_gtid= Master_info::USE_GTID_CURRENT_POS; + else if (val == Master_info::USE_GTID_SLAVE_POS) + mi->using_gtid= Master_info::USE_GTID_SLAVE_POS; + else + mi->using_gtid= Master_info::USE_GTID_NO; + } + } + } } #ifndef HAVE_OPENSSL @@ -407,7 +505,7 @@ file '%s')", fname); mi->master_log_name, (ulong) mi->master_log_pos)); - mi->rli.mi = mi; + mi->rli.mi= mi; if (init_relay_log_info(&mi->rli, slave_info_fname)) goto err; @@ -488,7 +586,7 @@ int flush_master_info(Master_info* mi, char* ignore_server_ids_buf; { ignore_server_ids_buf= - (char *) my_malloc((sizeof(::server_id) * 3 + 1) * + (char *) my_malloc((sizeof(global_system_variables.server_id) * 3 + 1) * (1 + mi->ignore_server_ids.elements), MYF(MY_WME)); if (!ignore_server_ids_buf) DBUG_RETURN(1); @@ -522,14 +620,16 @@ int flush_master_info(Master_info* mi, sprintf(heartbeat_buf, "%.3f", mi->heartbeat_period); my_b_seek(file, 0L); my_b_printf(file, - "%u\n%s\n%s\n%s\n%s\n%s\n%d\n%d\n%d\n%s\n%s\n%s\n%s\n%s\n%d\n%s\n%s\n%s\n", + "%u\n%s\n%s\n%s\n%s\n%s\n%d\n%d\n%d\n%s\n%s\n%s\n%s\n%s\n%d\n%s\n%s\n%s\n" + "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + "using_gtid=%d\n", LINES_IN_MASTER_INFO, mi->master_log_name, llstr(mi->master_log_pos, lbuf), mi->host, mi->user, mi->password, mi->port, mi->connect_retry, (int)(mi->ssl), mi->ssl_ca, mi->ssl_capath, mi->ssl_cert, mi->ssl_cipher, mi->ssl_key, mi->ssl_verify_server_cert, - heartbeat_buf, "", ignore_server_ids_buf); + heartbeat_buf, "", ignore_server_ids_buf, mi->using_gtid); my_free(ignore_server_ids_buf); err= flush_io_cache(file); if (sync_masterinfo_period && !err && @@ -560,5 +660,656 @@ void end_master_info(Master_info* mi) DBUG_VOID_RETURN; } +/* Multi-Master By P.Linux */ +uchar *get_key_master_info(Master_info *mi, size_t *length, + my_bool not_used __attribute__((unused))) +{ + /* Return lower case name */ + *length= mi->cmp_connection_name.length; + return (uchar*) mi->cmp_connection_name.str; +} + +void free_key_master_info(Master_info *mi) +{ + DBUG_ENTER("free_key_master_info"); + terminate_slave_threads(mi,SLAVE_FORCE_ALL); + end_master_info(mi); + delete mi; + DBUG_VOID_RETURN; +} + +/** + Check if connection name for master_info is valid. + + It's valid if it's a valid system name of length less than + MAX_CONNECTION_NAME. + + @return + 0 ok + 1 error +*/ + +bool check_master_connection_name(LEX_STRING *name) +{ + if (name->length >= MAX_CONNECTION_NAME) + return 1; + return 0; +} + + +/** + Create a log file with a given suffix. + + @param + res_file_name Store result here + length Length of res_file_name buffer + info_file Original file name (prefix) + append 1 if we should add suffix last (not before ext) + suffix Suffix + + @note + The suffix is added before the extension of the file name prefixed with '-'. + The suffix is also converted to lower case and we transform + all not safe character, as we do with MySQL table names. + + If suffix is an empty string, then we don't add any suffix. + This is to allow one to use this function also to generate old + file names without a prefix. +*/ + +void create_logfile_name_with_suffix(char *res_file_name, size_t length, + const char *info_file, bool append, + LEX_STRING *suffix) +{ + char buff[MAX_CONNECTION_NAME+1], + res[MAX_CONNECTION_NAME * MAX_FILENAME_MBWIDTH+1], *p; + + p= strmake(res_file_name, info_file, length); + /* If not empty suffix and there is place left for some part of the suffix */ + if (suffix->length != 0 && p <= res_file_name + length -1) + { + const char *info_file_end= info_file + (p - res_file_name); + const char *ext= append ? info_file_end : fn_ext2(info_file); + size_t res_length, ext_pos; + uint errors; + + /* Create null terminated string */ + strmake(buff, suffix->str, suffix->length); + /* Convert to characters usable in a file name */ + res_length= strconvert(system_charset_info, buff, + &my_charset_filename, res, sizeof(res), &errors); + + ext_pos= (size_t) (ext - info_file); + length-= (suffix->length - ext_pos); /* Leave place for extension */ + p= res_file_name + ext_pos; + *p++= '-'; /* Add separator */ + p= strmake(p, res, min((size_t) (length - (p - res_file_name)), + res_length)); + /* Add back extension. We have checked above that there is space for it */ + strmov(p, ext); + } +} + +void copy_filter_setting(Rpl_filter* dst_filter, Rpl_filter* src_filter) +{ + char buf[256]; + String tmp(buf, sizeof(buf), &my_charset_bin); + + dst_filter->get_do_db(&tmp); + if (tmp.is_empty()) + { + src_filter->get_do_db(&tmp); + if (!tmp.is_empty()) + dst_filter->set_do_db(tmp.ptr()); + } + + dst_filter->get_do_table(&tmp); + if (tmp.is_empty()) + { + src_filter->get_do_table(&tmp); + if (!tmp.is_empty()) + dst_filter->set_do_table(tmp.ptr()); + } + + dst_filter->get_ignore_db(&tmp); + if (tmp.is_empty()) + { + src_filter->get_ignore_db(&tmp); + if (!tmp.is_empty()) + dst_filter->set_ignore_db(tmp.ptr()); + } + + dst_filter->get_ignore_table(&tmp); + if (tmp.is_empty()) + { + src_filter->get_ignore_table(&tmp); + if (!tmp.is_empty()) + dst_filter->set_ignore_table(tmp.ptr()); + } + + dst_filter->get_wild_do_table(&tmp); + if (tmp.is_empty()) + { + src_filter->get_wild_do_table(&tmp); + if (!tmp.is_empty()) + dst_filter->set_wild_do_table(tmp.ptr()); + } + + dst_filter->get_wild_ignore_table(&tmp); + if (tmp.is_empty()) + { + src_filter->get_wild_ignore_table(&tmp); + if (!tmp.is_empty()) + dst_filter->set_wild_ignore_table(tmp.ptr()); + } + + if (dst_filter->rewrite_db_is_empty()) + { + if (!src_filter->rewrite_db_is_empty()) + dst_filter->copy_rewrite_db(src_filter); + } +} + +Master_info_index::Master_info_index() +{ + size_t filename_length, dir_length; + /* + Create the Master_info index file by prepending 'multi-' before + the master_info_file file name. + */ + fn_format(index_file_name, master_info_file, mysql_data_home, + "", MY_UNPACK_FILENAME); + filename_length= strlen(index_file_name) + 1; /* Count 0 byte */ + dir_length= dirname_length(index_file_name); + bmove_upp((uchar*) index_file_name + filename_length + 6, + (uchar*) index_file_name + filename_length, + filename_length - dir_length); + memcpy(index_file_name + dir_length, "multi-", 6); + + bzero((char*) &index_file, sizeof(index_file)); +} + +Master_info_index::~Master_info_index() +{ + /* This will close connection for all objects in the cache */ + my_hash_free(&master_info_hash); + end_io_cache(&index_file); + if (index_file.file > 0) + my_close(index_file.file, MYF(MY_WME)); +} + + +/* Load All Master_info from master.info.index File + * RETURN: + * 0 - All Success + * 1 - All Fail + * 2 - Some Success, Some Fail + */ + +bool Master_info_index::init_all_master_info() +{ + int thread_mask; + int err_num= 0, succ_num= 0; // The number of success read Master_info + char sign[MAX_CONNECTION_NAME+1]; + File index_file_nr; + DBUG_ENTER("init_all_master_info"); + + if ((index_file_nr= my_open(index_file_name, + O_RDWR | O_CREAT | O_BINARY , + MYF(MY_WME | ME_NOREFRESH))) < 0 || + my_sync(index_file_nr, MYF(MY_WME)) || + init_io_cache(&index_file, index_file_nr, + IO_SIZE, READ_CACHE, + my_seek(index_file_nr,0L,MY_SEEK_END,MYF(0)), + 0, MYF(MY_WME | MY_WAIT_IF_FULL))) + { + if (index_file_nr >= 0) + my_close(index_file_nr,MYF(0)); + + sql_print_error("Creation of Master_info index file '%s' failed", + index_file_name); + DBUG_RETURN(1); + } + + /* Initialize Master_info Hash Table */ + if (my_hash_init(&master_info_hash, system_charset_info, + MAX_REPLICATION_THREAD, 0, 0, + (my_hash_get_key) get_key_master_info, + (my_hash_free_key)free_key_master_info, HASH_UNIQUE)) + { + sql_print_error("Initializing Master_info hash table failed"); + DBUG_RETURN(1); + } + + reinit_io_cache(&index_file, READ_CACHE, 0L,0,0); + while (!init_strvar_from_file(sign, sizeof(sign), + &index_file, NULL)) + { + LEX_STRING connection_name; + Master_info *mi; + char buf_master_info_file[FN_REFLEN]; + char buf_relay_log_info_file[FN_REFLEN]; + + connection_name.str= sign; + connection_name.length= strlen(sign); + if (!(mi= new Master_info(&connection_name, relay_log_recovery)) || + mi->error()) + { + delete mi; + DBUG_RETURN(1); + } + + lock_slave_threads(mi); + init_thread_mask(&thread_mask,mi,0 /*not inverse*/); + + create_logfile_name_with_suffix(buf_master_info_file, + sizeof(buf_master_info_file), + master_info_file, 0, + &mi->cmp_connection_name); + create_logfile_name_with_suffix(buf_relay_log_info_file, + sizeof(buf_relay_log_info_file), + relay_log_info_file, 0, + &mi->cmp_connection_name); + if (global_system_variables.log_warnings > 1) + sql_print_information("Reading Master_info: '%s' Relay_info:'%s'", + buf_master_info_file, buf_relay_log_info_file); + + if (init_master_info(mi, buf_master_info_file, buf_relay_log_info_file, + 0, thread_mask)) + { + err_num++; + sql_print_error("Initialized Master_info from '%s' failed", + buf_master_info_file); + if (!master_info_index->get_master_info(&connection_name, + MYSQL_ERROR::WARN_LEVEL_NOTE)) + { + /* Master_info is not in HASH; Add it */ + if (master_info_index->add_master_info(mi, FALSE)) + return 1; + succ_num++; + unlock_slave_threads(mi); + } + else + { + /* Master_info already in HASH */ + sql_print_error(ER(ER_CONNECTION_ALREADY_EXISTS), + (int) connection_name.length, connection_name.str); + unlock_slave_threads(mi); + delete mi; + } + continue; + } + else + { + /* Initialization of Master_info succeded. Add it to HASH */ + if (global_system_variables.log_warnings > 1) + sql_print_information("Initialized Master_info from '%s'", + buf_master_info_file); + if (master_info_index->get_master_info(&connection_name, + MYSQL_ERROR::WARN_LEVEL_NOTE)) + { + /* Master_info was already registered */ + sql_print_error(ER(ER_CONNECTION_ALREADY_EXISTS), + (int) connection_name.length, connection_name.str); + unlock_slave_threads(mi); + delete mi; + continue; + } + + /* Master_info was not registered; add it */ + if (master_info_index->add_master_info(mi, FALSE)) + return 1; + succ_num++; + unlock_slave_threads(mi); + + if (!opt_skip_slave_start) + { + if (start_slave_threads(1 /* need mutex */, + 0 /* no wait for start*/, + mi, + buf_master_info_file, + buf_relay_log_info_file, + SLAVE_IO | SLAVE_SQL)) + { + sql_print_error("Failed to create slave threads for connection '%.*s'", + (int) connection_name.length, + connection_name.str); + continue; + } + if (global_system_variables.log_warnings) + sql_print_information("Started replication for '%.*s'", + (int) connection_name.length, + connection_name.str); + } + } + } + + if (!err_num) // No Error on read Master_info + { + if (global_system_variables.log_warnings > 1) + sql_print_information("Reading of all Master_info entries succeded"); + DBUG_RETURN(0); + } + else if (succ_num) // Have some Error and some Success + { + sql_print_warning("Reading of some Master_info entries failed"); + DBUG_RETURN(2); + } + else // All failed + { + sql_print_error("Reading of all Master_info entries failed!"); + DBUG_RETURN(1); + } +} + + +/* Write new master.info to master.info.index File */ +bool Master_info_index::write_master_name_to_index_file(LEX_STRING *name, + bool do_sync) +{ + DBUG_ASSERT(my_b_inited(&index_file) != 0); + DBUG_ENTER("write_master_name_to_index_file"); + + /* Don't write default slave to master_info.index */ + if (name->length == 0) + DBUG_RETURN(0); + + reinit_io_cache(&index_file, WRITE_CACHE, + my_b_filelength(&index_file), 0, 0); + + if (my_b_write(&index_file, (uchar*) name->str, name->length) || + my_b_write(&index_file, (uchar*) "\n", 1) || + flush_io_cache(&index_file) || + (do_sync && my_sync(index_file.file, MYF(MY_WME)))) + { + sql_print_error("Write of new Master_info for '%.*s' to index file failed", + (int) name->length, name->str); + DBUG_RETURN(1); + } + + DBUG_RETURN(0); +} + + +/** + Get Master_info for a connection + + @param + connection_name Connection name + warning WARN_LEVEL_NOTE -> Don't print anything + WARN_LEVEL_WARN -> Issue warning if not exists + WARN_LEVEL_ERROR-> Issue error if not exists +*/ + +Master_info * +Master_info_index::get_master_info(LEX_STRING *connection_name, + MYSQL_ERROR::enum_warning_level warning) +{ + Master_info *mi; + char buff[MAX_CONNECTION_NAME+1], *res; + uint buff_length; + DBUG_ENTER("get_master_info"); + DBUG_PRINT("enter", + ("connection_name: '%.*s'", (int) connection_name->length, + connection_name->str)); + + /* Make name lower case for comparison */ + res= strmake(buff, connection_name->str, connection_name->length); + my_casedn_str(system_charset_info, buff); + buff_length= (size_t) (res-buff); + + mi= (Master_info*) my_hash_search(&master_info_hash, + (uchar*) buff, buff_length); + if (!mi && warning != MYSQL_ERROR::WARN_LEVEL_NOTE) + { + my_error(WARN_NO_MASTER_INFO, + MYF(warning == MYSQL_ERROR::WARN_LEVEL_WARN ? ME_JUST_WARNING : + 0), + (int) connection_name->length, + connection_name->str); + } + DBUG_RETURN(mi); +} + + +/* Check Master_host & Master_port is duplicated or not */ +bool Master_info_index::check_duplicate_master_info(LEX_STRING *name_arg, + const char *host, + uint port) +{ + Master_info *mi; + DBUG_ENTER("check_duplicate_master_info"); + + /* Get full host and port name */ + if ((mi= master_info_index->get_master_info(name_arg, + MYSQL_ERROR::WARN_LEVEL_NOTE))) + { + if (!host) + host= mi->host; + if (!port) + port= mi->port; + } + if (!host || !port) + DBUG_RETURN(FALSE); // Not comparable yet + + for (uint i= 0; i < master_info_hash.records; ++i) + { + Master_info *tmp_mi; + tmp_mi= (Master_info *) my_hash_element(&master_info_hash, i); + if (tmp_mi == mi) + continue; // Current connection + if (!strcasecmp(host, tmp_mi->host) && port == tmp_mi->port) + { + my_error(ER_CONNECTION_ALREADY_EXISTS, MYF(0), + (int) name_arg->length, + name_arg->str, + (int) tmp_mi->connection_name.length, + tmp_mi->connection_name.str); + DBUG_RETURN(TRUE); + } + } + DBUG_RETURN(FALSE); +} + + +/* Add a Master_info class to Hash Table */ +bool Master_info_index::add_master_info(Master_info *mi, bool write_to_file) +{ + if (!my_hash_insert(&master_info_hash, (uchar*) mi)) + { + if (global_system_variables.log_warnings > 1) + sql_print_information("Added new Master_info '%.*s' to hash table", + (int) mi->connection_name.length, + mi->connection_name.str); + if (write_to_file) + return write_master_name_to_index_file(&mi->connection_name, 1); + return FALSE; + } + + /* Impossible error (EOM) ? */ + sql_print_error("Adding new entry '%.*s' to master_info failed", + (int) mi->connection_name.length, + mi->connection_name.str); + return TRUE; +} + + +/** + Remove a Master_info class From Hash Table + + TODO: Change this to use my_rename() to make the file name creation + atomic +*/ + +bool Master_info_index::remove_master_info(LEX_STRING *name) +{ + Master_info* mi; + DBUG_ENTER("remove_master_info"); + + if ((mi= get_master_info(name, MYSQL_ERROR::WARN_LEVEL_WARN))) + { + // Delete Master_info and rewrite others to file + if (!my_hash_delete(&master_info_hash, (uchar*) mi)) + { + File index_file_nr; + + // Close IO_CACHE and FILE handler fisrt + end_io_cache(&index_file); + my_close(index_file.file, MYF(MY_WME)); + + // Reopen File and truncate it + if ((index_file_nr= my_open(index_file_name, + O_RDWR | O_CREAT | O_TRUNC | O_BINARY , + MYF(MY_WME))) < 0 || + init_io_cache(&index_file, index_file_nr, + IO_SIZE, WRITE_CACHE, + my_seek(index_file_nr,0L,MY_SEEK_END,MYF(0)), + 0, MYF(MY_WME | MY_WAIT_IF_FULL))) + { + int error= my_errno; + if (index_file_nr >= 0) + my_close(index_file_nr,MYF(0)); + + sql_print_error("Create of Master Info Index file '%s' failed with " + "error: %M", + index_file_name, error); + DBUG_RETURN(TRUE); + } + + // Rewrite Master_info.index + for (uint i= 0; i< master_info_hash.records; ++i) + { + Master_info *tmp_mi; + tmp_mi= (Master_info *) my_hash_element(&master_info_hash, i); + write_master_name_to_index_file(&tmp_mi->connection_name, 0); + } + my_sync(index_file_nr, MYF(MY_WME)); + } + } + DBUG_RETURN(FALSE); +} + + +/** + Master_info_index::give_error_if_slave_running() + + @return + TRUE If some slave is running. An error is printed + FALSE No slave is running +*/ + +bool Master_info_index::give_error_if_slave_running() +{ + DBUG_ENTER("warn_if_slave_running"); + mysql_mutex_assert_owner(&LOCK_active_mi); + + for (uint i= 0; i< master_info_hash.records; ++i) + { + Master_info *mi; + mi= (Master_info *) my_hash_element(&master_info_hash, i); + if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) + { + my_error(ER_SLAVE_MUST_STOP, MYF(0), (int) mi->connection_name.length, + mi->connection_name.str); + DBUG_RETURN(TRUE); + } + } + DBUG_RETURN(FALSE); +} + + +/** + Master_info_index::start_all_slaves() + + Start all slaves that was not running. + + @return + TRUE Error + FALSE Everything ok. +*/ + +bool Master_info_index::start_all_slaves(THD *thd) +{ + bool result= FALSE; + DBUG_ENTER("warn_if_slave_running"); + mysql_mutex_assert_owner(&LOCK_active_mi); + + for (uint i= 0; i< master_info_hash.records; ++i) + { + int error; + Master_info *mi; + mi= (Master_info *) my_hash_element(&master_info_hash, i); + + /* + Try to start all slaves that are configured (host is defined) + and are not already running + */ + if ((mi->slave_running != MYSQL_SLAVE_RUN_CONNECT || + !mi->rli.slave_running) && *mi->host) + { + if ((error= start_slave(thd, mi, 1))) + { + my_error(ER_CANT_START_STOP_SLAVE, MYF(0), + "START", + (int) mi->connection_name.length, + mi->connection_name.str); + result= 1; + if (error < 0) // fatal error + break; + } + else + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_SLAVE_STARTED, ER(ER_SLAVE_STARTED), + (int) mi->connection_name.length, + mi->connection_name.str); + } + } + DBUG_RETURN(result); +} + + +/** + Master_info_index::stop_all_slaves() + + Start all slaves that was not running. + + @return + TRUE Error + FALSE Everything ok. +*/ + +bool Master_info_index::stop_all_slaves(THD *thd) +{ + bool result= FALSE; + DBUG_ENTER("warn_if_slave_running"); + mysql_mutex_assert_owner(&LOCK_active_mi); + + for (uint i= 0; i< master_info_hash.records; ++i) + { + int error; + Master_info *mi; + mi= (Master_info *) my_hash_element(&master_info_hash, i); + if ((mi->slave_running != MYSQL_SLAVE_NOT_RUN || + mi->rli.slave_running)) + { + if ((error= stop_slave(thd, mi, 1))) + { + my_error(ER_CANT_START_STOP_SLAVE, MYF(0), + "STOP", + (int) mi->connection_name.length, + mi->connection_name.str); + result= 1; + if (error < 0) // Fatal error + break; + } + else + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_SLAVE_STOPPED, ER(ER_SLAVE_STOPPED), + (int) mi->connection_name.length, + mi->connection_name.str); + } + } + DBUG_RETURN(result); +} #endif /* HAVE_REPLICATION */ diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index f9c8c8ea5b2..1c34a57b1e3 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -21,6 +21,8 @@ #include "rpl_rli.h" #include "rpl_reporting.h" #include "my_sys.h" +#include "rpl_filter.h" +#include "keycaches.h" typedef struct st_mysql MYSQL; @@ -59,16 +61,28 @@ typedef struct st_mysql MYSQL; class Master_info : public Slave_reporting_capability { public: - Master_info(bool is_slave_recovery); + enum enum_using_gtid { + USE_GTID_NO= 0, USE_GTID_CURRENT_POS= 1, USE_GTID_SLAVE_POS= 2 + }; + + Master_info(LEX_STRING *connection_name, bool is_slave_recovery); ~Master_info(); bool shall_ignore_server_id(ulong s_id); void clear_in_memory_info(bool all); + bool error() + { + /* If malloc() in initialization failed */ + return connection_name.str == 0; + } + static const char *using_gtid_astext(enum enum_using_gtid arg); /* the variables below are needed because we can change masters on the fly */ - char master_log_name[FN_REFLEN]; + char master_log_name[FN_REFLEN+6]; /* Room for multi-*/ char host[HOSTNAME_LENGTH*SYSTEM_CHARSET_MBMAXLEN+1]; char user[USERNAME_LENGTH+1]; char password[MAX_PASSWORD_LENGTH*SYSTEM_CHARSET_MBMAXLEN+1]; + LEX_STRING connection_name; /* User supplied connection name */ + LEX_STRING cmp_connection_name; /* Connection name in lower case */ bool ssl; // enables use of SSL connection if true char ssl_ca[FN_REFLEN], ssl_capath[FN_REFLEN], ssl_cert[FN_REFLEN]; char ssl_cipher[FN_REFLEN], ssl_key[FN_REFLEN]; @@ -85,6 +99,7 @@ class Master_info : public Slave_reporting_capability uint32 file_id; /* for 3.23 load data infile */ Relay_log_info rli; uint port; + Rpl_filter* rpl_filter; /* Each replication can set its filter rule*/ /* to hold checksum alg in use until IO thread has received FD. Initialized to novalue, then set to the queried from master @@ -119,6 +134,43 @@ class Master_info : public Slave_reporting_capability ulonglong received_heartbeats; // counter of received heartbeat events DYNAMIC_ARRAY ignore_server_ids; ulong master_id; + /* + Which kind of GTID position (if any) is used when connecting to master. + + Note that you can not change the numeric values of these, they are used + in master.info. + */ + enum enum_using_gtid using_gtid; + + /* + This GTID position records how far we have fetched into the relay logs. + This is used to continue fetching when the IO thread reconnects to the + master. + + (Full slave stop/start does not use it, as it resets the relay logs). + */ + slave_connection_state gtid_current_pos; + /* + If events_queued_since_last_gtid is non-zero, it is the number of events + queued so far in the relaylog of a GTID-prefixed event group. + It is zero when no partial event group has been queued at the moment. + */ + uint64 events_queued_since_last_gtid; + /* + The GTID of the partially-queued event group, when + events_queued_since_last_gtid is non-zero. + */ + rpl_gtid last_queued_gtid; + /* Whether last_queued_gtid had the FL_STANDALONE flag set. */ + bool last_queued_gtid_standalone; + /* + When slave IO thread needs to reconnect, gtid_reconnect_event_skip_count + counts number of events to skip from the first GTID-prefixed event group, + to avoid duplicating events in the relay log. + */ + uint64 gtid_reconnect_event_skip_count; + /* gtid_event_seen is false until we receive first GTID event from master. */ + bool gtid_event_seen; }; int init_master_info(Master_info* mi, const char* master_info_fname, const char* slave_info_fname, @@ -129,6 +181,50 @@ int flush_master_info(Master_info* mi, bool flush_relay_log_cache, bool need_lock_relay_log); int change_master_server_id_cmp(ulong *id1, ulong *id2); +void copy_filter_setting(Rpl_filter* dst_filter, Rpl_filter* src_filter); + +/* + Multi master are handled trough this struct. + Changes to this needs to be protected by LOCK_active_mi; +*/ + +class Master_info_index +{ +private: + IO_CACHE index_file; + char index_file_name[FN_REFLEN]; + +public: + Master_info_index(); + ~Master_info_index(); + + HASH master_info_hash; + + bool init_all_master_info(); + bool write_master_name_to_index_file(LEX_STRING *connection_name, + bool do_sync); + + bool check_duplicate_master_info(LEX_STRING *connection_name, + const char *host, uint port); + bool add_master_info(Master_info *mi, bool write_to_file); + bool remove_master_info(LEX_STRING *connection_name); + Master_info *get_master_info(LEX_STRING *connection_name, + MYSQL_ERROR::enum_warning_level warning); + bool give_error_if_slave_running(); + bool start_all_slaves(THD *thd); + bool stop_all_slaves(THD *thd); +}; + +bool check_master_connection_name(LEX_STRING *name); +void create_logfile_name_with_suffix(char *res_file_name, size_t length, + const char *info_file, + bool append, + LEX_STRING *suffix); + +uchar *get_key_master_info(Master_info *mi, size_t *length, + my_bool not_used __attribute__((unused))); +void free_key_master_info(Master_info *mi); + #endif /* HAVE_REPLICATION */ #endif /* RPL_MI_H */ diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc new file mode 100644 index 00000000000..ef282611f70 --- /dev/null +++ b/sql/rpl_parallel.cc @@ -0,0 +1,1116 @@ +#include "my_global.h" +#include "rpl_parallel.h" +#include "slave.h" +#include "rpl_mi.h" +#include "debug_sync.h" + + +/* + Code for optional parallel execution of replicated events on the slave. + + ToDo list: + + - Retry of failed transactions is not yet implemented for the parallel case. + + - All the waits (eg. in struct wait_for_commit and in + rpl_parallel_thread_pool::get_thread()) need to be killable. And on kill, + everything needs to be correctly rolled back and stopped in all threads, + to ensure a consistent slave replication state. +*/ + +struct rpl_parallel_thread_pool global_rpl_thread_pool; + + +static int +rpt_handle_event(rpl_parallel_thread::queued_event *qev, + struct rpl_parallel_thread *rpt) +{ + int err; + rpl_group_info *rgi= qev->rgi; + Relay_log_info *rli= rgi->rli; + THD *thd= rgi->thd; + + thd->rgi_slave= rgi; + thd->rpl_filter = rli->mi->rpl_filter; + + /* ToDo: Access to thd, and what about rli, split out a parallel part? */ + mysql_mutex_lock(&rli->data_lock); + qev->ev->thd= thd; + strcpy(rgi->event_relay_log_name_buf, qev->event_relay_log_name); + rgi->event_relay_log_name= rgi->event_relay_log_name_buf; + rgi->event_relay_log_pos= qev->event_relay_log_pos; + rgi->future_event_relay_log_pos= qev->future_event_relay_log_pos; + strcpy(rgi->future_event_master_log_name, qev->future_event_master_log_name); + err= apply_event_and_update_pos(qev->ev, thd, rgi, rpt); + thd->rgi_slave= NULL; + + thread_safe_increment64(&rli->executed_entries, + &slave_executed_entries_lock); + /* ToDo: error handling. */ + return err; +} + + +static void +handle_queued_pos_update(THD *thd, rpl_parallel_thread::queued_event *qev) +{ + int cmp; + Relay_log_info *rli; + /* + Events that are not part of an event group, such as Format Description, + Stop, GTID List and such, are executed directly in the driver SQL thread, + to keep the relay log state up-to-date. But the associated position update + is done here, in sync with other normal events as they are queued to + worker threads. + */ + if ((thd->variables.option_bits & OPTION_BEGIN) && + opt_using_transactions) + return; + rli= qev->rgi->rli; + mysql_mutex_lock(&rli->data_lock); + cmp= strcmp(rli->group_relay_log_name, qev->event_relay_log_name); + if (cmp < 0) + { + rli->group_relay_log_pos= qev->future_event_relay_log_pos; + strmake_buf(rli->group_relay_log_name, qev->event_relay_log_name); + rli->notify_group_relay_log_name_update(); + } else if (cmp == 0 && + rli->group_relay_log_pos < qev->future_event_relay_log_pos) + rli->group_relay_log_pos= qev->future_event_relay_log_pos; + + cmp= strcmp(rli->group_master_log_name, qev->future_event_master_log_name); + if (cmp < 0) + { + strcpy(rli->group_master_log_name, qev->future_event_master_log_name); + rli->notify_group_master_log_name_update(); + rli->group_master_log_pos= qev->future_event_master_log_pos; + } + else if (cmp == 0 + && rli->group_master_log_pos < qev->future_event_master_log_pos) + rli->group_master_log_pos= qev->future_event_master_log_pos; + mysql_mutex_unlock(&rli->data_lock); + mysql_cond_broadcast(&rli->data_cond); +} + + +static bool +sql_worker_killed(THD *thd, rpl_group_info *rgi, bool in_event_group) +{ + if (!rgi->rli->abort_slave && !abort_loop) + return false; + + /* + Do not abort in the middle of an event group that cannot be rolled back. + */ + if ((thd->transaction.all.modified_non_trans_table || + (thd->variables.option_bits & OPTION_KEEP_LOG)) + && in_event_group) + return false; + /* ToDo: should we add some timeout like in sql_slave_killed? + if (rgi->last_event_start_time == 0) + rgi->last_event_start_time= my_time(0); + */ + + return true; +} + + +static void +finish_event_group(THD *thd, int err, uint64 sub_id, + rpl_parallel_entry *entry, wait_for_commit *wfc) +{ + /* + Remove any left-over registration to wait for a prior commit to + complete. Normally, such wait would already have been removed at + this point by wait_for_prior_commit() called from within COMMIT + processing. However, in case of MyISAM and no binlog, we might not + have any commit processing, and so we need to do the wait here, + before waking up any subsequent commits, to preserve correct + order of event execution. Also, in the error case we might have + skipped waiting and thus need to remove it explicitly. + + It is important in the non-error case to do a wait, not just an + unregister. Because we might be last in a group-commit that is + replicated in parallel, and the following event will then wait + for us to complete and rely on this also ensuring that any other + event in the group has completed. + + But in the error case, we have to abort anyway, and it seems best + to just complete as quickly as possible with unregister. Anyone + waiting for us will in any case receive the error back from their + wait_for_prior_commit() call. + */ + if (err) + wfc->unregister_wait_for_prior_commit(); + else + err= wfc->wait_for_prior_commit(thd); + thd->wait_for_commit_ptr= NULL; + + /* + Record that this event group has finished (eg. transaction is + committed, if transactional), so other event groups will no longer + attempt to wait for us to commit. Once we have increased + entry->last_committed_sub_id, no other threads will execute + register_wait_for_prior_commit() against us. Thus, by doing one + extra (usually redundant) wakeup_subsequent_commits() we can ensure + that no register_wait_for_prior_commit() can ever happen without a + subsequent wakeup_subsequent_commits() to wake it up. + + We can race here with the next transactions, but that is fine, as + long as we check that we do not decrease last_committed_sub_id. If + this commit is done, then any prior commits will also have been + done and also no longer need waiting for. + */ + mysql_mutex_lock(&entry->LOCK_parallel_entry); + if (entry->last_committed_sub_id < sub_id) + { + entry->last_committed_sub_id= sub_id; + mysql_cond_broadcast(&entry->COND_parallel_entry); + } + mysql_mutex_unlock(&entry->LOCK_parallel_entry); + + wfc->wakeup_subsequent_commits(err); +} + + +static void +signal_error_to_sql_driver_thread(THD *thd, rpl_group_info *rgi) +{ + rgi->is_error= true; + rgi->cleanup_context(thd, true); + rgi->rli->abort_slave= true; + mysql_mutex_lock(rgi->rli->relay_log.get_log_lock()); + mysql_mutex_unlock(rgi->rli->relay_log.get_log_lock()); + rgi->rli->relay_log.signal_update(); +} + + +pthread_handler_t +handle_rpl_parallel_thread(void *arg) +{ + THD *thd; + const char* old_msg; + struct rpl_parallel_thread::queued_event *events; + bool group_standalone= true; + bool in_event_group= false; + rpl_group_info *group_rgi= NULL; + uint64 event_gtid_sub_id= 0; + int err; + + struct rpl_parallel_thread *rpt= (struct rpl_parallel_thread *)arg; + + my_thread_init(); + thd = new THD; + thd->thread_stack = (char*)&thd; + mysql_mutex_lock(&LOCK_thread_count); + thd->thread_id= thd->variables.pseudo_thread_id= thread_id++; + threads.append(thd); + mysql_mutex_unlock(&LOCK_thread_count); + set_current_thd(thd); + pthread_detach_this_thread(); + thd->init_for_queries(); + thd->variables.binlog_annotate_row_events= 0; + init_thr_lock(); + thd->store_globals(); + thd->system_thread= SYSTEM_THREAD_SLAVE_SQL; + thd->security_ctx->skip_grants(); + thd->variables.max_allowed_packet= slave_max_allowed_packet; + thd->slave_thread= 1; + thd->enable_slow_log= opt_log_slow_slave_statements; + thd->variables.log_slow_filter= global_system_variables.log_slow_filter; + set_slave_thread_options(thd); + thd->client_capabilities = CLIENT_LOCAL_FILES; + thd->net.reading_or_writing= 0; + thd_proc_info(thd, "Waiting for work from main SQL threads"); + thd->set_time(); + thd->variables.lock_wait_timeout= LONG_TIMEOUT; + + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + rpt->thd= thd; + + while (rpt->delay_start) + mysql_cond_wait(&rpt->COND_rpl_thread, &rpt->LOCK_rpl_thread); + + rpt->running= true; + mysql_cond_signal(&rpt->COND_rpl_thread); + + while (!rpt->stop && !thd->killed) + { + rpl_parallel_thread *list; + + old_msg= thd->proc_info; + thd->enter_cond(&rpt->COND_rpl_thread, &rpt->LOCK_rpl_thread, + "Waiting for work from SQL thread"); + while (!(events= rpt->event_queue) && !rpt->stop && !thd->killed && + !(rpt->current_entry && rpt->current_entry->force_abort)) + mysql_cond_wait(&rpt->COND_rpl_thread, &rpt->LOCK_rpl_thread); + rpt->dequeue(events); + thd->exit_cond(old_msg); + mysql_cond_signal(&rpt->COND_rpl_thread); + + more_events: + while (events) + { + struct rpl_parallel_thread::queued_event *next= events->next; + Log_event_type event_type; + rpl_group_info *rgi= events->rgi; + rpl_parallel_entry *entry= rgi->parallel_entry; + uint64 wait_for_sub_id; + uint64 wait_start_sub_id; + bool end_of_group; + + if (!events->ev) + { + handle_queued_pos_update(thd, events); + my_free(events); + events= next; + continue; + } + + err= 0; + group_rgi= rgi; + /* Handle a new event group, which will be initiated by a GTID event. */ + if ((event_type= events->ev->get_type_code()) == GTID_EVENT) + { + in_event_group= true; + /* + If the standalone flag is set, then this event group consists of a + single statement (possibly preceeded by some Intvar_log_event and + similar), without any terminating COMMIT/ROLLBACK/XID. + */ + group_standalone= + (0 != (static_cast<Gtid_log_event *>(events->ev)->flags2 & + Gtid_log_event::FL_STANDALONE)); + + /* Save this, as it gets cleared when the event group commits. */ + event_gtid_sub_id= rgi->gtid_sub_id; + + rgi->thd= thd; + + /* + Register ourself to wait for the previous commit, if we need to do + such registration _and_ that previous commit has not already + occured. + + Also do not start parallel execution of this event group until all + prior groups have committed that are not safe to run in parallel with. + */ + wait_for_sub_id= rgi->wait_commit_sub_id; + wait_start_sub_id= rgi->wait_start_sub_id; + if (wait_for_sub_id || wait_start_sub_id) + { + bool did_enter_cond= false; + const char *old_msg= NULL; + + mysql_mutex_lock(&entry->LOCK_parallel_entry); + if (wait_start_sub_id) + { + old_msg= thd->enter_cond(&entry->COND_parallel_entry, + &entry->LOCK_parallel_entry, + "Waiting for prior transaction to commit " + "before starting next transaction"); + did_enter_cond= true; + DEBUG_SYNC(thd, "rpl_parallel_start_waiting_for_prior"); + while (wait_start_sub_id > entry->last_committed_sub_id && + !thd->check_killed()) + mysql_cond_wait(&entry->COND_parallel_entry, + &entry->LOCK_parallel_entry); + if (wait_start_sub_id > entry->last_committed_sub_id) + { + /* The thread got a kill signal. */ + DEBUG_SYNC(thd, "rpl_parallel_start_waiting_for_prior_killed"); + thd->send_kill_message(); + slave_output_error_info(rgi->rli, thd); + signal_error_to_sql_driver_thread(thd, rgi); + } + rgi->wait_start_sub_id= 0; /* No need to check again. */ + } + if (wait_for_sub_id > entry->last_committed_sub_id) + { + wait_for_commit *waitee= + &rgi->wait_commit_group_info->commit_orderer; + rgi->commit_orderer.register_wait_for_prior_commit(waitee); + } + if (did_enter_cond) + thd->exit_cond(old_msg); + else + mysql_mutex_unlock(&entry->LOCK_parallel_entry); + } + + if(thd->wait_for_commit_ptr) + { + /* + This indicates that we get a new GTID event in the middle of + a not completed event group. This is corrupt binlog (the master + will never write such binlog), so it does not happen unless + someone tries to inject wrong crafted binlog, but let us still + try to handle it somewhat nicely. + */ + rgi->cleanup_context(thd, true); + thd->wait_for_commit_ptr->unregister_wait_for_prior_commit(); + thd->wait_for_commit_ptr->wakeup_subsequent_commits(err); + } + thd->wait_for_commit_ptr= &rgi->commit_orderer; + } + + /* + If the SQL thread is stopping, we just skip execution of all the + following event groups. We still do all the normal waiting and wakeup + processing between the event groups as a simple way to ensure that + everything is stopped and cleaned up correctly. + */ + if (!rgi->is_error && !sql_worker_killed(thd, rgi, in_event_group)) + err= rpt_handle_event(events, rpt); + else + err= thd->wait_for_prior_commit(); + + end_of_group= + in_event_group && + ((group_standalone && !Log_event::is_part_of_group(event_type)) || + event_type == XID_EVENT || + (event_type == QUERY_EVENT && + (((Query_log_event *)events->ev)->is_commit() || + ((Query_log_event *)events->ev)->is_rollback()))); + + delete_or_keep_event_post_apply(rgi, event_type, events->ev); + my_free(events); + + if (err) + { + slave_output_error_info(rgi->rli, thd); + signal_error_to_sql_driver_thread(thd, rgi); + } + if (end_of_group) + { + in_event_group= false; + finish_event_group(thd, err, event_gtid_sub_id, entry, + &rgi->commit_orderer); + delete rgi; + group_rgi= rgi= NULL; + DEBUG_SYNC(thd, "rpl_parallel_end_of_group"); + } + + events= next; + } + + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + if ((events= rpt->event_queue) != NULL) + { + /* + Take next group of events from the replication pool. + This is faster than having to wakeup the pool manager thread to give us + a new event. + */ + rpt->dequeue(events); + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + mysql_cond_signal(&rpt->COND_rpl_thread); + goto more_events; + } + + if (in_event_group && group_rgi->parallel_entry->force_abort) + { + /* + We are asked to abort, without getting the remaining events in the + current event group. + + We have to rollback the current transaction and update the last + sub_id value so that SQL thread will know we are done with the + half-processed event group. + */ + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + finish_event_group(thd, 1, group_rgi->gtid_sub_id, + group_rgi->parallel_entry, &group_rgi->commit_orderer); + signal_error_to_sql_driver_thread(thd, group_rgi); + in_event_group= false; + delete group_rgi; + group_rgi= NULL; + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + } + if (!in_event_group) + { + rpt->current_entry= NULL; + if (!rpt->stop) + { + mysql_mutex_lock(&rpt->pool->LOCK_rpl_thread_pool); + list= rpt->pool->free_list; + rpt->next= list; + rpt->pool->free_list= rpt; + if (!list) + mysql_cond_broadcast(&rpt->pool->COND_rpl_thread_pool); + mysql_mutex_unlock(&rpt->pool->LOCK_rpl_thread_pool); + } + } + } + + rpt->thd= NULL; + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + + thd->clear_error(); + thd->catalog= 0; + thd->reset_query(); + thd->reset_db(NULL, 0); + thd_proc_info(thd, "Slave worker thread exiting"); + thd->temporary_tables= 0; + mysql_mutex_lock(&LOCK_thread_count); + THD_CHECK_SENTRY(thd); + delete thd; + mysql_mutex_unlock(&LOCK_thread_count); + + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + rpt->running= false; + mysql_cond_signal(&rpt->COND_rpl_thread); + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + + my_thread_end(); + + return NULL; +} + + +int +rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, + uint32 new_count, bool skip_check) +{ + uint32 i; + rpl_parallel_thread **new_list= NULL; + rpl_parallel_thread *new_free_list= NULL; + rpl_parallel_thread *rpt_array= NULL; + + /* + Allocate the new list of threads up-front. + That way, if we fail half-way, we only need to free whatever we managed + to allocate, and will not be left with a half-functional thread pool. + */ + if (new_count && + !my_multi_malloc(MYF(MY_WME|MY_ZEROFILL), + &new_list, new_count*sizeof(*new_list), + &rpt_array, new_count*sizeof(*rpt_array), + NULL)) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int(new_count*sizeof(*new_list) + + new_count*sizeof(*rpt_array)))); + goto err;; + } + + for (i= 0; i < new_count; ++i) + { + pthread_t th; + + new_list[i]= &rpt_array[i]; + new_list[i]->delay_start= true; + mysql_mutex_init(key_LOCK_rpl_thread, &new_list[i]->LOCK_rpl_thread, + MY_MUTEX_INIT_SLOW); + mysql_cond_init(key_COND_rpl_thread, &new_list[i]->COND_rpl_thread, NULL); + new_list[i]->pool= pool; + if (mysql_thread_create(key_rpl_parallel_thread, &th, NULL, + handle_rpl_parallel_thread, new_list[i])) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + goto err; + } + new_list[i]->next= new_free_list; + new_free_list= new_list[i]; + } + + if (!skip_check) + { + mysql_mutex_lock(&LOCK_active_mi); + if (master_info_index->give_error_if_slave_running()) + { + mysql_mutex_unlock(&LOCK_active_mi); + goto err; + } + if (pool->changing) + { + mysql_mutex_unlock(&LOCK_active_mi); + my_error(ER_CHANGE_SLAVE_PARALLEL_THREADS_ACTIVE, MYF(0)); + goto err; + } + pool->changing= true; + mysql_mutex_unlock(&LOCK_active_mi); + } + + /* + Grab each old thread in turn, and signal it to stop. + + Note that since we require all replication threads to be stopped before + changing the parallel replication worker thread pool, all the threads will + be already idle and will terminate immediately. + */ + for (i= 0; i < pool->count; ++i) + { + rpl_parallel_thread *rpt= pool->get_thread(NULL); + rpt->stop= true; + mysql_cond_signal(&rpt->COND_rpl_thread); + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + } + + for (i= 0; i < pool->count; ++i) + { + rpl_parallel_thread *rpt= pool->threads[i]; + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + while (rpt->running) + mysql_cond_wait(&rpt->COND_rpl_thread, &rpt->LOCK_rpl_thread); + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + mysql_mutex_destroy(&rpt->LOCK_rpl_thread); + mysql_cond_destroy(&rpt->COND_rpl_thread); + } + + my_free(pool->threads); + pool->threads= new_list; + pool->free_list= new_free_list; + pool->count= new_count; + for (i= 0; i < pool->count; ++i) + { + mysql_mutex_lock(&pool->threads[i]->LOCK_rpl_thread); + pool->threads[i]->delay_start= false; + mysql_cond_signal(&pool->threads[i]->COND_rpl_thread); + while (!pool->threads[i]->running) + mysql_cond_wait(&pool->threads[i]->COND_rpl_thread, + &pool->threads[i]->LOCK_rpl_thread); + mysql_mutex_unlock(&pool->threads[i]->LOCK_rpl_thread); + } + + if (!skip_check) + { + mysql_mutex_lock(&LOCK_active_mi); + pool->changing= false; + mysql_mutex_unlock(&LOCK_active_mi); + } + return 0; + +err: + if (new_list) + { + while (new_free_list) + { + mysql_mutex_lock(&new_free_list->LOCK_rpl_thread); + new_free_list->delay_start= false; + new_free_list->stop= true; + mysql_cond_signal(&new_free_list->COND_rpl_thread); + while (!new_free_list->running) + mysql_cond_wait(&new_free_list->COND_rpl_thread, + &new_free_list->LOCK_rpl_thread); + while (new_free_list->running) + mysql_cond_wait(&new_free_list->COND_rpl_thread, + &new_free_list->LOCK_rpl_thread); + mysql_mutex_unlock(&new_free_list->LOCK_rpl_thread); + new_free_list= new_free_list->next; + } + my_free(new_list); + } + if (!skip_check) + { + mysql_mutex_lock(&LOCK_active_mi); + pool->changing= false; + mysql_mutex_unlock(&LOCK_active_mi); + } + return 1; +} + + +rpl_parallel_thread_pool::rpl_parallel_thread_pool() + : count(0), threads(0), free_list(0), changing(false), inited(false) +{ +} + + +int +rpl_parallel_thread_pool::init(uint32 size) +{ + count= 0; + threads= NULL; + free_list= NULL; + + mysql_mutex_init(key_LOCK_rpl_thread_pool, &LOCK_rpl_thread_pool, + MY_MUTEX_INIT_SLOW); + mysql_cond_init(key_COND_rpl_thread_pool, &COND_rpl_thread_pool, NULL); + changing= false; + inited= true; + + return rpl_parallel_change_thread_count(this, size, true); +} + + +void +rpl_parallel_thread_pool::destroy() +{ + if (!inited) + return; + rpl_parallel_change_thread_count(this, 0, true); + mysql_mutex_destroy(&LOCK_rpl_thread_pool); + mysql_cond_destroy(&COND_rpl_thread_pool); + inited= false; +} + + +/* + Wait for a worker thread to become idle. When one does, grab the thread for + our use and return it. + + Note that we return with the worker threads's LOCK_rpl_thread mutex locked. +*/ +struct rpl_parallel_thread * +rpl_parallel_thread_pool::get_thread(rpl_parallel_entry *entry) +{ + rpl_parallel_thread *rpt; + + mysql_mutex_lock(&LOCK_rpl_thread_pool); + while ((rpt= free_list) == NULL) + mysql_cond_wait(&COND_rpl_thread_pool, &LOCK_rpl_thread_pool); + free_list= rpt->next; + mysql_mutex_unlock(&LOCK_rpl_thread_pool); + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + rpt->current_entry= entry; + + return rpt; +} + + +static void +free_rpl_parallel_entry(void *element) +{ + rpl_parallel_entry *e= (rpl_parallel_entry *)element; + mysql_cond_destroy(&e->COND_parallel_entry); + mysql_mutex_destroy(&e->LOCK_parallel_entry); + my_free(e); +} + + +rpl_parallel::rpl_parallel() : + current(NULL), sql_thread_stopping(false) +{ + my_hash_init(&domain_hash, &my_charset_bin, 32, + offsetof(rpl_parallel_entry, domain_id), sizeof(uint32), + NULL, free_rpl_parallel_entry, HASH_UNIQUE); +} + + +void +rpl_parallel::reset() +{ + my_hash_reset(&domain_hash); + current= NULL; + sql_thread_stopping= false; +} + + +rpl_parallel::~rpl_parallel() +{ + my_hash_free(&domain_hash); +} + + +rpl_parallel_entry * +rpl_parallel::find(uint32 domain_id) +{ + struct rpl_parallel_entry *e; + + if (!(e= (rpl_parallel_entry *)my_hash_search(&domain_hash, + (const uchar *)&domain_id, 0))) + { + /* Allocate a new, empty one. */ + if (!(e= (struct rpl_parallel_entry *)my_malloc(sizeof(*e), + MYF(MY_ZEROFILL)))) + return NULL; + e->domain_id= domain_id; + if (my_hash_insert(&domain_hash, (uchar *)e)) + { + my_free(e); + return NULL; + } + mysql_mutex_init(key_LOCK_parallel_entry, &e->LOCK_parallel_entry, + MY_MUTEX_INIT_FAST); + mysql_cond_init(key_COND_parallel_entry, &e->COND_parallel_entry, NULL); + } + else + e->force_abort= false; + + return e; +} + + +void +rpl_parallel::wait_for_done() +{ + struct rpl_parallel_entry *e; + uint32 i; + + /* + First signal all workers that they must force quit; no more events will + be queued to complete any partial event groups executed. + */ + for (i= 0; i < domain_hash.records; ++i) + { + rpl_parallel_thread *rpt; + + e= (struct rpl_parallel_entry *)my_hash_element(&domain_hash, i); + e->force_abort= true; + if ((rpt= e->rpl_thread)) + { + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + if (rpt->current_entry == e) + mysql_cond_signal(&rpt->COND_rpl_thread); + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + } + } + + for (i= 0; i < domain_hash.records; ++i) + { + e= (struct rpl_parallel_entry *)my_hash_element(&domain_hash, i); + mysql_mutex_lock(&e->LOCK_parallel_entry); + while (e->current_sub_id > e->last_committed_sub_id) + mysql_cond_wait(&e->COND_parallel_entry, &e->LOCK_parallel_entry); + mysql_mutex_unlock(&e->LOCK_parallel_entry); + } +} + + +bool +rpl_parallel::workers_idle() +{ + struct rpl_parallel_entry *e; + uint32 i, max_i; + + max_i= domain_hash.records; + for (i= 0; i < max_i; ++i) + { + bool active; + e= (struct rpl_parallel_entry *)my_hash_element(&domain_hash, i); + mysql_mutex_lock(&e->LOCK_parallel_entry); + active= e->current_sub_id > e->last_committed_sub_id; + mysql_mutex_unlock(&e->LOCK_parallel_entry); + if (active) + break; + } + return (i == max_i); +} + + +/* + do_event() is executed by the sql_driver_thd thread. + It's main purpose is to find a thread that can execute the query. + + @retval false ok, event was accepted + @retval true error +*/ + +bool +rpl_parallel::do_event(rpl_group_info *serial_rgi, Log_event *ev, + ulonglong event_size) +{ + rpl_parallel_entry *e; + rpl_parallel_thread *cur_thread; + rpl_parallel_thread::queued_event *qev; + rpl_group_info *rgi= NULL; + Relay_log_info *rli= serial_rgi->rli; + enum Log_event_type typ; + bool is_group_event; + bool did_enter_cond= false; + const char *old_msg= NULL; + + /* ToDo: what to do with this lock?!? */ + mysql_mutex_unlock(&rli->data_lock); + + /* + Stop queueing additional event groups once the SQL thread is requested to + stop. + */ + if (((typ= ev->get_type_code()) == GTID_EVENT || + !(is_group_event= Log_event::is_group_event(typ))) && + rli->abort_slave) + sql_thread_stopping= true; + if (sql_thread_stopping) + { + delete ev; + /* QQ: Need a better comment why we return false here */ + return false; + } + + if (!(qev= (rpl_parallel_thread::queued_event *)my_malloc(sizeof(*qev), + MYF(0)))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + delete ev; + return true; + } + qev->ev= ev; + qev->event_size= event_size; + qev->next= NULL; + strcpy(qev->event_relay_log_name, rli->event_relay_log_name); + qev->event_relay_log_pos= rli->event_relay_log_pos; + qev->future_event_relay_log_pos= rli->future_event_relay_log_pos; + strcpy(qev->future_event_master_log_name, rli->future_event_master_log_name); + + if (typ == GTID_EVENT) + { + Gtid_log_event *gtid_ev= static_cast<Gtid_log_event *>(ev); + uint32 domain_id= (rli->mi->using_gtid == Master_info::USE_GTID_NO ? + 0 : gtid_ev->domain_id); + + if (!(e= find(domain_id)) || + !(rgi= new rpl_group_info(rli)) || + event_group_new_gtid(rgi, gtid_ev)) + { + my_error(ER_OUT_OF_RESOURCES, MYF(MY_WME)); + delete rgi; + my_free(qev); + delete ev; + return true; + } + rgi->is_parallel_exec = true; + if ((rgi->deferred_events_collecting= rli->mi->rpl_filter->is_on())) + rgi->deferred_events= new Deferred_log_events(rli); + + if ((gtid_ev->flags2 & Gtid_log_event::FL_GROUP_COMMIT_ID) && + e->last_commit_id == gtid_ev->commit_id) + { + /* + We are already executing something else in this domain. But the two + event groups were committed together in the same group commit on the + master, so we can still do them in parallel here on the slave. + + However, the commit of this event must wait for the commit of the prior + event, to preserve binlog commit order and visibility across all + servers in the replication hierarchy. + + In addition, we must not start executing this event until we have + finished the previous collection of event groups that group-committed + together; we use rgi->wait_start_sub_id to control this. + */ + rpl_parallel_thread *rpt= global_rpl_thread_pool.get_thread(e); + rgi->wait_commit_sub_id= e->current_sub_id; + rgi->wait_commit_group_info= e->current_group_info; + rgi->wait_start_sub_id= e->prev_groupcommit_sub_id; + e->rpl_thread= cur_thread= rpt; + /* get_thread() returns with the LOCK_rpl_thread locked. */ + } + else + { + /* + Check if we already have a worker thread for this entry. + + We continue to queue more events up for the worker thread while it is + still executing the first ones, to be able to start executing a large + event group without having to wait for the end to be fetched from the + master. And we continue to queue up more events after the first group, + so that we can continue to process subsequent parts of the relay log in + parallel without having to wait for previous long-running events to + complete. + + But if the worker thread is idle at any point, it may return to the + idle list or start servicing a different request. So check this, and + allocate a new thread if the old one is no longer processing for us. + */ + cur_thread= e->rpl_thread; + if (cur_thread) + { + mysql_mutex_lock(&cur_thread->LOCK_rpl_thread); + for (;;) + { + if (cur_thread->current_entry != e) + { + /* + The worker thread became idle, and returned to the free list and + possibly was allocated to a different request. This also means + that everything previously queued has already been executed, + else the worker thread would not have become idle. So we should + allocate a new worker thread. + */ + mysql_mutex_unlock(&cur_thread->LOCK_rpl_thread); + e->rpl_thread= cur_thread= NULL; + break; + } + else if (cur_thread->queued_size <= opt_slave_parallel_max_queued) + break; // The thread is ready to queue into + else if (rli->sql_driver_thd->check_killed()) + { + mysql_mutex_unlock(&cur_thread->LOCK_rpl_thread); + my_error(ER_CONNECTION_KILLED, MYF(0)); + delete rgi; + my_free(qev); + delete ev; + DBUG_EXECUTE_IF("rpl_parallel_wait_queue_max", + { + debug_sync_set_action(rli->sql_driver_thd, + STRING_WITH_LEN("now SIGNAL wait_queue_killed")); + };); + slave_output_error_info(rli, rli->sql_driver_thd); + return true; + } + else + { + /* + We have reached the limit of how much memory we are allowed to + use for queuing events, so wait for the thread to consume some + of its queue. + */ + if (!did_enter_cond) + { + old_msg= rli->sql_driver_thd->enter_cond + (&cur_thread->COND_rpl_thread, &cur_thread->LOCK_rpl_thread, + "Waiting for room in worker thread event queue"); + did_enter_cond= true; + DBUG_EXECUTE_IF("rpl_parallel_wait_queue_max", + { + debug_sync_set_action(rli->sql_driver_thd, + STRING_WITH_LEN("now SIGNAL wait_queue_ready")); + };); + } + mysql_cond_wait(&cur_thread->COND_rpl_thread, + &cur_thread->LOCK_rpl_thread); + } + } + } + + if (!cur_thread) + { + /* + Nothing else is currently running in this domain. We can + spawn a new thread to do this event group in parallel with + anything else that might be running in other domains. + */ + cur_thread= e->rpl_thread= global_rpl_thread_pool.get_thread(e); + /* get_thread() returns with the LOCK_rpl_thread locked. */ + } + else + { + /* + We are still executing the previous event group for this replication + domain, and we have to wait for that to finish before we can start on + the next one. So just re-use the thread. + */ + } + + rgi->wait_commit_sub_id= 0; + rgi->wait_start_sub_id= 0; + e->prev_groupcommit_sub_id= e->current_sub_id; + } + + if (gtid_ev->flags2 & Gtid_log_event::FL_GROUP_COMMIT_ID) + { + e->last_server_id= gtid_ev->server_id; + e->last_seq_no= gtid_ev->seq_no; + e->last_commit_id= gtid_ev->commit_id; + } + else + { + e->last_server_id= 0; + e->last_seq_no= 0; + e->last_commit_id= 0; + } + + qev->rgi= e->current_group_info= rgi; + e->current_sub_id= rgi->gtid_sub_id; + current= rgi->parallel_entry= e; + } + else if (!is_group_event || !current) + { + my_off_t log_pos; + int err; + bool tmp; + /* + Events like ROTATE and FORMAT_DESCRIPTION. Do not run in worker thread. + Same for events not preceeded by GTID (we should not see those normally, + but they might be from an old master). + + The varuable `current' is NULL for the case where the master did not + have GTID, like a MariaDB 5.5 or MySQL master. + */ + qev->rgi= serial_rgi; + /* Handle master log name change, seen in Rotate_log_event. */ + if (typ == ROTATE_EVENT) + { + Rotate_log_event *rev= static_cast<Rotate_log_event *>(qev->ev); + if ((rev->server_id != global_system_variables.server_id || + rli->replicate_same_server_id) && + !rev->is_relay_log_event() && + !rli->is_in_group()) + { + memcpy(rli->future_event_master_log_name, + rev->new_log_ident, rev->ident_len+1); + } + } + + tmp= serial_rgi->is_parallel_exec; + serial_rgi->is_parallel_exec= true; + err= rpt_handle_event(qev, NULL); + serial_rgi->is_parallel_exec= tmp; + log_pos= qev->ev->log_pos; + delete_or_keep_event_post_apply(serial_rgi, typ, qev->ev); + + if (err) + { + my_free(qev); + return true; + } + qev->ev= NULL; + qev->future_event_master_log_pos= log_pos; + if (!current) + { + rli->event_relay_log_pos= rli->future_event_relay_log_pos; + handle_queued_pos_update(rli->sql_driver_thd, qev); + my_free(qev); + return false; + } + /* + Queue an empty event, so that the position will be updated in a + reasonable way relative to other events: + + - If the currently executing events are queued serially for a single + thread, the position will only be updated when everything before has + completed. + + - If we are executing multiple independent events in parallel, then at + least the position will not be updated until one of them has reached + the current point. + */ + cur_thread= current->rpl_thread; + if (cur_thread) + { + mysql_mutex_lock(&cur_thread->LOCK_rpl_thread); + if (cur_thread->current_entry != current) + { + /* Not ours anymore, we need to grab a new one. */ + mysql_mutex_unlock(&cur_thread->LOCK_rpl_thread); + cur_thread= NULL; + } + } + if (!cur_thread) + cur_thread= current->rpl_thread= + global_rpl_thread_pool.get_thread(current); + } + else + { + cur_thread= current->rpl_thread; + if (cur_thread) + { + mysql_mutex_lock(&cur_thread->LOCK_rpl_thread); + if (cur_thread->current_entry != current) + { + /* Not ours anymore, we need to grab a new one. */ + mysql_mutex_unlock(&cur_thread->LOCK_rpl_thread); + cur_thread= NULL; + } + } + if (!cur_thread) + { + cur_thread= current->rpl_thread= + global_rpl_thread_pool.get_thread(current); + } + qev->rgi= current->current_group_info; + } + + /* + Queue the event for processing. + */ + rli->event_relay_log_pos= rli->future_event_relay_log_pos; + cur_thread->enqueue(qev); + if (did_enter_cond) + rli->sql_driver_thd->exit_cond(old_msg); + else + mysql_mutex_unlock(&cur_thread->LOCK_rpl_thread); + mysql_cond_signal(&cur_thread->COND_rpl_thread); + + return false; +} diff --git a/sql/rpl_parallel.h b/sql/rpl_parallel.h new file mode 100644 index 00000000000..019a354c57d --- /dev/null +++ b/sql/rpl_parallel.h @@ -0,0 +1,133 @@ +#ifndef RPL_PARALLEL_H +#define RPL_PARALLEL_H + +#include "log_event.h" + + +struct rpl_parallel; +struct rpl_parallel_entry; +struct rpl_parallel_thread_pool; + +class Relay_log_info; +struct rpl_parallel_thread { + bool delay_start; + bool running; + bool stop; + mysql_mutex_t LOCK_rpl_thread; + mysql_cond_t COND_rpl_thread; + struct rpl_parallel_thread *next; /* For free list. */ + struct rpl_parallel_thread_pool *pool; + THD *thd; + struct rpl_parallel_entry *current_entry; + struct queued_event { + queued_event *next; + Log_event *ev; + rpl_group_info *rgi; + ulonglong future_event_relay_log_pos; + char event_relay_log_name[FN_REFLEN]; + char future_event_master_log_name[FN_REFLEN]; + ulonglong event_relay_log_pos; + my_off_t future_event_master_log_pos; + size_t event_size; + } *event_queue, *last_in_queue; + uint64 queued_size; + + void enqueue(queued_event *qev) + { + if (last_in_queue) + last_in_queue->next= qev; + else + event_queue= qev; + last_in_queue= qev; + queued_size+= qev->event_size; + } + + void dequeue(queued_event *list) + { + queued_event *tmp; + + DBUG_ASSERT(list == event_queue); + event_queue= last_in_queue= NULL; + for (tmp= list; tmp; tmp= tmp->next) + queued_size-= tmp->event_size; + } +}; + + +struct rpl_parallel_thread_pool { + uint32 count; + struct rpl_parallel_thread **threads; + struct rpl_parallel_thread *free_list; + mysql_mutex_t LOCK_rpl_thread_pool; + mysql_cond_t COND_rpl_thread_pool; + bool changing; + bool inited; + + rpl_parallel_thread_pool(); + int init(uint32 size); + void destroy(); + struct rpl_parallel_thread *get_thread(rpl_parallel_entry *entry); +}; + + +struct rpl_parallel_entry { + uint32 domain_id; + uint32 last_server_id; + uint64 last_seq_no; + uint64 last_commit_id; + bool active; + /* + Set when SQL thread is shutting down, and no more events can be processed, + so worker threads must force abort any current transactions without + waiting for event groups to complete. + */ + bool force_abort; + + rpl_parallel_thread *rpl_thread; + /* + The sub_id of the last transaction to commit within this domain_id. + Must be accessed under LOCK_parallel_entry protection. + */ + uint64 last_committed_sub_id; + mysql_mutex_t LOCK_parallel_entry; + mysql_cond_t COND_parallel_entry; + /* + The sub_id of the last event group in this replication domain that was + queued for execution by a worker thread. + */ + uint64 current_sub_id; + rpl_group_info *current_group_info; + /* + The sub_id of the last event group in the previous batch of group-committed + transactions. + + When we spawn parallel worker threads for the next group-committed batch, + they first need to wait for this sub_id to be committed before it is safe + to start executing them. + */ + uint64 prev_groupcommit_sub_id; +}; +struct rpl_parallel { + HASH domain_hash; + rpl_parallel_entry *current; + bool sql_thread_stopping; + + rpl_parallel(); + ~rpl_parallel(); + void reset(); + rpl_parallel_entry *find(uint32 domain_id); + void wait_for_done(); + bool workers_idle(); + bool do_event(rpl_group_info *serial_rgi, Log_event *ev, + ulonglong event_size); +}; + + +extern struct rpl_parallel_thread_pool global_rpl_thread_pool; + + +extern int rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, + uint32 new_count, + bool skip_check= false); + +#endif /* RPL_PARALLEL_H */ diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc index db65cfa8757..1d4eedbb017 100644 --- a/sql/rpl_record.cc +++ b/sql/rpl_record.cc @@ -189,7 +189,7 @@ pack_row(TABLE *table, MY_BITMAP const* cols, */ #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) int -unpack_row(Relay_log_info const *rli, +unpack_row(rpl_group_info *rgi, TABLE *table, uint const colcnt, uchar const *const row_data, MY_BITMAP const *cols, uchar const **const current_row_end, ulong *const master_reclength, @@ -217,18 +217,18 @@ unpack_row(Relay_log_info const *rli, uint i= 0; table_def *tabledef= NULL; TABLE *conv_table= NULL; - bool table_found= rli && rli->get_table_data(table, &tabledef, &conv_table); + bool table_found= rgi && rgi->get_table_data(table, &tabledef, &conv_table); DBUG_PRINT("debug", ("Table data: table_found: %d, tabldef: %p, conv_table: %p", table_found, tabledef, conv_table)); DBUG_ASSERT(table_found); /* - If rli is NULL it means that there is no source table and that the + If rgi is NULL it means that there is no source table and that the row shall just be unpacked without doing any checks. This feature is used by MySQL Backup, but can be used for other purposes as well. */ - if (rli && !table_found) + if (rgi && !table_found) DBUG_RETURN(HA_ERR_GENERIC); for (field_ptr= begin_ptr ; field_ptr < end_ptr && *field_ptr ; ++field_ptr) @@ -316,7 +316,7 @@ unpack_row(Relay_log_info const *rli, (int) (pack_ptr - old_pack_ptr))); if (!pack_ptr) { - rli->report(ERROR_LEVEL, ER_SLAVE_CORRUPT_EVENT, + rgi->rli->report(ERROR_LEVEL, ER_SLAVE_CORRUPT_EVENT, "Could not read field '%s' of table '%s.%s'", f->field_name, table->s->db.str, table->s->table_name.str); diff --git a/sql/rpl_record.h b/sql/rpl_record.h index d4eb8986846..c10eb8225b0 100644 --- a/sql/rpl_record.h +++ b/sql/rpl_record.h @@ -20,7 +20,7 @@ #include <rpl_reporting.h> #include "my_global.h" /* uchar */ -class Relay_log_info; +struct rpl_group_info; struct TABLE; typedef struct st_bitmap MY_BITMAP; @@ -30,7 +30,7 @@ size_t pack_row(TABLE* table, MY_BITMAP const* cols, #endif #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) -int unpack_row(Relay_log_info const *rli, +int unpack_row(rpl_group_info *rgi, TABLE *table, uint const colcnt, uchar const *const row_data, MY_BITMAP const *cols, uchar const **const curr_row_end, ulong *const master_reclength, diff --git a/sql/rpl_record_old.cc b/sql/rpl_record_old.cc index fa0c49b413c..5afa529a63c 100644 --- a/sql/rpl_record_old.cc +++ b/sql/rpl_record_old.cc @@ -88,7 +88,7 @@ pack_row_old(TABLE *table, MY_BITMAP const* cols, */ #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) int -unpack_row_old(Relay_log_info *rli, +unpack_row_old(rpl_group_info *rgi, TABLE *table, uint const colcnt, uchar *record, uchar const *row, const uchar *row_buffer_end, MY_BITMAP const *cols, @@ -141,7 +141,7 @@ unpack_row_old(Relay_log_info *rli, f->move_field_offset(-offset); if (!ptr) { - rli->report(ERROR_LEVEL, ER_SLAVE_CORRUPT_EVENT, + rgi->rli->report(ERROR_LEVEL, ER_SLAVE_CORRUPT_EVENT, "Could not read field `%s` of table `%s`.`%s`", f->field_name, table->s->db.str, table->s->table_name.str); @@ -183,7 +183,7 @@ unpack_row_old(Relay_log_info *rli, if (event_type == WRITE_ROWS_EVENT && ((*field_ptr)->flags & mask) == mask) { - rli->report(ERROR_LEVEL, ER_NO_DEFAULT_FOR_FIELD, + rgi->rli->report(ERROR_LEVEL, ER_NO_DEFAULT_FOR_FIELD, "Field `%s` of table `%s`.`%s` " "has no default value and cannot be NULL", (*field_ptr)->field_name, table->s->db.str, diff --git a/sql/rpl_record_old.h b/sql/rpl_record_old.h index ea981fb23c3..34ef9f11c47 100644 --- a/sql/rpl_record_old.h +++ b/sql/rpl_record_old.h @@ -23,7 +23,7 @@ size_t pack_row_old(TABLE *table, MY_BITMAP const* cols, uchar *row_data, const uchar *record); #ifdef HAVE_REPLICATION -int unpack_row_old(Relay_log_info *rli, +int unpack_row_old(rpl_group_info *rgi, TABLE *table, uint const colcnt, uchar *record, uchar const *row, uchar const *row_buffer_end, MY_BITMAP const *cols, diff --git a/sql/rpl_reporting.cc b/sql/rpl_reporting.cc index f442f3a37c0..96fe6242ac3 100644 --- a/sql/rpl_reporting.cc +++ b/sql/rpl_reporting.cc @@ -67,7 +67,7 @@ Slave_reporting_capability::report(loglevel level, int err_code, va_end(args); /* If the msg string ends with '.', do not add a ',' it would be ugly */ - report_function("Slave %s: %s%s Error_code: %d", + report_function("Slave %s: %s%s Internal MariaDB error code: %d", m_thread_name, pbuff, (pbuff[0] && *(strend(pbuff)-1) == '.') ? "" : ",", err_code); diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index b01f74408a6..797f5681ec5 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -32,6 +32,15 @@ static int count_relay_log_space(Relay_log_info* rli); +/** + Current replication state (hash of last GTID executed, per replication + domain). +*/ +rpl_slave_state rpl_global_gtid_slave_state; +/* Object used for MASTER_GTID_WAIT(). */ +gtid_waiting rpl_global_gtid_waiting; + + // Defined in slave.cc int init_intvar_from_file(int* var, IO_CACHE* f, int default_val); int init_strvar_from_file(char *var, int max_size, IO_CACHE *f, @@ -42,23 +51,22 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) no_storage(FALSE), replicate_same_server_id(::replicate_same_server_id), info_fd(-1), cur_log_fd(-1), relay_log(&sync_relaylog_period), sync_counter(0), is_relay_log_recovery(is_slave_recovery), - save_temporary_tables(0), cur_log_old_open_count(0), group_relay_log_pos(0), + save_temporary_tables(0), mi(0), + cur_log_old_open_count(0), group_relay_log_pos(0), event_relay_log_pos(0), #if HAVE_valgrind is_fake(FALSE), #endif group_master_log_pos(0), log_space_total(0), ignore_log_space_limit(0), - last_master_timestamp(0), slave_skip_counter(0), - abort_pos_wait(0), slave_run_id(0), sql_thd(0), + last_master_timestamp(0), sql_thread_caught_up(true), slave_skip_counter(0), + abort_pos_wait(0), slave_run_id(0), sql_driver_thd(), inited(0), abort_slave(0), slave_running(0), until_condition(UNTIL_NONE), - until_log_pos(0), retried_trans(0), - tables_to_lock(0), tables_to_lock_count(0), - last_event_start_time(0), deferred_events(NULL),m_flags(0), - row_stmt_start_timestamp(0), long_find_row_note_printed(false), - m_annotate_event(0) + until_log_pos(0), retried_trans(0), executed_entries(0), + m_flags(0) { DBUG_ENTER("Relay_log_info::Relay_log_info"); + relay_log.is_relay_log= TRUE; #ifdef HAVE_PSI_INTERFACE relay_log.set_psi_keys(key_RELAYLOG_LOCK_index, key_RELAYLOG_update_cond, @@ -70,6 +78,7 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) group_relay_log_name[0]= event_relay_log_name[0]= group_master_log_name[0]= 0; until_log_name[0]= ign_master_log_name_end[0]= 0; + max_relay_log_size= global_system_variables.max_relay_log_size; bzero((char*) &info_file, sizeof(info_file)); bzero((char*) &cache_buf, sizeof(cache_buf)); cached_charset_invalidate(); @@ -78,12 +87,10 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) &data_lock, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_relay_log_info_log_space_lock, &log_space_lock, MY_MUTEX_INIT_FAST); - mysql_mutex_init(key_relay_log_info_sleep_lock, &sleep_lock, MY_MUTEX_INIT_FAST); mysql_cond_init(key_relay_log_info_data_cond, &data_cond, NULL); mysql_cond_init(key_relay_log_info_start_cond, &start_cond, NULL); mysql_cond_init(key_relay_log_info_stop_cond, &stop_cond, NULL); mysql_cond_init(key_relay_log_info_log_space_cond, &log_space_cond, NULL); - mysql_cond_init(key_relay_log_info_sleep_cond, &sleep_cond, NULL); relay_log.init_pthread_objects(); DBUG_VOID_RETURN; } @@ -96,14 +103,11 @@ Relay_log_info::~Relay_log_info() mysql_mutex_destroy(&run_lock); mysql_mutex_destroy(&data_lock); mysql_mutex_destroy(&log_space_lock); - mysql_mutex_destroy(&sleep_lock); mysql_cond_destroy(&data_cond); mysql_cond_destroy(&start_cond); mysql_cond_destroy(&stop_cond); mysql_cond_destroy(&log_space_cond); - mysql_cond_destroy(&sleep_cond); relay_log.cleanup(); - free_annotate_event(); DBUG_VOID_RETURN; } @@ -128,8 +132,6 @@ int init_relay_log_info(Relay_log_info* rli, rli->abort_pos_wait=0; rli->log_space_limit= relay_log_space_limit; rli->log_space_total= 0; - rli->tables_to_lock= 0; - rli->tables_to_lock_count= 0; char pattern[FN_REFLEN]; (void) my_realpath(pattern, slave_load_tmpdir, 0); @@ -150,15 +152,6 @@ int init_relay_log_info(Relay_log_info* rli, event, in flush_master_info(mi, 1, ?). */ - /* - For the maximum log size, we choose max_relay_log_size if it is - non-zero, max_binlog_size otherwise. If later the user does SET - GLOBAL on one of these variables, fix_max_binlog_size and - fix_max_relay_log_size will reconsider the choice (for example - if the user changes max_relay_log_size to zero, we have to - switch to using max_binlog_size for the relay log) and update - rli->relay_log.max_size (and mysql_bin_log.max_size). - */ { /* Reports an error and returns, if the --relay-log's path is a directory.*/ @@ -208,19 +201,35 @@ a file name for --relay-log-index option", opt_relaylog_index_name); name_warning_sent= 1; } - rli->relay_log.is_relay_log= TRUE; + /* For multimaster, add connection name to relay log filenames */ + Master_info* mi= rli->mi; + char buf_relay_logname[FN_REFLEN], buf_relaylog_index_name_buff[FN_REFLEN]; + char *buf_relaylog_index_name= opt_relaylog_index_name; + + create_logfile_name_with_suffix(buf_relay_logname, + sizeof(buf_relay_logname), + ln, 1, &mi->cmp_connection_name); + ln= buf_relay_logname; + + if (opt_relaylog_index_name) + { + buf_relaylog_index_name= buf_relaylog_index_name_buff; + create_logfile_name_with_suffix(buf_relaylog_index_name_buff, + sizeof(buf_relaylog_index_name_buff), + opt_relaylog_index_name, 0, + &mi->cmp_connection_name); + } /* note, that if open() fails, we'll still have index file open but a destructor will take care of that */ - if (rli->relay_log.open_index_file(opt_relaylog_index_name, ln, TRUE) || - rli->relay_log.open(ln, LOG_BIN, 0, SEQ_READ_APPEND, 0, - (max_relay_log_size ? max_relay_log_size : - max_binlog_size), 1, TRUE)) + if (rli->relay_log.open_index_file(buf_relaylog_index_name, ln, TRUE) || + rli->relay_log.open(ln, LOG_BIN, 0, SEQ_READ_APPEND, + mi->rli.max_relay_log_size, 1, TRUE)) { mysql_mutex_unlock(&rli->data_lock); - sql_print_error("Failed in open_log() called from init_relay_log_info()"); + sql_print_error("Failed when trying to open logs for '%s' in init_relay_log_info(). Error: %M", ln, my_errno); DBUG_RETURN(1); } } @@ -512,6 +521,8 @@ int init_relay_log_pos(Relay_log_info* rli,const char* log, } rli->group_relay_log_pos = rli->event_relay_log_pos = pos; + rli->clear_flag(Relay_log_info::IN_STMT); + rli->clear_flag(Relay_log_info::IN_TRANSACTION); /* Test to see if the previous run was with the skip of purging @@ -861,17 +872,54 @@ improper_arguments: %d timed_out: %d", void Relay_log_info::inc_group_relay_log_pos(ulonglong log_pos, - bool skip_lock) + rpl_group_info *rgi, + bool skip_lock) { DBUG_ENTER("Relay_log_info::inc_group_relay_log_pos"); if (!skip_lock) mysql_mutex_lock(&data_lock); - inc_event_relay_log_pos(); - group_relay_log_pos= event_relay_log_pos; - strmake_buf(group_relay_log_name,event_relay_log_name); + rgi->inc_event_relay_log_pos(); + DBUG_PRINT("info", ("log_pos: %lu group_master_log_pos: %lu", + (long) log_pos, (long) group_master_log_pos)); + if (rgi->is_parallel_exec) + { + /* In case of parallel replication, do not update the position backwards. */ + int cmp= strcmp(group_relay_log_name, event_relay_log_name); + if (cmp < 0) + { + group_relay_log_pos= rgi->future_event_relay_log_pos; + strmake_buf(group_relay_log_name, event_relay_log_name); + notify_group_relay_log_name_update(); + } else if (cmp == 0 && group_relay_log_pos < rgi->future_event_relay_log_pos) + group_relay_log_pos= rgi->future_event_relay_log_pos; - notify_group_relay_log_name_update(); + /* + In the parallel case we need to update the master_log_name here, rather + than in Rotate_log_event::do_update_pos(). + */ + cmp= strcmp(group_master_log_name, rgi->future_event_master_log_name); + if (cmp <= 0) + { + if (cmp < 0) + { + strcpy(group_master_log_name, rgi->future_event_master_log_name); + notify_group_master_log_name_update(); + group_master_log_pos= log_pos; + } + else if (group_master_log_pos < log_pos) + group_master_log_pos= log_pos; + } + } + else + { + /* Non-parallel case. */ + group_relay_log_pos= event_relay_log_pos; + strmake_buf(group_relay_log_name, event_relay_log_name); + notify_group_relay_log_name_update(); + if (log_pos) // 3.23 binlogs don't have log_posx + group_master_log_pos= log_pos; + } /* If the slave does not support transactions and replicates a transaction, @@ -903,12 +951,6 @@ void Relay_log_info::inc_group_relay_log_pos(ulonglong log_pos, the relay log is not "val". With the end_log_pos solution, we avoid computations involving lengthes. */ - DBUG_PRINT("info", ("log_pos: %lu group_master_log_pos: %lu", - (long) log_pos, (long) group_master_log_pos)); - if (log_pos) // 3.23 binlogs don't have log_posx - { - group_master_log_pos= log_pos; - } mysql_cond_broadcast(&data_cond); if (!skip_lock) mysql_mutex_unlock(&data_lock); @@ -924,6 +966,9 @@ void Relay_log_info::close_temporary_tables() for (table=save_temporary_tables ; table ; table=next) { next=table->next; + + /* Reset in_use as the table may have been created by another thd */ + table->in_use=0; /* Don't ask for disk deletion. For now, anyway they will be deleted when slave restarts, but it is a better intention to not delete them. @@ -995,26 +1040,35 @@ int purge_relay_logs(Relay_log_info* rli, THD *thd, bool just_reset, rli->cur_log_fd= -1; } - if (rli->relay_log.reset_logs(thd)) + if (rli->relay_log.reset_logs(thd, !just_reset, NULL, 0)) { *errmsg = "Failed during log reset"; error=1; goto err; } - /* Save name of used relay log file */ - strmake_buf(rli->group_relay_log_name, rli->relay_log.get_log_fname()); - strmake_buf(rli->event_relay_log_name, rli->relay_log.get_log_fname()); - rli->group_relay_log_pos= rli->event_relay_log_pos= BIN_LOG_HEADER_SIZE; - if (count_relay_log_space(rli)) - { - *errmsg= "Error counting relay log space"; - error=1; - goto err; - } if (!just_reset) + { + /* Save name of used relay log file */ + strmake_buf(rli->group_relay_log_name, rli->relay_log.get_log_fname()); + strmake_buf(rli->event_relay_log_name, rli->relay_log.get_log_fname()); + rli->group_relay_log_pos= rli->event_relay_log_pos= BIN_LOG_HEADER_SIZE; + rli->log_space_total= 0; + + if (count_relay_log_space(rli)) + { + *errmsg= "Error counting relay log space"; + error=1; + goto err; + } error= init_relay_log_pos(rli, rli->group_relay_log_name, rli->group_relay_log_pos, 0 /* do not need data lock */, errmsg, 0); + } + else + { + /* Ensure relay log names are not used */ + rli->group_relay_log_name[0]= rli->event_relay_log_name[0]= 0; + } err: #ifndef DBUG_OFF @@ -1065,16 +1119,18 @@ bool Relay_log_info::is_until_satisfied(THD *thd, Log_event *ev) ulonglong log_pos; DBUG_ENTER("Relay_log_info::is_until_satisfied"); - DBUG_ASSERT(until_condition != UNTIL_NONE); + DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || + until_condition == UNTIL_RELAY_POS); if (until_condition == UNTIL_MASTER_POS) { - if (ev && ev->server_id == (uint32) ::server_id && !replicate_same_server_id) + if (ev && ev->server_id == (uint32) global_system_variables.server_id && + !replicate_same_server_id) DBUG_RETURN(FALSE); log_name= group_master_log_name; - log_pos= (!ev)? group_master_log_pos : - ((thd->variables.option_bits & OPTION_BEGIN || !ev->log_pos) ? - group_master_log_pos : ev->log_pos - ev->data_written); + log_pos= ((!ev)? group_master_log_pos : + (get_flag(IN_TRANSACTION) || !ev->log_pos) ? + group_master_log_pos : ev->log_pos - ev->data_written); } else { /* until_condition == UNTIL_RELAY_POS */ @@ -1167,19 +1223,24 @@ bool Relay_log_info::cached_charset_compare(char *charset) const void Relay_log_info::stmt_done(my_off_t event_master_log_pos, - time_t event_creation_time) + time_t event_creation_time, THD *thd, + rpl_group_info *rgi) { #ifndef DBUG_OFF extern uint debug_not_change_ts_if_art_event; #endif - clear_flag(IN_STMT); + DBUG_ENTER("Relay_log_info::stmt_done"); + DBUG_ASSERT(rgi->rli == this); /* If in a transaction, and if the slave supports transactions, just inc_event_relay_log_pos(). We only have to check for OPTION_BEGIN (not OPTION_NOT_AUTOCOMMIT) as transactions are logged with BEGIN/COMMIT, not with SET AUTOCOMMIT= . + We can't use rgi->rli->get_flag(IN_TRANSACTION) here as OPTION_BEGIN + is also used for single row transactions. + CAUTION: opt_using_transactions means innodb || bdb ; suppose the master supports InnoDB and BDB, but the slave supports only BDB, problems will arise: - suppose an InnoDB table is created on the @@ -1197,12 +1258,30 @@ void Relay_log_info::stmt_done(my_off_t event_master_log_pos, middle of the "transaction". START SLAVE will resume at BEGIN while the MyISAM table has already been updated. */ - if ((sql_thd->variables.option_bits & OPTION_BEGIN) && opt_using_transactions) - inc_event_relay_log_pos(); + if ((rgi->thd->variables.option_bits & OPTION_BEGIN) && + opt_using_transactions) + rgi->inc_event_relay_log_pos(); else { - inc_group_relay_log_pos(event_master_log_pos); - flush_relay_log_info(this); + inc_group_relay_log_pos(event_master_log_pos, rgi); + if (rpl_global_gtid_slave_state.record_and_update_gtid(thd, rgi)) + { + report(WARNING_LEVEL, ER_CANNOT_UPDATE_GTID_STATE, + "Failed to update GTID state in %s.%s, slave state may become " + "inconsistent: %d: %s", + "mysql", rpl_gtid_slave_state_table_name.str, + thd->stmt_da->sql_errno(), thd->stmt_da->message()); + /* + At this point we are not in a transaction (for example after DDL), + so we can not roll back. Anyway, normally updates to the slave + state table should not fail, and if they do, at least we made the + DBA aware of the problem in the error log. + */ + } + DBUG_EXECUTE_IF("inject_crash_before_flush_rli", DBUG_SUICIDE();); + if (mi->using_gtid == Master_info::USE_GTID_NO) + flush_relay_log_info(this); + DBUG_EXECUTE_IF("inject_crash_after_flush_rli", DBUG_SUICIDE();); /* Note that Rotate_log_event::do_apply_event() does not call this function, so there is no chance that a fake rotate event resets @@ -1210,19 +1289,289 @@ void Relay_log_info::stmt_done(my_off_t event_master_log_pos, (probably ok - except in some very rare cases, only consequence is that value may take some time to display in Seconds_Behind_Master - not critical). + + In parallel replication, we take care to not set last_master_timestamp + backwards, in case of out-of-order calls here. */ if (!(event_creation_time == 0 && - IF_DBUG(debug_not_change_ts_if_art_event > 0, 1))) + IF_DBUG(debug_not_change_ts_if_art_event > 0, 1)) && + !(rgi->is_parallel_exec && event_creation_time <= last_master_timestamp) + ) last_master_timestamp= event_creation_time; } + DBUG_VOID_RETURN; } #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) -void Relay_log_info::cleanup_context(THD *thd, bool error) +int +rpl_load_gtid_slave_state(THD *thd) { - DBUG_ENTER("Relay_log_info::cleanup_context"); + TABLE_LIST tlist; + TABLE *table; + bool table_opened= false; + bool table_scanned= false; + bool array_inited= false; + struct local_element { uint64 sub_id; rpl_gtid gtid; }; + struct local_element tmp_entry, *entry; + HASH hash; + DYNAMIC_ARRAY array; + int err= 0; + uint32 i; + DBUG_ENTER("rpl_load_gtid_slave_state"); + + mysql_mutex_lock(&rpl_global_gtid_slave_state.LOCK_slave_state); + bool loaded= rpl_global_gtid_slave_state.loaded; + mysql_mutex_unlock(&rpl_global_gtid_slave_state.LOCK_slave_state); + if (loaded) + DBUG_RETURN(0); + + my_hash_init(&hash, &my_charset_bin, 32, + offsetof(local_element, gtid) + offsetof(rpl_gtid, domain_id), + sizeof(uint32), NULL, my_free, HASH_UNIQUE); + if ((err= my_init_dynamic_array(&array, sizeof(local_element), 0, 0, MYF(0)))) + goto end; + array_inited= true; + + mysql_reset_thd_for_next_command(thd); + + tlist.init_one_table(STRING_WITH_LEN("mysql"), + rpl_gtid_slave_state_table_name.str, + rpl_gtid_slave_state_table_name.length, + NULL, TL_READ); + if ((err= open_and_lock_tables(thd, &tlist, FALSE, 0))) + goto end; + table_opened= true; + table= tlist.table; + + if ((err= gtid_check_rpl_slave_state_table(table))) + goto end; + + bitmap_set_all(table->read_set); + if ((err= table->file->ha_rnd_init_with_error(1))) + { + table->file->print_error(err, MYF(0)); + goto end; + } + table_scanned= true; + for (;;) + { + uint32 domain_id, server_id; + uint64 sub_id, seq_no; + uchar *rec; + + if ((err= table->file->ha_rnd_next(table->record[0]))) + { + if (err == HA_ERR_RECORD_DELETED) + continue; + else if (err == HA_ERR_END_OF_FILE) + break; + else + { + table->file->print_error(err, MYF(0)); + goto end; + } + } + domain_id= (ulonglong)table->field[0]->val_int(); + sub_id= (ulonglong)table->field[1]->val_int(); + server_id= (ulonglong)table->field[2]->val_int(); + seq_no= (ulonglong)table->field[3]->val_int(); + DBUG_PRINT("info", ("Read slave state row: %u-%u-%lu sub_id=%lu\n", + (unsigned)domain_id, (unsigned)server_id, + (ulong)seq_no, (ulong)sub_id)); + + tmp_entry.sub_id= sub_id; + tmp_entry.gtid.domain_id= domain_id; + tmp_entry.gtid.server_id= server_id; + tmp_entry.gtid.seq_no= seq_no; + if ((err= insert_dynamic(&array, (uchar *)&tmp_entry))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + goto end; + } + + if ((rec= my_hash_search(&hash, (const uchar *)&domain_id, 0))) + { + entry= (struct local_element *)rec; + if (entry->sub_id >= sub_id) + continue; + entry->sub_id= sub_id; + DBUG_ASSERT(entry->gtid.domain_id == domain_id); + entry->gtid.server_id= server_id; + entry->gtid.seq_no= seq_no; + } + else + { + if (!(entry= (struct local_element *)my_malloc(sizeof(*entry), + MYF(MY_WME)))) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int)sizeof(*entry)); + err= 1; + goto end; + } + entry->sub_id= sub_id; + entry->gtid.domain_id= domain_id; + entry->gtid.server_id= server_id; + entry->gtid.seq_no= seq_no; + if ((err= my_hash_insert(&hash, (uchar *)entry))) + { + my_free(entry); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + goto end; + } + } + } + + mysql_mutex_lock(&rpl_global_gtid_slave_state.LOCK_slave_state); + if (rpl_global_gtid_slave_state.loaded) + { + mysql_mutex_unlock(&rpl_global_gtid_slave_state.LOCK_slave_state); + goto end; + } + + for (i= 0; i < array.elements; ++i) + { + get_dynamic(&array, (uchar *)&tmp_entry, i); + if ((err= rpl_global_gtid_slave_state.update(tmp_entry.gtid.domain_id, + tmp_entry.gtid.server_id, + tmp_entry.sub_id, + tmp_entry.gtid.seq_no))) + { + mysql_mutex_unlock(&rpl_global_gtid_slave_state.LOCK_slave_state); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + goto end; + } + } + + for (i= 0; i < hash.records; ++i) + { + entry= (struct local_element *)my_hash_element(&hash, i); + if (opt_bin_log && + mysql_bin_log.bump_seq_no_counter_if_needed(entry->gtid.domain_id, + entry->gtid.seq_no)) + { + mysql_mutex_unlock(&rpl_global_gtid_slave_state.LOCK_slave_state); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + goto end; + } + } + + rpl_global_gtid_slave_state.loaded= true; + mysql_mutex_unlock(&rpl_global_gtid_slave_state.LOCK_slave_state); - DBUG_ASSERT(sql_thd == thd); + err= 0; /* Clear HA_ERR_END_OF_FILE */ + +end: + if (table_scanned) + { + table->file->ha_index_or_rnd_end(); + ha_commit_trans(thd, FALSE); + ha_commit_trans(thd, TRUE); + } + if (table_opened) + { + close_thread_tables(thd); + thd->mdl_context.release_transactional_locks(); + } + if (array_inited) + delete_dynamic(&array); + my_hash_free(&hash); + DBUG_RETURN(err); +} + + +rpl_group_info::rpl_group_info(Relay_log_info *rli_) + : rli(rli_), thd(0), gtid_sub_id(0), wait_commit_sub_id(0), + wait_commit_group_info(0), wait_start_sub_id(0), parallel_entry(0), + deferred_events(NULL), m_annotate_event(0), tables_to_lock(0), + tables_to_lock_count(0), trans_retries(0), last_event_start_time(0), + is_parallel_exec(false), is_error(false), + row_stmt_start_timestamp(0), long_find_row_note_printed(false) +{ + bzero(¤t_gtid, sizeof(current_gtid)); + mysql_mutex_init(key_rpl_group_info_sleep_lock, &sleep_lock, + MY_MUTEX_INIT_FAST); + mysql_cond_init(key_rpl_group_info_sleep_cond, &sleep_cond, NULL); +} + + +rpl_group_info::~rpl_group_info() +{ + free_annotate_event(); + delete deferred_events; + mysql_mutex_destroy(&sleep_lock); + mysql_cond_destroy(&sleep_cond); +} + + +int +event_group_new_gtid(rpl_group_info *rgi, Gtid_log_event *gev) +{ + uint64 sub_id= rpl_global_gtid_slave_state.next_sub_id(gev->domain_id); + if (!sub_id) + { + /* Out of memory caused hash insertion to fail. */ + return 1; + } + rgi->gtid_sub_id= sub_id; + rgi->current_gtid.server_id= gev->server_id; + rgi->current_gtid.domain_id= gev->domain_id; + rgi->current_gtid.seq_no= gev->seq_no; + return 0; +} + + +void +delete_or_keep_event_post_apply(rpl_group_info *rgi, + Log_event_type typ, Log_event *ev) +{ + /* + ToDo: This needs to work on rpl_group_info, not Relay_log_info, to be + thread-safe for parallel replication. + */ + + switch (typ) { + case FORMAT_DESCRIPTION_EVENT: + /* + Format_description_log_event should not be deleted because it + will be used to read info about the relay log's format; + it will be deleted when the SQL thread does not need it, + i.e. when this thread terminates. + */ + break; + case ANNOTATE_ROWS_EVENT: + /* + Annotate_rows event should not be deleted because after it has + been applied, thd->query points to the string inside this event. + The thd->query will be used to generate new Annotate_rows event + during applying the subsequent Rows events. + */ + rgi->set_annotate_event((Annotate_rows_log_event*) ev); + break; + case DELETE_ROWS_EVENT: + case UPDATE_ROWS_EVENT: + case WRITE_ROWS_EVENT: + /* + After the last Rows event has been applied, the saved Annotate_rows + event (if any) is not needed anymore and can be deleted. + */ + if (((Rows_log_event*)ev)->get_flags(Rows_log_event::STMT_END_F)) + rgi->free_annotate_event(); + /* fall through */ + default: + DBUG_PRINT("info", ("Deleting the event after it has been executed")); + if (!rgi->is_deferred_event(ev)) + delete ev; + break; + } +} + + +void rpl_group_info::cleanup_context(THD *thd, bool error) +{ + DBUG_ENTER("Relay_log_info::cleanup_context"); + DBUG_PRINT("enter", ("error: %d", (int) error)); + + DBUG_ASSERT(this->thd == thd); /* 1) Instances of Table_map_log_event, if ::do_apply_event() was called on them, may have opened tables, which we cannot be sure have been closed (because @@ -1243,8 +1592,20 @@ void Relay_log_info::cleanup_context(THD *thd, bool error) m_table_map.clear_tables(); slave_close_thread_tables(thd); if (error) + { thd->mdl_context.release_transactional_locks(); - clear_flag(IN_STMT); + + if (thd == rli->sql_driver_thd) + { + /* + Reset flags. This is needed to handle incident events and errors in + the relay log noticed by the sql driver thread. + */ + rli->clear_flag(Relay_log_info::IN_STMT); + rli->clear_flag(Relay_log_info::IN_TRANSACTION); + } + } + /* Cleanup for the flags that have been set at do_apply_event. */ @@ -1262,7 +1623,8 @@ void Relay_log_info::cleanup_context(THD *thd, bool error) DBUG_VOID_RETURN; } -void Relay_log_info::clear_tables_to_lock() + +void rpl_group_info::clear_tables_to_lock() { DBUG_ENTER("Relay_log_info::clear_tables_to_lock()"); #ifndef DBUG_OFF @@ -1308,7 +1670,8 @@ void Relay_log_info::clear_tables_to_lock() DBUG_VOID_RETURN; } -void Relay_log_info::slave_close_thread_tables(THD *thd) + +void rpl_group_info::slave_close_thread_tables(THD *thd) { DBUG_ENTER("Relay_log_info::slave_close_thread_tables(THD *thd)"); thd->stmt_da->can_overwrite_status= TRUE; @@ -1341,4 +1704,6 @@ void Relay_log_info::slave_close_thread_tables(THD *thd) clear_tables_to_lock(); DBUG_VOID_RETURN; } + + #endif diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index b989283deb4..3f95849a926 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -22,10 +22,10 @@ #include "log.h" /* LOG_INFO, MYSQL_BIN_LOG */ #include "sql_class.h" /* THD */ #include "log_event.h" +#include "rpl_parallel.h" struct RPL_TABLE_LIST; class Master_info; -extern uint sql_slave_skip_counter; /**************************************************************************** @@ -53,18 +53,20 @@ extern uint sql_slave_skip_counter; *****************************************************************************/ +struct rpl_group_info; + class Relay_log_info : public Slave_reporting_capability { public: /** - Flags for the state of the replication. - */ + Flags for the state of reading the relay log. Note that these are + bit masks. + */ enum enum_state_flag { - /** The replication thread is inside a statement */ - IN_STMT, - - /** Flag counter. Should always be last */ - STATE_FLAGS_COUNT + /** We are inside a group of events forming a statement */ + IN_STMT=1, + /** We have inside a transaction */ + IN_TRANSACTION=2 }; /* @@ -129,9 +131,14 @@ public: IO_CACHE info_file; /* - When we restart slave thread we need to have access to the previously - created temporary tables. Modified only on init/end and by the SQL - thread, read only by SQL thread. + List of temporary tables used by this connection. + This is updated when a temporary table is created or dropped by + a replication thread. + + Not reset when replication ends, to allow one to access the tables + when replication restarts. + + Protected by data_lock. */ TABLE *save_temporary_tables; @@ -139,13 +146,13 @@ public: standard lock acquisition order to avoid deadlocks: run_lock, data_lock, relay_log.LOCK_log, relay_log.LOCK_index */ - mysql_mutex_t data_lock, run_lock, sleep_lock; + mysql_mutex_t data_lock, run_lock; /* start_cond is broadcast when SQL thread is started stop_cond - when stopped data_cond - when data protected by data_lock changes */ - mysql_cond_t start_cond, stop_cond, data_cond, sleep_cond; + mysql_cond_t start_cond, stop_cond, data_cond; /* parent Master_info structure */ Master_info *mi; @@ -162,8 +169,8 @@ public: - an autocommiting query + its associated events (INSERT_ID, TIMESTAMP...) We need these rli coordinates : - - relay log name and position of the beginning of the group we currently are - executing. Needed to know where we have to restart when replication has + - relay log name and position of the beginning of the group we currently + are executing. Needed to know where we have to restart when replication has stopped in the middle of a group (which has been rolled back by the slave). - relay log name and position just after the event we have just executed. This event is part of the current group. @@ -178,6 +185,10 @@ public: char event_relay_log_name[FN_REFLEN]; ulonglong event_relay_log_pos; ulonglong future_event_relay_log_pos; + /* + The master log name for current event. Only used in parallel replication. + */ + char future_event_master_log_name[FN_REFLEN]; #ifdef HAVE_valgrind bool is_fake; /* Mark that this is a fake relay log info structure */ @@ -209,19 +220,13 @@ public: */ bool sql_force_rotate_relay; + time_t last_master_timestamp; /* - When it commits, InnoDB internally stores the master log position it has - processed so far; the position to store is the one of the end of the - committing event (the COMMIT query event, or the event if in autocommit - mode). + The SQL driver thread sets this true while it is waiting at the end of the + relay log for more events to arrive. SHOW SLAVE STATUS uses this to report + Seconds_Behind_Master as zero while the SQL thread is so waiting. */ -#if MYSQL_VERSION_ID < 40100 - ulonglong future_master_log_pos; -#else - ulonglong future_group_master_log_pos; -#endif - - time_t last_master_timestamp; + bool sql_thread_caught_up; void clear_until_condition(); @@ -229,13 +234,21 @@ public: Needed for problems when slave stops and we want to restart it skipping one or more events in the master log that have caused errors, and have been manually applied by DBA already. + Must be ulong as it's refered to from set_var.cc */ - volatile uint32 slave_skip_counter; + volatile ulong slave_skip_counter; volatile ulong abort_pos_wait; /* Incremented on change master */ volatile ulong slave_run_id; /* Incremented on slave start */ + ulong max_relay_log_size; mysql_mutex_t log_space_lock; mysql_cond_t log_space_cond; - THD * sql_thd; + /* + THD for the main sql thread, the one that starts threads to process + slave requests. If there is only one thread, then this THD is also + used for SQL processing. + A kill sent to this THD will kill the replication. + */ + THD *sql_driver_thd; #ifndef DBUG_OFF int events_till_abort; #endif @@ -262,7 +275,9 @@ public: thread is running). */ - enum {UNTIL_NONE= 0, UNTIL_MASTER_POS, UNTIL_RELAY_POS} until_condition; + enum { + UNTIL_NONE= 0, UNTIL_MASTER_POS, UNTIL_RELAY_POS, UNTIL_GTID + } until_condition; char until_log_name[FN_REFLEN]; ulonglong until_log_pos; /* extension extracted from log_name and converted to int */ @@ -276,16 +291,21 @@ public: UNTIL_LOG_NAMES_CMP_UNKNOWN= -2, UNTIL_LOG_NAMES_CMP_LESS= -1, UNTIL_LOG_NAMES_CMP_EQUAL= 0, UNTIL_LOG_NAMES_CMP_GREATER= 1 } until_log_names_cmp_result; + /* Condition for UNTIL master_gtid_pos. */ + slave_connection_state until_gtid_pos; char cached_charset[6]; /* - trans_retries varies between 0 to slave_transaction_retries and counts how - many times the slave has retried the present transaction; gets reset to 0 - when the transaction finally succeeds. retried_trans is a cumulative - counter: how many times the slave has retried a transaction (any) since - slave started. + retried_trans is a cumulative counter: how many times the slave + has retried a transaction (any) since slave started. + Protected by data_lock. + */ + ulong retried_trans; + /* + Number of executed events for SLAVE STATUS. + Protected by slave_executed_entries_lock */ - ulong trans_retries, retried_trans; + int64 executed_entries; /* If the end of the hot relay log is made of master's events ignored by the @@ -297,6 +317,8 @@ public: */ char ign_master_log_name_end[FN_REFLEN]; ulonglong ign_master_log_pos_end; + /* Similar for ignored GTID events. */ + slave_connection_state ign_gtids; /* Indentifies where the SQL Thread should create temporary files for the @@ -305,6 +327,8 @@ public: char slave_patternload_file[FN_REFLEN]; size_t slave_patternload_file_size; + rpl_parallel parallel; + Relay_log_info(bool is_slave_recovery); ~Relay_log_info(); @@ -327,13 +351,9 @@ public: if (until_condition==UNTIL_MASTER_POS) until_log_names_cmp_result= UNTIL_LOG_NAMES_CMP_UNKNOWN; } - - inline void inc_event_relay_log_pos() - { - event_relay_log_pos= future_event_relay_log_pos; - } void inc_group_relay_log_pos(ulonglong log_pos, + rpl_group_info *rgi, bool skip_lock=0); int wait_for_pos(THD* thd, String* log_name, longlong log_pos, @@ -344,31 +364,12 @@ public: bool is_until_satisfied(THD *thd, Log_event *ev); inline ulonglong until_pos() { + DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || + until_condition == UNTIL_RELAY_POS); return ((until_condition == UNTIL_MASTER_POS) ? group_master_log_pos : group_relay_log_pos); } - RPL_TABLE_LIST *tables_to_lock; /* RBR: Tables to lock */ - uint tables_to_lock_count; /* RBR: Count of tables to lock */ - table_mapping m_table_map; /* RBR: Mapping table-id to table */ - - bool get_table_data(TABLE *table_arg, table_def **tabledef_var, TABLE **conv_table_var) const - { - DBUG_ASSERT(tabledef_var && conv_table_var); - for (TABLE_LIST *ptr= tables_to_lock ; ptr != NULL ; ptr= ptr->next_global) - if (ptr->table == table_arg) - { - *tabledef_var= &static_cast<RPL_TABLE_LIST*>(ptr)->m_tabledef; - *conv_table_var= static_cast<RPL_TABLE_LIST*>(ptr)->m_conv_table; - DBUG_PRINT("debug", ("Fetching table data for table %s.%s:" - " tabledef: %p, conv_table: %p", - table_arg->s->db.str, table_arg->s->table_name.str, - *tabledef_var, *conv_table_var)); - return true; - } - return false; - } - /* Last charset (6 bytes) seen by slave SQL thread is cached here; it helps the thread save 3 get_charset() per Query_log_event if the charset is not @@ -378,52 +379,6 @@ public: void cached_charset_invalidate(); bool cached_charset_compare(char *charset) const; - void cleanup_context(THD *, bool); - void slave_close_thread_tables(THD *); - void clear_tables_to_lock(); - - /* - Used to defer stopping the SQL thread to give it a chance - to finish up the current group of events. - The timestamp is set and reset in @c sql_slave_killed(). - */ - time_t last_event_start_time; - - /* - A container to hold on Intvar-, Rand-, Uservar- log-events in case - the slave is configured with table filtering rules. - The withhold events are executed when their parent Query destiny is - determined for execution as well. - */ - Deferred_log_events *deferred_events; - - /* - State of the container: true stands for IRU events gathering, - false does for execution, either deferred or direct. - */ - bool deferred_events_collecting; - - /* - Returns true if the argument event resides in the containter; - more specifically, the checking is done against the last added event. - */ - bool is_deferred_event(Log_event * ev) - { - return deferred_events_collecting ? deferred_events->is_last(ev) : false; - }; - /* The general cleanup that slave applier may need at the end of query. */ - inline void cleanup_after_query() - { - if (deferred_events) - deferred_events->rewind(); - }; - /* The general cleanup that slave applier may need at the end of session. */ - void cleanup_after_session() - { - if (deferred_events) - delete deferred_events; - }; - /** Helper function to do after statement completion. @@ -443,8 +398,28 @@ public: the <code>Seconds_behind_master</code> field. */ void stmt_done(my_off_t event_log_pos, - time_t event_creation_time); + time_t event_creation_time, THD *thd, + rpl_group_info *rgi); + + /** + Is the replication inside a group? + The reader of the relay log is inside a group if either: + - The IN_TRANSACTION flag is set, meaning we're inside a transaction + - The IN_STMT flag is set, meaning we have read at least one row from + a multi-event entry. + + This flag reflects the state of the log 'just now', ie after the last + read event would be executed. + This allow us to test if we can stop replication before reading + the next entry. + + @retval true Replication thread is currently inside a group + @retval false Replication thread is currently not inside a group + */ + bool is_in_group() const { + return (m_flags & (IN_STMT | IN_TRANSACTION)); + } /** Set the value of a replication state flag. @@ -453,7 +428,7 @@ public: */ void set_flag(enum_state_flag flag) { - m_flags |= (1UL << flag); + m_flags|= flag; } /** @@ -465,7 +440,7 @@ public: */ bool get_flag(enum_state_flag flag) { - return m_flags & (1UL << flag); + return m_flags & flag; } /** @@ -475,23 +450,159 @@ public: */ void clear_flag(enum_state_flag flag) { - m_flags &= ~(1UL << flag); + m_flags&= ~flag; } - /** - Is the replication inside a group? +private: - Replication is inside a group if either: - - The OPTION_BEGIN flag is set, meaning we're inside a transaction - - The RLI_IN_STMT flag is set, meaning we're inside a statement + /* + Holds the state of the data in the relay log. + We need this to ensure that we are not in the middle of a + statement or inside BEGIN ... COMMIT when should rotate the + relay log. + */ + uint32 m_flags; +}; - @retval true Replication thread is currently inside a group - @retval false Replication thread is currently not inside a group + +/* + This is data for various state needed to be kept for the processing of + one event group (transaction) during replication. + + In single-threaded replication, there will be one global rpl_group_info and + one global Relay_log_info per master connection. They will be linked + together. + + In parallel replication, there will be one rpl_group_info object for + each running sql thread, each having their own thd. + + All rpl_group_info will share the same Relay_log_info. +*/ + +struct rpl_group_info +{ + Relay_log_info *rli; + THD *thd; + /* + Current GTID being processed. + The sub_id gives the binlog order within one domain_id. A zero sub_id + means that there is no active GTID. + */ + uint64 gtid_sub_id; + rpl_gtid current_gtid; + /* + This is used to keep transaction commit order. + We will signal this when we commit, and can register it to wait for the + commit_orderer of the previous commit to signal us. + */ + wait_for_commit commit_orderer; + /* + If non-zero, the sub_id of a prior event group whose commit we have to wait + for before committing ourselves. Then wait_commit_group_info points to the + event group to wait for. + + Before using this, rpl_parallel_entry::last_committed_sub_id should be + compared against wait_commit_sub_id. Only if last_committed_sub_id is + smaller than wait_commit_sub_id must the wait be done (otherwise the + waited-for transaction is already committed, so we would otherwise wait + for the wrong commit). + */ + uint64 wait_commit_sub_id; + rpl_group_info *wait_commit_group_info; + /* + If non-zero, the event group must wait for this sub_id to be committed + before the execution of the event group is allowed to start. + + (When we execute in parallel the transactions that group committed + together on the master, we still need to wait for any prior transactions + to have commtted). + */ + uint64 wait_start_sub_id; + + struct rpl_parallel_entry *parallel_entry; + + /* + A container to hold on Intvar-, Rand-, Uservar- log-events in case + the slave is configured with table filtering rules. + The withhold events are executed when their parent Query destiny is + determined for execution as well. + */ + Deferred_log_events *deferred_events; + + /* + State of the container: true stands for IRU events gathering, + false does for execution, either deferred or direct. + */ + bool deferred_events_collecting; + + Annotate_rows_log_event *m_annotate_event; + + RPL_TABLE_LIST *tables_to_lock; /* RBR: Tables to lock */ + uint tables_to_lock_count; /* RBR: Count of tables to lock */ + table_mapping m_table_map; /* RBR: Mapping table-id to table */ + mysql_mutex_t sleep_lock; + mysql_cond_t sleep_cond; + + /* + trans_retries varies between 0 to slave_transaction_retries and counts how + many times the slave has retried the present transaction; gets reset to 0 + when the transaction finally succeeds. + */ + ulong trans_retries; + + /* + Used to defer stopping the SQL thread to give it a chance + to finish up the current group of events. + The timestamp is set and reset in @c sql_slave_killed(). + */ + time_t last_event_start_time; + + char *event_relay_log_name; + char event_relay_log_name_buf[FN_REFLEN]; + ulonglong event_relay_log_pos; + ulonglong future_event_relay_log_pos; + /* + The master log name for current event. Only used in parallel replication. + */ + char future_event_master_log_name[FN_REFLEN]; + bool is_parallel_exec; + bool is_error; + +private: + /* + Runtime state for printing a note when slave is taking + too long while processing a row event. */ - bool is_in_group() const { - return (sql_thd->variables.option_bits & OPTION_BEGIN) || - (m_flags & (1UL << IN_STMT)); - } + time_t row_stmt_start_timestamp; + bool long_find_row_note_printed; +public: + + rpl_group_info(Relay_log_info *rli_); + ~rpl_group_info(); + + /* + Returns true if the argument event resides in the containter; + more specifically, the checking is done against the last added event. + */ + bool is_deferred_event(Log_event * ev) + { + return deferred_events_collecting ? deferred_events->is_last(ev) : false; + }; + /* The general cleanup that slave applier may need at the end of query. */ + inline void cleanup_after_query() + { + if (deferred_events) + deferred_events->rewind(); + }; + /* The general cleanup that slave applier may need at the end of session. */ + void cleanup_after_session() + { + if (deferred_events) + { + delete deferred_events; + deferred_events= NULL; + } + }; /** Save pointer to Annotate_rows event and switch on the @@ -502,7 +613,7 @@ public: { free_annotate_event(); m_annotate_event= event; - sql_thd->variables.binlog_annotate_row_events= 1; + this->thd->variables.binlog_annotate_row_events= 1; } /** @@ -524,12 +635,33 @@ public: { if (m_annotate_event) { - sql_thd->variables.binlog_annotate_row_events= 0; + this->thd->variables.binlog_annotate_row_events= 0; delete m_annotate_event; m_annotate_event= 0; } } + bool get_table_data(TABLE *table_arg, table_def **tabledef_var, TABLE **conv_table_var) const + { + DBUG_ASSERT(tabledef_var && conv_table_var); + for (TABLE_LIST *ptr= tables_to_lock ; ptr != NULL ; ptr= ptr->next_global) + if (ptr->table == table_arg) + { + *tabledef_var= &static_cast<RPL_TABLE_LIST*>(ptr)->m_tabledef; + *conv_table_var= static_cast<RPL_TABLE_LIST*>(ptr)->m_conv_table; + DBUG_PRINT("debug", ("Fetching table data for table %s.%s:" + " tabledef: %p, conv_table: %p", + table_arg->s->db.str, table_arg->s->table_name.str, + *tabledef_var, *conv_table_var)); + return true; + } + return false; + } + + void clear_tables_to_lock(); + void cleanup_context(THD *, bool); + void slave_close_thread_tables(THD *); + time_t get_row_stmt_start_timestamp() { return row_stmt_start_timestamp; @@ -563,18 +695,11 @@ public: return long_find_row_note_printed; } -private: - - uint32 m_flags; - - /* - Runtime state for printing a note when slave is taking - too long while processing a row event. - */ - time_t row_stmt_start_timestamp; - bool long_find_row_note_printed; - - Annotate_rows_log_event *m_annotate_event; + inline void inc_event_relay_log_pos() + { + if (!is_parallel_exec) + rli->event_relay_log_pos= future_event_relay_log_pos; + } }; @@ -582,4 +707,12 @@ private: int init_relay_log_info(Relay_log_info* rli, const char* info_fname); +extern struct rpl_slave_state rpl_global_gtid_slave_state; +extern gtid_waiting rpl_global_gtid_waiting; + +int rpl_load_gtid_slave_state(THD *thd); +int event_group_new_gtid(rpl_group_info *rgi, Gtid_log_event *gev); +void delete_or_keep_event_post_apply(rpl_group_info *rgi, + Log_event_type typ, Log_event *ev); + #endif /* RPL_RLI_H */ diff --git a/sql/rpl_tblmap.cc b/sql/rpl_tblmap.cc index b7ac1b2d091..7b55911d887 100644 --- a/sql/rpl_tblmap.cc +++ b/sql/rpl_tblmap.cc @@ -46,7 +46,7 @@ table_mapping::table_mapping() offsetof(entry,table_id),sizeof(ulong), 0,0,0); /* We don't preallocate any block, this is consistent with m_free=0 above */ - init_alloc_root(&m_mem_root, TABLE_ID_HASH_SIZE*sizeof(entry), 0); + init_alloc_root(&m_mem_root, TABLE_ID_HASH_SIZE*sizeof(entry), 0, MYF(0)); DBUG_VOID_RETURN; } diff --git a/sql/rpl_utility.cc b/sql/rpl_utility.cc index 22241da1348..678a2bf978d 100644 --- a/sql/rpl_utility.cc +++ b/sql/rpl_utility.cc @@ -508,9 +508,9 @@ bool is_conversion_ok(int order, Relay_log_info *rli) bool allow_non_lossy, allow_lossy; allow_non_lossy = slave_type_conversions_options & - (ULL(1) << SLAVE_TYPE_CONVERSIONS_ALL_NON_LOSSY); + (1ULL << SLAVE_TYPE_CONVERSIONS_ALL_NON_LOSSY); allow_lossy= slave_type_conversions_options & - (ULL(1) << SLAVE_TYPE_CONVERSIONS_ALL_LOSSY); + (1ULL << SLAVE_TYPE_CONVERSIONS_ALL_LOSSY); DBUG_PRINT("enter", ("order: %d, flags:%s%s", order, allow_non_lossy ? " ALL_NON_LOSSY" : "", @@ -1117,7 +1117,7 @@ bool event_checksum_test(uchar *event_buf, ulong event_len, uint8 alg) Deferred_log_events::Deferred_log_events(Relay_log_info *rli) : last_added(NULL) { - my_init_dynamic_array(&array, sizeof(Log_event *), 32, 16); + my_init_dynamic_array(&array, sizeof(Log_event *), 32, 16, MYF(0)); } Deferred_log_events::~Deferred_log_events() @@ -1137,20 +1137,20 @@ bool Deferred_log_events::is_empty() return array.elements == 0; } -bool Deferred_log_events::execute(Relay_log_info *rli) +bool Deferred_log_events::execute(rpl_group_info *rgi) { bool res= false; DBUG_ENTER("Deferred_log_events::execute"); - DBUG_ASSERT(rli->deferred_events_collecting); + DBUG_ASSERT(rgi->deferred_events_collecting); - rli->deferred_events_collecting= false; + rgi->deferred_events_collecting= false; for (uint i= 0; !res && i < array.elements; i++) { Log_event *ev= (* (Log_event **) dynamic_array_ptr(&array, i)); - res= ev->apply_event(rli); + res= ev->apply_event(rgi); } - rli->deferred_events_collecting= true; + rgi->deferred_events_collecting= true; DBUG_RETURN(res); } diff --git a/sql/rpl_utility.h b/sql/rpl_utility.h index 79f4517c492..893cc8d3e04 100644 --- a/sql/rpl_utility.h +++ b/sql/rpl_utility.h @@ -275,7 +275,7 @@ public: /* queue for exection at Query-log-event time prior the Query */ int add(Log_event *ev); bool is_empty(); - bool execute(Relay_log_info *rli); + bool execute(struct rpl_group_info *rgi); void rewind(); bool is_last(Log_event *ev) { return ev == last_added; }; }; diff --git a/sql/scheduler.cc b/sql/scheduler.cc index 06e8ffb2b5e..4b544090dcd 100644 --- a/sql/scheduler.cc +++ b/sql/scheduler.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2007, 2011, Oracle and/or its affiliates. All rights reserved. - Copyright (c) 2012, Monty Program Ab + Copyright (c) 2012, 2013, Monty Program Ab This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -138,52 +138,3 @@ void one_thread_scheduler(scheduler_functions *func) func->end_thread= no_threads_end; } - - -/* - no pluggable schedulers in mariadb. - when we'll want it, we'll do it properly -*/ -#if 0 - -static scheduler_functions *saved_thread_scheduler; -static uint saved_thread_handling; - -extern "C" -int my_thread_scheduler_set(scheduler_functions *scheduler) -{ - DBUG_ASSERT(scheduler != 0); - - if (scheduler == NULL) - return 1; - - saved_thread_scheduler= thread_scheduler; - saved_thread_handling= thread_handling; - thread_scheduler= scheduler; - // Scheduler loaded dynamically - thread_handling= SCHEDULER_TYPES_COUNT; - return 0; -} - - -extern "C" -int my_thread_scheduler_reset() -{ - DBUG_ASSERT(saved_thread_scheduler != NULL); - - if (saved_thread_scheduler == NULL) - return 1; - - thread_scheduler= saved_thread_scheduler; - thread_handling= saved_thread_handling; - saved_thread_scheduler= 0; - return 0; -} -#else -extern "C" int my_thread_scheduler_set(scheduler_functions *scheduler) -{ return 1; } - -extern "C" int my_thread_scheduler_reset() -{ return 1; } -#endif - diff --git a/sql/set_var.cc b/sql/set_var.cc index 9efa29d8041..a266b24165b 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -264,6 +264,114 @@ bool sys_var::set_default(THD *thd, set_var* var) return check(thd, var) || update(thd, var); } + +#define do_num_val(T,CMD) \ +do { \ + mysql_mutex_lock(&LOCK_global_system_variables); \ + T val= *(T*) value_ptr(thd, type, base); \ + mysql_mutex_unlock(&LOCK_global_system_variables); \ + CMD; \ +} while (0) + +#define case_for_integers(CMD) \ + case SHOW_SINT: do_num_val (int,CMD); \ + case SHOW_SLONG: do_num_val (long,CMD); \ + case SHOW_SLONGLONG:do_num_val (longlong,CMD); \ + case SHOW_UINT: do_num_val (uint,CMD); \ + case SHOW_ULONG: do_num_val (ulong,CMD); \ + case SHOW_ULONGLONG:do_num_val (ulonglong,CMD); \ + case SHOW_HA_ROWS: do_num_val (ha_rows,CMD); \ + case SHOW_BOOL: do_num_val (bool,CMD); \ + case SHOW_MY_BOOL: do_num_val (my_bool,CMD) + +#define case_for_double(CMD) \ + case SHOW_DOUBLE: do_num_val (double,CMD) + +#define case_get_string_as_lex_string \ + case SHOW_CHAR: \ + mysql_mutex_lock(&LOCK_global_system_variables); \ + sval.str= (char*) value_ptr(thd, type, base); \ + sval.length= sval.str ? strlen(sval.str) : 0; \ + break; \ + case SHOW_CHAR_PTR: \ + mysql_mutex_lock(&LOCK_global_system_variables); \ + sval.str= *(char**) value_ptr(thd, type, base); \ + sval.length= sval.str ? strlen(sval.str) : 0; \ + break; \ + case SHOW_LEX_STRING: \ + mysql_mutex_lock(&LOCK_global_system_variables); \ + sval= *(LEX_STRING *) value_ptr(thd, type, base); \ + break + +longlong sys_var::val_int(bool *is_null, + THD *thd, enum_var_type type, LEX_STRING *base) +{ + LEX_STRING sval; + *is_null= false; + switch (show_type()) + { + case_get_string_as_lex_string; + case_for_integers(return val); + case_for_double(return (longlong) val); + default: + my_error(ER_VAR_CANT_BE_READ, MYF(0), name.str); + return 0; + } + + longlong ret= 0; + if (!(*is_null= !sval.str)) + ret= longlong_from_string_with_check(system_charset_info, + sval.str, sval.str + sval.length); + mysql_mutex_unlock(&LOCK_global_system_variables); + return ret; +} + + +String *sys_var::val_str(String *str, + THD *thd, enum_var_type type, LEX_STRING *base) +{ + LEX_STRING sval; + switch (show_type()) + { + case_get_string_as_lex_string; + case_for_integers(return str->set((ulonglong)val, system_charset_info) ? 0 : str); + case_for_double(return str->set_real(val, 6, system_charset_info) ? 0 : str); + default: + my_error(ER_VAR_CANT_BE_READ, MYF(0), name.str); + return 0; + } + + if (!sval.str || str->copy(sval.str, sval.length, system_charset_info)) + str= NULL; + mysql_mutex_unlock(&LOCK_global_system_variables); + return str; +} + + +double sys_var::val_real(bool *is_null, + THD *thd, enum_var_type type, LEX_STRING *base) +{ + LEX_STRING sval; + *is_null= false; + switch (show_type()) + { + case_get_string_as_lex_string; + case_for_integers(return val); + case_for_double(return val); + default: + my_error(ER_VAR_CANT_BE_READ, MYF(0), name.str); + return 0; + } + + double ret= 0; + if (!(*is_null= !sval.str)) + ret= double_from_string_with_check(system_charset_info, + sval.str, sval.str + sval.length); + mysql_mutex_unlock(&LOCK_global_system_variables); + return ret; +} + + void sys_var::do_deprecated_warning(THD *thd) { if (deprecation_substitute != NULL) @@ -724,26 +832,7 @@ int set_var_user::update(THD *thd) int set_var_password::check(THD *thd) { #ifndef NO_EMBEDDED_ACCESS_CHECKS - if (!user->host.str) - { - DBUG_ASSERT(thd->security_ctx->priv_host); - if (*thd->security_ctx->priv_host != 0) - { - user->host.str= (char *) thd->security_ctx->priv_host; - user->host.length= strlen(thd->security_ctx->priv_host); - } - else - { - user->host.str= (char *)"%"; - user->host.length= 1; - } - } - if (!user->user.str) - { - DBUG_ASSERT(thd->security_ctx->user); - user->user.str= (char *) thd->security_ctx->user; - user->user.length= strlen(thd->security_ctx->user); - } + user= get_current_user(thd, user); /* Returns 1 as the function sends error to client */ return check_change_password(thd, user->host.str, user->user.str, password, strlen(password)) ? 1 : 0; @@ -764,6 +853,31 @@ int set_var_password::update(THD *thd) } /***************************************************************************** + Functions to handle SET ROLE +*****************************************************************************/ +int set_var_role::check(THD *thd) +{ +#ifndef NO_EMBEDDED_ACCESS_CHECKS + ulonglong access; + int status= acl_check_setrole(thd, base.str, &access); + save_result.ulonglong_value= access; + return status; +#else + return 0; +#endif +} + +int set_var_role::update(THD *thd) +{ +#ifndef NO_EMBEDDED_ACCESS_CHECKS + return acl_setrole(thd, base.str, save_result.ulonglong_value); +#else + return 0; +#endif +} + + +/***************************************************************************** Functions to handle SET NAMES and SET CHARACTER SET *****************************************************************************/ diff --git a/sql/set_var.h b/sql/set_var.h index 32207e834d9..e25109385e3 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -69,13 +69,14 @@ public: enum binlog_status_enum { VARIABLE_NOT_IN_BINLOG, SESSION_VARIABLE_IN_BINLOG } binlog_status; + my_option option; ///< min, max, default values are stored here + protected: typedef bool (*on_check_function)(sys_var *self, THD *thd, set_var *var); typedef bool (*on_update_function)(sys_var *self, THD *thd, enum_var_type type); int flags; ///< or'ed flag_enum values const SHOW_TYPE show_val_type; ///< what value_ptr() returns for sql_show.cc - my_option option; ///< min, max, default values are stored here PolyLock *guard; ///< *second* lock that protects the variable ptrdiff_t offset; ///< offset to the value from global_system_variables on_check_function on_check; @@ -114,6 +115,10 @@ public: bool set_default(THD *thd, set_var *var); bool update(THD *thd, set_var *var); + longlong val_int(bool *is_null, THD *thd, enum_var_type type, LEX_STRING *base); + String *val_str(String *str, THD *thd, enum_var_type type, LEX_STRING *base); + double val_real(bool *is_null, THD *thd, enum_var_type type, LEX_STRING *base); + SHOW_TYPE show_type() { return show_val_type; } int scope() const { return flags & SCOPE_MASK; } CHARSET_INFO *charset(THD *thd); @@ -273,6 +278,17 @@ public: int update(THD *thd); }; +/* For SET ROLE */ + +class set_var_role: public set_var +{ +public: + set_var_role(LEX_STRING role_arg) : + set_var(OPT_SESSION, NULL, &role_arg, NULL){}; + int check(THD *thd); + int update(THD *thd); +}; + /* For SET NAMES and SET CHARACTER SET */ @@ -304,6 +320,7 @@ extern SHOW_COMP_OPTION have_query_cache; extern SHOW_COMP_OPTION have_geometry, have_rtree_keys; extern SHOW_COMP_OPTION have_crypt; extern SHOW_COMP_OPTION have_compress; +extern SHOW_COMP_OPTION have_openssl; /* Prototypes for helper functions diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index e8d44071a43..92a5b6a2ae9 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -51,79 +51,79 @@ ER_YES spa "SI" ukr "ТАК" ER_CANT_CREATE_FILE - cze "Nemohu vytvo-Břit soubor '%-.200s' (chybový kód: %d)" - dan "Kan ikke oprette filen '%-.200s' (Fejlkode: %d)" - nla "Kan file '%-.200s' niet aanmaken (Errcode: %d)" - eng "Can't create file '%-.200s' (errno: %d)" - est "Ei suuda luua faili '%-.200s' (veakood: %d)" - fre "Ne peut créer le fichier '%-.200s' (Errcode: %d)" - ger "Kann Datei '%-.200s' nicht erzeugen (Fehler: %d)" - greek "Αδύνατη η δημιουργία του αρχείου '%-.200s' (κωδικός λάθους: %d)" - hun "A '%-.200s' file nem hozhato letre (hibakod: %d)" - ita "Impossibile creare il file '%-.200s' (errno: %d)" - jpn "'%-.200s' ファイルが作れません (errno: %d)" - kor "화일 '%-.200s'를 만들지 못했습니다. (에러번호: %d)" - nor "Kan ikke opprette fila '%-.200s' (Feilkode: %d)" - norwegian-ny "Kan ikkje opprette fila '%-.200s' (Feilkode: %d)" - pol "Nie można stworzyć pliku '%-.200s' (Kod błędu: %d)" - por "Não pode criar o arquivo '%-.200s' (erro no. %d)" - rum "Nu pot sa creez fisierul '%-.200s' (Eroare: %d)" - rus "Невозможно создать файл '%-.200s' (ошибка: %d)" - serbian "Ne mogu da kreiram file '%-.200s' (errno: %d)" - slo "Nemôžem vytvoriť súbor '%-.200s' (chybový kód: %d)" - spa "No puedo crear archivo '%-.200s' (Error: %d)" - swe "Kan inte skapa filen '%-.200s' (Felkod: %d)" - ukr "Не можу створити файл '%-.200s' (помилка: %d)" + cze "Nemohu vytvo-Břit soubor '%-.200s' (chybový kód: %M)" + dan "Kan ikke oprette filen '%-.200s' (Fejlkode: %M)" + nla "Kan file '%-.200s' niet aanmaken (Errcode: %M)" + eng "Can't create file '%-.200s' (errno: %M)" + est "Ei suuda luua faili '%-.200s' (veakood: %M)" + fre "Ne peut créer le fichier '%-.200s' (Errcode: %M)" + ger "Kann Datei '%-.200s' nicht erzeugen (Fehler: %M)" + greek "Αδύνατη η δημιουργία του αρχείου '%-.200s' (κωδικός λάθους: %M)" + hun "A '%-.200s' file nem hozhato letre (hibakod: %M)" + ita "Impossibile creare il file '%-.200s' (errno: %M)" + jpn "'%-.200s' ファイルが作れません (errno: %M)" + kor "화일 '%-.200s'를 만들지 못했습니다. (에러번호: %M)" + nor "Kan ikke opprette fila '%-.200s' (Feilkode: %M)" + norwegian-ny "Kan ikkje opprette fila '%-.200s' (Feilkode: %M)" + pol "Nie można stworzyć pliku '%-.200s' (Kod błędu: %M)" + por "Não pode criar o arquivo '%-.200s' (erro no. %M)" + rum "Nu pot sa creez fisierul '%-.200s' (Eroare: %M)" + rus "Невозможно создать файл '%-.200s' (ошибка: %M)" + serbian "Ne mogu da kreiram file '%-.200s' (errno: %M)" + slo "Nemôžem vytvoriť súbor '%-.200s' (chybový kód: %M)" + spa "No puedo crear archivo '%-.200s' (Error: %M)" + swe "Kan inte skapa filen '%-.200s' (Felkod: %M)" + ukr "Не можу створити файл '%-.200s' (помилка: %M)" ER_CANT_CREATE_TABLE - cze "Nemohu vytvo-Břit tabulku '%-.200s' (chybový kód: %d)" - dan "Kan ikke oprette tabellen '%-.200s' (Fejlkode: %d)" - nla "Kan tabel '%-.200s' niet aanmaken (Errcode: %d)" - eng "Can't create table '%-.200s' (errno: %d)" - jps "'%-.200s' テーブルが作れません.(errno: %d)", - est "Ei suuda luua tabelit '%-.200s' (veakood: %d)" - fre "Ne peut créer la table '%-.200s' (Errcode: %d)" - ger "Kann Tabelle '%-.200s' nicht erzeugen (Fehler: %d)" - greek "Αδύνατη η δημιουργία του πίνακα '%-.200s' (κωδικός λάθους: %d)" - hun "A '%-.200s' tabla nem hozhato letre (hibakod: %d)" - ita "Impossibile creare la tabella '%-.200s' (errno: %d)" - jpn "'%-.200s' テーブルが作れません.(errno: %d)" - kor "테이블 '%-.200s'를 만들지 못했습니다. (에러번호: %d)" - nor "Kan ikke opprette tabellen '%-.200s' (Feilkode: %d)" - norwegian-ny "Kan ikkje opprette tabellen '%-.200s' (Feilkode: %d)" - pol "Nie można stworzyć tabeli '%-.200s' (Kod błędu: %d)" - por "Não pode criar a tabela '%-.200s' (erro no. %d)" - rum "Nu pot sa creez tabla '%-.200s' (Eroare: %d)" - rus "Невозможно создать таблицу '%-.200s' (ошибка: %d)" - serbian "Ne mogu da kreiram tabelu '%-.200s' (errno: %d)" - slo "Nemôžem vytvoriť tabuľku '%-.200s' (chybový kód: %d)" - spa "No puedo crear tabla '%-.200s' (Error: %d)" - swe "Kan inte skapa tabellen '%-.200s' (Felkod: %d)" - ukr "Не можу створити таблицю '%-.200s' (помилка: %d)" + cze "Nemohu vytvo-Břit tabulku %`s.%`s (chybový kód: %M)" + dan "Kan ikke oprette tabellen %`s.%`s (Fejlkode: %M)" + nla "Kan tabel %`s.%`s niet aanmaken (Errcode: %M)" + eng "Can't create table %`s.%`s (errno: %M)" + jps "%`s.%`s テーブルが作れません.(errno: %M)", + est "Ei suuda luua tabelit %`s.%`s (veakood: %M)" + fre "Ne peut créer la table %`s.%`s (Errcode: %M)" + ger "Kann Tabelle %`s.%`s nicht erzeugen (Fehler: %M)" + greek "Αδύνατη η δημιουργία του πίνακα %`s.%`s (κωδικός λάθους: %M)" + hun "A %`s.%`s tabla nem hozhato letre (hibakod: %M)" + ita "Impossibile creare la tabella %`s.%`s (errno: %M)" + jpn "%`s.%`s テーブルが作れません.(errno: %M)" + kor "테이블 %`s.%`s를 만들지 못했습니다. (에러번호: %M)" + nor "Kan ikke opprette tabellen %`s.%`s (Feilkode: %M)" + norwegian-ny "Kan ikkje opprette tabellen %`s.%`s (Feilkode: %M)" + pol "Nie można stworzyć tabeli %`s.%`s (Kod błędu: %M)" + por "Não pode criar a tabela %`s.%`s (erro no. %M)" + rum "Nu pot sa creez tabla %`s.%`s (Eroare: %M)" + rus "Невозможно создать таблицу %`s.%`s (ошибка: %M)" + serbian "Ne mogu da kreiram tabelu %`s.%`s (errno: %M)" + slo "Nemôžem vytvoriť tabuľku %`s.%`s (chybový kód: %M)" + spa "No puedo crear tabla %`s.%`s (Error: %M)" + swe "Kan inte skapa tabellen %`s.%`s (Felkod: %M)" + ukr "Не можу створити таблицю %`s.%`s (помилка: %M)" ER_CANT_CREATE_DB - cze "Nemohu vytvo-Břit databázi '%-.192s' (chybový kód: %d)" - dan "Kan ikke oprette databasen '%-.192s' (Fejlkode: %d)" - nla "Kan database '%-.192s' niet aanmaken (Errcode: %d)" - eng "Can't create database '%-.192s' (errno: %d)" - jps "'%-.192s' データベースが作れません (errno: %d)", - est "Ei suuda luua andmebaasi '%-.192s' (veakood: %d)" - fre "Ne peut créer la base '%-.192s' (Erreur %d)" - ger "Kann Datenbank '%-.192s' nicht erzeugen (Fehler: %d)" - greek "Αδύνατη η δημιουργία της βάσης δεδομένων '%-.192s' (κωδικός λάθους: %d)" - hun "Az '%-.192s' adatbazis nem hozhato letre (hibakod: %d)" - ita "Impossibile creare il database '%-.192s' (errno: %d)" - jpn "'%-.192s' データベースが作れません (errno: %d)" - kor "데이타베이스 '%-.192s'를 만들지 못했습니다.. (에러번호: %d)" - nor "Kan ikke opprette databasen '%-.192s' (Feilkode: %d)" - norwegian-ny "Kan ikkje opprette databasen '%-.192s' (Feilkode: %d)" - pol "Nie można stworzyć bazy danych '%-.192s' (Kod błędu: %d)" - por "Não pode criar o banco de dados '%-.192s' (erro no. %d)" - rum "Nu pot sa creez baza de date '%-.192s' (Eroare: %d)" - rus "Невозможно создать базу данных '%-.192s' (ошибка: %d)" - serbian "Ne mogu da kreiram bazu '%-.192s' (errno: %d)" - slo "Nemôžem vytvoriť databázu '%-.192s' (chybový kód: %d)" - spa "No puedo crear base de datos '%-.192s' (Error: %d)" - swe "Kan inte skapa databasen '%-.192s' (Felkod: %d)" - ukr "Не можу створити базу данних '%-.192s' (помилка: %d)" + cze "Nemohu vytvo-Břit databázi '%-.192s' (chybový kód: %M)" + dan "Kan ikke oprette databasen '%-.192s' (Fejlkode: %M)" + nla "Kan database '%-.192s' niet aanmaken (Errcode: %M)" + eng "Can't create database '%-.192s' (errno: %M)" + jps "'%-.192s' データベースが作れません (errno: %M)", + est "Ei suuda luua andmebaasi '%-.192s' (veakood: %M)" + fre "Ne peut créer la base '%-.192s' (Erreur %M)" + ger "Kann Datenbank '%-.192s' nicht erzeugen (Fehler: %M)" + greek "Αδύνατη η δημιουργία της βάσης δεδομένων '%-.192s' (κωδικός λάθους: %M)" + hun "Az '%-.192s' adatbazis nem hozhato letre (hibakod: %M)" + ita "Impossibile creare il database '%-.192s' (errno: %M)" + jpn "'%-.192s' データベースが作れません (errno: %M)" + kor "데이타베이스 '%-.192s'를 만들지 못했습니다.. (에러번호: %M)" + nor "Kan ikke opprette databasen '%-.192s' (Feilkode: %M)" + norwegian-ny "Kan ikkje opprette databasen '%-.192s' (Feilkode: %M)" + pol "Nie można stworzyć bazy danych '%-.192s' (Kod błędu: %M)" + por "Não pode criar o banco de dados '%-.192s' (erro no. %M)" + rum "Nu pot sa creez baza de date '%-.192s' (Eroare: %M)" + rus "Невозможно создать базу данных '%-.192s' (ошибка: %M)" + serbian "Ne mogu da kreiram bazu '%-.192s' (errno: %M)" + slo "Nemôžem vytvoriť databázu '%-.192s' (chybový kód: %M)" + spa "No puedo crear base de datos '%-.192s' (Error: %M)" + swe "Kan inte skapa databasen '%-.192s' (Felkod: %M)" + ukr "Не можу створити базу данних '%-.192s' (помилка: %M)" ER_DB_CREATE_EXISTS cze "Nemohu vytvo-Břit databázi '%-.192s'; databáze již existuje" dan "Kan ikke oprette databasen '%-.192s'; databasen eksisterer" @@ -175,80 +175,80 @@ ER_DB_DROP_EXISTS swe "Kan inte radera databasen '%-.192s'; databasen finns inte" ukr "Не можу видалити базу данних '%-.192s'. База данних не існує" ER_DB_DROP_DELETE - cze "Chyba p-Bři rušení databáze (nemohu vymazat '%-.192s', chyba %d)" - dan "Fejl ved sletning (drop) af databasen (kan ikke slette '%-.192s', Fejlkode %d)" - nla "Fout bij verwijderen database (kan '%-.192s' niet verwijderen, Errcode: %d)" - eng "Error dropping database (can't delete '%-.192s', errno: %d)" - jps "データベース破棄エラー ('%-.192s' を削除できません, errno: %d)", - est "Viga andmebaasi kustutamisel (ei suuda kustutada faili '%-.192s', veakood: %d)" - fre "Ne peut effacer la base '%-.192s' (erreur %d)" - ger "Fehler beim Löschen der Datenbank ('%-.192s' kann nicht gelöscht werden, Fehler: %d)" - greek "Παρουσιάστηκε πρόβλημα κατά τη διαγραφή της βάσης δεδομένων (αδύνατη η διαγραφή '%-.192s', κωδικός λάθους: %d)" - hun "Adatbazis megszuntetesi hiba ('%-.192s' nem torolheto, hibakod: %d)" - ita "Errore durante la cancellazione del database (impossibile cancellare '%-.192s', errno: %d)" - jpn "データベース破棄エラー ('%-.192s' を削除できません, errno: %d)" - kor "데이타베이스 제거 에러('%-.192s'를 삭제할 수 없읍니다, 에러번호: %d)" - nor "Feil ved fjerning (drop) av databasen (kan ikke slette '%-.192s', feil %d)" - norwegian-ny "Feil ved fjerning (drop) av databasen (kan ikkje slette '%-.192s', feil %d)" - pol "Bł?d podczas usuwania bazy danych (nie można usun?ć '%-.192s', bł?d %d)" - por "Erro ao eliminar banco de dados (não pode eliminar '%-.192s' - erro no. %d)" - rum "Eroare dropuind baza de date (nu pot sa sterg '%-.192s', Eroare: %d)" - rus "Ошибка при удалении базы данных (невозможно удалить '%-.192s', ошибка: %d)" - serbian "Ne mogu da izbrišem bazu (ne mogu da izbrišem '%-.192s', errno: %d)" - slo "Chyba pri mazaní databázy (nemôžem zmazať '%-.192s', chybový kód: %d)" - spa "Error eliminando la base de datos(no puedo borrar '%-.192s', error %d)" - swe "Fel vid radering av databasen (Kan inte radera '%-.192s'. Felkod: %d)" - ukr "Не можу видалити базу данних (Не можу видалити '%-.192s', помилка: %d)" + cze "Chyba p-Bři rušení databáze (nemohu vymazat '%-.192s', chyba %M)" + dan "Fejl ved sletning (drop) af databasen (kan ikke slette '%-.192s', Fejlkode %M)" + nla "Fout bij verwijderen database (kan '%-.192s' niet verwijderen, Errcode: %M)" + eng "Error dropping database (can't delete '%-.192s', errno: %M)" + jps "データベース破棄エラー ('%-.192s' を削除できません, errno: %M)", + est "Viga andmebaasi kustutamisel (ei suuda kustutada faili '%-.192s', veakood: %M)" + fre "Ne peut effacer la base '%-.192s' (erreur %M)" + ger "Fehler beim Löschen der Datenbank ('%-.192s' kann nicht gelöscht werden, Fehler: %M)" + greek "Παρουσιάστηκε πρόβλημα κατά τη διαγραφή της βάσης δεδομένων (αδύνατη η διαγραφή '%-.192s', κωδικός λάθους: %M)" + hun "Adatbazis megszuntetesi hiba ('%-.192s' nem torolheto, hibakod: %M)" + ita "Errore durante la cancellazione del database (impossibile cancellare '%-.192s', errno: %M)" + jpn "データベース破棄エラー ('%-.192s' を削除できません, errno: %M)" + kor "데이타베이스 제거 에러('%-.192s'를 삭제할 수 없읍니다, 에러번호: %M)" + nor "Feil ved fjerning (drop) av databasen (kan ikke slette '%-.192s', feil %M)" + norwegian-ny "Feil ved fjerning (drop) av databasen (kan ikkje slette '%-.192s', feil %M)" + pol "Bł?d podczas usuwania bazy danych (nie można usun?ć '%-.192s', bł?d %M)" + por "Erro ao eliminar banco de dados (não pode eliminar '%-.192s' - erro no. %M)" + rum "Eroare dropuind baza de date (nu pot sa sterg '%-.192s', Eroare: %M)" + rus "Ошибка при удалении базы данных (невозможно удалить '%-.192s', ошибка: %M)" + serbian "Ne mogu da izbrišem bazu (ne mogu da izbrišem '%-.192s', errno: %M)" + slo "Chyba pri mazaní databázy (nemôžem zmazať '%-.192s', chybový kód: %M)" + spa "Error eliminando la base de datos(no puedo borrar '%-.192s', error %M)" + swe "Fel vid radering av databasen (Kan inte radera '%-.192s'. Felkod: %M)" + ukr "Не можу видалити базу данних (Не можу видалити '%-.192s', помилка: %M)" ER_DB_DROP_RMDIR - cze "Chyba p-Bři rušení databáze (nemohu vymazat adresář '%-.192s', chyba %d)" - dan "Fejl ved sletting af database (kan ikke slette folderen '%-.192s', Fejlkode %d)" - nla "Fout bij verwijderen database (kan rmdir '%-.192s' niet uitvoeren, Errcode: %d)" - eng "Error dropping database (can't rmdir '%-.192s', errno: %d)" - jps "データベース破棄エラー ('%-.192s' を rmdir できません, errno: %d)", - est "Viga andmebaasi kustutamisel (ei suuda kustutada kataloogi '%-.192s', veakood: %d)" - fre "Erreur en effaçant la base (rmdir '%-.192s', erreur %d)" - ger "Fehler beim Löschen der Datenbank (Verzeichnis '%-.192s' kann nicht gelöscht werden, Fehler: %d)" - greek "Παρουσιάστηκε πρόβλημα κατά τη διαγραφή της βάσης δεδομένων (αδύνατη η διαγραφή του φακέλλου '%-.192s', κωδικός λάθους: %d)" - hun "Adatbazis megszuntetesi hiba ('%-.192s' nem szuntetheto meg, hibakod: %d)" - ita "Errore durante la cancellazione del database (impossibile rmdir '%-.192s', errno: %d)" - jpn "データベース破棄エラー ('%-.192s' を rmdir できません, errno: %d)" - kor "데이타베이스 제거 에러(rmdir '%-.192s'를 할 수 없읍니다, 에러번호: %d)" - nor "Feil ved sletting av database (kan ikke slette katalogen '%-.192s', feil %d)" - norwegian-ny "Feil ved sletting av database (kan ikkje slette katalogen '%-.192s', feil %d)" - pol "Bł?d podczas usuwania bazy danych (nie można wykonać rmdir '%-.192s', bł?d %d)" - por "Erro ao eliminar banco de dados (não pode remover diretório '%-.192s' - erro no. %d)" - rum "Eroare dropuind baza de date (nu pot sa rmdir '%-.192s', Eroare: %d)" - rus "Невозможно удалить базу данных (невозможно удалить каталог '%-.192s', ошибка: %d)" - serbian "Ne mogu da izbrišem bazu (ne mogu da izbrišem direktorijum '%-.192s', errno: %d)" - slo "Chyba pri mazaní databázy (nemôžem vymazať adresár '%-.192s', chybový kód: %d)" - spa "Error eliminando la base de datos (No puedo borrar directorio '%-.192s', error %d)" - swe "Fel vid radering av databasen (Kan inte radera biblioteket '%-.192s'. Felkod: %d)" - ukr "Не можу видалити базу данних (Не можу видалити теку '%-.192s', помилка: %d)" + cze "Chyba p-Bři rušení databáze (nemohu vymazat adresář '%-.192s', chyba %M)" + dan "Fejl ved sletting af database (kan ikke slette folderen '%-.192s', Fejlkode %M)" + nla "Fout bij verwijderen database (kan rmdir '%-.192s' niet uitvoeren, Errcode: %M)" + eng "Error dropping database (can't rmdir '%-.192s', errno: %M)" + jps "データベース破棄エラー ('%-.192s' を rmdir できません, errno: %M)", + est "Viga andmebaasi kustutamisel (ei suuda kustutada kataloogi '%-.192s', veakood: %M)" + fre "Erreur en effaçant la base (rmdir '%-.192s', erreur %M)" + ger "Fehler beim Löschen der Datenbank (Verzeichnis '%-.192s' kann nicht gelöscht werden, Fehler: %M)" + greek "Παρουσιάστηκε πρόβλημα κατά τη διαγραφή της βάσης δεδομένων (αδύνατη η διαγραφή του φακέλλου '%-.192s', κωδικός λάθους: %M)" + hun "Adatbazis megszuntetesi hiba ('%-.192s' nem szuntetheto meg, hibakod: %M)" + ita "Errore durante la cancellazione del database (impossibile rmdir '%-.192s', errno: %M)" + jpn "データベース破棄エラー ('%-.192s' を rmdir できません, errno: %M)" + kor "데이타베이스 제거 에러(rmdir '%-.192s'를 할 수 없읍니다, 에러번호: %M)" + nor "Feil ved sletting av database (kan ikke slette katalogen '%-.192s', feil %M)" + norwegian-ny "Feil ved sletting av database (kan ikkje slette katalogen '%-.192s', feil %M)" + pol "Bł?d podczas usuwania bazy danych (nie można wykonać rmdir '%-.192s', bł?d %M)" + por "Erro ao eliminar banco de dados (não pode remover diretório '%-.192s' - erro no. %M)" + rum "Eroare dropuind baza de date (nu pot sa rmdir '%-.192s', Eroare: %M)" + rus "Невозможно удалить базу данных (невозможно удалить каталог '%-.192s', ошибка: %M)" + serbian "Ne mogu da izbrišem bazu (ne mogu da izbrišem direktorijum '%-.192s', errno: %M)" + slo "Chyba pri mazaní databázy (nemôžem vymazať adresár '%-.192s', chybový kód: %M)" + spa "Error eliminando la base de datos (No puedo borrar directorio '%-.192s', error %M)" + swe "Fel vid radering av databasen (Kan inte radera biblioteket '%-.192s'. Felkod: %M)" + ukr "Не можу видалити базу данних (Не можу видалити теку '%-.192s', помилка: %M)" ER_CANT_DELETE_FILE - cze "Chyba p-Bři výmazu '%-.192s' (chybový kód: %d)" - dan "Fejl ved sletning af '%-.192s' (Fejlkode: %d)" - nla "Fout bij het verwijderen van '%-.192s' (Errcode: %d)" - eng "Error on delete of '%-.192s' (errno: %d)" - jps "'%-.192s' の削除がエラー (errno: %d)", - est "Viga '%-.192s' kustutamisel (veakood: %d)" - fre "Erreur en effaçant '%-.192s' (Errcode: %d)" - ger "Fehler beim Löschen von '%-.192s' (Fehler: %d)" - greek "Παρουσιάστηκε πρόβλημα κατά τη διαγραφή '%-.192s' (κωδικός λάθους: %d)" - hun "Torlesi hiba: '%-.192s' (hibakod: %d)" - ita "Errore durante la cancellazione di '%-.192s' (errno: %d)" - jpn "'%-.192s' の削除がエラー (errno: %d)" - kor "'%-.192s' 삭제 중 에러 (에러번호: %d)" - nor "Feil ved sletting av '%-.192s' (Feilkode: %d)" - norwegian-ny "Feil ved sletting av '%-.192s' (Feilkode: %d)" - pol "Bł?d podczas usuwania '%-.192s' (Kod błędu: %d)" - por "Erro na remoção de '%-.192s' (erro no. %d)" - rum "Eroare incercind sa delete '%-.192s' (Eroare: %d)" - rus "Ошибка при удалении '%-.192s' (ошибка: %d)" - serbian "Greška pri brisanju '%-.192s' (errno: %d)" - slo "Chyba pri mazaní '%-.192s' (chybový kód: %d)" - spa "Error en el borrado de '%-.192s' (Error: %d)" - swe "Kan inte radera filen '%-.192s' (Felkod: %d)" - ukr "Не можу видалити '%-.192s' (помилка: %d)" + cze "Chyba p-Bři výmazu '%-.192s' (chybový kód: %M)" + dan "Fejl ved sletning af '%-.192s' (Fejlkode: %M)" + nla "Fout bij het verwijderen van '%-.192s' (Errcode: %M)" + eng "Error on delete of '%-.192s' (errno: %M)" + jps "'%-.192s' の削除がエラー (errno: %M)", + est "Viga '%-.192s' kustutamisel (veakood: %M)" + fre "Erreur en effaçant '%-.192s' (Errcode: %M)" + ger "Fehler beim Löschen von '%-.192s' (Fehler: %M)" + greek "Παρουσιάστηκε πρόβλημα κατά τη διαγραφή '%-.192s' (κωδικός λάθους: %M)" + hun "Torlesi hiba: '%-.192s' (hibakod: %M)" + ita "Errore durante la cancellazione di '%-.192s' (errno: %M)" + jpn "'%-.192s' の削除がエラー (errno: %M)" + kor "'%-.192s' 삭제 중 에러 (에러번호: %M)" + nor "Feil ved sletting av '%-.192s' (Feilkode: %M)" + norwegian-ny "Feil ved sletting av '%-.192s' (Feilkode: %M)" + pol "Bł?d podczas usuwania '%-.192s' (Kod błędu: %M)" + por "Erro na remoção de '%-.192s' (erro no. %M)" + rum "Eroare incercind sa delete '%-.192s' (Eroare: %M)" + rus "Ошибка при удалении '%-.192s' (ошибка: %M)" + serbian "Greška pri brisanju '%-.192s' (errno: %M)" + slo "Chyba pri mazaní '%-.192s' (chybový kód: %M)" + spa "Error en el borrado de '%-.192s' (Error: %M)" + swe "Kan inte radera filen '%-.192s' (Felkod: %M)" + ukr "Не можу видалити '%-.192s' (помилка: %M)" ER_CANT_FIND_SYSTEM_REC cze "Nemohu -Bčíst záznam v systémové tabulce" dan "Kan ikke læse posten i systemfolderen" @@ -275,180 +275,180 @@ ER_CANT_FIND_SYSTEM_REC swe "Hittar inte posten i systemregistret" ukr "Не можу зчитати запис з системної таблиці" ER_CANT_GET_STAT - cze "Nemohu z-Bískat stav '%-.200s' (chybový kód: %d)" - dan "Kan ikke læse status af '%-.200s' (Fejlkode: %d)" - nla "Kan de status niet krijgen van '%-.200s' (Errcode: %d)" - eng "Can't get status of '%-.200s' (errno: %d)" - jps "'%-.200s' のステイタスが得られません. (errno: %d)", - est "Ei suuda lugeda '%-.200s' olekut (veakood: %d)" - fre "Ne peut obtenir le status de '%-.200s' (Errcode: %d)" - ger "Kann Status von '%-.200s' nicht ermitteln (Fehler: %d)" - greek "Αδύνατη η λήψη πληροφοριών για την κατάσταση του '%-.200s' (κωδικός λάθους: %d)" - hun "A(z) '%-.200s' statusza nem allapithato meg (hibakod: %d)" - ita "Impossibile leggere lo stato di '%-.200s' (errno: %d)" - jpn "'%-.200s' のステイタスが得られません. (errno: %d)" - kor "'%-.200s'의 상태를 얻지 못했습니다. (에러번호: %d)" - nor "Kan ikke lese statusen til '%-.200s' (Feilkode: %d)" - norwegian-ny "Kan ikkje lese statusen til '%-.200s' (Feilkode: %d)" - pol "Nie można otrzymać statusu '%-.200s' (Kod błędu: %d)" - por "Não pode obter o status de '%-.200s' (erro no. %d)" - rum "Nu pot sa obtin statusul lui '%-.200s' (Eroare: %d)" - rus "Невозможно получить статусную информацию о '%-.200s' (ошибка: %d)" - serbian "Ne mogu da dobijem stanje file-a '%-.200s' (errno: %d)" - slo "Nemôžem zistiť stav '%-.200s' (chybový kód: %d)" - spa "No puedo obtener el estado de '%-.200s' (Error: %d)" - swe "Kan inte läsa filinformationen (stat) från '%-.200s' (Felkod: %d)" - ukr "Не можу отримати статус '%-.200s' (помилка: %d)" + cze "Nemohu z-Bískat stav '%-.200s' (chybový kód: %M)" + dan "Kan ikke læse status af '%-.200s' (Fejlkode: %M)" + nla "Kan de status niet krijgen van '%-.200s' (Errcode: %M)" + eng "Can't get status of '%-.200s' (errno: %M)" + jps "'%-.200s' のステイタスが得られません. (errno: %M)", + est "Ei suuda lugeda '%-.200s' olekut (veakood: %M)" + fre "Ne peut obtenir le status de '%-.200s' (Errcode: %M)" + ger "Kann Status von '%-.200s' nicht ermitteln (Fehler: %M)" + greek "Αδύνατη η λήψη πληροφοριών για την κατάσταση του '%-.200s' (κωδικός λάθους: %M)" + hun "A(z) '%-.200s' statusza nem allapithato meg (hibakod: %M)" + ita "Impossibile leggere lo stato di '%-.200s' (errno: %M)" + jpn "'%-.200s' のステイタスが得られません. (errno: %M)" + kor "'%-.200s'의 상태를 얻지 못했습니다. (에러번호: %M)" + nor "Kan ikke lese statusen til '%-.200s' (Feilkode: %M)" + norwegian-ny "Kan ikkje lese statusen til '%-.200s' (Feilkode: %M)" + pol "Nie można otrzymać statusu '%-.200s' (Kod błędu: %M)" + por "Não pode obter o status de '%-.200s' (erro no. %M)" + rum "Nu pot sa obtin statusul lui '%-.200s' (Eroare: %M)" + rus "Невозможно получить статусную информацию о '%-.200s' (ошибка: %M)" + serbian "Ne mogu da dobijem stanje file-a '%-.200s' (errno: %M)" + slo "Nemôžem zistiť stav '%-.200s' (chybový kód: %M)" + spa "No puedo obtener el estado de '%-.200s' (Error: %M)" + swe "Kan inte läsa filinformationen (stat) från '%-.200s' (Felkod: %M)" + ukr "Не можу отримати статус '%-.200s' (помилка: %M)" ER_CANT_GET_WD - cze "Chyba p-Bři zjišťování pracovní adresář (chybový kód: %d)" - dan "Kan ikke læse aktive folder (Fejlkode: %d)" - nla "Kan de werkdirectory niet krijgen (Errcode: %d)" - eng "Can't get working directory (errno: %d)" - jps "working directory を得る事ができませんでした (errno: %d)", - est "Ei suuda identifitseerida jooksvat kataloogi (veakood: %d)" - fre "Ne peut obtenir le répertoire de travail (Errcode: %d)" - ger "Kann Arbeitsverzeichnis nicht ermitteln (Fehler: %d)" - greek "Ο φάκελλος εργασίας δεν βρέθηκε (κωδικός λάθους: %d)" - hun "A munkakonyvtar nem allapithato meg (hibakod: %d)" - ita "Impossibile leggere la directory di lavoro (errno: %d)" - jpn "working directory を得る事ができませんでした (errno: %d)" - kor "수행 디렉토리를 찾지 못했습니다. (에러번호: %d)" - nor "Kan ikke lese aktiv katalog(Feilkode: %d)" - norwegian-ny "Kan ikkje lese aktiv katalog(Feilkode: %d)" - pol "Nie można rozpoznać aktualnego katalogu (Kod błędu: %d)" - por "Não pode obter o diretório corrente (erro no. %d)" - rum "Nu pot sa obtin directorul current (working directory) (Eroare: %d)" - rus "Невозможно определить рабочий каталог (ошибка: %d)" - serbian "Ne mogu da dobijem trenutni direktorijum (errno: %d)" - slo "Nemôžem zistiť pracovný adresár (chybový kód: %d)" - spa "No puedo acceder al directorio (Error: %d)" - swe "Kan inte inte läsa aktivt bibliotek. (Felkod: %d)" - ukr "Не можу визначити робочу теку (помилка: %d)" + cze "Chyba p-Bři zjišťování pracovní adresář (chybový kód: %M)" + dan "Kan ikke læse aktive folder (Fejlkode: %M)" + nla "Kan de werkdirectory niet krijgen (Errcode: %M)" + eng "Can't get working directory (errno: %M)" + jps "working directory を得る事ができませんでした (errno: %M)", + est "Ei suuda identifitseerida jooksvat kataloogi (veakood: %M)" + fre "Ne peut obtenir le répertoire de travail (Errcode: %M)" + ger "Kann Arbeitsverzeichnis nicht ermitteln (Fehler: %M)" + greek "Ο φάκελλος εργασίας δεν βρέθηκε (κωδικός λάθους: %M)" + hun "A munkakonyvtar nem allapithato meg (hibakod: %M)" + ita "Impossibile leggere la directory di lavoro (errno: %M)" + jpn "working directory を得る事ができませんでした (errno: %M)" + kor "수행 디렉토리를 찾지 못했습니다. (에러번호: %M)" + nor "Kan ikke lese aktiv katalog(Feilkode: %M)" + norwegian-ny "Kan ikkje lese aktiv katalog(Feilkode: %M)" + pol "Nie można rozpoznać aktualnego katalogu (Kod błędu: %M)" + por "Não pode obter o diretório corrente (erro no. %M)" + rum "Nu pot sa obtin directorul current (working directory) (Eroare: %M)" + rus "Невозможно определить рабочий каталог (ошибка: %M)" + serbian "Ne mogu da dobijem trenutni direktorijum (errno: %M)" + slo "Nemôžem zistiť pracovný adresár (chybový kód: %M)" + spa "No puedo acceder al directorio (Error: %M)" + swe "Kan inte inte läsa aktivt bibliotek. (Felkod: %M)" + ukr "Не можу визначити робочу теку (помилка: %M)" ER_CANT_LOCK - cze "Nemohu uzamknout soubor (chybov-Bý kód: %d)" - dan "Kan ikke låse fil (Fejlkode: %d)" - nla "Kan de file niet blokeren (Errcode: %d)" - eng "Can't lock file (errno: %d)" - jps "ファイルをロックできません (errno: %d)", - est "Ei suuda lukustada faili (veakood: %d)" - fre "Ne peut verrouiller le fichier (Errcode: %d)" - ger "Datei kann nicht gesperrt werden (Fehler: %d)" - greek "Το αρχείο δεν μπορεί να κλειδωθεί (κωδικός λάθους: %d)" - hun "A file nem zarolhato. (hibakod: %d)" - ita "Impossibile il locking il file (errno: %d)" - jpn "ファイルをロックできません (errno: %d)" - kor "화일을 잠그지(lock) 못했습니다. (에러번호: %d)" - nor "Kan ikke låse fila (Feilkode: %d)" - norwegian-ny "Kan ikkje låse fila (Feilkode: %d)" - pol "Nie można zablokować pliku (Kod błędu: %d)" - por "Não pode travar o arquivo (erro no. %d)" - rum "Nu pot sa lock fisierul (Eroare: %d)" - rus "Невозможно поставить блокировку на файле (ошибка: %d)" - serbian "Ne mogu da zaključam file (errno: %d)" - slo "Nemôžem zamknúť súbor (chybový kód: %d)" - spa "No puedo bloquear archivo: (Error: %d)" - swe "Kan inte låsa filen. (Felkod: %d)" - ukr "Не можу заблокувати файл (помилка: %d)" + cze "Nemohu uzamknout soubor (chybov-Bý kód: %M)" + dan "Kan ikke låse fil (Fejlkode: %M)" + nla "Kan de file niet blokeren (Errcode: %M)" + eng "Can't lock file (errno: %M)" + jps "ファイルをロックできません (errno: %M)", + est "Ei suuda lukustada faili (veakood: %M)" + fre "Ne peut verrouiller le fichier (Errcode: %M)" + ger "Datei kann nicht gesperrt werden (Fehler: %M)" + greek "Το αρχείο δεν μπορεί να κλειδωθεί (κωδικός λάθους: %M)" + hun "A file nem zarolhato. (hibakod: %M)" + ita "Impossibile il locking il file (errno: %M)" + jpn "ファイルをロックできません (errno: %M)" + kor "화일을 잠그지(lock) 못했습니다. (에러번호: %M)" + nor "Kan ikke låse fila (Feilkode: %M)" + norwegian-ny "Kan ikkje låse fila (Feilkode: %M)" + pol "Nie można zablokować pliku (Kod błędu: %M)" + por "Não pode travar o arquivo (erro no. %M)" + rum "Nu pot sa lock fisierul (Eroare: %M)" + rus "Невозможно поставить блокировку на файле (ошибка: %M)" + serbian "Ne mogu da zaključam file (errno: %M)" + slo "Nemôžem zamknúť súbor (chybový kód: %M)" + spa "No puedo bloquear archivo: (Error: %M)" + swe "Kan inte låsa filen. (Felkod: %M)" + ukr "Не можу заблокувати файл (помилка: %M)" ER_CANT_OPEN_FILE - cze "Nemohu otev-Břít soubor '%-.200s' (chybový kód: %d)" - dan "Kan ikke åbne fil: '%-.200s' (Fejlkode: %d)" - nla "Kan de file '%-.200s' niet openen (Errcode: %d)" - eng "Can't open file: '%-.200s' (errno: %d)" - jps "'%-.200s' ファイルを開く事ができません (errno: %d)", - est "Ei suuda avada faili '%-.200s' (veakood: %d)" - fre "Ne peut ouvrir le fichier: '%-.200s' (Errcode: %d)" - ger "Kann Datei '%-.200s' nicht öffnen (Fehler: %d)" - greek "Δεν είναι δυνατό να ανοιχτεί το αρχείο: '%-.200s' (κωδικός λάθους: %d)" - hun "A '%-.200s' file nem nyithato meg (hibakod: %d)" - ita "Impossibile aprire il file: '%-.200s' (errno: %d)" - jpn "'%-.200s' ファイルを開く事ができません (errno: %d)" - kor "화일을 열지 못했습니다.: '%-.200s' (에러번호: %d)" - nor "Kan ikke åpne fila: '%-.200s' (Feilkode: %d)" - norwegian-ny "Kan ikkje åpne fila: '%-.200s' (Feilkode: %d)" - pol "Nie można otworzyć pliku: '%-.200s' (Kod błędu: %d)" - por "Não pode abrir o arquivo '%-.200s' (erro no. %d)" - rum "Nu pot sa deschid fisierul: '%-.200s' (Eroare: %d)" - rus "Невозможно открыть файл: '%-.200s' (ошибка: %d)" - serbian "Ne mogu da otvorim file: '%-.200s' (errno: %d)" - slo "Nemôžem otvoriť súbor: '%-.200s' (chybový kód: %d)" - spa "No puedo abrir archivo: '%-.200s' (Error: %d)" - swe "Kan inte använda '%-.200s' (Felkod: %d)" - ukr "Не можу відкрити файл: '%-.200s' (помилка: %d)" + cze "Nemohu otev-Břít soubor '%-.200s' (chybový kód: %M)" + dan "Kan ikke åbne fil: '%-.200s' (Fejlkode: %M)" + nla "Kan de file '%-.200s' niet openen (Errcode: %M)" + eng "Can't open file: '%-.200s' (errno: %M)" + jps "'%-.200s' ファイルを開く事ができません (errno: %M)", + est "Ei suuda avada faili '%-.200s' (veakood: %M)" + fre "Ne peut ouvrir le fichier: '%-.200s' (Errcode: %M)" + ger "Kann Datei '%-.200s' nicht öffnen (Fehler: %M)" + greek "Δεν είναι δυνατό να ανοιχτεί το αρχείο: '%-.200s' (κωδικός λάθους: %M)" + hun "A '%-.200s' file nem nyithato meg (hibakod: %M)" + ita "Impossibile aprire il file: '%-.200s' (errno: %M)" + jpn "'%-.200s' ファイルを開く事ができません (errno: %M)" + kor "화일을 열지 못했습니다.: '%-.200s' (에러번호: %M)" + nor "Kan ikke åpne fila: '%-.200s' (Feilkode: %M)" + norwegian-ny "Kan ikkje åpne fila: '%-.200s' (Feilkode: %M)" + pol "Nie można otworzyć pliku: '%-.200s' (Kod błędu: %M)" + por "Não pode abrir o arquivo '%-.200s' (erro no. %M)" + rum "Nu pot sa deschid fisierul: '%-.200s' (Eroare: %M)" + rus "Невозможно открыть файл: '%-.200s' (ошибка: %M)" + serbian "Ne mogu da otvorim file: '%-.200s' (errno: %M)" + slo "Nemôžem otvoriť súbor: '%-.200s' (chybový kód: %M)" + spa "No puedo abrir archivo: '%-.200s' (Error: %M)" + swe "Kan inte använda '%-.200s' (Felkod: %M)" + ukr "Не можу відкрити файл: '%-.200s' (помилка: %M)" ER_FILE_NOT_FOUND - cze "Nemohu naj-Bít soubor '%-.200s' (chybový kód: %d)" - dan "Kan ikke finde fila: '%-.200s' (Fejlkode: %d)" - nla "Kan de file: '%-.200s' niet vinden (Errcode: %d)" - eng "Can't find file: '%-.200s' (errno: %d)" - jps "'%-.200s' ファイルを見付ける事ができません.(errno: %d)", - est "Ei suuda leida faili '%-.200s' (veakood: %d)" - fre "Ne peut trouver le fichier: '%-.200s' (Errcode: %d)" - ger "Kann Datei '%-.200s' nicht finden (Fehler: %d)" - greek "Δεν βρέθηκε το αρχείο: '%-.200s' (κωδικός λάθους: %d)" - hun "A(z) '%-.200s' file nem talalhato (hibakod: %d)" - ita "Impossibile trovare il file: '%-.200s' (errno: %d)" - jpn "'%-.200s' ファイルを見付ける事ができません.(errno: %d)" - kor "화일을 찾지 못했습니다.: '%-.200s' (에러번호: %d)" - nor "Kan ikke finne fila: '%-.200s' (Feilkode: %d)" - norwegian-ny "Kan ikkje finne fila: '%-.200s' (Feilkode: %d)" - pol "Nie można znaleĽć pliku: '%-.200s' (Kod błędu: %d)" - por "Não pode encontrar o arquivo '%-.200s' (erro no. %d)" - rum "Nu pot sa gasesc fisierul: '%-.200s' (Eroare: %d)" - rus "Невозможно найти файл: '%-.200s' (ошибка: %d)" - serbian "Ne mogu da pronađem file: '%-.200s' (errno: %d)" - slo "Nemôžem nájsť súbor: '%-.200s' (chybový kód: %d)" - spa "No puedo encontrar archivo: '%-.200s' (Error: %d)" - swe "Hittar inte filen '%-.200s' (Felkod: %d)" - ukr "Не можу знайти файл: '%-.200s' (помилка: %d)" + cze "Nemohu naj-Bít soubor '%-.200s' (chybový kód: %M)" + dan "Kan ikke finde fila: '%-.200s' (Fejlkode: %M)" + nla "Kan de file: '%-.200s' niet vinden (Errcode: %M)" + eng "Can't find file: '%-.200s' (errno: %M)" + jps "'%-.200s' ファイルを見付ける事ができません.(errno: %M)", + est "Ei suuda leida faili '%-.200s' (veakood: %M)" + fre "Ne peut trouver le fichier: '%-.200s' (Errcode: %M)" + ger "Kann Datei '%-.200s' nicht finden (Fehler: %M)" + greek "Δεν βρέθηκε το αρχείο: '%-.200s' (κωδικός λάθους: %M)" + hun "A(z) '%-.200s' file nem talalhato (hibakod: %M)" + ita "Impossibile trovare il file: '%-.200s' (errno: %M)" + jpn "'%-.200s' ファイルを見付ける事ができません.(errno: %M)" + kor "화일을 찾지 못했습니다.: '%-.200s' (에러번호: %M)" + nor "Kan ikke finne fila: '%-.200s' (Feilkode: %M)" + norwegian-ny "Kan ikkje finne fila: '%-.200s' (Feilkode: %M)" + pol "Nie można znaleĽć pliku: '%-.200s' (Kod błędu: %M)" + por "Não pode encontrar o arquivo '%-.200s' (erro no. %M)" + rum "Nu pot sa gasesc fisierul: '%-.200s' (Eroare: %M)" + rus "Невозможно найти файл: '%-.200s' (ошибка: %M)" + serbian "Ne mogu da pronađem file: '%-.200s' (errno: %M)" + slo "Nemôžem nájsť súbor: '%-.200s' (chybový kód: %M)" + spa "No puedo encontrar archivo: '%-.200s' (Error: %M)" + swe "Hittar inte filen '%-.200s' (Felkod: %M)" + ukr "Не можу знайти файл: '%-.200s' (помилка: %M)" ER_CANT_READ_DIR - cze "Nemohu -Bčíst adresář '%-.192s' (chybový kód: %d)" - dan "Kan ikke læse folder '%-.192s' (Fejlkode: %d)" - nla "Kan de directory niet lezen van '%-.192s' (Errcode: %d)" - eng "Can't read dir of '%-.192s' (errno: %d)" - jps "'%-.192s' ディレクトリが読めません.(errno: %d)", - est "Ei suuda lugeda kataloogi '%-.192s' (veakood: %d)" - fre "Ne peut lire le répertoire de '%-.192s' (Errcode: %d)" - ger "Verzeichnis von '%-.192s' nicht lesbar (Fehler: %d)" - greek "Δεν είναι δυνατό να διαβαστεί ο φάκελλος του '%-.192s' (κωδικός λάθους: %d)" - hun "A(z) '%-.192s' konyvtar nem olvashato. (hibakod: %d)" - ita "Impossibile leggere la directory di '%-.192s' (errno: %d)" - jpn "'%-.192s' ディレクトリが読めません.(errno: %d)" - kor "'%-.192s'디렉토리를 읽지 못했습니다. (에러번호: %d)" - nor "Kan ikke lese katalogen '%-.192s' (Feilkode: %d)" - norwegian-ny "Kan ikkje lese katalogen '%-.192s' (Feilkode: %d)" - pol "Nie można odczytać katalogu '%-.192s' (Kod błędu: %d)" - por "Não pode ler o diretório de '%-.192s' (erro no. %d)" - rum "Nu pot sa citesc directorul '%-.192s' (Eroare: %d)" - rus "Невозможно прочитать каталог '%-.192s' (ошибка: %d)" - serbian "Ne mogu da pročitam direktorijum '%-.192s' (errno: %d)" - slo "Nemôžem čítať adresár '%-.192s' (chybový kód: %d)" - spa "No puedo leer el directorio de '%-.192s' (Error: %d)" - swe "Kan inte läsa från bibliotek '%-.192s' (Felkod: %d)" - ukr "Не можу прочитати теку '%-.192s' (помилка: %d)" + cze "Nemohu -Bčíst adresář '%-.192s' (chybový kód: %M)" + dan "Kan ikke læse folder '%-.192s' (Fejlkode: %M)" + nla "Kan de directory niet lezen van '%-.192s' (Errcode: %M)" + eng "Can't read dir of '%-.192s' (errno: %M)" + jps "'%-.192s' ディレクトリが読めません.(errno: %M)", + est "Ei suuda lugeda kataloogi '%-.192s' (veakood: %M)" + fre "Ne peut lire le répertoire de '%-.192s' (Errcode: %M)" + ger "Verzeichnis von '%-.192s' nicht lesbar (Fehler: %M)" + greek "Δεν είναι δυνατό να διαβαστεί ο φάκελλος του '%-.192s' (κωδικός λάθους: %M)" + hun "A(z) '%-.192s' konyvtar nem olvashato. (hibakod: %M)" + ita "Impossibile leggere la directory di '%-.192s' (errno: %M)" + jpn "'%-.192s' ディレクトリが読めません.(errno: %M)" + kor "'%-.192s'디렉토리를 읽지 못했습니다. (에러번호: %M)" + nor "Kan ikke lese katalogen '%-.192s' (Feilkode: %M)" + norwegian-ny "Kan ikkje lese katalogen '%-.192s' (Feilkode: %M)" + pol "Nie można odczytać katalogu '%-.192s' (Kod błędu: %M)" + por "Não pode ler o diretório de '%-.192s' (erro no. %M)" + rum "Nu pot sa citesc directorul '%-.192s' (Eroare: %M)" + rus "Невозможно прочитать каталог '%-.192s' (ошибка: %M)" + serbian "Ne mogu da pročitam direktorijum '%-.192s' (errno: %M)" + slo "Nemôžem čítať adresár '%-.192s' (chybový kód: %M)" + spa "No puedo leer el directorio de '%-.192s' (Error: %M)" + swe "Kan inte läsa från bibliotek '%-.192s' (Felkod: %M)" + ukr "Не можу прочитати теку '%-.192s' (помилка: %M)" ER_CANT_SET_WD - cze "Nemohu zm-Běnit adresář na '%-.192s' (chybový kód: %d)" - dan "Kan ikke skifte folder til '%-.192s' (Fejlkode: %d)" - nla "Kan de directory niet veranderen naar '%-.192s' (Errcode: %d)" - eng "Can't change dir to '%-.192s' (errno: %d)" - jps "'%-.192s' ディレクトリに chdir できません.(errno: %d)", - est "Ei suuda siseneda kataloogi '%-.192s' (veakood: %d)" - fre "Ne peut changer le répertoire pour '%-.192s' (Errcode: %d)" - ger "Kann nicht in das Verzeichnis '%-.192s' wechseln (Fehler: %d)" - greek "Αδύνατη η αλλαγή του τρέχοντος καταλόγου σε '%-.192s' (κωδικός λάθους: %d)" - hun "Konyvtarvaltas nem lehetseges a(z) '%-.192s'-ba. (hibakod: %d)" - ita "Impossibile cambiare la directory in '%-.192s' (errno: %d)" - jpn "'%-.192s' ディレクトリに chdir できません.(errno: %d)" - kor "'%-.192s'디렉토리로 이동할 수 없었습니다. (에러번호: %d)" - nor "Kan ikke skifte katalog til '%-.192s' (Feilkode: %d)" - norwegian-ny "Kan ikkje skifte katalog til '%-.192s' (Feilkode: %d)" - pol "Nie można zmienić katalogu na '%-.192s' (Kod błędu: %d)" - por "Não pode mudar para o diretório '%-.192s' (erro no. %d)" - rum "Nu pot sa schimb directorul '%-.192s' (Eroare: %d)" - rus "Невозможно перейти в каталог '%-.192s' (ошибка: %d)" - serbian "Ne mogu da promenim direktorijum na '%-.192s' (errno: %d)" - slo "Nemôžem vojsť do adresára '%-.192s' (chybový kód: %d)" - spa "No puedo cambiar al directorio de '%-.192s' (Error: %d)" - swe "Kan inte byta till '%-.192s' (Felkod: %d)" - ukr "Не можу перейти у теку '%-.192s' (помилка: %d)" + cze "Nemohu zm-Běnit adresář na '%-.192s' (chybový kód: %M)" + dan "Kan ikke skifte folder til '%-.192s' (Fejlkode: %M)" + nla "Kan de directory niet veranderen naar '%-.192s' (Errcode: %M)" + eng "Can't change dir to '%-.192s' (errno: %M)" + jps "'%-.192s' ディレクトリに chdir できません.(errno: %M)", + est "Ei suuda siseneda kataloogi '%-.192s' (veakood: %M)" + fre "Ne peut changer le répertoire pour '%-.192s' (Errcode: %M)" + ger "Kann nicht in das Verzeichnis '%-.192s' wechseln (Fehler: %M)" + greek "Αδύνατη η αλλαγή του τρέχοντος καταλόγου σε '%-.192s' (κωδικός λάθους: %M)" + hun "Konyvtarvaltas nem lehetseges a(z) '%-.192s'-ba. (hibakod: %M)" + ita "Impossibile cambiare la directory in '%-.192s' (errno: %M)" + jpn "'%-.192s' ディレクトリに chdir できません.(errno: %M)" + kor "'%-.192s'디렉토리로 이동할 수 없었습니다. (에러번호: %M)" + nor "Kan ikke skifte katalog til '%-.192s' (Feilkode: %M)" + norwegian-ny "Kan ikkje skifte katalog til '%-.192s' (Feilkode: %M)" + pol "Nie można zmienić katalogu na '%-.192s' (Kod błędu: %M)" + por "Não pode mudar para o diretório '%-.192s' (erro no. %M)" + rum "Nu pot sa schimb directorul '%-.192s' (Eroare: %M)" + rus "Невозможно перейти в каталог '%-.192s' (ошибка: %M)" + serbian "Ne mogu da promenim direktorijum na '%-.192s' (errno: %M)" + slo "Nemôžem vojsť do adresára '%-.192s' (chybový kód: %M)" + spa "No puedo cambiar al directorio de '%-.192s' (Error: %M)" + swe "Kan inte byta till '%-.192s' (Felkod: %M)" + ukr "Не можу перейти у теку '%-.192s' (помилка: %M)" ER_CHECKREAD cze "Z-Báznam byl změněn od posledního čtení v tabulce '%-.192s'" dan "Posten er ændret siden sidste læsning '%-.192s'" @@ -523,103 +523,103 @@ ER_DUP_KEY 23000 swe "Kan inte skriva, dubbel söknyckel i register '%-.192s'" ukr "Не можу записати, дублюючийся ключ в таблиці '%-.192s'" ER_ERROR_ON_CLOSE - cze "Chyba p-Bři zavírání '%-.192s' (chybový kód: %d)" - dan "Fejl ved lukning af '%-.192s' (Fejlkode: %d)" - nla "Fout bij het sluiten van '%-.192s' (Errcode: %d)" - eng "Error on close of '%-.192s' (errno: %d)" - est "Viga faili '%-.192s' sulgemisel (veakood: %d)" - fre "Erreur a la fermeture de '%-.192s' (Errcode: %d)" - ger "Fehler beim Schließen von '%-.192s' (Fehler: %d)" - greek "Παρουσιάστηκε πρόβλημα κλείνοντας το '%-.192s' (κωδικός λάθους: %d)" - hun "Hiba a(z) '%-.192s' zarasakor. (hibakod: %d)" - ita "Errore durante la chiusura di '%-.192s' (errno: %d)" - kor "'%-.192s'닫는 중 에러 (에러번호: %d)" - nor "Feil ved lukking av '%-.192s' (Feilkode: %d)" - norwegian-ny "Feil ved lukking av '%-.192s' (Feilkode: %d)" - pol "Bł?d podczas zamykania '%-.192s' (Kod błędu: %d)" - por "Erro ao fechar '%-.192s' (erro no. %d)" - rum "Eroare inchizind '%-.192s' (errno: %d)" - rus "Ошибка при закрытии '%-.192s' (ошибка: %d)" - serbian "Greška pri zatvaranju '%-.192s' (errno: %d)" - slo "Chyba pri zatváraní '%-.192s' (chybový kód: %d)" - spa "Error en el cierre de '%-.192s' (Error: %d)" - swe "Fick fel vid stängning av '%-.192s' (Felkod: %d)" - ukr "Не можу закрити '%-.192s' (помилка: %d)" + cze "Chyba p-Bři zavírání '%-.192s' (chybový kód: %M)" + dan "Fejl ved lukning af '%-.192s' (Fejlkode: %M)" + nla "Fout bij het sluiten van '%-.192s' (Errcode: %M)" + eng "Error on close of '%-.192s' (errno: %M)" + est "Viga faili '%-.192s' sulgemisel (veakood: %M)" + fre "Erreur a la fermeture de '%-.192s' (Errcode: %M)" + ger "Fehler beim Schließen von '%-.192s' (Fehler: %M)" + greek "Παρουσιάστηκε πρόβλημα κλείνοντας το '%-.192s' (κωδικός λάθους: %M)" + hun "Hiba a(z) '%-.192s' zarasakor. (hibakod: %M)" + ita "Errore durante la chiusura di '%-.192s' (errno: %M)" + kor "'%-.192s'닫는 중 에러 (에러번호: %M)" + nor "Feil ved lukking av '%-.192s' (Feilkode: %M)" + norwegian-ny "Feil ved lukking av '%-.192s' (Feilkode: %M)" + pol "Bł?d podczas zamykania '%-.192s' (Kod błędu: %M)" + por "Erro ao fechar '%-.192s' (erro no. %M)" + rum "Eroare inchizind '%-.192s' (errno: %M)" + rus "Ошибка при закрытии '%-.192s' (ошибка: %M)" + serbian "Greška pri zatvaranju '%-.192s' (errno: %M)" + slo "Chyba pri zatváraní '%-.192s' (chybový kód: %M)" + spa "Error en el cierre de '%-.192s' (Error: %M)" + swe "Fick fel vid stängning av '%-.192s' (Felkod: %M)" + ukr "Не можу закрити '%-.192s' (помилка: %M)" ER_ERROR_ON_READ - cze "Chyba p-Bři čtení souboru '%-.200s' (chybový kód: %d)" - dan "Fejl ved læsning af '%-.200s' (Fejlkode: %d)" - nla "Fout bij het lezen van file '%-.200s' (Errcode: %d)" - eng "Error reading file '%-.200s' (errno: %d)" - jps "'%-.200s' ファイルの読み込みエラー (errno: %d)", - est "Viga faili '%-.200s' lugemisel (veakood: %d)" - fre "Erreur en lecture du fichier '%-.200s' (Errcode: %d)" - ger "Fehler beim Lesen der Datei '%-.200s' (Fehler: %d)" - greek "Πρόβλημα κατά την ανάγνωση του αρχείου '%-.200s' (κωδικός λάθους: %d)" - hun "Hiba a '%-.200s'file olvasasakor. (hibakod: %d)" - ita "Errore durante la lettura del file '%-.200s' (errno: %d)" - jpn "'%-.200s' ファイルの読み込みエラー (errno: %d)" - kor "'%-.200s'화일 읽기 에러 (에러번호: %d)" - nor "Feil ved lesing av '%-.200s' (Feilkode: %d)" - norwegian-ny "Feil ved lesing av '%-.200s' (Feilkode: %d)" - pol "Bł?d podczas odczytu pliku '%-.200s' (Kod błędu: %d)" - por "Erro ao ler arquivo '%-.200s' (erro no. %d)" - rum "Eroare citind fisierul '%-.200s' (errno: %d)" - rus "Ошибка чтения файла '%-.200s' (ошибка: %d)" - serbian "Greška pri čitanju file-a '%-.200s' (errno: %d)" - slo "Chyba pri čítaní súboru '%-.200s' (chybový kód: %d)" - spa "Error leyendo el fichero '%-.200s' (Error: %d)" - swe "Fick fel vid läsning av '%-.200s' (Felkod %d)" - ukr "Не можу прочитати файл '%-.200s' (помилка: %d)" + cze "Chyba p-Bři čtení souboru '%-.200s' (chybový kód: %M)" + dan "Fejl ved læsning af '%-.200s' (Fejlkode: %M)" + nla "Fout bij het lezen van file '%-.200s' (Errcode: %M)" + eng "Error reading file '%-.200s' (errno: %M)" + jps "'%-.200s' ファイルの読み込みエラー (errno: %M)", + est "Viga faili '%-.200s' lugemisel (veakood: %M)" + fre "Erreur en lecture du fichier '%-.200s' (Errcode: %M)" + ger "Fehler beim Lesen der Datei '%-.200s' (Fehler: %M)" + greek "Πρόβλημα κατά την ανάγνωση του αρχείου '%-.200s' (κωδικός λάθους: %M)" + hun "Hiba a '%-.200s'file olvasasakor. (hibakod: %M)" + ita "Errore durante la lettura del file '%-.200s' (errno: %M)" + jpn "'%-.200s' ファイルの読み込みエラー (errno: %M)" + kor "'%-.200s'화일 읽기 에러 (에러번호: %M)" + nor "Feil ved lesing av '%-.200s' (Feilkode: %M)" + norwegian-ny "Feil ved lesing av '%-.200s' (Feilkode: %M)" + pol "Bł?d podczas odczytu pliku '%-.200s' (Kod błędu: %M)" + por "Erro ao ler arquivo '%-.200s' (erro no. %M)" + rum "Eroare citind fisierul '%-.200s' (errno: %M)" + rus "Ошибка чтения файла '%-.200s' (ошибка: %M)" + serbian "Greška pri čitanju file-a '%-.200s' (errno: %M)" + slo "Chyba pri čítaní súboru '%-.200s' (chybový kód: %M)" + spa "Error leyendo el fichero '%-.200s' (Error: %M)" + swe "Fick fel vid läsning av '%-.200s' (Felkod %M)" + ukr "Не можу прочитати файл '%-.200s' (помилка: %M)" ER_ERROR_ON_RENAME - cze "Chyba p-Bři přejmenování '%-.210s' na '%-.210s' (chybový kód: %d)" - dan "Fejl ved omdøbning af '%-.210s' til '%-.210s' (Fejlkode: %d)" - nla "Fout bij het hernoemen van '%-.210s' naar '%-.210s' (Errcode: %d)" - eng "Error on rename of '%-.210s' to '%-.210s' (errno: %d)" - jps "'%-.210s' を '%-.210s' に rename できません (errno: %d)", - est "Viga faili '%-.210s' ümbernimetamisel '%-.210s'-ks (veakood: %d)" - fre "Erreur en renommant '%-.210s' en '%-.210s' (Errcode: %d)" - ger "Fehler beim Umbenennen von '%-.210s' in '%-.210s' (Fehler: %d)" - greek "Πρόβλημα κατά την μετονομασία του αρχείου '%-.210s' to '%-.210s' (κωδικός λάθους: %d)" - hun "Hiba a '%-.210s' file atnevezesekor '%-.210s'. (hibakod: %d)" - ita "Errore durante la rinominazione da '%-.210s' a '%-.210s' (errno: %d)" - jpn "'%-.210s' を '%-.210s' に rename できません (errno: %d)" - kor "'%-.210s'를 '%-.210s'로 이름 변경중 에러 (에러번호: %d)" - nor "Feil ved omdøping av '%-.210s' til '%-.210s' (Feilkode: %d)" - norwegian-ny "Feil ved omdøyping av '%-.210s' til '%-.210s' (Feilkode: %d)" - pol "Bł?d podczas zmieniania nazwy '%-.210s' na '%-.210s' (Kod błędu: %d)" - por "Erro ao renomear '%-.210s' para '%-.210s' (erro no. %d)" - rum "Eroare incercind sa renumesc '%-.210s' in '%-.210s' (errno: %d)" - rus "Ошибка при переименовании '%-.210s' в '%-.210s' (ошибка: %d)" - serbian "Greška pri promeni imena '%-.210s' na '%-.210s' (errno: %d)" - slo "Chyba pri premenovávaní '%-.210s' na '%-.210s' (chybový kód: %d)" - spa "Error en el renombrado de '%-.210s' a '%-.210s' (Error: %d)" - swe "Kan inte byta namn från '%-.210s' till '%-.210s' (Felkod: %d)" - ukr "Не можу перейменувати '%-.210s' у '%-.210s' (помилка: %d)" + cze "Chyba p-Bři přejmenování '%-.210s' na '%-.210s' (chybový kód: %M)" + dan "Fejl ved omdøbning af '%-.210s' til '%-.210s' (Fejlkode: %M)" + nla "Fout bij het hernoemen van '%-.210s' naar '%-.210s' (Errcode: %M)" + eng "Error on rename of '%-.210s' to '%-.210s' (errno: %M)" + jps "'%-.210s' を '%-.210s' に rename できません (errno: %M)", + est "Viga faili '%-.210s' ümbernimetamisel '%-.210s'-ks (veakood: %M)" + fre "Erreur en renommant '%-.210s' en '%-.210s' (Errcode: %M)" + ger "Fehler beim Umbenennen von '%-.210s' in '%-.210s' (Fehler: %M)" + greek "Πρόβλημα κατά την μετονομασία του αρχείου '%-.210s' to '%-.210s' (κωδικός λάθους: %M)" + hun "Hiba a '%-.210s' file atnevezesekor '%-.210s'. (hibakod: %M)" + ita "Errore durante la rinominazione da '%-.210s' a '%-.210s' (errno: %M)" + jpn "'%-.210s' を '%-.210s' に rename できません (errno: %M)" + kor "'%-.210s'를 '%-.210s'로 이름 변경중 에러 (에러번호: %M)" + nor "Feil ved omdøping av '%-.210s' til '%-.210s' (Feilkode: %M)" + norwegian-ny "Feil ved omdøyping av '%-.210s' til '%-.210s' (Feilkode: %M)" + pol "Bł?d podczas zmieniania nazwy '%-.210s' na '%-.210s' (Kod błędu: %M)" + por "Erro ao renomear '%-.210s' para '%-.210s' (erro no. %M)" + rum "Eroare incercind sa renumesc '%-.210s' in '%-.210s' (errno: %M)" + rus "Ошибка при переименовании '%-.210s' в '%-.210s' (ошибка: %M)" + serbian "Greška pri promeni imena '%-.210s' na '%-.210s' (errno: %M)" + slo "Chyba pri premenovávaní '%-.210s' na '%-.210s' (chybový kód: %M)" + spa "Error en el renombrado de '%-.210s' a '%-.210s' (Error: %M)" + swe "Kan inte byta namn från '%-.210s' till '%-.210s' (Felkod: %M)" + ukr "Не можу перейменувати '%-.210s' у '%-.210s' (помилка: %M)" ER_ERROR_ON_WRITE - cze "Chyba p-Bři zápisu do souboru '%-.200s' (chybový kód: %d)" - dan "Fejl ved skriving av filen '%-.200s' (Fejlkode: %d)" - nla "Fout bij het wegschrijven van file '%-.200s' (Errcode: %d)" - eng "Error writing file '%-.200s' (errno: %d)" - jps "'%-.200s' ファイルを書く事ができません (errno: %d)", - est "Viga faili '%-.200s' kirjutamisel (veakood: %d)" - fre "Erreur d'écriture du fichier '%-.200s' (Errcode: %d)" - ger "Fehler beim Speichern der Datei '%-.200s' (Fehler: %d)" - greek "Πρόβλημα κατά την αποθήκευση του αρχείου '%-.200s' (κωδικός λάθους: %d)" - hun "Hiba a '%-.200s' file irasakor. (hibakod: %d)" - ita "Errore durante la scrittura del file '%-.200s' (errno: %d)" - jpn "'%-.200s' ファイルを書く事ができません (errno: %d)" - kor "'%-.200s'화일 기록 중 에러 (에러번호: %d)" - nor "Feil ved skriving av fila '%-.200s' (Feilkode: %d)" - norwegian-ny "Feil ved skriving av fila '%-.200s' (Feilkode: %d)" - pol "Bł?d podczas zapisywania pliku '%-.200s' (Kod błędu: %d)" - por "Erro ao gravar arquivo '%-.200s' (erro no. %d)" - rum "Eroare scriind fisierul '%-.200s' (errno: %d)" - rus "Ошибка записи в файл '%-.200s' (ошибка: %d)" - serbian "Greška pri upisu '%-.200s' (errno: %d)" - slo "Chyba pri zápise do súboru '%-.200s' (chybový kód: %d)" - spa "Error escribiendo el archivo '%-.200s' (Error: %d)" - swe "Fick fel vid skrivning till '%-.200s' (Felkod %d)" - ukr "Не можу записати файл '%-.200s' (помилка: %d)" + cze "Chyba p-Bři zápisu do souboru '%-.200s' (chybový kód: %M)" + dan "Fejl ved skriving av filen '%-.200s' (Fejlkode: %M)" + nla "Fout bij het wegschrijven van file '%-.200s' (Errcode: %M)" + eng "Error writing file '%-.200s' (errno: %M)" + jps "'%-.200s' ファイルを書く事ができません (errno: %M)", + est "Viga faili '%-.200s' kirjutamisel (veakood: %M)" + fre "Erreur d'écriture du fichier '%-.200s' (Errcode: %M)" + ger "Fehler beim Speichern der Datei '%-.200s' (Fehler: %M)" + greek "Πρόβλημα κατά την αποθήκευση του αρχείου '%-.200s' (κωδικός λάθους: %M)" + hun "Hiba a '%-.200s' file irasakor. (hibakod: %M)" + ita "Errore durante la scrittura del file '%-.200s' (errno: %M)" + jpn "'%-.200s' ファイルを書く事ができません (errno: %M)" + kor "'%-.200s'화일 기록 중 에러 (에러번호: %M)" + nor "Feil ved skriving av fila '%-.200s' (Feilkode: %M)" + norwegian-ny "Feil ved skriving av fila '%-.200s' (Feilkode: %M)" + pol "Bł?d podczas zapisywania pliku '%-.200s' (Kod błędu: %M)" + por "Erro ao gravar arquivo '%-.200s' (erro no. %M)" + rum "Eroare scriind fisierul '%-.200s' (errno: %M)" + rus "Ошибка записи в файл '%-.200s' (ошибка: %M)" + serbian "Greška pri upisu '%-.200s' (errno: %M)" + slo "Chyba pri zápise do súboru '%-.200s' (chybový kód: %M)" + spa "Error escribiendo el archivo '%-.200s' (Error: %M)" + swe "Fick fel vid skrivning till '%-.200s' (Felkod %M)" + ukr "Не можу записати файл '%-.200s' (помилка: %M)" ER_FILE_USED cze "'%-.192s' je zam-Bčen proti změnám" dan "'%-.192s' er låst mod opdateringer" @@ -696,53 +696,26 @@ ER_FORM_NOT_FOUND swe "Formulär '%-.192s' finns inte i '%-.192s'" ukr "Вигляд '%-.192s' не існує для '%-.192s'" ER_GET_ERRNO - cze "Obsluha tabulky vr-Bátila chybu %d" - dan "Modtog fejl %d fra tabel håndteringen" - nla "Fout %d van tabel handler" - eng "Got error %d from storage engine" - est "Tabeli handler tagastas vea %d" - fre "Reçu l'erreur %d du handler de la table" - ger "Fehler %d (Speicher-Engine)" - greek "Ελήφθη μήνυμα λάθους %d από τον χειριστή πίνακα (table handler)" - hun "%d hibajelzes a tablakezelotol" - ita "Rilevato l'errore %d dal gestore delle tabelle" - jpn "Got error %d from table handler" - kor "테이블 handler에서 %d 에러가 발생 하였습니다." - nor "Mottok feil %d fra tabell håndterer" - norwegian-ny "Mottok feil %d fra tabell handterar" - pol "Otrzymano bł?d %d z obsługi tabeli" - por "Obteve erro %d no manipulador de tabelas" - rum "Eroarea %d obtinuta din handlerul tabelei" - rus "Получена ошибка %d от обработчика таблиц" - serbian "Handler tabela je vratio grešku %d" - slo "Obsluha tabuľky vrátila chybu %d" - spa "Error %d desde el manejador de la tabla" - swe "Fick felkod %d från databashanteraren" - ukr "Отримано помилку %d від дескриптора таблиці" + nla "Fout %M van tabel handler %s" + eng "Got error %M from storage engine %s" + fre "Reçu l'erreur %M du handler de la table %s" + ger "Fehler %M von Speicher-Engine %s" + greek "Ελήφθη μήνυμα λάθους %M από τον χειριστή πίνακα (table handler) %s" + ita "Rilevato l'errore %M dal gestore delle tabelle %s" + nor "Mottok feil %M fra tabell håndterer %s" + norwegian-ny "Mottok feil %M fra tabell handterar %s" + pol "Otrzymano bł?d %M z obsługi tabeli %s" + por "Obteve erro %M no manipulador de tabelas %s" + rum "Eroarea %M obtinuta din handlerul tabelei %s" + rus "Получена ошибка %M от обработчика таблиц %s" + spa "Error %M desde el manejador de la tabla %s" + swe "Fick felkod %M från databashanteraren %s" + ukr "Отримано помилку %M від дескриптора таблиці %s" ER_ILLEGAL_HA - cze "Obsluha tabulky '%-.192s' nem-Bá tento parametr" - dan "Denne mulighed eksisterer ikke for tabeltypen '%-.192s'" - nla "Tabel handler voor '%-.192s' heeft deze optie niet" - eng "Table storage engine for '%-.192s' doesn't have this option" - est "Tabeli '%-.192s' handler ei toeta antud operatsiooni" - fre "Le handler de la table '%-.192s' n'a pas cette option" - ger "Diese Option gibt es nicht (Speicher-Engine für '%-.192s')" - greek "Ο χειριστής πίνακα (table handler) για '%-.192s' δεν διαθέτει αυτή την επιλογή" - hun "A(z) '%-.192s' tablakezelonek nincs ilyen opcioja" - ita "Il gestore delle tabelle per '%-.192s' non ha questa opzione" - jpn "Table handler for '%-.192s' doesn't have this option" - kor "'%-.192s'의 테이블 handler는 이러한 옵션을 제공하지 않읍니다." - nor "Tabell håndtereren for '%-.192s' har ikke denne muligheten" - norwegian-ny "Tabell håndteraren for '%-.192s' har ikkje denne moglegheita" - pol "Obsługa tabeli '%-.192s' nie posiada tej opcji" - por "Manipulador de tabela para '%-.192s' não tem esta opção" - rum "Handlerul tabelei pentru '%-.192s' nu are aceasta optiune" - rus "Обработчик таблицы '%-.192s' не поддерживает эту возможность" - serbian "Handler tabela za '%-.192s' nema ovu opciju" - slo "Obsluha tabuľky '%-.192s' nemá tento parameter" - spa "El manejador de la tabla de '%-.192s' no tiene esta opcion" - swe "Tabellhanteraren for tabell '%-.192s' stödjer ej detta" - ukr "Дескриптор таблиці '%-.192s' не має цієї властивості" + eng "Storage engine %s of the table %`s.%`s doesn't have this option" + ger "Diese Option gibt es nicht in Speicher-Engine %s für %`s.%`s" + rus "Обработчик %s таблицы %`s.%`s не поддерживает эту возможность" + ukr "Дескриптор %s таблиці %`s.%`s не має цієї властивості" ER_KEY_NOT_FOUND cze "Nemohu naj-Bít záznam v '%-.192s'" dan "Kan ikke finde posten i '%-.192s'" @@ -919,30 +892,30 @@ ER_OUT_OF_SORTMEMORY HY001 S1001 swe "Sorteringsbufferten räcker inte till. Kontrollera startparametrarna" ukr "Брак пам'яті для сортування. Треба збільшити розмір буфера сортування у сервера" ER_UNEXPECTED_EOF - cze "Neo-Bčekávaný konec souboru při čtení '%-.192s' (chybový kód: %d)" - dan "Uventet afslutning på fil (eof) ved læsning af filen '%-.192s' (Fejlkode: %d)" - nla "Onverwachte eof gevonden tijdens het lezen van file '%-.192s' (Errcode: %d)" - eng "Unexpected EOF found when reading file '%-.192s' (errno: %d)" - jps "'%-.192s' ファイルを読み込み中に EOF が予期せぬ所で現れました. (errno: %d)", - est "Ootamatu faililõpumärgend faili '%-.192s' lugemisel (veakood: %d)" - fre "Fin de fichier inattendue en lisant '%-.192s' (Errcode: %d)" - ger "Unerwartetes Ende beim Lesen der Datei '%-.192s' (Fehler: %d)" - greek "Κατά τη διάρκεια της ανάγνωσης, βρέθηκε απροσδόκητα το τέλος του αρχείου '%-.192s' (κωδικός λάθους: %d)" - hun "Varatlan filevege-jel a '%-.192s'olvasasakor. (hibakod: %d)" - ita "Fine del file inaspettata durante la lettura del file '%-.192s' (errno: %d)" - jpn "'%-.192s' ファイルを読み込み中に EOF が予期せぬ所で現れました. (errno: %d)" - kor "'%-.192s' 화일을 읽는 도중 잘못된 eof을 발견 (에러번호: %d)" - nor "Uventet slutt på fil (eof) ved lesing av filen '%-.192s' (Feilkode: %d)" - norwegian-ny "Uventa slutt på fil (eof) ved lesing av fila '%-.192s' (Feilkode: %d)" - pol "Nieoczekiwany 'eof' napotkany podczas czytania z pliku '%-.192s' (Kod błędu: %d)" - por "Encontrado fim de arquivo inesperado ao ler arquivo '%-.192s' (erro no. %d)" - rum "Sfirsit de fisier neasteptat in citirea fisierului '%-.192s' (errno: %d)" - rus "Неожиданный конец файла '%-.192s' (ошибка: %d)" - serbian "Neočekivani kraj pri čitanju file-a '%-.192s' (errno: %d)" - slo "Neočakávaný koniec súboru pri čítaní '%-.192s' (chybový kód: %d)" - spa "Inesperado fin de ficheroU mientras leiamos el archivo '%-.192s' (Error: %d)" - swe "Oväntat filslut vid läsning från '%-.192s' (Felkod: %d)" - ukr "Хибний кінець файлу '%-.192s' (помилка: %d)" + cze "Neo-Bčekávaný konec souboru při čtení '%-.192s' (chybový kód: %M)" + dan "Uventet afslutning på fil (eof) ved læsning af filen '%-.192s' (Fejlkode: %M)" + nla "Onverwachte eof gevonden tijdens het lezen van file '%-.192s' (Errcode: %M)" + eng "Unexpected EOF found when reading file '%-.192s' (errno: %M)" + jps "'%-.192s' ファイルを読み込み中に EOF が予期せぬ所で現れました. (errno: %M)", + est "Ootamatu faililõpumärgend faili '%-.192s' lugemisel (veakood: %M)" + fre "Fin de fichier inattendue en lisant '%-.192s' (Errcode: %M)" + ger "Unerwartetes Ende beim Lesen der Datei '%-.192s' (Fehler: %M)" + greek "Κατά τη διάρκεια της ανάγνωσης, βρέθηκε απροσδόκητα το τέλος του αρχείου '%-.192s' (κωδικός λάθους: %M)" + hun "Varatlan filevege-jel a '%-.192s'olvasasakor. (hibakod: %M)" + ita "Fine del file inaspettata durante la lettura del file '%-.192s' (errno: %M)" + jpn "'%-.192s' ファイルを読み込み中に EOF が予期せぬ所で現れました. (errno: %M)" + kor "'%-.192s' 화일을 읽는 도중 잘못된 eof을 발견 (에러번호: %M)" + nor "Uventet slutt på fil (eof) ved lesing av filen '%-.192s' (Feilkode: %M)" + norwegian-ny "Uventa slutt på fil (eof) ved lesing av fila '%-.192s' (Feilkode: %M)" + pol "Nieoczekiwany 'eof' napotkany podczas czytania z pliku '%-.192s' (Kod błędu: %M)" + por "Encontrado fim de arquivo inesperado ao ler arquivo '%-.192s' (erro no. %M)" + rum "Sfirsit de fisier neasteptat in citirea fisierului '%-.192s' (errno: %M)" + rus "Неожиданный конец файла '%-.192s' (ошибка: %M)" + serbian "Neočekivani kraj pri čitanju file-a '%-.192s' (errno: %M)" + slo "Neočakávaný koniec súboru pri čítaní '%-.192s' (chybový kód: %M)" + spa "Inesperado fin de ficheroU mientras leiamos el archivo '%-.192s' (Error: %M)" + swe "Oväntat filslut vid läsning från '%-.192s' (Felkod: %M)" + ukr "Хибний кінець файлу '%-.192s' (помилка: %M)" ER_CON_COUNT_ERROR 08004 cze "P-Bříliš mnoho spojení" dan "For mange forbindelser (connections)" @@ -1750,28 +1723,10 @@ ER_KEY_COLUMN_DOES_NOT_EXITS 42000 S1009 swe "Nyckelkolumn '%-.192s' finns inte" ukr "Ключовий стовбець '%-.192s' не існує у таблиці" ER_BLOB_USED_AS_KEY 42000 S1009 - cze "Blob sloupec '%-.192s' nem-Bůže být použit jako klíč" - dan "BLOB feltet '%-.192s' kan ikke bruges ved specifikation af indeks" - nla "BLOB kolom '%-.192s' kan niet gebruikt worden bij zoeksleutel specificatie" - eng "BLOB column '%-.192s' can't be used in key specification with the used table type" - est "BLOB-tüüpi tulpa '%-.192s' ei saa kasutada võtmena" - fre "Champ BLOB '%-.192s' ne peut être utilisé dans une clé" - ger "BLOB-Feld '%-.192s' kann beim verwendeten Tabellentyp nicht als Schlüssel verwendet werden" - greek "Πεδίο τύπου Blob '%-.192s' δεν μπορεί να χρησιμοποιηθεί στον ορισμό ενός κλειδιού (key specification)" - hun "Blob objektum '%-.192s' nem hasznalhato kulcskent" - ita "La colonna BLOB '%-.192s' non puo` essere usata nella specifica della chiave" - kor "BLOB 칼럼 '%-.192s'는 키 정의에서 사용될 수 없습니다." - nor "Blob felt '%-.192s' kan ikke brukes ved spesifikasjon av nøkler" - norwegian-ny "Blob kolonne '%-.192s' kan ikkje brukast ved spesifikasjon av nyklar" - pol "Kolumna typu Blob '%-.192s' nie może być użyta w specyfikacji klucza" - por "Coluna BLOB '%-.192s' não pode ser utilizada na especificação de chave para o tipo de tabela usado" - rum "Coloana de tip BLOB '%-.192s' nu poate fi folosita in specificarea cheii cu tipul de tabla folosit" - rus "Столбец типа BLOB '%-.192s' не может быть использован как значение ключа в таблице такого типа" - serbian "BLOB kolona '%-.192s' ne može biti upotrebljena za navođenje ključa sa tipom tabele koji se trenutno koristi" - slo "Blob pole '%-.192s' nemôže byť použité ako kľúč" - spa "La columna Blob '%-.192s' no puede ser usada en una declaracion de clave" - swe "En BLOB '%-.192s' kan inte vara nyckel med den använda tabelltypen" - ukr "BLOB стовбець '%-.192s' не може бути використаний у визначенні ключа в цьому типі таблиці" + eng "BLOB column %`s can't be used in key specification in the %s table" + ger "BLOB-Feld %`s kann beim %s Tabellen nicht als Schlüssel verwendet werden" + rus "Столбец типа BLOB %`s не может быть использован как значение ключа в %s таблице" + ukr "BLOB стовбець %`s не може бути використаний у визначенні ключа в %s таблиці" ER_TOO_BIG_FIELDLENGTH 42000 S1009 cze "P-Bříliš velká délka sloupce '%-.192s' (nejvíce %lu). Použijte BLOB" dan "For stor feltlængde for kolonne '%-.192s' (maks = %lu). Brug BLOB i stedet" @@ -2785,7 +2740,7 @@ ER_TOO_BIG_ROWSIZE 42000 cze "-BŘádek je příliš velký. Maximální velikost řádku, nepočítaje položky blob, je %ld. Musíte změnit některé položky na blob" dan "For store poster. Max post størrelse, uden BLOB's, er %ld. Du må lave nogle felter til BLOB's" nla "Rij-grootte is groter dan toegestaan. Maximale rij grootte, blobs niet meegeteld, is %ld. U dient sommige velden in blobs te veranderen." - eng "Row size too large. The maximum row size for the used table type, not counting BLOBs, is %ld. You have to change some columns to TEXT or BLOBs" + eng "Row size too large. The maximum row size for the used table type, not counting BLOBs, is %ld. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs" jps "row size が大きすぎます. BLOB を含まない場合の row size の最大は %ld です. いくつかの field を BLOB に変えてください.", est "Liiga pikk kirje. Kirje maksimumpikkus arvestamata BLOB-tüüpi välju on %ld. Muuda mõned väljad BLOB-tüüpi väljadeks" fre "Ligne trop grande. Le taille maximale d'une ligne, sauf les BLOBs, est %ld. Changez le type de quelques colonnes en BLOB" @@ -3087,7 +3042,7 @@ ER_PASSWORD_NOT_ALLOWED 42000 spa "Tu debes de tener permiso para actualizar tablas en la base de datos mysql para cambiar las claves para otros" swe "För att ändra lösenord för andra måste du ha rättigheter att uppdatera mysql-databasen" ukr "Ви повині мати право на оновлення таблиць у базі данних mysql, аби мати можливість змінювати пароль іншим" -ER_PASSWORD_NO_MATCH 42000 +ER_PASSWORD_NO_MATCH 28000 cze "V tabulce user nen-Bí žádný odpovídající řádek" dan "Kan ikke finde nogen tilsvarende poster i bruger tabellen" nla "Kan geen enkele passende rij vinden in de gebruikers tabel" @@ -3127,28 +3082,28 @@ ER_UPDATE_INFO swe "Rader: %ld Uppdaterade: %ld Varningar: %ld" ukr "Записів відповідає: %ld Змінено: %ld Застережень: %ld" ER_CANT_CREATE_THREAD - cze "Nemohu vytvo-Břit nový thread (errno %d). Pokud je ještě nějaká volná paměť, podívejte se do manuálu na část o chybách specifických pro jednotlivé operační systémy" - dan "Kan ikke danne en ny tråd (fejl nr. %d). Hvis computeren ikke er løbet tør for hukommelse, kan du se i brugervejledningen for en mulig operativ-system - afhængig fejl" - nla "Kan geen nieuwe thread aanmaken (Errcode: %d). Indien er geen tekort aan geheugen is kunt u de handleiding consulteren over een mogelijke OS afhankelijke fout" - eng "Can't create a new thread (errno %d); if you are not out of available memory, you can consult the manual for a possible OS-dependent bug" - jps "新規にスレッドが作れませんでした (errno %d). もし最大使用許可メモリー数を越えていないのにエラーが発生しているなら, マニュアルの中から 'possible OS-dependent bug' という文字を探してくみてださい.", - est "Ei suuda luua uut lõime (veakood %d). Kui mälu ei ole otsas, on tõenäoliselt tegemist operatsioonisüsteemispetsiifilise veaga" - fre "Impossible de créer une nouvelle tâche (errno %d). S'il reste de la mémoire libre, consultez le manual pour trouver un éventuel bug dépendant de l'OS" - ger "Kann keinen neuen Thread erzeugen (Fehler: %d). Sollte noch Speicher verfügbar sein, bitte im Handbuch wegen möglicher Fehler im Betriebssystem nachschlagen" - hun "Uj thread letrehozasa nem lehetseges (Hibakod: %d). Amenyiben van meg szabad memoria, olvassa el a kezikonyv operacios rendszerfuggo hibalehetosegekrol szolo reszet" - ita "Impossibile creare un nuovo thread (errno %d). Se non ci sono problemi di memoria disponibile puoi consultare il manuale per controllare possibili problemi dipendenti dal SO" - jpn "新規にスレッドが作れませんでした (errno %d). もし最大使用許可メモリー数を越えていないのにエラーが発生しているなら, マニュアルの中から 'possible OS-dependent bug' という文字を探してくみてださい." - kor "새로운 쓰레드를 만들 수 없습니다.(에러번호 %d). 만약 여유메모리가 있다면 OS-dependent버그 의 메뉴얼 부분을 찾아보시오." - nor "Can't create a new thread (errno %d); if you are not out of available memory you can consult the manual for any possible OS dependent bug" - norwegian-ny "Can't create a new thread (errno %d); if you are not out of available memory you can consult the manual for any possible OS dependent bug" - pol "Can't create a new thread (errno %d); if you are not out of available memory you can consult the manual for any possible OS dependent bug" - por "Não pode criar uma nova 'thread' (erro no. %d). Se você não estiver sem memória disponível, você pode consultar o manual sobre um possível 'bug' dependente do sistema operacional" - rum "Nu pot crea un thread nou (Eroare %d). Daca mai aveti memorie disponibila in sistem, puteti consulta manualul - ar putea exista un potential bug in legatura cu sistemul de operare" - rus "Невозможно создать новый поток (ошибка %d). Если это не ситуация, связанная с нехваткой памяти, то вам следует изучить документацию на предмет описания возможной ошибки работы в конкретной ОС" - serbian "Ne mogu da kreiram novi thread (errno %d). Ako imate još slobodne memorije, trebali biste da pogledate u priručniku da li je ovo specifična greška vašeg operativnog sistema" - spa "No puedo crear un nuevo thread (errno %d). Si tu está con falta de memoria disponible, tu puedes consultar el Manual para posibles problemas con SO" - swe "Kan inte skapa en ny tråd (errno %d)" - ukr "Не можу створити нову гілку (помилка %d). Якщо ви не використали усю пам'ять, то прочитайте документацію до вашої ОС - можливо це помилка ОС" + cze "Nemohu vytvo-Břit nový thread (errno %M). Pokud je ještě nějaká volná paměť, podívejte se do manuálu na část o chybách specifických pro jednotlivé operační systémy" + dan "Kan ikke danne en ny tråd (fejl nr. %M). Hvis computeren ikke er løbet tør for hukommelse, kan du se i brugervejledningen for en mulig operativ-system - afhængig fejl" + nla "Kan geen nieuwe thread aanmaken (Errcode: %M). Indien er geen tekort aan geheugen is kunt u de handleiding consulteren over een mogelijke OS afhankelijke fout" + eng "Can't create a new thread (errno %M); if you are not out of available memory, you can consult the manual for a possible OS-dependent bug" + jps "新規にスレッドが作れませんでした (errno %M). もし最大使用許可メモリー数を越えていないのにエラーが発生しているなら, マニュアルの中から 'possible OS-dependent bug' という文字を探してくみてださい.", + est "Ei suuda luua uut lõime (veakood %M). Kui mälu ei ole otsas, on tõenäoliselt tegemist operatsioonisüsteemispetsiifilise veaga" + fre "Impossible de créer une nouvelle tâche (errno %M). S'il reste de la mémoire libre, consultez le manual pour trouver un éventuel bug dépendant de l'OS" + ger "Kann keinen neuen Thread erzeugen (Fehler: %M). Sollte noch Speicher verfügbar sein, bitte im Handbuch wegen möglicher Fehler im Betriebssystem nachschlagen" + hun "Uj thread letrehozasa nem lehetseges (Hibakod: %M). Amenyiben van meg szabad memoria, olvassa el a kezikonyv operacios rendszerfuggo hibalehetosegekrol szolo reszet" + ita "Impossibile creare un nuovo thread (errno %M). Se non ci sono problemi di memoria disponibile puoi consultare il manuale per controllare possibili problemi dipendenti dal SO" + jpn "新規にスレッドが作れませんでした (errno %M). もし最大使用許可メモリー数を越えていないのにエラーが発生しているなら, マニュアルの中から 'possible OS-dependent bug' という文字を探してくみてださい." + kor "새로운 쓰레드를 만들 수 없습니다.(에러번호 %M). 만약 여유메모리가 있다면 OS-dependent버그 의 메뉴얼 부분을 찾아보시오." + nor "Can't create a new thread (errno %M); if you are not out of available memory you can consult the manual for any possible OS dependent bug" + norwegian-ny "Can't create a new thread (errno %M); if you are not out of available memory you can consult the manual for any possible OS dependent bug" + pol "Can't create a new thread (errno %M); if you are not out of available memory you can consult the manual for any possible OS dependent bug" + por "Não pode criar uma nova 'thread' (erro no. %M). Se você não estiver sem memória disponível, você pode consultar o manual sobre um possível 'bug' dependente do sistema operacional" + rum "Nu pot crea un thread nou (Eroare %M). Daca mai aveti memorie disponibila in sistem, puteti consulta manualul - ar putea exista un potential bug in legatura cu sistemul de operare" + rus "Невозможно создать новый поток (ошибка %M). Если это не ситуация, связанная с нехваткой памяти, то вам следует изучить документацию на предмет описания возможной ошибки работы в конкретной ОС" + serbian "Ne mogu da kreiram novi thread (errno %M). Ako imate još slobodne memorije, trebali biste da pogledate u priručniku da li je ovo specifična greška vašeg operativnog sistema" + spa "No puedo crear un nuevo thread (errno %M). Si tu está con falta de memoria disponible, tu puedes consultar el Manual para posibles problemas con SO" + swe "Kan inte skapa en ny tråd (errno %M)" + ukr "Не можу створити нову гілку (помилка %M). Якщо ви не використали усю пам'ять, то прочитайте документацію до вашої ОС - можливо це помилка ОС" ER_WRONG_VALUE_COUNT_ON_ROW 21S01 cze "Po-Bčet sloupců neodpovídá počtu hodnot na řádku %lu" dan "Kolonne antallet stemmer ikke overens med antallet af værdier i post %lu" @@ -3265,45 +3220,45 @@ ER_NONEXISTING_GRANT 42000 swe "Det finns inget privilegium definierat för användare '%-.48s' på '%-.64s'" ukr "Повноважень не визначено для користувача '%-.48s' з хосту '%-.64s'" ER_TABLEACCESS_DENIED_ERROR 42000 - cze "%-.16s p-Bříkaz nepřístupný pro uživatele: '%s'@'%s' pro tabulku '%-.192s'" - dan "%-.16s-kommandoen er ikke tilladt for brugeren '%s'@'%s' for tabellen '%-.192s'" - nla "%-.16s commando geweigerd voor gebruiker: '%s'@'%s' voor tabel '%-.192s'" - eng "%-.16s command denied to user '%s'@'%s' for table '%-.192s'" - jps "コマンド %-.16s は ユーザー '%s'@'%s' ,テーブル '%-.192s' に対して許可されていません", - est "%-.16s käsk ei ole lubatud kasutajale '%s'@'%s' tabelis '%-.192s'" - fre "La commande '%-.16s' est interdite à l'utilisateur: '%s'@'%s' sur la table '%-.192s'" - ger "%-.16s Befehl nicht erlaubt für Benutzer '%s'@'%s' auf Tabelle '%-.192s'" - hun "%-.16s parancs a '%s'@'%s' felhasznalo szamara nem engedelyezett a '%-.192s' tablaban" - ita "Comando %-.16s negato per l'utente: '%s'@'%s' sulla tabella '%-.192s'" - jpn "コマンド %-.16s は ユーザー '%s'@'%s' ,テーブル '%-.192s' に対して許可されていません" - kor "'%-.16s' 명령은 다음 사용자에게 거부되었습니다. : '%s'@'%s' for 테이블 '%-.192s'" - por "Comando '%-.16s' negado para o usuário '%s'@'%s' na tabela '%-.192s'" - rum "Comanda %-.16s interzisa utilizatorului: '%s'@'%s' pentru tabela '%-.192s'" - rus "Команда %-.16s запрещена пользователю '%s'@'%s' для таблицы '%-.192s'" - serbian "%-.16s komanda zabranjena za korisnika '%s'@'%s' za tabelu '%-.192s'" - spa "%-.16s comando negado para usuario: '%s'@'%s' para tabla '%-.192s'" - swe "%-.16s ej tillåtet för '%s'@'%s' för tabell '%-.192s'" - ukr "%-.16s команда заборонена користувачу: '%s'@'%s' у таблиці '%-.192s'" + cze "%-.32s p-Bříkaz nepřístupný pro uživatele: '%s'@'%s' pro tabulku '%-.192s'" + dan "%-.32s-kommandoen er ikke tilladt for brugeren '%s'@'%s' for tabellen '%-.192s'" + nla "%-.32s commando geweigerd voor gebruiker: '%s'@'%s' voor tabel '%-.192s'" + eng "%-.32s command denied to user '%s'@'%s' for table '%-.192s'" + jps "コマンド %-.32s は ユーザー '%s'@'%s' ,テーブル '%-.192s' に対して許可されていません", + est "%-.32s käsk ei ole lubatud kasutajale '%s'@'%s' tabelis '%-.192s'" + fre "La commande '%-.32s' est interdite à l'utilisateur: '%s'@'%s' sur la table '%-.192s'" + ger "%-.32s Befehl nicht erlaubt für Benutzer '%s'@'%s' auf Tabelle '%-.192s'" + hun "%-.32s parancs a '%s'@'%s' felhasznalo szamara nem engedelyezett a '%-.192s' tablaban" + ita "Comando %-.32s negato per l'utente: '%s'@'%s' sulla tabella '%-.192s'" + jpn "コマンド %-.32s は ユーザー '%s'@'%s' ,テーブル '%-.192s' に対して許可されていません" + kor "'%-.32s' 명령은 다음 사용자에게 거부되었습니다. : '%s'@'%s' for 테이블 '%-.192s'" + por "Comando '%-.32s' negado para o usuário '%s'@'%s' na tabela '%-.192s'" + rum "Comanda %-.32s interzisa utilizatorului: '%s'@'%s' pentru tabela '%-.192s'" + rus "Команда %-.32s запрещена пользователю '%s'@'%s' для таблицы '%-.192s'" + serbian "%-.32s komanda zabranjena za korisnika '%s'@'%s' za tabelu '%-.192s'" + spa "%-.32s comando negado para usuario: '%s'@'%s' para tabla '%-.192s'" + swe "%-.32s ej tillåtet för '%s'@'%s' för tabell '%-.192s'" + ukr "%-.32s команда заборонена користувачу: '%s'@'%s' у таблиці '%-.192s'" ER_COLUMNACCESS_DENIED_ERROR 42000 - cze "%-.16s p-Bříkaz nepřístupný pro uživatele: '%s'@'%s' pro sloupec '%-.192s' v tabulce '%-.192s'" - dan "%-.16s-kommandoen er ikke tilladt for brugeren '%s'@'%s' for kolonne '%-.192s' in tabellen '%-.192s'" - nla "%-.16s commando geweigerd voor gebruiker: '%s'@'%s' voor kolom '%-.192s' in tabel '%-.192s'" - eng "%-.16s command denied to user '%s'@'%s' for column '%-.192s' in table '%-.192s'" - jps "コマンド %-.16s は ユーザー '%s'@'%s'¥n カラム '%-.192s' テーブル '%-.192s' に対して許可されていません", - est "%-.16s käsk ei ole lubatud kasutajale '%s'@'%s' tulbale '%-.192s' tabelis '%-.192s'" - fre "La commande '%-.16s' est interdite à l'utilisateur: '%s'@'%s' sur la colonne '%-.192s' de la table '%-.192s'" - ger "%-.16s Befehl nicht erlaubt für Benutzer '%s'@'%s' und Feld '%-.192s' in Tabelle '%-.192s'" - hun "%-.16s parancs a '%s'@'%s' felhasznalo szamara nem engedelyezett a '%-.192s' mezo eseten a '%-.192s' tablaban" - ita "Comando %-.16s negato per l'utente: '%s'@'%s' sulla colonna '%-.192s' della tabella '%-.192s'" - jpn "コマンド %-.16s は ユーザー '%s'@'%s'\n カラム '%-.192s' テーブル '%-.192s' に対して許可されていません" - kor "'%-.16s' 명령은 다음 사용자에게 거부되었습니다. : '%s'@'%s' for 칼럼 '%-.192s' in 테이블 '%-.192s'" - por "Comando '%-.16s' negado para o usuário '%s'@'%s' na coluna '%-.192s', na tabela '%-.192s'" - rum "Comanda %-.16s interzisa utilizatorului: '%s'@'%s' pentru coloana '%-.192s' in tabela '%-.192s'" - rus "Команда %-.16s запрещена пользователю '%s'@'%s' для столбца '%-.192s' в таблице '%-.192s'" - serbian "%-.16s komanda zabranjena za korisnika '%s'@'%s' za kolonu '%-.192s' iz tabele '%-.192s'" - spa "%-.16s comando negado para usuario: '%s'@'%s' para columna '%-.192s' en la tabla '%-.192s'" - swe "%-.16s ej tillåtet för '%s'@'%s' för kolumn '%-.192s' i tabell '%-.192s'" - ukr "%-.16s команда заборонена користувачу: '%s'@'%s' для стовбця '%-.192s' у таблиці '%-.192s'" + cze "%-.32s p-Bříkaz nepřístupný pro uživatele: '%s'@'%s' pro sloupec '%-.192s' v tabulce '%-.192s'" + dan "%-.32s-kommandoen er ikke tilladt for brugeren '%s'@'%s' for kolonne '%-.192s' in tabellen '%-.192s'" + nla "%-.32s commando geweigerd voor gebruiker: '%s'@'%s' voor kolom '%-.192s' in tabel '%-.192s'" + eng "%-.32s command denied to user '%s'@'%s' for column '%-.192s' in table '%-.192s'" + jps "コマンド %-.32s は ユーザー '%s'@'%s'¥n カラム '%-.192s' テーブル '%-.192s' に対して許可されていません", + est "%-.32s käsk ei ole lubatud kasutajale '%s'@'%s' tulbale '%-.192s' tabelis '%-.192s'" + fre "La commande '%-.32s' est interdite à l'utilisateur: '%s'@'%s' sur la colonne '%-.192s' de la table '%-.192s'" + ger "%-.32s Befehl nicht erlaubt für Benutzer '%s'@'%s' und Feld '%-.192s' in Tabelle '%-.192s'" + hun "%-.32s parancs a '%s'@'%s' felhasznalo szamara nem engedelyezett a '%-.192s' mezo eseten a '%-.192s' tablaban" + ita "Comando %-.32s negato per l'utente: '%s'@'%s' sulla colonna '%-.192s' della tabella '%-.192s'" + jpn "コマンド %-.32s は ユーザー '%s'@'%s'\n カラム '%-.192s' テーブル '%-.192s' に対して許可されていません" + kor "'%-.32s' 명령은 다음 사용자에게 거부되었습니다. : '%s'@'%s' for 칼럼 '%-.192s' in 테이블 '%-.192s'" + por "Comando '%-.32s' negado para o usuário '%s'@'%s' na coluna '%-.192s', na tabela '%-.192s'" + rum "Comanda %-.32s interzisa utilizatorului: '%s'@'%s' pentru coloana '%-.192s' in tabela '%-.192s'" + rus "Команда %-.32s запрещена пользователю '%s'@'%s' для столбца '%-.192s' в таблице '%-.192s'" + serbian "%-.32s komanda zabranjena za korisnika '%s'@'%s' za kolonu '%-.192s' iz tabele '%-.192s'" + spa "%-.32s comando negado para usuario: '%s'@'%s' para columna '%-.192s' en la tabla '%-.192s'" + swe "%-.32s ej tillåtet för '%s'@'%s' för kolumn '%-.192s' i tabell '%-.192s'" + ukr "%-.32s команда заборонена користувачу: '%s'@'%s' для стовбця '%-.192s' у таблиці '%-.192s'" ER_ILLEGAL_GRANT_FOR_TABLE 42000 cze "Neplatn-Bý příkaz GRANT/REVOKE. Prosím, přečtěte si v manuálu, jaká privilegia je možné použít." dan "Forkert GRANT/REVOKE kommando. Se i brugervejledningen hvilke privilegier der kan specificeres." @@ -3668,39 +3623,39 @@ ER_TOO_LONG_STRING 42000 swe "Resultatsträngen är längre än max_allowed_packet" ukr "Строка результату довша ніж max_allowed_packet" ER_TABLE_CANT_HANDLE_BLOB 42000 - cze "Typ pou-Bžité tabulky nepodporuje BLOB/TEXT sloupce" - dan "Denne tabeltype understøtter ikke brug af BLOB og TEXT kolonner" - nla "Het gebruikte tabel type ondersteunt geen BLOB/TEXT kolommen" - eng "The used table type doesn't support BLOB/TEXT columns" - est "Valitud tabelitüüp ei toeta BLOB/TEXT tüüpi välju" - fre "Ce type de table ne supporte pas les colonnes BLOB/TEXT" - ger "Der verwendete Tabellentyp unterstützt keine BLOB- und TEXT-Felder" - hun "A hasznalt tabla tipus nem tamogatja a BLOB/TEXT mezoket" - ita "Il tipo di tabella usata non supporta colonne di tipo BLOB/TEXT" - por "Tipo de tabela usado não permite colunas BLOB/TEXT" - rum "Tipul de tabela folosit nu suporta coloane de tip BLOB/TEXT" - rus "Используемая таблица не поддерживает типы BLOB/TEXT" - serbian "Iskorišteni tip tabele ne podržava kolone tipa 'BLOB' odnosno 'TEXT'" - spa "El tipo de tabla usada no permite soporte para columnas BLOB/TEXT" - swe "Den använda tabelltypen kan inte hantera BLOB/TEXT-kolumner" - ukr "Використаний тип таблиці не підтримує BLOB/TEXT стовбці" + cze "Typ pou-Bžité tabulky (%s) nepodporuje BLOB/TEXT sloupce" + dan "Denne tabeltype (%s) understøtter ikke brug af BLOB og TEXT kolonner" + nla "Het gebruikte tabel type (%s) ondersteunt geen BLOB/TEXT kolommen" + eng "Storage engine %s doesn't support BLOB/TEXT columns" + est "Valitud tabelitüüp (%s) ei toeta BLOB/TEXT tüüpi välju" + fre "Ce type de table (%s) ne supporte pas les colonnes BLOB/TEXT" + ger "Der verwendete Tabellentyp (%s) unterstützt keine BLOB- und TEXT-Felder" + hun "A hasznalt tabla tipus (%s) nem tamogatja a BLOB/TEXT mezoket" + ita "Il tipo di tabella usata (%s) non supporta colonne di tipo BLOB/TEXT" + por "Tipo de tabela usado (%s) não permite colunas BLOB/TEXT" + rum "Tipul de tabela folosit (%s) nu suporta coloane de tip BLOB/TEXT" + rus "%s таблицы не поддерживают типы BLOB/TEXT" + serbian "Iskorišteni tip tabele (%s) ne podržava kolone tipa 'BLOB' odnosno 'TEXT'" + spa "El tipo de tabla usada (%s) no permite soporte para columnas BLOB/TEXT" + swe "Den använda tabelltypen (%s) kan inte hantera BLOB/TEXT-kolumner" + ukr "%s таблиці не підтримують BLOB/TEXT стовбці" ER_TABLE_CANT_HANDLE_AUTO_INCREMENT 42000 - cze "Typ pou-Bžité tabulky nepodporuje AUTO_INCREMENT sloupce" - dan "Denne tabeltype understøtter ikke brug af AUTO_INCREMENT kolonner" - nla "Het gebruikte tabel type ondersteunt geen AUTO_INCREMENT kolommen" - eng "The used table type doesn't support AUTO_INCREMENT columns" - est "Valitud tabelitüüp ei toeta AUTO_INCREMENT tüüpi välju" - fre "Ce type de table ne supporte pas les colonnes AUTO_INCREMENT" - ger "Der verwendete Tabellentyp unterstützt keine AUTO_INCREMENT-Felder" - hun "A hasznalt tabla tipus nem tamogatja az AUTO_INCREMENT tipusu mezoket" - ita "Il tipo di tabella usata non supporta colonne di tipo AUTO_INCREMENT" - por "Tipo de tabela usado não permite colunas AUTO_INCREMENT" - rum "Tipul de tabela folosit nu suporta coloane de tip AUTO_INCREMENT" - rus "Используемая таблица не поддерживает автоинкрементные столбцы" - serbian "Iskorišteni tip tabele ne podržava kolone tipa 'AUTO_INCREMENT'" - spa "El tipo de tabla usada no permite soporte para columnas AUTO_INCREMENT" - swe "Den använda tabelltypen kan inte hantera AUTO_INCREMENT-kolumner" - ukr "Використаний тип таблиці не підтримує AUTO_INCREMENT стовбці" + cze "Typ pou-Bžité tabulky (%s) nepodporuje AUTO_INCREMENT sloupce" + dan "Denne tabeltype understøtter (%s) ikke brug af AUTO_INCREMENT kolonner" + nla "Het gebruikte tabel type (%s) ondersteunt geen AUTO_INCREMENT kolommen" + eng "Storage engine %s doesn't support AUTO_INCREMENT columns" + est "Valitud tabelitüüp (%s) ei toeta AUTO_INCREMENT tüüpi välju" + fre "Ce type de table (%s) ne supporte pas les colonnes AUTO_INCREMENT" + ger "Der verwendete Tabellentyp (%s) unterstützt keine AUTO_INCREMENT-Felder" + hun "A hasznalt tabla tipus (%s) nem tamogatja az AUTO_INCREMENT tipusu mezoket" + ita "Il tipo di tabella usata (%s) non supporta colonne di tipo AUTO_INCREMENT" + por "Tipo de tabela usado (%s) não permite colunas AUTO_INCREMENT" + rum "Tipul de tabela folosit (%s) nu suporta coloane de tip AUTO_INCREMENT" + rus "%s таблицы не поддерживают автоинкрементные столбцы" + serbian "Iskorišteni tip tabele (%s) ne podržava kolone tipa 'AUTO_INCREMENT'" + spa "El tipo de tabla usada (%s) no permite soporte para columnas AUTO_INCREMENT" + swe "Den använda tabelltypen (%s) kan inte hantera AUTO_INCREMENT-kolumner" + ukr "%s таблиці не підтримують AUTO_INCREMENT стовбці" ER_DELAYED_INSERT_TABLE_LOCKED cze "INSERT DELAYED nen-Bí možno s tabulkou '%-.192s' použít, protože je zamčená pomocí LOCK TABLES" dan "INSERT DELAYED kan ikke bruges med tabellen '%-.192s', fordi tabellen er låst med LOCK TABLES" @@ -3743,29 +3698,10 @@ ER_WRONG_COLUMN_NAME 42000 swe "Felaktigt kolumnnamn '%-.100s'" ukr "Невірне ім'я стовбця '%-.100s'" ER_WRONG_KEY_COLUMN 42000 - cze "Handler pou-Bžité tabulky neumí indexovat sloupce '%-.192s'" - dan "Den brugte tabeltype kan ikke indeksere kolonnen '%-.192s'" - nla "De gebruikte tabel 'handler' kan kolom '%-.192s' niet indexeren" - eng "The used storage engine can't index column '%-.192s'" - est "Tabelihandler ei oska indekseerida tulpa '%-.192s'" - fre "Le handler de la table ne peut indexé la colonne '%-.192s'" - ger "Die verwendete Speicher-Engine kann die Spalte '%-.192s' nicht indizieren" - greek "The used table handler can't index column '%-.192s'" - hun "A hasznalt tablakezelo nem tudja a '%-.192s' mezot indexelni" - ita "Il gestore delle tabelle non puo` indicizzare la colonna '%-.192s'" - jpn "The used table handler can't index column '%-.192s'" - kor "The used table handler can't index column '%-.192s'" - nor "The used table handler can't index column '%-.192s'" - norwegian-ny "The used table handler can't index column '%-.192s'" - pol "The used table handler can't index column '%-.192s'" - por "O manipulador de tabela usado não pode indexar a coluna '%-.192s'" - rum "Handler-ul tabelei folosite nu poate indexa coloana '%-.192s'" - rus "Использованный обработчик таблицы не может проиндексировать столбец '%-.192s'" - serbian "Handler tabele ne može da indeksira kolonu '%-.192s'" - slo "The used table handler can't index column '%-.192s'" - spa "El manipulador de tabla usado no puede indexar columna '%-.192s'" - swe "Den använda tabelltypen kan inte indexera kolumn '%-.192s'" - ukr "Використаний вказівник таблиці не може індексувати стовбець '%-.192s'" + eng "The storage engine %s can't index column %`s" + ger "Die Speicher-Engine %s kann die Spalte %`s nicht indizieren" + rus "Обработчик таблиц %s не может проиндексировать столбец %`s" + ukr "Вказівник таблиц %s не може індексувати стовбець %`s" ER_WRONG_MRG_TABLE cze "V-Bšechny tabulky v MERGE tabulce nejsou definovány stejně" dan "Tabellerne i MERGE er ikke defineret ens" @@ -3987,69 +3923,69 @@ ER_CANT_DO_THIS_DURING_AN_TRANSACTION 25000 swe "Du får inte utföra detta kommando i en transaktion" ukr "Вам не дозволено виконувати цю команду в транзакції" ER_ERROR_DURING_COMMIT - cze "Chyba %d p-Bři COMMIT" - dan "Modtog fejl %d mens kommandoen COMMIT blev udført" - nla "Kreeg fout %d tijdens COMMIT" - eng "Got error %d during COMMIT" - est "Viga %d käsu COMMIT täitmisel" - fre "Erreur %d lors du COMMIT" - ger "Fehler %d beim COMMIT" - hun "%d hiba a COMMIT vegrehajtasa soran" - ita "Rilevato l'errore %d durante il COMMIT" - por "Obteve erro %d durante COMMIT" - rus "Получена ошибка %d в процессе COMMIT" - serbian "Greška %d za vreme izvršavanja komande 'COMMIT'" - spa "Obtenido error %d durante COMMIT" - swe "Fick fel %d vid COMMIT" - ukr "Отримано помилку %d під час COMMIT" + cze "Chyba %M p-Bři COMMIT" + dan "Modtog fejl %M mens kommandoen COMMIT blev udført" + nla "Kreeg fout %M tijdens COMMIT" + eng "Got error %M during COMMIT" + est "Viga %M käsu COMMIT täitmisel" + fre "Erreur %M lors du COMMIT" + ger "Fehler %M beim COMMIT" + hun "%M hiba a COMMIT vegrehajtasa soran" + ita "Rilevato l'errore %M durante il COMMIT" + por "Obteve erro %M durante COMMIT" + rus "Получена ошибка %M в процессе COMMIT" + serbian "Greška %M za vreme izvršavanja komande 'COMMIT'" + spa "Obtenido error %M durante COMMIT" + swe "Fick fel %M vid COMMIT" + ukr "Отримано помилку %M під час COMMIT" ER_ERROR_DURING_ROLLBACK - cze "Chyba %d p-Bři ROLLBACK" - dan "Modtog fejl %d mens kommandoen ROLLBACK blev udført" - nla "Kreeg fout %d tijdens ROLLBACK" - eng "Got error %d during ROLLBACK" - est "Viga %d käsu ROLLBACK täitmisel" - fre "Erreur %d lors du ROLLBACK" - ger "Fehler %d beim ROLLBACK" - hun "%d hiba a ROLLBACK vegrehajtasa soran" - ita "Rilevato l'errore %d durante il ROLLBACK" - por "Obteve erro %d durante ROLLBACK" - rus "Получена ошибка %d в процессе ROLLBACK" - serbian "Greška %d za vreme izvršavanja komande 'ROLLBACK'" - spa "Obtenido error %d durante ROLLBACK" - swe "Fick fel %d vid ROLLBACK" - ukr "Отримано помилку %d під час ROLLBACK" + cze "Chyba %M p-Bři ROLLBACK" + dan "Modtog fejl %M mens kommandoen ROLLBACK blev udført" + nla "Kreeg fout %M tijdens ROLLBACK" + eng "Got error %M during ROLLBACK" + est "Viga %M käsu ROLLBACK täitmisel" + fre "Erreur %M lors du ROLLBACK" + ger "Fehler %M beim ROLLBACK" + hun "%M hiba a ROLLBACK vegrehajtasa soran" + ita "Rilevato l'errore %M durante il ROLLBACK" + por "Obteve erro %M durante ROLLBACK" + rus "Получена ошибка %M в процессе ROLLBACK" + serbian "Greška %M za vreme izvršavanja komande 'ROLLBACK'" + spa "Obtenido error %M durante ROLLBACK" + swe "Fick fel %M vid ROLLBACK" + ukr "Отримано помилку %M під час ROLLBACK" ER_ERROR_DURING_FLUSH_LOGS - cze "Chyba %d p-Bři FLUSH_LOGS" - dan "Modtog fejl %d mens kommandoen FLUSH_LOGS blev udført" - nla "Kreeg fout %d tijdens FLUSH_LOGS" - eng "Got error %d during FLUSH_LOGS" - est "Viga %d käsu FLUSH_LOGS täitmisel" - fre "Erreur %d lors du FLUSH_LOGS" - ger "Fehler %d bei FLUSH_LOGS" - hun "%d hiba a FLUSH_LOGS vegrehajtasa soran" - ita "Rilevato l'errore %d durante il FLUSH_LOGS" - por "Obteve erro %d durante FLUSH_LOGS" - rus "Получена ошибка %d в процессе FLUSH_LOGS" - serbian "Greška %d za vreme izvršavanja komande 'FLUSH_LOGS'" - spa "Obtenido error %d durante FLUSH_LOGS" - swe "Fick fel %d vid FLUSH_LOGS" - ukr "Отримано помилку %d під час FLUSH_LOGS" + cze "Chyba %M p-Bři FLUSH_LOGS" + dan "Modtog fejl %M mens kommandoen FLUSH_LOGS blev udført" + nla "Kreeg fout %M tijdens FLUSH_LOGS" + eng "Got error %M during FLUSH_LOGS" + est "Viga %M käsu FLUSH_LOGS täitmisel" + fre "Erreur %M lors du FLUSH_LOGS" + ger "Fehler %M bei FLUSH_LOGS" + hun "%M hiba a FLUSH_LOGS vegrehajtasa soran" + ita "Rilevato l'errore %M durante il FLUSH_LOGS" + por "Obteve erro %M durante FLUSH_LOGS" + rus "Получена ошибка %M в процессе FLUSH_LOGS" + serbian "Greška %M za vreme izvršavanja komande 'FLUSH_LOGS'" + spa "Obtenido error %M durante FLUSH_LOGS" + swe "Fick fel %M vid FLUSH_LOGS" + ukr "Отримано помилку %M під час FLUSH_LOGS" ER_ERROR_DURING_CHECKPOINT - cze "Chyba %d p-Bři CHECKPOINT" - dan "Modtog fejl %d mens kommandoen CHECKPOINT blev udført" - nla "Kreeg fout %d tijdens CHECKPOINT" - eng "Got error %d during CHECKPOINT" - est "Viga %d käsu CHECKPOINT täitmisel" - fre "Erreur %d lors du CHECKPOINT" - ger "Fehler %d bei CHECKPOINT" - hun "%d hiba a CHECKPOINT vegrehajtasa soran" - ita "Rilevato l'errore %d durante il CHECKPOINT" - por "Obteve erro %d durante CHECKPOINT" - rus "Получена ошибка %d в процессе CHECKPOINT" - serbian "Greška %d za vreme izvršavanja komande 'CHECKPOINT'" - spa "Obtenido error %d durante CHECKPOINT" - swe "Fick fel %d vid CHECKPOINT" - ukr "Отримано помилку %d під час CHECKPOINT" + cze "Chyba %M p-Bři CHECKPOINT" + dan "Modtog fejl %M mens kommandoen CHECKPOINT blev udført" + nla "Kreeg fout %M tijdens CHECKPOINT" + eng "Got error %M during CHECKPOINT" + est "Viga %M käsu CHECKPOINT täitmisel" + fre "Erreur %M lors du CHECKPOINT" + ger "Fehler %M bei CHECKPOINT" + hun "%M hiba a CHECKPOINT vegrehajtasa soran" + ita "Rilevato l'errore %M durante il CHECKPOINT" + por "Obteve erro %M durante CHECKPOINT" + rus "Получена ошибка %M в процессе CHECKPOINT" + serbian "Greška %M za vreme izvršavanja komande 'CHECKPOINT'" + spa "Obtenido error %M durante CHECKPOINT" + swe "Fick fel %M vid CHECKPOINT" + ukr "Отримано помилку %M під час CHECKPOINT" ER_NEW_ABORTING_CONNECTION 08S01 cze "Spojen-Bí %ld do databáze: '%-.192s' uživatel: '%-.48s' stroj: '%-.64s' (%-.64s) bylo přerušeno" dan "Afbrød forbindelsen %ld til databasen '%-.192s' bruger: '%-.48s' vært: '%-.64s' (%-.64s)" @@ -4065,22 +4001,8 @@ ER_NEW_ABORTING_CONNECTION 08S01 spa "Abortada conexión %ld para db: '%-.192s' usuario: '%-.48s' servidor: '%-.64s' (%-.64s)" swe "Avbröt länken för tråd %ld till db '%-.192s', användare '%-.48s', host '%-.64s' (%-.64s)" ukr "Перервано з'єднання %ld до бази данних: '%-.192s' користувач: '%-.48s' хост: '%-.64s' (%-.64s)" -ER_DUMP_NOT_IMPLEMENTED - cze "Handler tabulky nepodporuje bin-Bární dump" - dan "Denne tabeltype unserstøtter ikke binært tabeldump" - nla "De 'handler' voor de tabel ondersteund geen binaire tabel dump" - eng "The storage engine for the table does not support binary table dump" - fre "Ce type de table ne supporte pas les copies binaires" - ger "Die Speicher-Engine für die Tabelle unterstützt keinen binären Tabellen-Dump" - ita "Il gestore per la tabella non supporta il dump binario" - jpn "The handler for the table does not support binary table dump" - por "O manipulador de tabela não suporta 'dump' binário de tabela" - rum "The handler for the table does not support binary table dump" - rus "Обработчик этой таблицы не поддерживает двоичного сохранения образа таблицы (dump)" - serbian "Handler tabele ne podržava binarni dump tabele" - spa "El manipulador de tabla no soporta dump para tabla binaria" - swe "Tabellhanteraren klarar inte en binär kopiering av tabellen" - ukr "Цей тип таблиці не підтримує бінарну передачу таблиці" +ER_unused_2 + eng "You should never see it" ER_FLUSH_MASTER_BINLOG_CLOSED eng "Binlog closed, cannot RESET MASTER" ger "Binlog geschlossen. Kann RESET MASTER nicht ausführen" @@ -4248,18 +4170,18 @@ ER_TRANS_CACHE_FULL swe "Transaktionen krävde mera än 'max_binlog_cache_size' minne. Öka denna mysqld-variabel och försök på nytt" ukr "Транзакція з багатьма виразами вимагає більше ніж 'max_binlog_cache_size' байтів для зберігання. Збільште цю змінну mysqld та спробуйте знову" ER_SLAVE_MUST_STOP - dan "Denne handling kunne ikke udføres med kørende slave, brug først kommandoen STOP SLAVE" - nla "Deze operatie kan niet worden uitgevoerd met een actieve slave, doe eerst STOP SLAVE" - eng "This operation cannot be performed with a running slave; run STOP SLAVE first" - fre "Cette opération ne peut être réalisée avec un esclave actif, faites STOP SLAVE d'abord" - ger "Diese Operation kann bei einem aktiven Slave nicht durchgeführt werden. Bitte zuerst STOP SLAVE ausführen" - ita "Questa operazione non puo' essere eseguita con un database 'slave' che gira, lanciare prima STOP SLAVE" - por "Esta operação não pode ser realizada com um 'slave' em execução. Execute STOP SLAVE primeiro" - rus "Эту операцию невозможно выполнить при работающем потоке подчиненного сервера. Сначала выполните STOP SLAVE" - serbian "Ova operacija ne može biti izvršena dok je aktivan podređeni server. Zadajte prvo komandu 'STOP SLAVE' da zaustavite podređeni server." - spa "Esta operación no puede ser hecha con el esclavo funcionando, primero use STOP SLAVE" - swe "Denna operation kan inte göras under replikering; Gör STOP SLAVE först" - ukr "Операція не може бути виконана з запущеним підлеглим, спочатку виконайте STOP SLAVE" + dan "Denne handling kunne ikke udføres med kørende slave '%2$*1$s', brug først kommandoen STOP SLAVE '%2$*1$s'" + nla "Deze operatie kan niet worden uitgevoerd met een actieve slave '%2$*1$s', doe eerst STOP SLAVE '%2$*1$s'" + eng "This operation cannot be performed as you have a running slave '%2$*1$s'; run STOP SLAVE '%2$*1$s' first" + fre "Cette opération ne peut être réalisée avec un esclave '%2$*1$s' actif, faites STOP SLAVE '%2$*1$s' d'abord" + ger "Diese Operation kann bei einem aktiven Slave '%2$*1$s' nicht durchgeführt werden. Bitte zuerst STOP SLAVE '%2$*1$s' ausführen" + ita "Questa operazione non puo' essere eseguita con un database 'slave' '%2$*1$s' che gira, lanciare prima STOP SLAVE '%2$*1$s'" + por "Esta operação não pode ser realizada com um 'slave' '%2$*1$s' em execução. Execute STOP SLAVE '%2$*1$s' primeiro" + rus "Эту операцию невозможно выполнить при работающем потоке подчиненного сервера %2$*1$s. Сначала выполните STOP SLAVE '%2$*1$s'" + serbian "Ova operacija ne može biti izvršena dok je aktivan podređeni '%2$*1$s' server. Zadajte prvo komandu 'STOP SLAVE '%2$*1$s'' da zaustavite podređeni server." + spa "Esta operación no puede ser hecha con el esclavo '%2$*1$s' funcionando, primero use STOP SLAVE '%2$*1$s'" + swe "Denna operation kan inte göras under replikering; Du har en aktiv förbindelse till '%2$*1$s'. Gör STOP SLAVE '%2$*1$s' först" + ukr "Операція не може бути виконана з запущеним підлеглим '%2$*1$s', спочатку виконайте STOP SLAVE '%2$*1$s'" ER_SLAVE_NOT_RUNNING dan "Denne handling kræver en kørende slave. Konfigurer en slave og brug kommandoen START SLAVE" nla "Deze operatie vereist een actieve slave, configureer slave en doe dan START SLAVE" @@ -4287,11 +4209,11 @@ ER_BAD_SLAVE swe "Servern är inte konfigurerade som en replikationsslav. Ändra konfigurationsfilen eller gör CHANGE MASTER TO" ukr "Сервер не зконфігуровано як підлеглий, виправте це у файлі конфігурації або з CHANGE MASTER TO" ER_MASTER_INFO - eng "Could not initialize master info structure; more error messages can be found in the MariaDB error log" - fre "Impossible d'initialiser les structures d'information de maître, vous trouverez des messages d'erreur supplémentaires dans le journal des erreurs de MariaDB" - ger "Konnte Master-Info-Struktur nicht initialisieren. Weitere Fehlermeldungen können im MariaDB-Error-Log eingesehen werden" - serbian "Nisam mogao da inicijalizujem informacionu strukturu glavnog servera, proverite da li imam privilegije potrebne za pristup file-u 'master.info'" - swe "Kunde inte initialisera replikationsstrukturerna. See MariaDB fel fil för mera information" + eng "Could not initialize master info structure for '%.*s'; more error messages can be found in the MariaDB error log" + fre "Impossible d'initialiser les structures d'information de maître '%.*s', vous trouverez des messages d'erreur supplémentaires dans le journal des erreurs de MariaDB" + ger "Konnte Master-Info-Struktur '%.*s' nicht initialisieren. Weitere Fehlermeldungen können im MariaDB-Error-Log eingesehen werden" + serbian "Nisam mogao da inicijalizujem informacionu strukturu glavnog servera, proverite da li imam privilegije potrebne za pristup file-u 'master.info' '%.*s'" + swe "Kunde inte initialisera replikationsstrukturerna för '%.*s'. See MariaDB fel fil för mera information" ER_SLAVE_THREAD dan "Kunne ikke danne en slave-tråd; check systemressourcerne" nla "Kon slave thread niet aanmaken, controleer systeem resources" @@ -4322,13 +4244,13 @@ ER_TOO_MANY_USER_CONNECTIONS 42000 ER_SET_CONSTANTS_ONLY dan "Du må kun bruge konstantudtryk med SET" nla "U mag alleen constante expressies gebruiken bij SET" - eng "You may only use constant expressions with SET" + eng "You may only use constant expressions in this statement" est "Ainult konstantsed suurused on lubatud SET klauslis" fre "Seules les expressions constantes sont autorisées avec SET" - ger "Bei SET dürfen nur konstante Ausdrücke verwendet werden" + ger "Bei diesem Befehl dürfen nur konstante Ausdrücke verwendet werden" ita "Si possono usare solo espressioni costanti con SET" por "Você pode usar apenas expressões constantes com SET" - rus "Вы можете использовать в SET только константные выражения" + rus "С этой командой вы можете использовать только константные выражения" serbian "Možete upotrebiti samo konstantan iskaz sa komandom 'SET'" spa "Tu solo debes usar expresiones constantes con SET" swe "Man kan endast använda konstantuttryck med SET" @@ -4454,18 +4376,18 @@ ER_LOCK_DEADLOCK 40001 spa "Encontrado deadlock cuando tentando obtener el bloqueo; Tente recomenzar la transición" swe "Fick 'DEADLOCK' vid låsförsök av block/rad. Försök att starta om transaktionen" ER_TABLE_CANT_HANDLE_FT - nla "Het gebruikte tabel type ondersteund geen FULLTEXT indexen" - eng "The used table type doesn't support FULLTEXT indexes" - est "Antud tabelitüüp ei toeta FULLTEXT indekseid" - fre "Le type de table utilisé ne supporte pas les index FULLTEXT" - ger "Der verwendete Tabellentyp unterstützt keine FULLTEXT-Indizes" - ita "La tabella usata non supporta gli indici FULLTEXT" - por "O tipo de tabela utilizado não suporta índices de texto completo (fulltext indexes)" - rus "Используемый тип таблиц не поддерживает полнотекстовых индексов" - serbian "Upotrebljeni tip tabele ne podržava 'FULLTEXT' indekse" - spa "El tipo de tabla usada no soporta índices FULLTEXT" - swe "Tabelltypen har inte hantering av FULLTEXT-index" - ukr "Використаний тип таблиці не підтримує FULLTEXT індексів" + nla "Het gebruikte tabel type (%s) ondersteund geen FULLTEXT indexen" + eng "The storage engine %s doesn't support FULLTEXT indexes" + est "Antud tabelitüüp (%s) ei toeta FULLTEXT indekseid" + fre "Le type de table utilisé (%s) ne supporte pas les index FULLTEXT" + ger "Der verwendete Tabellentyp (%s) unterstützt keine FULLTEXT-Indizes" + ita "La tabella usata (%s) non supporta gli indici FULLTEXT" + por "O tipo de tabela utilizado (%s) não suporta índices de texto completo (fulltext indexes)" + rus "Используемый тип таблиц (%s) не поддерживает полнотекстовых индексов" + serbian "Upotrebljeni tip tabele (%s) ne podržava 'FULLTEXT' indekse" + spa "El tipo de tabla usada (%s) no soporta índices FULLTEXT" + swe "Tabelltypen (%s) har inte hantering av FULLTEXT-index" + ukr "Використаний тип таблиці (%s) не підтримує FULLTEXT індексів" ER_CANNOT_ADD_FOREIGN nla "Kan foreign key beperking niet toevoegen" eng "Cannot add foreign key constraint" @@ -5307,8 +5229,8 @@ ER_VIEW_CHECK_FAILED rus "проверка CHECK OPTION для VIEW '%-.192s.%-.192s' провалилась" ukr "Перевірка CHECK OPTION для VIEW '%-.192s.%-.192s' не пройшла" ER_PROCACCESS_DENIED_ERROR 42000 - eng "%-.16s command denied to user '%s'@'%s' for routine '%-.192s'" - ger "Befehl %-.16s nicht zulässig für Benutzer '%s'@'%s' in Routine '%-.192s'" + eng "%-.32s command denied to user '%s'@'%s' for routine '%-.192s'" + ger "Befehl %-.32s nicht zulässig für Benutzer '%s'@'%s' in Routine '%-.192s'" ER_RELAY_LOG_FAIL eng "Failed purging old relay logs: %s" ger "Bereinigen alter Relais-Logs fehlgeschlagen: %s" @@ -5361,8 +5283,8 @@ ER_LOGGING_PROHIBIT_CHANGING_OF eng "Binary logging and replication forbid changing the global server %s" ger "Binärlogs und Replikation verhindern Wechsel des globalen Servers %s" ER_NO_FILE_MAPPING - eng "Can't map file: %-.200s, errno: %d" - ger "Kann Datei nicht abbilden: %-.200s, Fehler: %d" + eng "Can't map file: %-.200s, errno: %M" + ger "Kann Datei nicht abbilden: %-.200s, Fehler: %M" ER_WRONG_MAGIC eng "Wrong magic in %-.64s" ger "Falsche magische Zahlen in %-.64s" @@ -5525,7 +5447,7 @@ ER_WARN_CANT_DROP_DEFAULT_KEYCACHE ger "Der vorgabemäßige Schlüssel-Cache kann nicht gelöscht werden" ER_TOO_BIG_DISPLAYWIDTH 42000 S1009 eng "Display width out of range for '%-.192s' (max = %lu)" - ger "Anzeigebreite außerhalb des zulässigen Bereichs für '%-.192s' (Maximum: %lu)" + ger "Anzeigebreite außerhalb des zulässigen Bereichs für '%-.192s' (Maximum = %lu)" ER_XAER_DUPID XAE08 eng "XAER_DUPID: The XID already exists" ger "XAER_DUPID: Die XID existiert bereits" @@ -5544,9 +5466,8 @@ ER_PS_NO_RECURSION ER_SP_CANT_SET_AUTOCOMMIT eng "Not allowed to set autocommit from a stored function or trigger" ger "Es ist nicht erlaubt, innerhalb einer gespeicherten Funktion oder eines Triggers AUTOCOMMIT zu setzen" -ER_MALFORMED_DEFINER - eng "Definer is not fully qualified" - ger "Definierer des View ist nicht vollständig spezifiziert" +ER_MALFORMED_DEFINER 0L000 + eng "Invalid definer" ER_VIEW_FRM_NO_USER eng "View '%-.192s'.'%-.192s' has no definer information (old table format). Current user is used as definer. Please recreate the view!" ger "View '%-.192s'.'%-.192s' hat keine Definierer-Information (altes Tabellenformat). Der aktuelle Benutzer wird als Definierer verwendet. Bitte erstellen Sie den View neu" @@ -5599,8 +5520,8 @@ ER_NON_GROUPING_FIELD_USED 42000 eng "Non-grouping field '%-.192s' is used in %-.64s clause" ger "In der %-.192s-Klausel wird das die Nicht-Gruppierungsspalte '%-.64s' verwendet" ER_TABLE_CANT_HANDLE_SPKEYS - eng "The used table type doesn't support SPATIAL indexes" - ger "Der verwendete Tabellentyp unterstützt keine SPATIAL-Indizes" + eng "The storage engine %s doesn't support SPATIAL indexes" + ger "Der verwendete Tabellentyp (%s) unterstützt keine SPATIAL-Indizes" ER_NO_TRIGGERS_ON_SYSTEM_SCHEMA eng "Triggers can not be created on system tables" ger "Trigger können nicht auf Systemtabellen erzeugt werden" @@ -5863,8 +5784,8 @@ ER_EVENT_ALREADY_EXISTS eng "Event '%-.192s' already exists" ger "Event '%-.192s' existiert bereits" ER_EVENT_STORE_FAILED - eng "Failed to store event %s. Error code %d from storage engine." - ger "Speichern von Event %s fehlgeschlagen. Fehlercode der Speicher-Engine: %d" + eng "Failed to store event %s. Error code %M from storage engine." + ger "Speichern von Event %s fehlgeschlagen. Fehlercode der Speicher-Engine: %M" ER_EVENT_DOES_NOT_EXIST eng "Unknown event '%-.192s'" ger "Unbekanntes Event '%-.192s'" @@ -5968,8 +5889,8 @@ ER_EVENT_MODIFY_QUEUE_ERROR eng "Internal scheduler error %d" ger "Interner Scheduler-Fehler %d" ER_EVENT_SET_VAR_ERROR - eng "Error during starting/stopping of the scheduler. Error code %u" - ger "Fehler während des Startens oder Anhalten des Schedulers. Fehlercode %u" + eng "Error during starting/stopping of the scheduler. Error code %M" + ger "Fehler während des Startens oder Anhalten des Schedulers. Fehlercode %M" ER_PARTITION_MERGE_ERROR eng "Engine cannot be used in partitioned tables" ger "Engine kann in partitionierten Tabellen nicht verwendet werden" @@ -5994,8 +5915,8 @@ ER_ONLY_INTEGERS_ALLOWED eng "Only integers allowed as number here" ger "An dieser Stelle sind nur Ganzzahlen zulässig" ER_UNSUPORTED_LOG_ENGINE - eng "This storage engine cannot be used for log tables" - ger "Diese Speicher-Engine kann für Logtabellen nicht verwendet werden" + eng "Storage engine %s cannot be used for log tables" + ger "Speicher-Engine %s kann für Logtabellen nicht verwendet werden" ER_BAD_LOG_STATEMENT eng "You cannot '%s' a log table if logging is enabled" ger "Sie können eine Logtabelle nicht '%s', wenn Loggen angeschaltet ist" @@ -6133,8 +6054,8 @@ ER_DELAYED_NOT_SUPPORTED eng "DELAYED option not supported for table '%-.192s'" ger "Die DELAYED-Option wird für Tabelle '%-.192s' nicht unterstützt" WARN_NO_MASTER_INFO - eng "The master info structure does not exist" - ger "Die Master-Info-Struktur existiert nicht" + eng "There is no master connection '%.*s'" + ger "Die Master-Info-Struktur existiert nicht '%.*s'" WARN_OPTION_IGNORED eng "<%-.64s> option ignored" ger "Option <%-.64s> ignoriert" @@ -6165,13 +6086,13 @@ ER_EXCEPTIONS_WRITE_ERROR eng "Write to exceptions table failed. Message: %-.128s"" ger "Schreiben in Ausnahme-Tabelle fehlgeschlagen. Meldung: %-.128s"" ER_TOO_LONG_TABLE_COMMENT - eng "Comment for table '%-.64s' is too long (max = %lu)" - por "Comentário para a tabela '%-.64s' é longo demais (max = %lu)" - ger "Kommentar für Tabelle '%-.64s' ist zu lang (max = %lu)" + eng "Comment for table '%-.64s' is too long (max = %u)" + por "Comentário para a tabela '%-.64s' é longo demais (max = %u)" + ger "Kommentar für Tabelle '%-.64s' ist zu lang (max = %u)" ER_TOO_LONG_FIELD_COMMENT - eng "Comment for field '%-.64s' is too long (max = %lu)" - por "Comentário para o campo '%-.64s' é longo demais (max = %lu)" - ger "Kommentar für Feld '%-.64s' ist zu lang (max = %lu)" + eng "Comment for field '%-.64s' is too long (max = %u)" + por "Comentário para o campo '%-.64s' é longo demais (max = %u)" + ger "Kommentar für Feld '%-.64s' ist zu lang (max = %u)" ER_FUNC_INEXISTENT_NAME_COLLISION 42000 eng "FUNCTION %s does not exist. Check the 'Function Name Parsing and Resolution' section in the Reference Manual" ger "FUNCTION %s existiert nicht. Erläuterungen im Abschnitt 'Function Name Parsing and Resolution' im Referenzhandbuch" @@ -6475,7 +6396,7 @@ ER_BINLOG_UNSAFE_INSERT_TWO_KEYS ER_TABLE_IN_FK_CHECK eng "Table is being used in foreign key check." -ER_UNUSED_1 +ER_unused_1 eng "You should never see it" ER_BINLOG_UNSAFE_AUTOINC_NOT_FIRST @@ -6565,3 +6486,74 @@ ER_QUERY_EXCEEDED_ROWS_EXAMINED_LIMIT ER_NO_SUCH_TABLE_IN_ENGINE 42S02 eng "Table '%-.192s.%-.192s' doesn't exist in engine" swe "Det finns ingen tabell som heter '%-.192s.%-.192s' i handlern" +ER_TARGET_NOT_EXPLAINABLE + eng "Target is not running an EXPLAINable command" +ER_CONNECTION_ALREADY_EXISTS + eng "Connection '%.*s' conflicts with existing connection '%.*s'" +ER_MASTER_LOG_PREFIX + eng "Master '%.*s': " +ER_CANT_START_STOP_SLAVE + eng "Can't %s SLAVE '%.*s'" +ER_SLAVE_STARTED + eng "SLAVE '%.*s' started" +ER_SLAVE_STOPPED + eng "SLAVE '%.*s' stopped" +ER_SQL_DISCOVER_ERROR + eng "Engine %s failed to discover table %`-.192s.%`-.192s with '%s'" +ER_FAILED_GTID_STATE_INIT + eng "Failed initializing replication GTID state" +ER_INCORRECT_GTID_STATE + eng "Could not parse GTID list" +ER_CANNOT_UPDATE_GTID_STATE + eng "Could not update replication slave gtid state" +ER_DUPLICATE_GTID_DOMAIN + eng "GTID %u-%u-%llu and %u-%u-%llu conflict (duplicate domain id %u)" +ER_GTID_OPEN_TABLE_FAILED + eng "Failed to open %s.%s" + ger "Öffnen von %s.%s fehlgeschlagen" +ER_GTID_POSITION_NOT_FOUND_IN_BINLOG + eng "Connecting slave requested to start from GTID %u-%u-%llu, which is not in the master's binlog" +ER_CANNOT_LOAD_SLAVE_GTID_STATE + eng "Failed to load replication slave GTID position from table %s.%s" +ER_MASTER_GTID_POS_CONFLICTS_WITH_BINLOG + eng "Specified GTID %u-%u-%llu conflicts with the binary log which contains a more recent GTID %u-%u-%llu. If MASTER_GTID_POS=CURRENT_POS is used, the binlog position will override the new value of @@gtid_slave_pos." +ER_MASTER_GTID_POS_MISSING_DOMAIN + eng "Specified value for @@gtid_slave_pos contains no value for replication domain %u. This conflicts with the binary log which contains GTID %u-%u-%llu. If MASTER_GTID_POS=CURRENT_POS is used, the binlog position will override the new value of @@gtid_slave_pos." +ER_UNTIL_REQUIRES_USING_GTID + eng "START SLAVE UNTIL master_gtid_pos requires that slave is using GTID" +ER_GTID_STRICT_OUT_OF_ORDER + eng "An attempt was made to binlog GTID %u-%u-%llu which would create an out-of-order sequence number with existing GTID %u-%u-%llu, and gtid strict mode is enabled." +ER_GTID_START_FROM_BINLOG_HOLE + eng "The binlog on the master is missing the GTID %u-%u-%llu requested by the slave (even though a subsequent sequence number does exist), and GTID strict mode is enabled" +ER_SLAVE_UNEXPECTED_MASTER_SWITCH + eng "Unexpected GTID received from master after reconnect. This normally indicates that the master server was replaced without restarting the slave threads. %s" +ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_GTID_DOMAIN_ID_SEQ_NO + eng "Cannot modify @@session.gtid_domain_id or @@session.gtid_seq_no inside a transaction" +ER_STORED_FUNCTION_PREVENTS_SWITCH_GTID_DOMAIN_ID_SEQ_NO + eng "Cannot modify @@session.gtid_domain_id or @@session.gtid_seq_no inside a stored function or trigger" +ER_GTID_POSITION_NOT_FOUND_IN_BINLOG2 + eng "Connecting slave requested to start from GTID %u-%u-%llu, which is not in the master's binlog. Since the master's binlog contains GTIDs with higher sequence numbers, it probably means that the slave has diverged due to executing extra errorneous transactions" +ER_BINLOG_MUST_BE_EMPTY + eng "This operation is not allowed if any GTID has been logged to the binary log. Run RESET MASTER first to erase the log" +ER_NO_SUCH_QUERY + eng "Unknown query id: %lld" + ger "Unbekannte Abfrage-ID: %lld" + rus "Неизвестный номер запроса: %lld" +ER_INVALID_ROLE OP000 + eng "Invalid role specification %`s." + rum "Rolul %`s este invalid." +ER_INVALID_CURRENT_USER 0L000 + eng "The current user is invalid." + rum "Utilizatorul curent este invalid." +ER_CANNOT_GRANT_ROLE + eng "Cannot grant role '%s' to: %s." + rum "Rolul '%s' nu poate fi acordat catre: %s." +ER_CANNOT_REVOKE_ROLE + eng "Cannot revoke role '%s' from: %s." + rum "Rolul '%s' nu poate fi revocat de la: %s." +ER_CHANGE_SLAVE_PARALLEL_THREADS_ACTIVE + eng "Cannot change @@slave_parallel_threads while another change is in progress" +ER_PRIOR_COMMIT_FAILED + eng "Commit failed due to failure of an earlier commit on which this one depends" +ER_SLAVE_SKIP_NOT_IN_GTID + eng "When using GTID, @@sql_slave_skip_counter can not be used. Instead, setting @@gtid_slave_pos explicitly can be used to skip to after a given GTID position." diff --git a/sql/slave.cc b/sql/slave.cc index 0d762199323..a4c2f470bb1 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -57,6 +57,8 @@ #include "rpl_tblmap.h" #include "debug_sync.h" +#include "rpl_parallel.h" + #define FLAGSTR(V,F) ((V)&(F)?#F" ":"") @@ -71,8 +73,10 @@ char slave_skip_error_names[SHOW_VAR_FUNC_BUFF_SIZE]; char* slave_load_tmpdir = 0; Master_info *active_mi= 0; +Master_info_index *master_info_index; my_bool replicate_same_server_id; ulonglong relay_log_space_limit = 0; +LEX_STRING default_master_connection_name= { (char*) "", 0 }; /* When slave thread exits, we need to remember the temporary tables so we @@ -112,7 +116,7 @@ static const char *reconnect_messages[SLAVE_RECON_ACT_MAX][SLAVE_RECON_MSG_MAX]= registration on master", "Reconnecting after a failed registration on master", "failed registering on master, reconnecting to try again, \ -log '%s' at position %s", +log '%s' at position %s%s", "COM_REGISTER_SLAVE", "Slave I/O thread killed during or after reconnect" }, @@ -120,7 +124,7 @@ log '%s' at position %s", "Waiting to reconnect after a failed binlog dump request", "Slave I/O thread killed while retrying master dump", "Reconnecting after a failed binlog dump request", - "failed dump request, reconnecting to try again, log '%s' at position %s", + "failed dump request, reconnecting to try again, log '%s' at position %s%s", "COM_BINLOG_DUMP", "Slave I/O thread killed during or after reconnect" }, @@ -129,7 +133,7 @@ log '%s' at position %s", "Slave I/O thread killed while waiting to reconnect after a failed read", "Reconnecting after a failed master event read", "Slave I/O thread: Failed reading log event, reconnecting to retry, \ -log '%s' at position %s", +log '%s' at position %s%s", "", "Slave I/O thread killed during or after a reconnect done to recover from \ failed read" @@ -142,23 +146,28 @@ typedef enum { SLAVE_THD_IO, SLAVE_THD_SQL} SLAVE_THD_TYPE; static int process_io_rotate(Master_info* mi, Rotate_log_event* rev); static int process_io_create_file(Master_info* mi, Create_file_log_event* cev); static bool wait_for_relay_log_space(Relay_log_info* rli); -static inline bool io_slave_killed(THD* thd,Master_info* mi); -static inline bool sql_slave_killed(THD* thd,Relay_log_info* rli); -static int init_slave_thread(THD* thd, SLAVE_THD_TYPE thd_type); +static bool io_slave_killed(Master_info* mi); +static bool sql_slave_killed(rpl_group_info *rgi); +static int init_slave_thread(THD* thd, Master_info *mi, + SLAVE_THD_TYPE thd_type); static void print_slave_skip_errors(void); static int safe_connect(THD* thd, MYSQL* mysql, Master_info* mi); static int safe_reconnect(THD* thd, MYSQL* mysql, Master_info* mi, bool suppress_warnings); static int connect_to_master(THD* thd, MYSQL* mysql, Master_info* mi, bool reconnect, bool suppress_warnings); -static Log_event* next_event(Relay_log_info* rli); +static Log_event* next_event(rpl_group_info* rgi, ulonglong *event_size); static int queue_event(Master_info* mi,const char* buf,ulong event_len); static int terminate_slave_thread(THD *thd, mysql_mutex_t *term_lock, mysql_cond_t *term_cond, volatile uint *slave_running, bool skip_lock); -static bool check_io_slave_killed(THD *thd, Master_info *mi, const char *info); +static bool check_io_slave_killed(Master_info *mi, const char *info); +static bool send_show_master_info_header(THD *thd, bool full, + size_t gtid_pos_length); +static bool send_show_master_info_data(THD *thd, Master_info *mi, bool full, + String *gtid_pos); /* Function to set the slave's max_allowed_packet based on the value of slave_max_allowed_packet. @@ -277,6 +286,66 @@ static void init_slave_psi_keys(void) } #endif /* HAVE_PSI_INTERFACE */ + +static bool slave_init_thread_running; + + +pthread_handler_t +handle_slave_init(void *arg __attribute__((unused))) +{ + THD *thd; + + my_thread_init(); + thd= new THD; + thd->thread_stack= (char*) &thd; /* Set approximate stack start */ + mysql_mutex_lock(&LOCK_thread_count); + thd->thread_id= thread_id++; + mysql_mutex_unlock(&LOCK_thread_count); + thd->store_globals(); + + thd_proc_info(thd, "Loading slave GTID position from table"); + if (rpl_load_gtid_slave_state(thd)) + sql_print_warning("Failed to load slave replication state from table " + "%s.%s: %u: %s", "mysql", + rpl_gtid_slave_state_table_name.str, + thd->stmt_da->sql_errno(), thd->stmt_da->message()); + + mysql_mutex_lock(&LOCK_thread_count); + delete thd; + mysql_mutex_unlock(&LOCK_thread_count); + my_thread_end(); + + mysql_mutex_lock(&LOCK_thread_count); + slave_init_thread_running= false; + mysql_cond_signal(&COND_thread_count); + mysql_mutex_unlock(&LOCK_thread_count); + + return 0; +} + + +static int +run_slave_init_thread() +{ + pthread_t th; + + slave_init_thread_running= true; + if (mysql_thread_create(key_thread_slave_init, &th, NULL, + handle_slave_init, NULL)) + { + sql_print_error("Failed to create thread while initialising slave"); + return 1; + } + + mysql_mutex_lock(&LOCK_thread_count); + while (slave_init_thread_running) + mysql_cond_wait(&COND_thread_count, &LOCK_thread_count); + mysql_mutex_unlock(&LOCK_thread_count); + + return 0; +} + + /* Initialize slave structures */ int init_slave() @@ -288,21 +357,45 @@ int init_slave() init_slave_psi_keys(); #endif + if (run_slave_init_thread()) + return 1; + /* This is called when mysqld starts. Before client connections are accepted. However bootstrap may conflict with us if it does START SLAVE. So it's safer to take the lock. */ mysql_mutex_lock(&LOCK_active_mi); - /* - TODO: re-write this to interate through the list of files - for multi-master - */ - active_mi= new Master_info(relay_log_recovery); if (pthread_key_create(&RPL_MASTER_INFO, NULL)) goto err; + master_info_index= new Master_info_index; + if (!master_info_index || master_info_index->init_all_master_info()) + { + sql_print_error("Failed to initialize multi master structures"); + mysql_mutex_unlock(&LOCK_active_mi); + DBUG_RETURN(1); + } + if (!(active_mi= new Master_info(&default_master_connection_name, + relay_log_recovery)) || + active_mi->error()) + { + delete active_mi; + active_mi= 0; + goto err; + } + + if (master_info_index->add_master_info(active_mi, FALSE)) + { + delete active_mi; + active_mi= 0; + goto err; + } + + if (global_rpl_thread_pool.init(opt_slave_parallel_threads)) + return 1; + /* If --slave-skip-errors=... was not used, the string value for the system variable has not been set up yet. Do it now. @@ -317,18 +410,11 @@ int init_slave() If master_host is specified, create the master_info file if it doesn't exists. */ - if (!active_mi) - { - sql_print_error("Failed to allocate memory for the master info structure"); - error= 1; - goto err; - } if (init_master_info(active_mi,master_info_file,relay_log_info_file, 1, (SLAVE_IO | SLAVE_SQL))) { sql_print_error("Failed to initialize the master info structure"); - error= 1; goto err; } @@ -344,14 +430,18 @@ int init_slave() SLAVE_IO | SLAVE_SQL)) { sql_print_error("Failed to create slave threads"); - error= 1; goto err; } } -err: +end: mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(error); + +err: + sql_print_error("Failed to allocate memory for the Master Info structure"); + error= 1; + goto end; } /* @@ -403,6 +493,7 @@ int init_recovery(Master_info* mi, const char** errmsg) DBUG_RETURN(0); } + /** Convert slave skip errors bitmap into a printable string. @@ -510,13 +601,6 @@ void init_slave_skip_errors(const char* arg) DBUG_VOID_RETURN; } -static void set_thd_in_use_temporary_tables(Relay_log_info *rli) -{ - TABLE *table; - - for (table= rli->save_temporary_tables ; table ; table= table->next) - table->in_use= rli->sql_thd; -} int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) { @@ -532,7 +616,7 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) { DBUG_PRINT("info",("Terminating SQL thread")); mi->rli.abort_slave=1; - if ((error=terminate_slave_thread(mi->rli.sql_thd, sql_lock, + if ((error=terminate_slave_thread(mi->rli.sql_driver_thd, sql_lock, &mi->rli.stop_cond, &mi->rli.slave_running, skip_lock)) && @@ -715,7 +799,7 @@ int start_slave_thread( if (start_lock) mysql_mutex_lock(start_lock); - if (!server_id) + if (!global_system_variables.server_id) { if (start_cond) mysql_cond_broadcast(start_cond); @@ -792,6 +876,7 @@ int start_slave_threads(bool need_slave_mutex, bool wait_for_start, mysql_mutex_t *lock_io=0, *lock_sql=0, *lock_cond_io=0, *lock_cond_sql=0; mysql_cond_t* cond_io=0, *cond_sql=0; int error=0; + const char *errmsg; DBUG_ENTER("start_slave_threads"); if (need_slave_mutex) @@ -807,7 +892,40 @@ int start_slave_threads(bool need_slave_mutex, bool wait_for_start, lock_cond_sql = &mi->rli.run_lock; } - if (thread_mask & SLAVE_IO) + /* + If we are using GTID and both SQL and IO threads are stopped, then get + rid of all relay logs. + + Relay logs are not very useful when using GTID, except as a buffer + between the fetch in the IO thread and the apply in SQL thread. However + while one of the threads is running, they are in use and cannot be + removed. + */ + if (mi->using_gtid != Master_info::USE_GTID_NO && + !mi->slave_running && !mi->rli.slave_running) + { + /* + purge_relay_logs() clears the mi->rli.group_master_log_pos. + So save and restore them, like we do in CHANGE MASTER. + (We are not going to use them for GTID, but it might be worth to + keep them in case connection with GTID fails and user wants to go + back and continue with previous old-style replication coordinates). + */ + mi->master_log_pos = max(BIN_LOG_HEADER_SIZE, mi->rli.group_master_log_pos); + strmake(mi->master_log_name, mi->rli.group_master_log_name, + sizeof(mi->master_log_name)-1); + purge_relay_logs(&mi->rli, NULL, 0, &errmsg); + mi->rli.group_master_log_pos= mi->master_log_pos; + strmake(mi->rli.group_master_log_name, mi->master_log_name, + sizeof(mi->rli.group_master_log_name)-1); + + error= rpl_load_gtid_state(&mi->gtid_current_pos, mi->using_gtid == + Master_info::USE_GTID_CURRENT_POS); + mi->events_queued_since_last_gtid= 0; + mi->gtid_reconnect_event_skip_count= 0; + } + + if (!error && (thread_mask & SLAVE_IO)) error= start_slave_thread( #ifdef HAVE_PSI_INTERFACE key_thread_slave_io, @@ -850,49 +968,27 @@ void end_slave() running presently. If a START SLAVE was in progress, the mutex lock below will make us wait until slave threads have started, and START SLAVE returns, then we terminate them here. + + We can also be called by cleanup(), which only happens if some + startup parameter to the server was wrong. */ mysql_mutex_lock(&LOCK_active_mi); - if (active_mi) - { - /* - TODO: replace the line below with - list_walk(&master_list, (list_walk_action)end_slave_on_walk,0); - once multi-master code is ready. - */ - terminate_slave_threads(active_mi,SLAVE_FORCE_ALL); - } + /* This will call terminate_slave_threads() on all connections */ + delete master_info_index; + master_info_index= 0; + active_mi= 0; mysql_mutex_unlock(&LOCK_active_mi); + global_rpl_thread_pool.destroy(); + free_all_rpl_filters(); DBUG_VOID_RETURN; } -/** - Free all resources used by slave threads at time of executing shutdown. - The routine must be called after all possible users of @c active_mi - have left. - - SYNOPSIS - close_active_mi() - -*/ -void close_active_mi() -{ - mysql_mutex_lock(&LOCK_active_mi); - if (active_mi) - { - end_master_info(active_mi); - delete active_mi; - active_mi= 0; - } - mysql_mutex_unlock(&LOCK_active_mi); -} - -static bool io_slave_killed(THD* thd, Master_info* mi) +static bool io_slave_killed(Master_info* mi) { DBUG_ENTER("io_slave_killed"); - DBUG_ASSERT(mi->io_thd == thd); DBUG_ASSERT(mi->slave_running); // tracking buffer overrun - DBUG_RETURN(mi->abort_slave || abort_loop || thd->killed); + DBUG_RETURN(mi->abort_slave || abort_loop || mi->io_thd->killed); } /** @@ -908,26 +1004,36 @@ static bool io_slave_killed(THD* thd, Master_info* mi) @return TRUE the killed status is recognized, FALSE a possible killed status is deferred. */ -static bool sql_slave_killed(THD* thd, Relay_log_info* rli) +static bool sql_slave_killed(rpl_group_info *rgi) { bool ret= FALSE; + Relay_log_info *rli= rgi->rli; + THD *thd= rgi->thd; DBUG_ENTER("sql_slave_killed"); - DBUG_ASSERT(rli->sql_thd == thd); + DBUG_ASSERT(rli->sql_driver_thd == thd); DBUG_ASSERT(rli->slave_running == 1);// tracking buffer overrun - if (abort_loop || thd->killed || rli->abort_slave) + if (abort_loop || rli->sql_driver_thd->killed || rli->abort_slave) { /* - The transaction should always be binlogged if OPTION_KEEP_LOG is set - (it implies that something can not be rolled back). And such case - should be regarded similarly as modifing a non-transactional table - because retrying of the transaction will lead to an error or inconsistency - as well. - Example: OPTION_KEEP_LOG is set if a temporary table is created or dropped. + The transaction should always be binlogged if OPTION_KEEP_LOG is + set (it implies that something can not be rolled back). And such + case should be regarded similarly as modifing a + non-transactional table because retrying of the transaction will + lead to an error or inconsistency as well. + + Example: OPTION_KEEP_LOG is set if a temporary table is created + or dropped. + + Note that transaction.all.modified_non_trans_table may be 1 + if last statement was a single row transaction without begin/end. + Testing this flag must always be done in connection with + rli->is_in_group(). */ + if ((thd->transaction.all.modified_non_trans_table || - (thd->variables.option_bits & OPTION_KEEP_LOG)) - && rli->is_in_group()) + (thd->variables.option_bits & OPTION_KEEP_LOG)) && + rli->is_in_group()) { char msg_stopped[]= "... Slave SQL Thread stopped with incomplete event group " @@ -937,25 +1043,33 @@ static bool sql_slave_killed(THD* thd, Relay_log_info* rli) "ignores duplicate key, key not found, and similar errors (see " "documentation for details)."; + DBUG_PRINT("info", ("modified_non_trans_table: %d OPTION_BEGIN: %d " + "is_in_group: %d", + thd->transaction.all.modified_non_trans_table, + test(thd->variables.option_bits & OPTION_BEGIN), + rli->is_in_group())); + if (rli->abort_slave) { - DBUG_PRINT("info", ("Request to stop slave SQL Thread received while " - "applying a group that has non-transactional " - "changes; waiting for completion of the group ... ")); + DBUG_PRINT("info", + ("Request to stop slave SQL Thread received while " + "applying a group that has non-transactional " + "changes; waiting for completion of the group ... ")); /* - Slave sql thread shutdown in face of unfinished group modified - Non-trans table is handled via a timer. The slave may eventually - give out to complete the current group and in that case there - might be issues at consequent slave restart, see the error message. - WL#2975 offers a robust solution requiring to store the last exectuted - event's coordinates along with the group's coordianates - instead of waiting with @c last_event_start_time the timer. + Slave sql thread shutdown in face of unfinished group + modified Non-trans table is handled via a timer. The slave + may eventually give out to complete the current group and in + that case there might be issues at consequent slave restart, + see the error message. WL#2975 offers a robust solution + requiring to store the last exectuted event's coordinates + along with the group's coordianates instead of waiting with + @c last_event_start_time the timer. */ - if (rli->last_event_start_time == 0) - rli->last_event_start_time= my_time(0); - ret= difftime(my_time(0), rli->last_event_start_time) <= + if (rgi->last_event_start_time == 0) + rgi->last_event_start_time= my_time(0); + ret= difftime(my_time(0), rgi->last_event_start_time) <= SLAVE_WAIT_GROUP_DONE ? FALSE : TRUE; DBUG_EXECUTE_IF("stop_slave_middle_group", @@ -978,7 +1092,8 @@ static bool sql_slave_killed(THD* thd, Relay_log_info* rli) else { ret= TRUE; - rli->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, ER(ER_SLAVE_FATAL_ERROR), + rli->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, + ER(ER_SLAVE_FATAL_ERROR), msg_stopped); } } @@ -988,7 +1103,7 @@ static bool sql_slave_killed(THD* thd, Relay_log_info* rli) } } if (ret) - rli->last_event_start_time= 0; + rgi->last_event_start_time= 0; DBUG_RETURN(ret); } @@ -1241,7 +1356,7 @@ bool is_network_error(uint errorno) static int get_master_version_and_clock(MYSQL* mysql, Master_info* mi) { - char err_buff[MAX_SLAVE_ERRMSG]; + char err_buff[MAX_SLAVE_ERRMSG], err_buff2[MAX_SLAVE_ERRMSG]; const char* errmsg= 0; int err_code= 0; MYSQL_RES *master_res= 0; @@ -1258,23 +1373,28 @@ static int get_master_version_and_clock(MYSQL* mysql, Master_info* mi) if (!my_isdigit(&my_charset_bin,*mysql->server_version)) { - errmsg = "Master reported unrecognized MySQL version"; + errmsg= err_buff2; + snprintf(err_buff2, sizeof(err_buff2), + "Master reported unrecognized MySQL version: %s", + mysql->server_version); err_code= ER_SLAVE_FATAL_ERROR; - sprintf(err_buff, ER(err_code), errmsg); + sprintf(err_buff, ER(err_code), err_buff2); } else { /* Note the following switch will bug when we have MySQL branch 30 ;) */ - switch (version) - { + switch (version) { case 0: case 1: case 2: - errmsg = "Master reported unrecognized MySQL version"; + errmsg= err_buff2; + snprintf(err_buff2, sizeof(err_buff2), + "Master reported unrecognized MySQL version: %s", + mysql->server_version); err_code= ER_SLAVE_FATAL_ERROR; - sprintf(err_buff, ER(err_code), errmsg); + sprintf(err_buff, ER(err_code), err_buff2); break; case 3: mi->rli.relay_log.description_event_for_queue= new @@ -1385,7 +1505,7 @@ static int get_master_version_and_clock(MYSQL* mysql, Master_info* mi) mi->clock_diff_with_master= (long) (time((time_t*) 0) - strtoul(master_row[0], 0, 10)); } - else if (check_io_slave_killed(mi->io_thd, mi, NULL)) + else if (check_io_slave_killed(mi, NULL)) goto slave_killed_err; else if (is_network_error(mysql_errno(mysql))) { @@ -1435,7 +1555,8 @@ static int get_master_version_and_clock(MYSQL* mysql, Master_info* mi) (master_res= mysql_store_result(mysql)) && (master_row= mysql_fetch_row(master_res))) { - if ((::server_id == (mi->master_id= strtoul(master_row[1], 0, 10))) && + if ((global_system_variables.server_id == + (mi->master_id= strtoul(master_row[1], 0, 10))) && !mi->rli.replicate_same_server_id) { errmsg= "The slave I/O thread stops because master and slave have equal \ @@ -1449,7 +1570,7 @@ not always make sense; please check the manual before using it)."; } else if (mysql_errno(mysql)) { - if (check_io_slave_killed(mi->io_thd, mi, NULL)) + if (check_io_slave_killed(mi, NULL)) goto slave_killed_err; else if (is_network_error(mysql_errno(mysql))) { @@ -1522,7 +1643,7 @@ be equal for the Statement-format replication to work"; goto err; } } - else if (check_io_slave_killed(mi->io_thd, mi, NULL)) + else if (check_io_slave_killed(mi, NULL)) goto slave_killed_err; else if (is_network_error(mysql_errno(mysql))) { @@ -1585,7 +1706,7 @@ be equal for the Statement-format replication to work"; goto err; } } - else if (check_io_slave_killed(mi->io_thd, mi, NULL)) + else if (check_io_slave_killed(mi, NULL)) goto slave_killed_err; else if (is_network_error(err_code= mysql_errno(mysql))) { @@ -1630,7 +1751,7 @@ when it try to get the value of TIME_ZONE global variable from master."; sprintf(query, query_format, llbuf); if (mysql_real_query(mysql, query, strlen(query)) - && !check_io_slave_killed(mi->io_thd, mi, NULL)) + && !check_io_slave_killed(mi, NULL)) { errmsg= "The slave I/O thread stops because SET @master_heartbeat_period " "on master failed."; @@ -1665,7 +1786,7 @@ when it try to get the value of TIME_ZONE global variable from master."; rc= mysql_real_query(mysql, query, strlen(query)); if (rc != 0) { - if (check_io_slave_killed(mi->io_thd, mi, NULL)) + if (check_io_slave_killed(mi, NULL)) goto slave_killed_err; if (mysql_errno(mysql) == ER_UNKNOWN_SYSTEM_VARIABLE) @@ -1715,7 +1836,7 @@ when it try to get the value of TIME_ZONE global variable from master."; DBUG_ASSERT(mi->checksum_alg_before_fd == BINLOG_CHECKSUM_ALG_OFF || mi->checksum_alg_before_fd == BINLOG_CHECKSUM_ALG_CRC32); } - else if (check_io_slave_killed(mi->io_thd, mi, NULL)) + else if (check_io_slave_killed(mi, NULL)) goto slave_killed_err; else if (is_network_error(mysql_errno(mysql))) { @@ -1786,6 +1907,227 @@ past_checksum: } } } + + /* Announce MariaDB slave capabilities. */ + DBUG_EXECUTE_IF("simulate_slave_capability_none", goto after_set_capability;); + { + int rc= DBUG_EVALUATE_IF("simulate_slave_capability_old_53", + mysql_real_query(mysql, STRING_WITH_LEN("SET @mariadb_slave_capability=" + STRINGIFY_ARG(MARIA_SLAVE_CAPABILITY_ANNOTATE))), + mysql_real_query(mysql, STRING_WITH_LEN("SET @mariadb_slave_capability=" + STRINGIFY_ARG(MARIA_SLAVE_CAPABILITY_MINE)))); + if (rc) + { + err_code= mysql_errno(mysql); + if (is_network_error(err_code)) + { + mi->report(ERROR_LEVEL, err_code, + "Setting @mariadb_slave_capability failed with error: %s", + mysql_error(mysql)); + goto network_err; + } + else + { + /* Fatal error */ + errmsg= "The slave I/O thread stops because a fatal error is " + "encountered when it tries to set @mariadb_slave_capability."; + sprintf(err_buff, "%s Error: %s", errmsg, mysql_error(mysql)); + goto err; + } + } + } +#ifndef DBUG_OFF +after_set_capability: +#endif + + if (mi->using_gtid != Master_info::USE_GTID_NO) + { + /* Request dump to start from slave replication GTID state. */ + int rc; + char str_buf[256]; + String query_str(str_buf, sizeof(str_buf), system_charset_info); + query_str.length(0); + + /* + Read the master @@GLOBAL.gtid_domain_id variable. + This is mostly to check that master is GTID aware, but we could later + perhaps use it to check that different multi-source masters are correctly + configured with distinct domain_id. + */ + if (mysql_real_query(mysql, + STRING_WITH_LEN("SELECT @@GLOBAL.gtid_domain_id")) || + !(master_res= mysql_store_result(mysql)) || + !(master_row= mysql_fetch_row(master_res))) + { + err_code= mysql_errno(mysql); + errmsg= "The slave I/O thread stops because master does not support " + "MariaDB global transaction id. A fatal error is encountered when " + "it tries to SELECT @@GLOBAL.gtid_domain_id."; + sprintf(err_buff, "%s Error: %s", errmsg, mysql_error(mysql)); + goto err; + } + mysql_free_result(master_res); + master_res= NULL; + + query_str.append(STRING_WITH_LEN("SET @slave_connect_state='"), + system_charset_info); + if (mi->gtid_current_pos.append_to_string(&query_str)) + { + err_code= ER_OUTOFMEMORY; + errmsg= "The slave I/O thread stops because a fatal out-of-memory " + "error is encountered when it tries to compute @slave_connect_state."; + sprintf(err_buff, "%s Error: Out of memory", errmsg); + goto err; + } + query_str.append(STRING_WITH_LEN("'"), system_charset_info); + + rc= mysql_real_query(mysql, query_str.ptr(), query_str.length()); + if (rc) + { + err_code= mysql_errno(mysql); + if (is_network_error(err_code)) + { + mi->report(ERROR_LEVEL, err_code, + "Setting @slave_connect_state failed with error: %s", + mysql_error(mysql)); + goto network_err; + } + else + { + /* Fatal error */ + errmsg= "The slave I/O thread stops because a fatal error is " + "encountered when it tries to set @slave_connect_state."; + sprintf(err_buff, "%s Error: %s", errmsg, mysql_error(mysql)); + goto err; + } + } + + query_str.length(0); + if (query_str.append(STRING_WITH_LEN("SET @slave_gtid_strict_mode="), + system_charset_info) || + query_str.append_ulonglong(opt_gtid_strict_mode != false)) + { + err_code= ER_OUTOFMEMORY; + errmsg= "The slave I/O thread stops because a fatal out-of-memory " + "error is encountered when it tries to set @slave_gtid_strict_mode."; + sprintf(err_buff, "%s Error: Out of memory", errmsg); + goto err; + } + + rc= mysql_real_query(mysql, query_str.ptr(), query_str.length()); + if (rc) + { + err_code= mysql_errno(mysql); + if (is_network_error(err_code)) + { + mi->report(ERROR_LEVEL, err_code, + "Setting @slave_gtid_strict_mode failed with error: %s", + mysql_error(mysql)); + goto network_err; + } + else + { + /* Fatal error */ + errmsg= "The slave I/O thread stops because a fatal error is " + "encountered when it tries to set @slave_gtid_strict_mode."; + sprintf(err_buff, "%s Error: %s", errmsg, mysql_error(mysql)); + goto err; + } + } + + if (mi->rli.until_condition == Relay_log_info::UNTIL_GTID) + { + query_str.length(0); + query_str.append(STRING_WITH_LEN("SET @slave_until_gtid='"), + system_charset_info); + if (mi->rli.until_gtid_pos.append_to_string(&query_str)) + { + err_code= ER_OUTOFMEMORY; + errmsg= "The slave I/O thread stops because a fatal out-of-memory " + "error is encountered when it tries to compute @slave_until_gtid."; + sprintf(err_buff, "%s Error: Out of memory", errmsg); + goto err; + } + query_str.append(STRING_WITH_LEN("'"), system_charset_info); + + rc= mysql_real_query(mysql, query_str.ptr(), query_str.length()); + if (rc) + { + err_code= mysql_errno(mysql); + if (is_network_error(err_code)) + { + mi->report(ERROR_LEVEL, err_code, + "Setting @slave_until_gtid failed with error: %s", + mysql_error(mysql)); + goto network_err; + } + else + { + /* Fatal error */ + errmsg= "The slave I/O thread stops because a fatal error is " + "encountered when it tries to set @slave_until_gtid."; + sprintf(err_buff, "%s Error: %s", errmsg, mysql_error(mysql)); + goto err; + } + } + } + } + else + { + /* + If we are not using GTID to connect this time, then instead request + the corresponding GTID position from the master, so that the user + can reconnect the next time using MASTER_GTID_POS=AUTO. + */ + char quote_buf[2*sizeof(mi->master_log_name)+1]; + char str_buf[28+2*sizeof(mi->master_log_name)+10]; + String query(str_buf, sizeof(str_buf), system_charset_info); + query.length(0); + + query.append("SELECT binlog_gtid_pos('"); + escape_quotes_for_mysql(&my_charset_bin, quote_buf, sizeof(quote_buf), + mi->master_log_name, strlen(mi->master_log_name)); + query.append(quote_buf); + query.append("',"); + query.append_ulonglong(mi->master_log_pos); + query.append(")"); + + if (!mysql_real_query(mysql, query.c_ptr_safe(), query.length()) && + (master_res= mysql_store_result(mysql)) && + (master_row= mysql_fetch_row(master_res)) && + (master_row[0] != NULL)) + { + rpl_global_gtid_slave_state.load(mi->io_thd, master_row[0], + strlen(master_row[0]), false, false); + } + else if (check_io_slave_killed(mi, NULL)) + goto slave_killed_err; + else if (is_network_error(mysql_errno(mysql))) + { + mi->report(WARNING_LEVEL, mysql_errno(mysql), + "Get master GTID position failed with error: %s", mysql_error(mysql)); + goto network_err; + } + else + { + /* + ToDo: If the master does not have the binlog_gtid_pos() function, it + just means that it is an old master with no GTID support, so we should + do nothing. + + However, if binlog_gtid_pos() exists, but fails or returns NULL, then + it means that the requested position is not valid. We could use this + to catch attempts to replicate from within the middle of an event, + avoiding strange failures or possible corruption. + */ + } + if (master_res) + { + mysql_free_result(master_res); + master_res= NULL; + } + } + err: if (errmsg) { @@ -1824,7 +2166,7 @@ static bool wait_for_relay_log_space(Relay_log_info* rli) "\ Waiting for the slave SQL thread to free enough relay log space"); while (rli->log_space_limit < rli->log_space_total && - !(slave_killed=io_slave_killed(thd,mi)) && + !(slave_killed=io_slave_killed(mi)) && !rli->ignore_log_space_limit) mysql_cond_wait(&rli->log_space_cond, &rli->log_space_lock); @@ -1868,9 +2210,9 @@ Waiting for the slave SQL thread to free enough relay log space"); #endif if (rli->sql_force_rotate_relay) { - mysql_mutex_lock(&active_mi->data_lock); + mysql_mutex_lock(&mi->data_lock); rotate_relay_log(rli->mi); - mysql_mutex_unlock(&active_mi->data_lock); + mysql_mutex_unlock(&mi->data_lock); rli->sql_force_rotate_relay= false; } @@ -1903,34 +2245,66 @@ static void write_ignored_events_info_to_relay_log(THD *thd, Master_info *mi) DBUG_ASSERT(thd == mi->io_thd); mysql_mutex_lock(log_lock); - if (rli->ign_master_log_name_end[0]) - { - DBUG_PRINT("info",("writing a Rotate event to track down ignored events")); - Rotate_log_event *ev= new Rotate_log_event(rli->ign_master_log_name_end, - 0, rli->ign_master_log_pos_end, - Rotate_log_event::DUP_NAME); - rli->ign_master_log_name_end[0]= 0; - /* can unlock before writing as slave SQL thd will soon see our Rotate */ + if (rli->ign_master_log_name_end[0] || rli->ign_gtids.count()) + { + Rotate_log_event *rev= NULL; + Gtid_list_log_event *glev= NULL; + if (rli->ign_master_log_name_end[0]) + { + rev= new Rotate_log_event(rli->ign_master_log_name_end, + 0, rli->ign_master_log_pos_end, + Rotate_log_event::DUP_NAME); + rli->ign_master_log_name_end[0]= 0; + if (unlikely(!(bool)rev)) + mi->report(ERROR_LEVEL, ER_SLAVE_CREATE_EVENT_FAILURE, + ER(ER_SLAVE_CREATE_EVENT_FAILURE), + "Rotate_event (out of memory?)," + " SHOW SLAVE STATUS may be inaccurate"); + } + if (rli->ign_gtids.count()) + { + glev= new Gtid_list_log_event(&rli->ign_gtids, + Gtid_list_log_event::FLAG_IGN_GTIDS); + rli->ign_gtids.reset(); + if (unlikely(!(bool)glev)) + mi->report(ERROR_LEVEL, ER_SLAVE_CREATE_EVENT_FAILURE, + ER(ER_SLAVE_CREATE_EVENT_FAILURE), + "Gtid_list_event (out of memory?)," + " gtid_slave_pos may be inaccurate"); + } + + /* Can unlock before writing as slave SQL thd will soon see our event. */ mysql_mutex_unlock(log_lock); - if (likely((bool)ev)) + if (rev) { - ev->server_id= 0; // don't be ignored by slave SQL thread - if (unlikely(rli->relay_log.append(ev))) + DBUG_PRINT("info",("writing a Rotate event to track down ignored events")); + rev->server_id= 0; // don't be ignored by slave SQL thread + if (unlikely(rli->relay_log.append(rev))) mi->report(ERROR_LEVEL, ER_SLAVE_RELAY_LOG_WRITE_FAILURE, ER(ER_SLAVE_RELAY_LOG_WRITE_FAILURE), "failed to write a Rotate event" " to the relay log, SHOW SLAVE STATUS may be" " inaccurate"); + delete rev; + } + if (glev) + { + DBUG_PRINT("info",("writing a Gtid_list event to track down ignored events")); + glev->server_id= 0; // don't be ignored by slave SQL thread + glev->set_artificial_event(); // Don't mess up Exec_Master_Log_Pos + if (unlikely(rli->relay_log.append(glev))) + mi->report(ERROR_LEVEL, ER_SLAVE_RELAY_LOG_WRITE_FAILURE, + ER(ER_SLAVE_RELAY_LOG_WRITE_FAILURE), + "failed to write a Gtid_list event to the relay log, " + "gtid_slave_pos may be inaccurate"); + delete glev; + } + if (likely (rev || glev)) + { rli->relay_log.harvest_bytes_written(&rli->log_space_total); if (flush_master_info(mi, TRUE, TRUE)) sql_print_error("Failed to flush master info file"); - delete ev; } - else - mi->report(ERROR_LEVEL, ER_SLAVE_CREATE_EVENT_FAILURE, - ER(ER_SLAVE_CREATE_EVENT_FAILURE), - "Rotate_event (out of memory?)," - " SHOW SLAVE STATUS may be inaccurate"); } else mysql_mutex_unlock(log_lock); @@ -1979,7 +2353,7 @@ int register_slave_on_master(MYSQL* mysql, Master_info *mi, DBUG_RETURN(0); } - int4store(pos, server_id); pos+= 4; + int4store(pos, global_system_variables.server_id); pos+= 4; pos= net_store_data(pos, (uchar*) report_host, report_host_len); pos= net_store_data(pos, (uchar*) report_user, report_user_len); pos= net_store_data(pos, (uchar*) report_password, report_password_len); @@ -1999,7 +2373,7 @@ int register_slave_on_master(MYSQL* mysql, Master_info *mi, { *suppress_warnings= TRUE; // Suppress reconnect warning } - else if (!check_io_slave_killed(mi->io_thd, mi, NULL)) + else if (!check_io_slave_killed(mi, NULL)) { char buf[256]; my_snprintf(buf, sizeof(buf), "%s (Errno: %d)", mysql_error(mysql), @@ -2024,15 +2398,40 @@ int register_slave_on_master(MYSQL* mysql, Master_info *mi, @retval FALSE success @retval TRUE failure */ -bool show_master_info(THD* thd, Master_info* mi) + +bool show_master_info(THD *thd, Master_info *mi, bool full) +{ + DBUG_ENTER("show_master_info"); + String gtid_pos; + + if (full && rpl_global_gtid_slave_state.tostring(>id_pos, NULL, 0)) + DBUG_RETURN(TRUE); + if (send_show_master_info_header(thd, full, gtid_pos.length())) + DBUG_RETURN(TRUE); + if (send_show_master_info_data(thd, mi, full, >id_pos)) + DBUG_RETURN(TRUE); + my_eof(thd); + DBUG_RETURN(FALSE); +} + +static bool send_show_master_info_header(THD *thd, bool full, + size_t gtid_pos_length) { - // TODO: fix this for multi-master List<Item> field_list; Protocol *protocol= thd->protocol; - DBUG_ENTER("show_master_info"); + Master_info *mi; + DBUG_ENTER("show_master_info_header"); + + if (full) + { + field_list.push_back(new Item_empty_string("Connection_name", + MAX_CONNECTION_NAME)); + field_list.push_back(new Item_empty_string("Slave_SQL_State", + 30)); + } field_list.push_back(new Item_empty_string("Slave_IO_State", - 14)); + 30)); field_list.push_back(new Item_empty_string("Master_Host", sizeof(mi->host))); field_list.push_back(new Item_empty_string("Master_User", @@ -2095,22 +2494,65 @@ bool show_master_info(THD* thd, Master_info* mi) FN_REFLEN)); field_list.push_back(new Item_return_int("Master_Server_Id", sizeof(ulong), MYSQL_TYPE_LONG)); - + field_list.push_back(new Item_empty_string("Using_Gtid", + sizeof("Current_Pos")-1)); + field_list.push_back(new Item_empty_string("Gtid_IO_Pos", 30)); + if (full) + { + field_list.push_back(new Item_return_int("Retried_transactions", + 10, MYSQL_TYPE_LONG)); + field_list.push_back(new Item_return_int("Max_relay_log_size", + 10, MYSQL_TYPE_LONGLONG)); + field_list.push_back(new Item_return_int("Executed_log_entries", + 10, MYSQL_TYPE_LONG)); + field_list.push_back(new Item_return_int("Slave_received_heartbeats", + 10, MYSQL_TYPE_LONG)); + field_list.push_back(new Item_float("Slave_heartbeat_period", + 0.0, 3, 10)); + field_list.push_back(new Item_empty_string("Gtid_Slave_Pos", + gtid_pos_length)); + } if (protocol->send_result_set_metadata(&field_list, Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF)) DBUG_RETURN(TRUE); + DBUG_RETURN(FALSE); +} + + +static bool send_show_master_info_data(THD *thd, Master_info *mi, bool full, + String *gtid_pos) +{ + DBUG_ENTER("send_show_master_info_data"); if (mi->host[0]) { DBUG_PRINT("info",("host is set: '%s'", mi->host)); String *packet= &thd->packet; + Protocol *protocol= thd->protocol; + Rpl_filter *rpl_filter= mi->rpl_filter; + char buf[256]; + String tmp(buf, sizeof(buf), &my_charset_bin); + protocol->prepare_for_resend(); /* slave_running can be accessed without run_lock but not other non-volotile members like mi->io_thd, which is guarded by the mutex. */ + if (full) + protocol->store(mi->connection_name.str, mi->connection_name.length, + &my_charset_bin); mysql_mutex_lock(&mi->run_lock); + if (full) + { + /* + Show what the sql driver replication thread is doing + This is only meaningful if there is only one slave thread. + */ + protocol->store(mi->rli.sql_driver_thd ? + mi->rli.sql_driver_thd->proc_info : "", + &my_charset_bin); + } protocol->store(mi->io_thd ? mi->io_thd->proc_info : "", &my_charset_bin); mysql_mutex_unlock(&mi->run_lock); @@ -2136,8 +2578,6 @@ bool show_master_info(THD* thd, Master_info* mi) protocol->store(rpl_filter->get_do_db()); protocol->store(rpl_filter->get_ignore_db()); - char buf[256]; - String tmp(buf, sizeof(buf), &my_charset_bin); rpl_filter->get_do_table(&tmp); protocol->store(&tmp); rpl_filter->get_ignore_table(&tmp); @@ -2156,7 +2596,8 @@ bool show_master_info(THD* thd, Master_info* mi) protocol->store( mi->rli.until_condition==Relay_log_info::UNTIL_NONE ? "None": ( mi->rli.until_condition==Relay_log_info::UNTIL_MASTER_POS? "Master": - "Relay"), &my_charset_bin); + ( mi->rli.until_condition==Relay_log_info::UNTIL_RELAY_POS? "Relay": + "Gtid")), &my_charset_bin); protocol->store(mi->rli.until_log_name, &my_charset_bin); protocol->store((ulonglong) mi->rli.until_log_pos); @@ -2178,8 +2619,24 @@ bool show_master_info(THD* thd, Master_info* mi) if ((mi->slave_running == MYSQL_SLAVE_RUN_CONNECT) && mi->rli.slave_running) { - long time_diff= ((long)(time(0) - mi->rli.last_master_timestamp) - - mi->clock_diff_with_master); + long time_diff; + bool idle; + time_t stamp= mi->rli.last_master_timestamp; + + if (!stamp) + idle= true; + else + { + idle= mi->rli.sql_thread_caught_up; + if (opt_slave_parallel_threads > 0 && idle && + !mi->rli.parallel.workers_idle()) + idle= false; + } + if (idle) + time_diff= 0; + else + { + time_diff= ((long)(time(0) - stamp) - mi->clock_diff_with_master); /* Apparently on some systems time_diff can be <0. Here are possible reasons related to MySQL: @@ -2195,13 +2652,15 @@ bool show_master_info(THD* thd, Master_info* mi) slave is 2. At SHOW SLAVE STATUS time, assume that the difference between timestamp of slave and rli->last_master_timestamp is 0 (i.e. they are in the same second), then we get 0-(2-1)=-1 as a result. - This confuses users, so we don't go below 0: hence the max(). + This confuses users, so we don't go below 0. last_master_timestamp == 0 (an "impossible" timestamp 1970) is a special marker to say "consider we have caught up". */ - protocol->store((longlong)(mi->rli.last_master_timestamp ? - max(0, time_diff) : 0)); + if (time_diff < 0) + time_diff= 0; + } + protocol->store((longlong)time_diff); } else { @@ -2243,6 +2702,22 @@ bool show_master_info(THD* thd, Master_info* mi) } // Master_Server_id protocol->store((uint32) mi->master_id); + protocol->store(mi->using_gtid_astext(mi->using_gtid), &my_charset_bin); + { + char buff[30]; + String tmp(buff, sizeof(buff), system_charset_info); + mi->gtid_current_pos.to_string(&tmp); + protocol->store(tmp.ptr(), tmp.length(), &my_charset_bin); + } + if (full) + { + protocol->store((uint32) mi->rli.retried_trans); + protocol->store((ulonglong) mi->rli.max_relay_log_size); + protocol->store((uint32) mi->rli.executed_entries); + protocol->store((uint32) mi->received_heartbeats); + protocol->store((double) mi->heartbeat_period, 3, &tmp); + protocol->store(gtid_pos->ptr(), gtid_pos->length(), &my_charset_bin); + } mysql_mutex_unlock(&mi->rli.err_lock); mysql_mutex_unlock(&mi->err_lock); @@ -2252,6 +2727,77 @@ bool show_master_info(THD* thd, Master_info* mi) if (my_net_write(&thd->net, (uchar*) thd->packet.ptr(), packet->length())) DBUG_RETURN(TRUE); } + DBUG_RETURN(FALSE); +} + + +/* Used to sort connections by name */ + +static int cmp_mi_by_name(const Master_info **arg1, + const Master_info **arg2) +{ + return my_strcasecmp(system_charset_info, (*arg1)->connection_name.str, + (*arg2)->connection_name.str); +} + + +/** + Execute a SHOW FULL SLAVE STATUS statement. + + @param thd Pointer to THD object for the client thread executing the + statement. + + Elements are sorted according to the original connection_name. + + @retval FALSE success + @retval TRUE failure + + @note + master_info_index is protected by LOCK_active_mi. +*/ + +bool show_all_master_info(THD* thd) +{ + uint i, elements; + String gtid_pos; + Master_info **tmp; + DBUG_ENTER("show_master_info"); + mysql_mutex_assert_owner(&LOCK_active_mi); + + gtid_pos.length(0); + if (rpl_append_gtid_state(>id_pos, true)) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + DBUG_RETURN(TRUE); + } + + if (send_show_master_info_header(thd, 1, gtid_pos.length())) + DBUG_RETURN(TRUE); + + if (!(elements= master_info_index->master_info_hash.records)) + goto end; + + /* + Sort lines to get them into a predicted order + (needed for test cases and to not confuse users) + */ + if (!(tmp= (Master_info**) thd->alloc(sizeof(Master_info*) * elements))) + DBUG_RETURN(TRUE); + + for (i= 0; i < elements; i++) + { + tmp[i]= (Master_info *) my_hash_element(&master_info_index-> + master_info_hash, i); + } + my_qsort(tmp, elements, sizeof(Master_info*), (qsort_cmp) cmp_mi_by_name); + + for (i= 0; i < elements; i++) + { + if (send_show_master_info_data(thd, tmp[i], 1, >id_pos)) + DBUG_RETURN(TRUE); + } + +end: my_eof(thd); DBUG_RETURN(FALSE); } @@ -2305,36 +2851,37 @@ void set_slave_thread_default_charset(THD* thd, Relay_log_info const *rli) init_slave_thread() */ -static int init_slave_thread(THD* thd, SLAVE_THD_TYPE thd_type) +static int init_slave_thread(THD* thd, Master_info *mi, + SLAVE_THD_TYPE thd_type) { DBUG_ENTER("init_slave_thread"); -#if !defined(DBUG_OFF) - int simulate_error= 0; -#endif - thd->system_thread = (thd_type == SLAVE_THD_SQL) ? - SYSTEM_THREAD_SLAVE_SQL : SYSTEM_THREAD_SLAVE_IO; - thd->security_ctx->skip_grants(); - my_net_init(&thd->net, 0); - thd->slave_thread = 1; - thd->enable_slow_log= opt_log_slow_slave_statements; - thd->variables.log_slow_filter= global_system_variables.log_slow_filter; - set_slave_thread_options(thd); - thd->client_capabilities = CLIENT_LOCAL_FILES; - mysql_mutex_lock(&LOCK_thread_count); - thd->thread_id= thd->variables.pseudo_thread_id= thread_id++; - mysql_mutex_unlock(&LOCK_thread_count); - + int simulate_error __attribute__((unused))= 0; DBUG_EXECUTE_IF("simulate_io_slave_error_on_init", simulate_error|= (1 << SLAVE_THD_IO);); DBUG_EXECUTE_IF("simulate_sql_slave_error_on_init", simulate_error|= (1 << SLAVE_THD_SQL);); + /* We must call store_globals() before doing my_net_init() */ if (init_thr_lock() || thd->store_globals() || + my_net_init(&thd->net, 0, MYF(MY_THREAD_SPECIFIC)) || IF_DBUG(simulate_error & (1<< thd_type), 0)) { thd->cleanup(); DBUG_RETURN(-1); } + thd->system_thread = (thd_type == SLAVE_THD_SQL) ? + SYSTEM_THREAD_SLAVE_SQL : SYSTEM_THREAD_SLAVE_IO; + thd->security_ctx->skip_grants(); + thd->slave_thread= 1; + thd->connection_name= mi->connection_name; + thd->enable_slow_log= opt_log_slow_slave_statements; + thd->variables.log_slow_filter= global_system_variables.log_slow_filter; + set_slave_thread_options(thd); + thd->client_capabilities = CLIENT_LOCAL_FILES; + mysql_mutex_lock(&LOCK_thread_count); + thd->thread_id= thd->variables.pseudo_thread_id= thread_id++; + mysql_mutex_unlock(&LOCK_thread_count); + if (thd_type == SLAVE_THD_SQL) thd_proc_info(thd, "Waiting for the next event in relay log"); else @@ -2356,8 +2903,8 @@ static int init_slave_thread(THD* thd, SLAVE_THD_TYPE thd_type) @retval True if the thread has been killed, false otherwise. */ template <typename killed_func, typename rpl_info> -static inline bool slave_sleep(THD *thd, time_t seconds, - killed_func func, rpl_info info) +static bool slave_sleep(THD *thd, time_t seconds, + killed_func func, rpl_info info) { bool ret; @@ -2372,7 +2919,7 @@ static inline bool slave_sleep(THD *thd, time_t seconds, mysql_mutex_lock(lock); old_proc_info= thd->enter_cond(cond, lock, thd->proc_info); - while (! (ret= func(thd, info))) + while (! (ret= func(info))) { int error= mysql_cond_timedwait(cond, lock, &abstime); if (error == ETIMEDOUT || error == ETIME) @@ -2406,7 +2953,7 @@ static int request_dump(THD *thd, MYSQL* mysql, Master_info* mi, // TODO if big log files: Change next to int8store() int4store(buf, (ulong) mi->master_log_pos); int2store(buf + 4, binlog_flags); - int4store(buf + 6, server_id); + int4store(buf + 6, global_system_variables.server_id); len = (uint) strlen(logname); memcpy(buf + 10, logname,len); if (simple_command(mysql, COM_BINLOG_DUMP, buf, len + 10, 1)) @@ -2577,19 +3124,21 @@ static int has_temporary_error(THD *thd) @retval 2 No error calling ev->apply_event(), but error calling ev->update_pos(). */ -int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli) +int apply_event_and_update_pos(Log_event* ev, THD* thd, + rpl_group_info *rgi, + rpl_parallel_thread *rpt) { int exec_res= 0; - + Relay_log_info* rli= rgi->rli; DBUG_ENTER("apply_event_and_update_pos"); DBUG_PRINT("exec_event",("%s(type_code: %d; server_id: %d)", ev->get_type_str(), ev->get_type_code(), ev->server_id)); - DBUG_PRINT("info", ("thd->options: %s%s; rli->last_event_start_time: %lu", + DBUG_PRINT("info", ("thd->options: %s%s; rgi->last_event_start_time: %lu", FLAGSTR(thd->variables.option_bits, OPTION_NOT_AUTOCOMMIT), FLAGSTR(thd->variables.option_bits, OPTION_BEGIN), - (ulong) rli->last_event_start_time)); + (ulong) rgi->last_event_start_time)); /* Execute the event to change the database and update the binary @@ -2615,7 +3164,8 @@ int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli) has a Rotate etc). */ - thd->server_id = ev->server_id; // use the original server id for logging + /* Use the original server id for logging. */ + thd->variables.server_id = ev->server_id; thd->set_time(); // time the query thd->lex->current_select= 0; if (!ev->when) @@ -2629,12 +3179,21 @@ int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli) (ev->flags & LOG_EVENT_SKIP_REPLICATION_F ? OPTION_SKIP_REPLICATION : 0); ev->thd = thd; // because up to this point, ev->thd == 0 - int reason= ev->shall_skip(rli); + int reason= ev->shall_skip(rgi); if (reason == Log_event::EVENT_SKIP_COUNT) - sql_slave_skip_counter= --rli->slave_skip_counter; + { + DBUG_ASSERT(rli->slave_skip_counter > 0); + rli->slave_skip_counter--; + } mysql_mutex_unlock(&rli->data_lock); + DBUG_EXECUTE_IF("inject_slave_sql_before_apply_event", + { + DBUG_ASSERT(!debug_sync_set_action + (thd, STRING_WITH_LEN("now WAIT_FOR continue"))); + DBUG_SET_INITIAL("-d,inject_slave_sql_before_apply_event"); + };); if (reason == Log_event::EVENT_SKIP_NOT) - exec_res= ev->apply_event(rli); + exec_res= ev->apply_event(rgi); #ifndef DBUG_OFF /* @@ -2650,9 +3209,10 @@ int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli) // EVENT_SKIP_COUNT "skipped because event skip counter was non-zero" }; - DBUG_PRINT("info", ("OPTION_BEGIN: %d; IN_STMT: %d", + DBUG_PRINT("info", ("OPTION_BEGIN: %d IN_STMT: %d IN_TRANSACTION: %d", test(thd->variables.option_bits & OPTION_BEGIN), - rli->get_flag(Relay_log_info::IN_STMT))); + rli->get_flag(Relay_log_info::IN_STMT), + rli->get_flag(Relay_log_info::IN_TRANSACTION))); DBUG_PRINT("skip_event", ("%s event was %s", ev->get_type_str(), explain[reason])); #endif @@ -2660,7 +3220,7 @@ int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli) DBUG_PRINT("info", ("apply_event error = %d", exec_res)); if (exec_res == 0) { - int error= ev->update_pos(rli); + int error= ev->update_pos(rgi); #ifdef HAVE_valgrind if (!rli->is_fake) #endif @@ -2696,12 +3256,94 @@ int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli) DBUG_RETURN(2); } } + else + { + /* + Make sure we do not errorneously update gtid_slave_pos with a lingering + GTID from this failed event group (MDEV-4906). + */ + rgi->gtid_sub_id= 0; + } DBUG_RETURN(exec_res ? 1 : 0); } /** + Keep the relay log transaction state up to date. + + The state reflects how things are after the given event, that has just been + read from the relay log, is executed. + + This is only needed to ensure we: + - Don't abort the sql driver thread in the middle of an event group. + - Don't rotate the io thread in the middle of a statement or transaction. + The mechanism is that the io thread, when it needs to rotate the relay + log, will wait until the sql driver has read all the cached events + and then continue reading events one by one from the master until + the sql threads signals that log doesn't have an active group anymore. + + There are two possible cases. We keep them as 2 separate flags mainly + to make debugging easier. + + - IN_STMT is set when we have read an event that should be used + together with the next event. This is for example setting a + variable that is used when executing the next statement. + - IN_TRANSACTION is set when we are inside a BEGIN...COMMIT group + + To test the state one should use the is_in_group() function. +*/ + +inline void update_state_of_relay_log(Relay_log_info *rli, Log_event *ev) +{ + Log_event_type typ= ev->get_type_code(); + + /* check if we are in a multi part event */ + if (ev->is_part_of_group()) + rli->set_flag(Relay_log_info::IN_STMT); + else if (Log_event::is_group_event(typ)) + { + /* + If it was not a is_part_of_group() and not a group event (like + rotate) then we can reset the IN_STMT flag. We have the above + if only to allow us to have a rotate element anywhere. + */ + rli->clear_flag(Relay_log_info::IN_STMT); + } + + /* Check for an event that starts or stops a transaction */ + if (typ == QUERY_EVENT) + { + Query_log_event *qev= (Query_log_event*) ev; + /* + Trivial optimization to avoid the following somewhat expensive + checks. + */ + if (qev->q_len <= sizeof("ROLLBACK")) + { + if (qev->is_begin()) + rli->set_flag(Relay_log_info::IN_TRANSACTION); + if (qev->is_commit() || qev->is_rollback()) + rli->clear_flag(Relay_log_info::IN_TRANSACTION); + } + } + if (typ == XID_EVENT) + rli->clear_flag(Relay_log_info::IN_TRANSACTION); + if (typ == GTID_EVENT && + !(((Gtid_log_event*) ev)->flags2 & Gtid_log_event::FL_STANDALONE)) + { + /* This GTID_EVENT will generate a BEGIN event */ + rli->set_flag(Relay_log_info::IN_TRANSACTION); + } + + DBUG_PRINT("info", ("event: %u IN_STMT: %d IN_TRANSACTION: %d", + (uint) typ, + rli->get_flag(Relay_log_info::IN_STMT), + rli->get_flag(Relay_log_info::IN_TRANSACTION))); +} + + +/** Top-level function for executing the next event from the relay log. This function reads the event from the relay log, executes it, and @@ -2729,22 +3371,23 @@ int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli) @retval 1 The event was not applied. */ -static int exec_relay_log_event(THD* thd, Relay_log_info* rli) + +static int exec_relay_log_event(THD* thd, Relay_log_info* rli, + rpl_group_info *serial_rgi) { + ulonglong event_size; DBUG_ENTER("exec_relay_log_event"); /* - We acquire this mutex since we need it for all operations except - event execution. But we will release it in places where we will - wait for something for example inside of next_event(). - */ + We acquire this mutex since we need it for all operations except + event execution. But we will release it in places where we will + wait for something for example inside of next_event(). + */ mysql_mutex_lock(&rli->data_lock); - Log_event * ev = next_event(rli); + Log_event *ev= next_event(serial_rgi, &event_size); - DBUG_ASSERT(rli->sql_thd==thd); - - if (sql_slave_killed(thd,rli)) + if (sql_slave_killed(serial_rgi)) { mysql_mutex_unlock(&rli->data_lock); delete ev; @@ -2753,20 +3396,22 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli) if (ev) { int exec_res; + Log_event_type typ= ev->get_type_code(); /* This tests if the position of the beginning of the current event hits the UNTIL barrier. */ - if (rli->until_condition != Relay_log_info::UNTIL_NONE && + if ((rli->until_condition == Relay_log_info::UNTIL_MASTER_POS || + rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && rli->is_until_satisfied(thd, ev)) { char buf[22]; sql_print_information("Slave SQL thread stopped because it reached its" " UNTIL position %s", llstr(rli->until_pos(), buf)); /* - Setting abort_slave flag because we do not want additional message about - error in query execution to be printed. + Setting abort_slave flag because we do not want additional + message about error in query execution to be printed. */ rli->abort_slave= 1; mysql_mutex_unlock(&rli->data_lock); @@ -2781,56 +3426,49 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli) read hanging if the realy log does not have any more events. */ DBUG_EXECUTE_IF("incomplete_group_in_relay_log", - if ((ev->get_type_code() == XID_EVENT) || - ((ev->get_type_code() == QUERY_EVENT) && + if ((typ == XID_EVENT) || + ((typ == QUERY_EVENT) && strcmp("COMMIT", ((Query_log_event *) ev)->query) == 0)) { DBUG_ASSERT(thd->transaction.all.modified_non_trans_table); rli->abort_slave= 1; mysql_mutex_unlock(&rli->data_lock); delete ev; - rli->inc_event_relay_log_pos(); + serial_rgi->inc_event_relay_log_pos(); DBUG_RETURN(0); };); } - exec_res= apply_event_and_update_pos(ev, thd, rli); + update_state_of_relay_log(rli, ev); - switch (ev->get_type_code()) { - case FORMAT_DESCRIPTION_EVENT: - /* - Format_description_log_event should not be deleted because it - will be used to read info about the relay log's format; - it will be deleted when the SQL thread does not need it, - i.e. when this thread terminates. - */ - break; - case ANNOTATE_ROWS_EVENT: - /* - Annotate_rows event should not be deleted because after it has - been applied, thd->query points to the string inside this event. - The thd->query will be used to generate new Annotate_rows event - during applying the subsequent Rows events. - */ - rli->set_annotate_event((Annotate_rows_log_event*) ev); - break; - case DELETE_ROWS_EVENT: - case UPDATE_ROWS_EVENT: - case WRITE_ROWS_EVENT: - /* - After the last Rows event has been applied, the saved Annotate_rows - event (if any) is not needed anymore and can be deleted. - */ - if (((Rows_log_event*)ev)->get_flags(Rows_log_event::STMT_END_F)) - rli->free_annotate_event(); - /* fall through */ - default: - DBUG_PRINT("info", ("Deleting the event after it has been executed")); - if (!rli->is_deferred_event(ev)) - delete ev; - break; + /* + Execute queries in parallel, except if slave_skip_counter is set, + as it's is easier to skip queries in single threaded mode. + */ + + if (opt_slave_parallel_threads > 0 && rli->slave_skip_counter == 0) + DBUG_RETURN(rli->parallel.do_event(serial_rgi, ev, event_size)); + + /* + For GTID, allocate a new sub_id for the given domain_id. + The sub_id must be allocated in increasing order of binlog order. + */ + if (typ == GTID_EVENT && + event_group_new_gtid(serial_rgi, static_cast<Gtid_log_event *>(ev))) + { + sql_print_error("Error reading relay log event: %s", + "slave SQL thread aborted because of out-of-memory error"); + mysql_mutex_unlock(&rli->data_lock); + delete ev; + DBUG_RETURN(1); } + serial_rgi->future_event_relay_log_pos= rli->future_event_relay_log_pos; + serial_rgi->event_relay_log_name= rli->event_relay_log_name; + serial_rgi->event_relay_log_pos= rli->event_relay_log_pos; + exec_res= apply_event_and_update_pos(ev, thd, serial_rgi, NULL); + + delete_or_keep_event_post_apply(serial_rgi, typ, ev); /* update_log_pos failed: this should not happen, so we don't @@ -2846,6 +3484,7 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli) if (exec_res && (temp_err= has_temporary_error(thd))) { const char *errmsg; + rli->clear_error(); /* We were in a transaction which has been rolled back because of a temporary error; @@ -2853,14 +3492,16 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli) Note, if lock wait timeout (innodb_lock_wait_timeout exceeded) there is no rollback since 5.0.13 (ref: manual). We have to not only seek but also - a) init_master_info(), to seek back to hot relay log's start for later - (for when we will come back to this hot log after re-processing the - possibly existing old logs where BEGIN is: check_binlog_magic() will - then need the cache to be at position 0 (see comments at beginning of + + a) init_master_info(), to seek back to hot relay log's start + for later (for when we will come back to this hot log after + re-processing the possibly existing old logs where BEGIN is: + check_binlog_magic() will then need the cache to be at + position 0 (see comments at beginning of init_master_info()). b) init_relay_log_pos(), because the BEGIN may be an older relay log. */ - if (rli->trans_retries < slave_trans_retries) + if (serial_rgi->trans_retries < slave_trans_retries) { if (init_master_info(rli->mi, 0, 0, 0, SLAVE_SQL)) sql_print_error("Failed to initialize the master info structure"); @@ -2873,16 +3514,19 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli) else { exec_res= 0; - rli->cleanup_context(thd, 1); + serial_rgi->cleanup_context(thd, 1); /* chance for concurrent connection to get more locks */ - slave_sleep(thd, min(rli->trans_retries, MAX_SLAVE_RETRY_PAUSE), - sql_slave_killed, rli); + slave_sleep(thd, min(serial_rgi->trans_retries, + MAX_SLAVE_RETRY_PAUSE), + sql_slave_killed, serial_rgi); + serial_rgi->trans_retries++; mysql_mutex_lock(&rli->data_lock); // because of SHOW STATUS - rli->trans_retries++; rli->retried_trans++; + statistic_increment(slave_retried_transactions, LOCK_status); mysql_mutex_unlock(&rli->data_lock); DBUG_PRINT("info", ("Slave retries transaction " - "rli->trans_retries: %lu", rli->trans_retries)); + "rgi->trans_retries: %lu", + serial_rgi->trans_retries)); } } else @@ -2901,11 +3545,13 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli) event, the execution will proceed as usual; in the case of a non-transient error, the slave will stop with an error. */ - rli->trans_retries= 0; // restart from fresh - DBUG_PRINT("info", ("Resetting retry counter, rli->trans_retries: %lu", - rli->trans_retries)); + serial_rgi->trans_retries= 0; // restart from fresh + DBUG_PRINT("info", ("Resetting retry counter, rgi->trans_retries: %lu", + serial_rgi->trans_retries)); } } + thread_safe_increment64(&rli->executed_entries, + &slave_executed_entries_lock); DBUG_RETURN(exec_res); } mysql_mutex_unlock(&rli->data_lock); @@ -2923,9 +3569,9 @@ on this slave.\ } -static bool check_io_slave_killed(THD *thd, Master_info *mi, const char *info) +static bool check_io_slave_killed(Master_info *mi, const char *info) { - if (io_slave_killed(thd, mi)) + if (io_slave_killed(mi)) { if (info && global_system_variables.log_warnings) sql_print_information("%s", info); @@ -2976,14 +3622,28 @@ static int try_to_reconnect(THD *thd, MYSQL *mysql, Master_info *mi, return 1; // Don't retry forever slave_sleep(thd, mi->connect_retry, io_slave_killed, mi); } - if (check_io_slave_killed(thd, mi, messages[SLAVE_RECON_MSG_KILLED_WAITING])) + if (check_io_slave_killed(mi, messages[SLAVE_RECON_MSG_KILLED_WAITING])) return 1; thd->proc_info = messages[SLAVE_RECON_MSG_AFTER]; if (!suppress_warnings) { char buf[256], llbuff[22]; + String tmp; + if (mi->using_gtid != Master_info::USE_GTID_NO) + { + tmp.append(STRING_WITH_LEN("; GTID position '")); + mi->gtid_current_pos.append_to_string(&tmp); + if (mi->events_queued_since_last_gtid == 0) + tmp.append(STRING_WITH_LEN("'")); + else + { + tmp.append(STRING_WITH_LEN("', GTID event skip ")); + tmp.append_ulonglong((ulonglong)mi->events_queued_since_last_gtid); + } + } my_snprintf(buf, sizeof(buf), messages[SLAVE_RECON_MSG_FAILED], - IO_RPL_LOG_NAME, llstr(mi->master_log_pos, llbuff)); + IO_RPL_LOG_NAME, llstr(mi->master_log_pos, llbuff), + tmp.c_ptr_safe()); /* Raise a warining during registering on master/requesting dump. Log a message reading event. @@ -2999,7 +3659,7 @@ static int try_to_reconnect(THD *thd, MYSQL *mysql, Master_info *mi, sql_print_information("%s", buf); } } - if (safe_reconnect(thd, mysql, mi, 1) || io_slave_killed(thd, mi)) + if (safe_reconnect(thd, mysql, mi, 1) || io_slave_killed(mi)) { if (global_system_variables.log_warnings) sql_print_information("%s", messages[SLAVE_RECON_MSG_KILLED_AFTER]); @@ -3054,7 +3714,7 @@ pthread_handler_t handle_slave_io(void *arg) pthread_detach_this_thread(); thd->thread_stack= (char*) &thd; // remember where our stack is mi->clear_error(); - if (init_slave_thread(thd, SLAVE_THD_IO)) + if (init_slave_thread(thd, mi, SLAVE_THD_IO)) { mysql_cond_broadcast(&mi->start_cond); sql_print_error("Failed during slave I/O thread initialization"); @@ -3063,7 +3723,7 @@ pthread_handler_t handle_slave_io(void *arg) mysql_mutex_lock(&LOCK_thread_count); threads.append(thd); mysql_mutex_unlock(&LOCK_thread_count); - mi->slave_running = 1; + mi->slave_running = MYSQL_SLAVE_RUN_NOT_CONNECT; mi->abort_slave = 0; mysql_mutex_unlock(&mi->run_lock); mysql_cond_broadcast(&mi->start_cond); @@ -3075,6 +3735,22 @@ pthread_handler_t handle_slave_io(void *arg) /* This must be called before run any binlog_relay_io hooks */ my_pthread_setspecific_ptr(RPL_MASTER_INFO, mi); + /* Load the set of seen GTIDs, if we did not already. */ + if (rpl_load_gtid_slave_state(thd)) + { + mi->report(ERROR_LEVEL, thd->stmt_da->sql_errno(), + "Unable to load replication GTID slave state from mysql.%s: %s", + rpl_gtid_slave_state_table_name.str, thd->stmt_da->message()); + /* + If we are using old-style replication, we can continue, even though we + then will not be able to record the GTIDs we receive. But if using GTID, + we must give up. + */ + if (mi->using_gtid != Master_info::USE_GTID_NO || opt_gtid_strict_mode) + goto err; + } + + if (RUN_HOOK(binlog_relay_io, thread_start, (thd, mi))) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, @@ -3093,11 +3769,20 @@ pthread_handler_t handle_slave_io(void *arg) // we can get killed during safe_connect if (!safe_connect(thd, mysql, mi)) { - sql_print_information("Slave I/O thread: connected to master '%s@%s:%d'," - "replication started in log '%s' at position %s", - mi->user, mi->host, mi->port, - IO_RPL_LOG_NAME, - llstr(mi->master_log_pos,llbuff)); + if (mi->using_gtid == Master_info::USE_GTID_NO) + sql_print_information("Slave I/O thread: connected to master '%s@%s:%d'," + "replication started in log '%s' at position %s", + mi->user, mi->host, mi->port, + IO_RPL_LOG_NAME, + llstr(mi->master_log_pos,llbuff)); + else + { + String tmp; + mi->gtid_current_pos.to_string(&tmp); + sql_print_information("Slave I/O thread: connected to master '%s@%s:%d'," + "replication starts at GTID position '%s'", + mi->user, mi->host, mi->port, tmp.c_ptr_safe()); + } } else { @@ -3107,6 +3792,25 @@ pthread_handler_t handle_slave_io(void *arg) connected: + if (mi->using_gtid != Master_info::USE_GTID_NO) + { + /* + When the IO thread (re)connects to the master using GTID, it will + connect at the start of an event group. But the IO thread may have + previously logged part of the following event group to the relay + log. + + When the IO and SQL thread are started together, we erase any previous + relay logs, but this is not possible/desirable while the SQL thread is + running. To avoid duplicating partial event groups in the relay logs in + this case, we remember the count of events in any partially logged event + group before the reconnect, and then here at connect we set up a counter + to skip the already-logged part of the group. + */ + mi->gtid_reconnect_event_skip_count= mi->events_queued_since_last_gtid; + mi->gtid_event_seen= false; + } + #ifdef ENABLED_DEBUG_SYNC DBUG_EXECUTE_IF("dbug.before_get_running_status_yes", { @@ -3130,11 +3834,14 @@ connected: if (ret == 2) { - if (check_io_slave_killed(mi->io_thd, mi, "Slave I/O thread killed" + if (check_io_slave_killed(mi, "Slave I/O thread killed" "while calling get_master_version_and_clock(...)")) goto err; suppress_warnings= FALSE; - /* Try to reconnect because the error was caused by a transient network problem */ + /* + Try to reconnect because the error was caused by a transient network + problem + */ if (try_to_reconnect(thd, mysql, mi, &retry_count, suppress_warnings, reconnect_messages[SLAVE_RECON_ACT_REG])) goto err; @@ -3149,7 +3856,7 @@ connected: thd_proc_info(thd, "Registering slave on master"); if (register_slave_on_master(mysql, mi, &suppress_warnings)) { - if (!check_io_slave_killed(thd, mi, "Slave I/O thread killed " + if (!check_io_slave_killed(mi, "Slave I/O thread killed " "while registering slave on master")) { sql_print_error("Slave I/O thread couldn't register on master"); @@ -3174,13 +3881,13 @@ connected: } DBUG_PRINT("info",("Starting reading binary log from master")); - while (!io_slave_killed(thd,mi)) + while (!io_slave_killed(mi)) { thd_proc_info(thd, "Requesting binlog dump"); if (request_dump(thd, mysql, mi, &suppress_warnings)) { sql_print_error("Failed on request_dump()"); - if (check_io_slave_killed(thd, mi, "Slave I/O thread killed while \ + if (check_io_slave_killed(mi, "Slave I/O thread killed while \ requesting master dump") || try_to_reconnect(thd, mysql, mi, &retry_count, suppress_warnings, reconnect_messages[SLAVE_RECON_ACT_DUMP])) @@ -3200,7 +3907,7 @@ requesting master dump") || const char *event_buf; DBUG_ASSERT(mi->last_error().number == 0); - while (!io_slave_killed(thd,mi)) + while (!io_slave_killed(mi)) { ulong event_len; /* @@ -3211,7 +3918,7 @@ requesting master dump") || */ thd_proc_info(thd, "Waiting for master to send event"); event_len= read_event(mysql, mi, &suppress_warnings); - if (check_io_slave_killed(thd, mi, "Slave I/O thread killed while \ + if (check_io_slave_killed(mi, "Slave I/O thread killed while \ reading event")) goto err; DBUG_EXECUTE_IF("FORCE_SLAVE_TO_RECONNECT_EVENT", @@ -3289,7 +3996,8 @@ Stopping slave I/O thread due to out-of-memory error from master"); goto err; } - if (flush_master_info(mi, TRUE, TRUE)) + if (mi->using_gtid == Master_info::USE_GTID_NO && + flush_master_info(mi, TRUE, TRUE)) { sql_print_error("Failed to flush master info file"); goto err; @@ -3301,10 +4009,11 @@ Stopping slave I/O thread due to out-of-memory error from master"); - if mi->rli.ignore_log_space_limit is 1 but becomes 0 just after (so the clean value is 0), then we are reading only one more event as we should, and we'll block only at the next event. No big deal. - - if mi->rli.ignore_log_space_limit is 0 but becomes 1 just after (so - the clean value is 1), then we are going into wait_for_relay_log_space() - for no reason, but this function will do a clean read, notice the clean - value and exit immediately. + - if mi->rli.ignore_log_space_limit is 0 but becomes 1 just + after (so the clean value is 1), then we are going into + wait_for_relay_log_space() for no reason, but this function + will do a clean read, notice the clean value and exit + immediately. */ #ifndef DBUG_OFF { @@ -3332,8 +4041,19 @@ log space"); // error = 0; err: // print the current replication position - sql_print_information("Slave I/O thread exiting, read up to log '%s', position %s", - IO_RPL_LOG_NAME, llstr(mi->master_log_pos,llbuff)); + if (mi->using_gtid == Master_info::USE_GTID_NO) + sql_print_information("Slave I/O thread exiting, read up to log '%s', " + "position %s", + IO_RPL_LOG_NAME, llstr(mi->master_log_pos,llbuff)); + else + { + String tmp; + mi->gtid_current_pos.to_string(&tmp); + sql_print_information("Slave I/O thread exiting, read up to log '%s', " + "position %s; GTID position %s", + IO_RPL_LOG_NAME, llstr(mi->master_log_pos,llbuff), + tmp.c_ptr_safe()); + } RUN_HOOK(binlog_relay_io, thread_stop, (thd, mi)); thd->reset_query(); thd->reset_db(NULL, 0); @@ -3354,7 +4074,9 @@ err: mi->mysql=0; } write_ignored_events_info_to_relay_log(thd, mi); - thd_proc_info(thd, "Waiting for slave mutex on exit"); + if (mi->using_gtid != Master_info::USE_GTID_NO) + flush_master_info(mi, TRUE, TRUE); + thd_proc_info(thd, "Slave io thread waiting for slave mutex on exit"); mysql_mutex_lock(&mi->run_lock); err_during_init: @@ -3363,14 +4085,12 @@ err_during_init: mi->rli.relay_log.description_event_for_queue= 0; // TODO: make rpl_status part of Master_info change_rpl_status(RPL_ACTIVE_SLAVE,RPL_IDLE_SLAVE); - DBUG_ASSERT(thd->net.buff != 0); - net_end(&thd->net); // destructor will not free it, because net.vio is 0 mysql_mutex_lock(&LOCK_thread_count); THD_CHECK_SENTRY(thd); delete thd; mysql_mutex_unlock(&LOCK_thread_count); mi->abort_slave= 0; - mi->slave_running= 0; + mi->slave_running= MYSQL_SLAVE_NOT_RUN; mi->io_thd= 0; /* Note: the order of the two following calls (first broadcast, then unlock) @@ -3393,17 +4113,33 @@ err_during_init: /* Check the temporary directory used by commands like LOAD DATA INFILE. + + As the directory never changes during a mysqld run, we only + test this once and cache the result. This also resolve a race condition + when this can be run by multiple threads at the same time. */ + +static bool check_temp_dir_run= 0; +static int check_temp_dir_result= 0; + static int check_temp_dir(char* tmp_file) { - int fd; + File fd; + int result= 1; // Assume failure MY_DIR *dirp; char tmp_dir[FN_REFLEN]; size_t tmp_dir_size; - DBUG_ENTER("check_temp_dir"); + mysql_mutex_lock(&LOCK_thread_count); + if (check_temp_dir_run) + { + result= check_temp_dir_result; + goto end; + } + check_temp_dir_run= 1; + /* Get the directory from the temporary file. */ @@ -3413,27 +4149,119 @@ int check_temp_dir(char* tmp_file) Check if the directory exists. */ if (!(dirp=my_dir(tmp_dir,MYF(MY_WME)))) - DBUG_RETURN(1); + goto end; my_dirend(dirp); /* - Check permissions to create a file. + Check permissions to create a file. We use O_TRUNC to ensure that + things works even if we happen to have and old file laying around. */ if ((fd= mysql_file_create(key_file_misc, tmp_file, CREATE_MODE, - O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW, + O_WRONLY | O_BINARY | O_TRUNC | O_NOFOLLOW, MYF(MY_WME))) < 0) - DBUG_RETURN(1); + goto end; + result= 0; // Directory name ok /* Clean up. */ mysql_file_close(fd, MYF(0)); mysql_file_delete(key_file_misc, tmp_file, MYF(0)); - DBUG_RETURN(0); +end: + check_temp_dir_result= result; + mysql_mutex_unlock(&LOCK_thread_count); + DBUG_RETURN(result); +} + + +void +slave_output_error_info(Relay_log_info *rli, THD *thd) +{ + /* + retrieve as much info as possible from the thd and, error + codes and warnings and print this to the error log as to + allow the user to locate the error + */ + uint32 const last_errno= rli->last_error().number; + char llbuff[22]; + + if (thd->is_error()) + { + char const *const errmsg= thd->stmt_da->message(); + + DBUG_PRINT("info", + ("thd->stmt_da->sql_errno()=%d; rli->last_error.number=%d", + thd->stmt_da->sql_errno(), last_errno)); + if (last_errno == 0) + { + /* + This function is reporting an error which was not reported + while executing exec_relay_log_event(). + */ + rli->report(ERROR_LEVEL, thd->stmt_da->sql_errno(), "%s", errmsg); + } + else if (last_errno != thd->stmt_da->sql_errno()) + { + /* + * An error was reported while executing exec_relay_log_event() + * however the error code differs from what is in the thread. + * This function prints out more information to help finding + * what caused the problem. + */ + sql_print_error("Slave (additional info): %s Error_code: %d", + errmsg, thd->stmt_da->sql_errno()); + } + } + + /* Print any warnings issued */ + List_iterator_fast<MYSQL_ERROR> it(thd->warning_info->warn_list()); + MYSQL_ERROR *err; + /* + Added controlled slave thread cancel for replication + of user-defined variables. + */ + bool udf_error = false; + while ((err= it++)) + { + if (err->get_sql_errno() == ER_CANT_OPEN_LIBRARY) + udf_error = true; + sql_print_warning("Slave: %s Error_code: %d", err->get_message_text(), err->get_sql_errno()); + } + if (udf_error) + { + String tmp; + if (rli->mi->using_gtid != Master_info::USE_GTID_NO) + { + tmp.append(STRING_WITH_LEN("; GTID position '")); + rpl_append_gtid_state(&tmp, false); + tmp.append(STRING_WITH_LEN("'")); + } + sql_print_error("Error loading user-defined library, slave SQL " + "thread aborted. Install the missing library, and restart the " + "slave SQL thread with \"SLAVE START\". We stopped at log '%s' " + "position %s%s", RPL_LOG_NAME, llstr(rli->group_master_log_pos, + llbuff), tmp.c_ptr_safe()); + } + else + { + String tmp; + if (rli->mi->using_gtid != Master_info::USE_GTID_NO) + { + tmp.append(STRING_WITH_LEN("; GTID position '")); + rpl_append_gtid_state(&tmp, false); + tmp.append(STRING_WITH_LEN("'")); + } + sql_print_error("\ +Error running query, slave SQL thread aborted. Fix the problem, and restart \ +the slave SQL thread with \"SLAVE START\". We stopped at log \ +'%s' position %s%s", RPL_LOG_NAME, llstr(rli->group_master_log_pos, llbuff), + tmp.c_ptr_safe()); + } } + /** Slave SQL thread entry point. @@ -3451,8 +4279,10 @@ pthread_handler_t handle_slave_sql(void *arg) my_off_t UNINIT_VAR(saved_log_pos); my_off_t UNINIT_VAR(saved_master_log_pos); my_off_t saved_skip= 0; - Relay_log_info* rli = &((Master_info*)arg)->rli; + Master_info *mi= ((Master_info*)arg); + Relay_log_info* rli = &mi->rli; const char *errmsg; + rpl_group_info *serial_rgi; // needs to call my_thread_init(), otherwise we get a coredump in DBUG_ stuff my_thread_init(); @@ -3461,10 +4291,13 @@ pthread_handler_t handle_slave_sql(void *arg) LINT_INIT(saved_master_log_pos); LINT_INIT(saved_log_pos); + serial_rgi= new rpl_group_info(rli); thd = new THD; // note that contructor of THD uses DBUG_ ! thd->thread_stack = (char*)&thd; // remember where our stack is + thd->rpl_filter = mi->rpl_filter; DBUG_ASSERT(rli->inited); + DBUG_ASSERT(rli->mi == mi); mysql_mutex_lock(&rli->run_lock); DBUG_ASSERT(!rli->slave_running); errmsg= 0; @@ -3472,14 +4305,22 @@ pthread_handler_t handle_slave_sql(void *arg) rli->events_till_abort = abort_slave_event_count; #endif - rli->sql_thd= thd; + /* + THD for the sql driver thd. In parallel replication this is the thread + that reads things from the relay log and calls rpl_parallel::do_event() + to execute queries. + + In single thread replication this is the THD for the thread that is + executing SQL queries too. + */ + serial_rgi->thd= rli->sql_driver_thd= thd; /* Inform waiting threads that slave has started */ rli->slave_run_id++; - rli->slave_running = 1; + rli->slave_running= MYSQL_SLAVE_RUN_NOT_CONNECT; pthread_detach_this_thread(); - if (init_slave_thread(thd, SLAVE_THD_SQL)) + if (init_slave_thread(thd, mi, SLAVE_THD_SQL)) { /* TODO: this is currently broken - slave start and change master @@ -3491,14 +4332,12 @@ pthread_handler_t handle_slave_sql(void *arg) goto err_during_init; } thd->init_for_queries(); - thd->rli_slave= rli; - if ((rli->deferred_events_collecting= rpl_filter->is_on())) + thd->rgi_slave= serial_rgi; + if ((serial_rgi->deferred_events_collecting= mi->rpl_filter->is_on())) { - rli->deferred_events= new Deferred_log_events(rli); + serial_rgi->deferred_events= new Deferred_log_events(rli); } - thd->temporary_tables = rli->save_temporary_tables; // restore temp tables - set_thd_in_use_temporary_tables(rli); // (re)set sql_thd in use for saved temp tables /* binlog_annotate_row_events must be TRUE only after an Annotate_rows event has been recieved and only till the last corresponding rbr event has been @@ -3531,14 +4370,14 @@ pthread_handler_t handle_slave_sql(void *arg) But the master timestamp is reset by RESET SLAVE & CHANGE MASTER. */ rli->clear_error(); + rli->parallel.reset(); //tell the I/O thread to take relay_log_space_limit into account from now on mysql_mutex_lock(&rli->log_space_lock); rli->ignore_log_space_limit= 0; mysql_mutex_unlock(&rli->log_space_lock); - rli->trans_retries= 0; // start from "no error" - DBUG_PRINT("info", ("rli->trans_retries: %lu", rli->trans_retries)); + serial_rgi->gtid_sub_id= 0; if (init_relay_log_pos(rli, rli->group_relay_log_name, rli->group_relay_log_pos, @@ -3549,6 +4388,7 @@ pthread_handler_t handle_slave_sql(void *arg) "Error initializing relay log position: %s", errmsg); goto err; } + strcpy(rli->future_event_master_log_name, rli->group_master_log_name); THD_CHECK_SENTRY(thd); #ifndef DBUG_OFF { @@ -3574,16 +4414,25 @@ pthread_handler_t handle_slave_sql(void *arg) #endif } #endif - DBUG_ASSERT(rli->sql_thd == thd); DBUG_PRINT("master_info",("log_file_name: %s position: %s", rli->group_master_log_name, llstr(rli->group_master_log_pos,llbuff))); if (global_system_variables.log_warnings) + { + String tmp; + if (mi->using_gtid != Master_info::USE_GTID_NO) + { + tmp.append(STRING_WITH_LEN("; GTID position '")); + rpl_append_gtid_state(&tmp, + mi->using_gtid==Master_info::USE_GTID_CURRENT_POS); + tmp.append(STRING_WITH_LEN("'")); + } sql_print_information("Slave SQL thread initialized, starting replication in \ -log '%s' at position %s, relay log '%s' position: %s", RPL_LOG_NAME, +log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, llstr(rli->group_master_log_pos,llbuff),rli->group_relay_log_name, - llstr(rli->group_relay_log_pos,llbuff1)); + llstr(rli->group_relay_log_pos,llbuff1), tmp.c_ptr_safe()); + } if (check_temp_dir(rli->slave_patternload_file)) { @@ -3593,6 +4442,21 @@ log '%s' at position %s, relay log '%s' position: %s", RPL_LOG_NAME, goto err; } + /* Load the set of seen GTIDs, if we did not already. */ + if (rpl_load_gtid_slave_state(thd)) + { + rli->report(ERROR_LEVEL, thd->stmt_da->sql_errno(), + "Unable to load replication GTID slave state from mysql.%s: %s", + rpl_gtid_slave_state_table_name.str, thd->stmt_da->message()); + /* + If we are using old-style replication, we can continue, even though we + then will not be able to record the GTIDs we receive. But if using GTID, + we must give up. + */ + if (mi->using_gtid != Master_info::USE_GTID_NO || opt_gtid_strict_mode) + goto err; + } + /* execute init_slave variable */ if (opt_init_slave.length) { @@ -3618,7 +4482,8 @@ log '%s' at position %s, relay log '%s' position: %s", RPL_LOG_NAME, saved_master_log_pos= rli->group_master_log_pos; saved_skip= rli->slave_skip_counter; } - if (rli->until_condition != Relay_log_info::UNTIL_NONE && + if ((rli->until_condition == Relay_log_info::UNTIL_MASTER_POS || + rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && rli->is_until_satisfied(thd, NULL)) { char buf[22]; @@ -3631,10 +4496,9 @@ log '%s' at position %s, relay log '%s' position: %s", RPL_LOG_NAME, /* Read queries from the IO/THREAD until this thread is killed */ - while (!sql_slave_killed(thd,rli)) + while (!sql_slave_killed(serial_rgi)) { thd_proc_info(thd, "Reading event from the relay log"); - DBUG_ASSERT(rli->sql_thd == thd); THD_CHECK_SENTRY(thd); if (saved_skip && rli->slave_skip_counter == 0) @@ -3651,92 +4515,53 @@ log '%s' at position %s, relay log '%s' position: %s", RPL_LOG_NAME, saved_skip= 0; } - if (exec_relay_log_event(thd,rli)) + if (exec_relay_log_event(thd, rli, serial_rgi)) { DBUG_PRINT("info", ("exec_relay_log_event() failed")); // do not scare the user if SQL thread was simply killed or stopped - if (!sql_slave_killed(thd,rli)) - { - /* - retrieve as much info as possible from the thd and, error - codes and warnings and print this to the error log as to - allow the user to locate the error - */ - uint32 const last_errno= rli->last_error().number; - - if (thd->is_error()) - { - char const *const errmsg= thd->stmt_da->message(); - - DBUG_PRINT("info", - ("thd->stmt_da->sql_errno()=%d; rli->last_error.number=%d", - thd->stmt_da->sql_errno(), last_errno)); - if (last_errno == 0) - { - /* - This function is reporting an error which was not reported - while executing exec_relay_log_event(). - */ - rli->report(ERROR_LEVEL, thd->stmt_da->sql_errno(), "%s", errmsg); - } - else if (last_errno != thd->stmt_da->sql_errno()) - { - /* - * An error was reported while executing exec_relay_log_event() - * however the error code differs from what is in the thread. - * This function prints out more information to help finding - * what caused the problem. - */ - sql_print_error("Slave (additional info): %s Error_code: %d", - errmsg, thd->stmt_da->sql_errno()); - } - } - - /* Print any warnings issued */ - List_iterator_fast<MYSQL_ERROR> it(thd->warning_info->warn_list()); - MYSQL_ERROR *err; - /* - Added controlled slave thread cancel for replication - of user-defined variables. - */ - bool udf_error = false; - while ((err= it++)) - { - if (err->get_sql_errno() == ER_CANT_OPEN_LIBRARY) - udf_error = true; - sql_print_warning("Slave: %s Error_code: %d", err->get_message_text(), err->get_sql_errno()); - } - if (udf_error) - sql_print_error("Error loading user-defined library, slave SQL " - "thread aborted. Install the missing library, and restart the " - "slave SQL thread with \"SLAVE START\". We stopped at log '%s' " - "position %s", RPL_LOG_NAME, llstr(rli->group_master_log_pos, - llbuff)); - else - sql_print_error("\ -Error running query, slave SQL thread aborted. Fix the problem, and restart \ -the slave SQL thread with \"SLAVE START\". We stopped at log \ -'%s' position %s", RPL_LOG_NAME, llstr(rli->group_master_log_pos, llbuff)); - } + if (!sql_slave_killed(serial_rgi)) + slave_output_error_info(rli, thd); goto err; } } + if (opt_slave_parallel_threads > 0) + rli->parallel.wait_for_done(); + /* Thread stopped. Print the current replication position to the log */ - sql_print_information("Slave SQL thread exiting, replication stopped in log " - "'%s' at position %s", - RPL_LOG_NAME, llstr(rli->group_master_log_pos,llbuff)); + { + String tmp; + if (mi->using_gtid != Master_info::USE_GTID_NO) + { + tmp.append(STRING_WITH_LEN("; GTID position '")); + rpl_append_gtid_state(&tmp, false); + tmp.append(STRING_WITH_LEN("'")); + } + sql_print_information("Slave SQL thread exiting, replication stopped in " + "log '%s' at position %s%s", + RPL_LOG_NAME, + llstr(rli->group_master_log_pos,llbuff), + tmp.c_ptr_safe()); + } err: /* + Once again, in case we aborted with an error and skipped the first one. + (We want the first one to be before the printout of stop position to + get the correct position printed.) + */ + if (opt_slave_parallel_threads > 0) + rli->parallel.wait_for_done(); + + /* Some events set some playgrounds, which won't be cleared because thread stops. Stopping of this thread may not be known to these events ("stop" request is detected only by the present function, not by events), so we must "proactively" clear playgrounds: */ thd->clear_error(); - rli->cleanup_context(thd, 1); + serial_rgi->cleanup_context(thd, 1); /* Some extra safety, which should not been needed (normally, event deletion should already have done these assignments (each event which sets these @@ -3745,14 +4570,16 @@ the slave SQL thread with \"SLAVE START\". We stopped at log \ thd->catalog= 0; thd->reset_query(); thd->reset_db(NULL, 0); - thd_proc_info(thd, "Waiting for slave mutex on exit"); + if (rli->mi->using_gtid != Master_info::USE_GTID_NO) + flush_relay_log_info(rli); + thd_proc_info(thd, "Sql driver thread waiting for slave mutex on exit"); mysql_mutex_lock(&rli->run_lock); err_during_init: /* We need data_lock, at least to wake up any waiting master_pos_wait() */ mysql_mutex_lock(&rli->data_lock); - DBUG_ASSERT(rli->slave_running == 1); // tracking buffer overrun + DBUG_ASSERT(rli->slave_running == MYSQL_SLAVE_RUN_NOT_CONNECT); // tracking buffer overrun /* When master_pos_wait() wakes up it will check this and terminate */ - rli->slave_running= 0; + rli->slave_running= MYSQL_SLAVE_NOT_RUN; /* Forget the relay log's format */ delete rli->relay_log.description_event_for_exec; rli->relay_log.description_event_for_exec= 0; @@ -3763,21 +4590,18 @@ err_during_init: rli->ignore_log_space_limit= 0; /* don't need any lock */ /* we die so won't remember charset - re-update them on next thread start */ rli->cached_charset_invalidate(); - rli->save_temporary_tables = thd->temporary_tables; /* TODO: see if we can do this conditionally in next_event() instead to avoid unneeded position re-init */ thd->temporary_tables = 0; // remove tempation from destructor to close them - DBUG_ASSERT(thd->net.buff != 0); - net_end(&thd->net); // destructor will not free it, because we are weird - DBUG_ASSERT(rli->sql_thd == thd); THD_CHECK_SENTRY(thd); - rli->sql_thd= 0; - set_thd_in_use_temporary_tables(rli); // (re)set sql_thd in use for saved temp tables + serial_rgi->thd= rli->sql_driver_thd= 0; mysql_mutex_lock(&LOCK_thread_count); THD_CHECK_SENTRY(thd); + thd->rgi_fake= thd->rgi_slave= NULL; + delete serial_rgi; delete thd; mysql_mutex_unlock(&LOCK_thread_count); /* @@ -3815,14 +4639,14 @@ static int process_io_create_file(Master_info* mi, Create_file_log_event* cev) if (unlikely(!cev->is_valid())) DBUG_RETURN(1); - if (!rpl_filter->db_ok(cev->db)) + if (!mi->rpl_filter->db_ok(cev->db)) { skip_load_data_infile(net); DBUG_RETURN(0); } DBUG_ASSERT(cev->inited_from_old); thd->file_id = cev->file_id = mi->file_id++; - thd->server_id = cev->server_id; + thd->variables.server_id = cev->server_id; cev_not_written = 1; if (unlikely(net_request_file(net,cev->fname))) @@ -4205,6 +5029,10 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) mysql_mutex_t *log_lock= rli->relay_log.get_log_lock(); ulong s_id; bool unlock_data_lock= TRUE; + bool gtid_skip_enqueue= false; + bool got_gtid_event= false; + rpl_gtid event_gtid; + /* FD_q must have been prepared for the first R_a event inside get_master_version_and_clock() @@ -4267,8 +5095,6 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) goto err; } - LINT_INIT(inc_pos); - if (mi->rli.relay_log.description_event_for_queue->binlog_version<4 && (uchar)buf[EVENT_TYPE_OFFSET] != FORMAT_DESCRIPTION_EVENT /* a way to escape */) DBUG_RETURN(queue_old_event(mi,buf,event_len)); @@ -4392,6 +5218,19 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) mi->rli.relay_log.relay_log_checksum_alg= tmp->checksum_alg; /* + Do not queue any format description event that we receive after a + reconnect where we are skipping over a partial event group received + before the reconnect. + + (If we queued such an event, and it was the first format_description + event after master restart, the slave SQL thread would think that + the partial event group before it in the relay log was from a + previous master crash and should be rolled back). + */ + if (unlikely(mi->gtid_reconnect_event_skip_count && !mi->gtid_event_seen)) + gtid_skip_enqueue= true; + + /* Though this does some conversion to the slave's format, this will preserve the master's binlog format version, and number of event types. */ @@ -4434,16 +5273,18 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) Heartbeat is sent only after an event corresponding to the corrdinates the heartbeat carries. - Slave can not have a difference in coordinates except in the only + Slave can not have a higher coordinate except in the only special case when mi->master_log_name, master_log_pos have never been updated by Rotate event i.e when slave does not have any history with the master (and thereafter mi->master_log_pos is NULL). + Slave can have lower coordinates, if some event from master was omitted. + TODO: handling `when' for SHOW SLAVE STATUS' snds behind */ if ((memcmp(mi->master_log_name, hb.get_log_ident(), hb.get_ident_len()) && mi->master_log_name != NULL) - || mi->master_log_pos != hb.log_pos) + || mi->master_log_pos > hb.log_pos) { /* missed events of heartbeat from the past */ error= ER_SLAVE_HEARTBEAT_FAILURE; @@ -4459,7 +5300,139 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) } break; + case GTID_LIST_EVENT: + { + const char *errmsg; + Gtid_list_log_event *glev; + Log_event *tmp; + uint32 flags; + + if (!(tmp= Log_event::read_log_event(buf, event_len, &errmsg, + mi->rli.relay_log.description_event_for_queue, + opt_slave_sql_verify_checksum))) + { + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + goto err; + } + glev= static_cast<Gtid_list_log_event *>(tmp); + event_pos= glev->log_pos; + flags= glev->gl_flags; + delete glev; + + /* + We use fake Gtid_list events to update the old-style position (among + other things). + + Early code created fake Gtid_list events with zero log_pos, those should + not modify old-style position. + */ + if (event_pos == 0 || event_pos <= mi->master_log_pos) + inc_pos= 0; + else + inc_pos= event_pos - mi->master_log_pos; + + if (mi->rli.until_condition == Relay_log_info::UNTIL_GTID && + flags & Gtid_list_log_event::FLAG_UNTIL_REACHED) + { + char str_buf[128]; + String str(str_buf, sizeof(str_buf), system_charset_info); + mi->rli.until_gtid_pos.to_string(&str); + sql_print_information("Slave IO thread stops because it reached its" + " UNTIL master_gtid_pos %s", str.c_ptr_safe()); + mi->abort_slave= true; + } + } + break; + + case GTID_EVENT: + { + uchar gtid_flag; + + if (Gtid_log_event::peek(buf, event_len, checksum_alg, + &event_gtid.domain_id, &event_gtid.server_id, + &event_gtid.seq_no, >id_flag, + rli->relay_log.description_event_for_queue)) + { + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + goto err; + } + got_gtid_event= true; + if (mi->using_gtid == Master_info::USE_GTID_NO) + goto default_action; + if (unlikely(!mi->gtid_event_seen)) + { + mi->gtid_event_seen= true; + if (mi->gtid_reconnect_event_skip_count) + { + /* + If we are reconnecting, and we need to skip a partial event group + already queued to the relay log before the reconnect, then we check + that we actually get the same event group (same GTID) as before, so + we do not end up with half of one group and half another. + + The only way we should be able to receive a different GTID than what + we expect is if the binlog on the master (or more likely the whole + master server) was replaced with a different one, on the same IP + address, _and_ the new master happens to have domains in a different + order so we get the GTID from a different domain first. Still, it is + best to protect against this case. + */ + if (event_gtid.domain_id != mi->last_queued_gtid.domain_id || + event_gtid.server_id != mi->last_queued_gtid.server_id || + event_gtid.seq_no != mi->last_queued_gtid.seq_no) + { + bool first; + error= ER_SLAVE_UNEXPECTED_MASTER_SWITCH; + error_msg.append(STRING_WITH_LEN("Expected: ")); + first= true; + rpl_slave_state_tostring_helper(&error_msg, &mi->last_queued_gtid, + &first); + error_msg.append(STRING_WITH_LEN(", received: ")); + first= true; + rpl_slave_state_tostring_helper(&error_msg, &event_gtid, &first); + goto err; + } + } + } + + if (unlikely(mi->gtid_reconnect_event_skip_count)) + { + goto default_action; + } + + /* + We have successfully queued to relay log everything before this GTID, so + in case of reconnect we can start from after any previous GTID. + (Normally we would have updated gtid_current_pos earlier at the end of + the previous event group, but better leave an extra check here for + safety). + */ + if (mi->events_queued_since_last_gtid) + { + mi->gtid_current_pos.update(&mi->last_queued_gtid); + mi->events_queued_since_last_gtid= 0; + } + mi->last_queued_gtid= event_gtid; + mi->last_queued_gtid_standalone= + (gtid_flag & Gtid_log_event::FL_STANDALONE) != 0; + ++mi->events_queued_since_last_gtid; + inc_pos= event_len; + } + break; + default: + default_action: + if (mi->using_gtid != Master_info::USE_GTID_NO && mi->gtid_event_seen) + { + if (unlikely(mi->gtid_reconnect_event_skip_count)) + { + --mi->gtid_reconnect_event_skip_count; + gtid_skip_enqueue= true; + } + else if (mi->events_queued_since_last_gtid) + ++mi->events_queued_since_last_gtid; + } + inc_pos= event_len; break; } @@ -4495,7 +5468,51 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) mysql_mutex_lock(log_lock); s_id= uint4korr(buf + SERVER_ID_OFFSET); - if ((s_id == ::server_id && !mi->rli.replicate_same_server_id) || + /* + Write the event to the relay log, unless we reconnected in the middle + of an event group and now need to skip the initial part of the group that + we already wrote before reconnecting. + */ + if (unlikely(gtid_skip_enqueue)) + { + mi->master_log_pos+= inc_pos; + if ((uchar)buf[EVENT_TYPE_OFFSET] == FORMAT_DESCRIPTION_EVENT && + s_id == mi->master_id) + { + /* + If we write this master's description event in the middle of an event + group due to GTID reconnect, SQL thread will think that master crashed + in the middle of the group and roll back the first half, so we must not. + + But we still have to write an artificial copy of the masters description + event, to override the initial slave-version description event so that + SQL thread has the right information for parsing the events it reads. + */ + rli->relay_log.description_event_for_queue->created= 0; + rli->relay_log.description_event_for_queue->set_artificial_event(); + if (rli->relay_log.append_no_lock + (rli->relay_log.description_event_for_queue)) + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + else + rli->relay_log.harvest_bytes_written(&rli->log_space_total); + } + else if (mi->gtid_reconnect_event_skip_count == 0) + { + /* + Add a fake rotate event so that SQL thread can see the old-style + position where we re-connected in the middle of a GTID event group. + */ + Rotate_log_event fake_rev(mi->master_log_name, 0, mi->master_log_pos, 0); + fake_rev.server_id= mi->master_id; + if (rli->relay_log.append_no_lock(&fake_rev)) + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + else + rli->relay_log.harvest_bytes_written(&rli->log_space_total); + } + } + else + if ((s_id == global_system_variables.server_id && + !mi->rli.replicate_same_server_id) || /* the following conjunction deals with IGNORE_SERVER_IDS, if set If the master is on the ignore list, execution of @@ -4526,7 +5543,8 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) IGNORE_SERVER_IDS it increments mi->master_log_pos as well as rli->group_relay_log_pos. */ - if (!(s_id == ::server_id && !mi->rli.replicate_same_server_id) || + if (!(s_id == global_system_variables.server_id && + !mi->rli.replicate_same_server_id) || (buf[EVENT_TYPE_OFFSET] != FORMAT_DESCRIPTION_EVENT && buf[EVENT_TYPE_OFFSET] != ROTATE_EVENT && buf[EVENT_TYPE_OFFSET] != STOP_EVENT)) @@ -4535,6 +5553,8 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) memcpy(rli->ign_master_log_name_end, mi->master_log_name, FN_REFLEN); DBUG_ASSERT(rli->ign_master_log_name_end[0]); rli->ign_master_log_pos_end= mi->master_log_pos; + if (got_gtid_event) + rli->ign_gtids.update(&event_gtid); } rli->relay_log.signal_update(); // the slave SQL thread needs to re-check DBUG_PRINT("info", ("master_log_pos: %lu, event originating from %u server, ignored", @@ -4542,7 +5562,6 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) } else { - /* write the event to the relay log */ if (likely(!(rli->relay_log.appendv(buf,event_len,0)))) { mi->master_log_pos+= inc_pos; @@ -4554,11 +5573,33 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; } rli->ign_master_log_name_end[0]= 0; // last event is not ignored + if (got_gtid_event) + rli->ign_gtids.remove_if_present(&event_gtid); if (save_buf != NULL) buf= save_buf; } mysql_mutex_unlock(log_lock); + if (!error && + mi->using_gtid != Master_info::USE_GTID_NO && + mi->events_queued_since_last_gtid > 0 && + ( (mi->last_queued_gtid_standalone && + !Log_event::is_part_of_group((Log_event_type)(uchar) + buf[EVENT_TYPE_OFFSET])) || + (!mi->last_queued_gtid_standalone && + ((uchar)buf[EVENT_TYPE_OFFSET] == XID_EVENT || + ((uchar)buf[EVENT_TYPE_OFFSET] == QUERY_EVENT && + Query_log_event::peek_is_commit_rollback(buf, event_len, + checksum_alg)))))) + { + /* + The whole of the current event group is queued. So in case of + reconnect we can start from after the current GTID. + */ + mi->gtid_current_pos.update(&mi->last_queued_gtid); + mi->events_queued_since_last_gtid= 0; + } + skip_relay_logging: err: @@ -4667,6 +5708,7 @@ static int connect_to_master(THD* thd, MYSQL* mysql, Master_info* mi, int last_errno= -2; // impossible error ulong err_count=0; char llbuff[22]; + my_bool my_true= 1; DBUG_ENTER("connect_to_master"); set_slave_max_allowed_packet(thd, mysql); #ifndef DBUG_OFF @@ -4678,6 +5720,8 @@ static int connect_to_master(THD* thd, MYSQL* mysql, Master_info* mi, mysql_options(mysql, MYSQL_OPT_CONNECT_TIMEOUT, (char *) &slave_net_timeout); mysql_options(mysql, MYSQL_OPT_READ_TIMEOUT, (char *) &slave_net_timeout); + mysql_options(mysql, MYSQL_OPT_USE_THREAD_SPECIFIC_MEMORY, + (char*) &my_true); #ifdef HAVE_OPENSSL if (mi->ssl) @@ -4711,7 +5755,7 @@ static int connect_to_master(THD* thd, MYSQL* mysql, Master_info* mi, "terminated."); DBUG_RETURN(1); } - while (!(slave_was_killed = io_slave_killed(thd,mi)) && + while (!(slave_was_killed = io_slave_killed(mi)) && (reconnect ? mysql_reconnect(mysql) != 0 : mysql_real_connect(mysql, mi->host, mi->user, mi->password, 0, mi->port, 0, client_flag) == 0)) @@ -4789,18 +5833,20 @@ static int safe_reconnect(THD* thd, MYSQL* mysql, Master_info* mi, } +#ifdef NOT_USED MYSQL *rpl_connect_master(MYSQL *mysql) { - THD *thd= current_thd; Master_info *mi= my_pthread_getspecific_ptr(Master_info*, RPL_MASTER_INFO); + bool allocated= false; + my_bool my_true= 1; + THD *thd; + if (!mi) { sql_print_error("'rpl_connect_master' must be called in slave I/O thread context."); return NULL; } - - bool allocated= false; - + thd= mi->io_thd; if (!mysql) { if(!(mysql= mysql_init(NULL))) @@ -4820,6 +5866,8 @@ MYSQL *rpl_connect_master(MYSQL *mysql) */ mysql_options(mysql, MYSQL_OPT_CONNECT_TIMEOUT, (char *) &slave_net_timeout); mysql_options(mysql, MYSQL_OPT_READ_TIMEOUT, (char *) &slave_net_timeout); + mysql_options(mysql, MYSQL_OPT_USE_THREAD_SPECIFIC_MEMORY, + (char*) &my_true); #ifdef HAVE_OPENSSL if (mi->ssl) @@ -4841,11 +5889,11 @@ MYSQL *rpl_connect_master(MYSQL *mysql) if (mi->user == NULL || mi->user[0] == 0 - || io_slave_killed(thd, mi) + || io_slave_killed( mi) || !mysql_real_connect(mysql, mi->host, mi->user, mi->password, 0, mi->port, 0, 0)) { - if (!io_slave_killed(thd, mi)) + if (!io_slave_killed( mi)) sql_print_error("rpl_connect_master: error connecting to master: %s (server_error: %d)", mysql_error(mysql), mysql_errno(mysql)); @@ -4855,6 +5903,7 @@ MYSQL *rpl_connect_master(MYSQL *mysql) } return mysql; } +#endif /* Store the file and position where the execute-slave thread are in the @@ -4960,17 +6009,20 @@ static IO_CACHE *reopen_relay_log(Relay_log_info *rli, const char **errmsg) @return The event read, or NULL on error. If an error occurs, the error is reported through the sql_print_information() or sql_print_error() functions. + + The size of the read event (in bytes) is returned in *event_size. */ -static Log_event* next_event(Relay_log_info* rli) +static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) { Log_event* ev; + Relay_log_info *rli= rgi->rli; IO_CACHE* cur_log = rli->cur_log; mysql_mutex_t *log_lock = rli->relay_log.get_log_lock(); const char* errmsg=0; - THD* thd = rli->sql_thd; DBUG_ENTER("next_event"); - DBUG_ASSERT(thd != 0); + DBUG_ASSERT(rgi->thd != 0 && rgi->thd == rli->sql_driver_thd); + *event_size= 0; #ifndef DBUG_OFF if (abort_slave_event_count && !rli->events_till_abort--) @@ -4986,7 +6038,7 @@ static Log_event* next_event(Relay_log_info* rli) */ mysql_mutex_assert_owner(&rli->data_lock); - while (!sql_slave_killed(thd,rli)) + while (!sql_slave_killed(rgi)) { /* We can have two kinds of log reading: @@ -4999,6 +6051,7 @@ static Log_event* next_event(Relay_log_info* rli) The other case is much simpler: We just have a read only log that nobody else will be updating. */ + ulonglong old_pos; bool hot_log; if ((hot_log = (cur_log != &rli->cache_buf))) { @@ -5034,7 +6087,8 @@ static Log_event* next_event(Relay_log_info* rli) llstr(my_b_tell(cur_log),llbuf1), llstr(rli->event_relay_log_pos,llbuf2))); DBUG_ASSERT(my_b_tell(cur_log) >= BIN_LOG_HEADER_SIZE); - DBUG_ASSERT(my_b_tell(cur_log) == rli->event_relay_log_pos); + DBUG_ASSERT(opt_slave_parallel_threads > 0 || + my_b_tell(cur_log) == rli->event_relay_log_pos); } #endif /* @@ -5049,22 +6103,24 @@ static Log_event* next_event(Relay_log_info* rli) But if the relay log is created by new_file(): then the solution is: MYSQL_BIN_LOG::open() will write the buffered description event. */ + old_pos= rli->event_relay_log_pos; if ((ev= Log_event::read_log_event(cur_log,0, rli->relay_log.description_event_for_exec, opt_slave_sql_verify_checksum))) { - DBUG_ASSERT(thd==rli->sql_thd); /* read it while we have a lock, to avoid a mutex lock in inc_event_relay_log_pos() */ rli->future_event_relay_log_pos= my_b_tell(cur_log); + *event_size= rli->future_event_relay_log_pos - old_pos; + if (hot_log) mysql_mutex_unlock(log_lock); + rli->sql_thread_caught_up= false; DBUG_RETURN(ev); } - DBUG_ASSERT(thd==rli->sql_thd); if (opt_reckless_slave) // For mysql-test cur_log->error = 0; if (cur_log->error < 0) @@ -5100,12 +6156,10 @@ static Log_event* next_event(Relay_log_info* rli) Seconds_Behind_Master would be zero only when master has no more updates in binlog for slave. The heartbeat can be sent in a (small) fraction of slave_net_timeout. Until it's done - rli->last_master_timestamp is temporarely (for time of - waiting for the following event) reset whenever EOF is - reached. + rli->sql_thread_caught_up is temporarely (for time of waiting for + the following event) set whenever EOF is reached. */ - time_t save_timestamp= rli->last_master_timestamp; - rli->last_master_timestamp= 0; + rli->sql_thread_caught_up= true; DBUG_ASSERT(rli->relay_log.get_open_count() == rli->cur_log_old_open_count); @@ -5129,6 +6183,36 @@ static Log_event* next_event(Relay_log_info* rli) DBUG_RETURN(ev); } + if (rli->ign_gtids.count()) + { + /* We generate and return a Gtid_list, to update gtid_slave_pos. */ + DBUG_PRINT("info",("seeing ignored end gtids")); + ev= new Gtid_list_log_event(&rli->ign_gtids, + Gtid_list_log_event::FLAG_IGN_GTIDS); + rli->ign_gtids.reset(); + mysql_mutex_unlock(log_lock); + if (unlikely(!ev)) + { + errmsg= "Slave SQL thread failed to create a Gtid_list event " + "(out of memory?), gtid_slave_pos may be inaccurate"; + goto err; + } + ev->server_id= 0; // don't be ignored by slave SQL thread + ev->set_artificial_event(); // Don't mess up Exec_Master_Log_Pos + DBUG_RETURN(ev); + } + + /* + We have to check sql_slave_killed() here an extra time. + Otherwise we may miss a wakeup, since last check was done + without holding LOCK_log. + */ + if (sql_slave_killed(rgi)) + { + mysql_mutex_unlock(log_lock); + break; + } + /* We can, and should release data_lock while we are waiting for update. If we do not, show slave status will block @@ -5152,14 +6236,15 @@ static Log_event* next_event(Relay_log_info* rli) and reads one more event and starts honoring log_space_limit again. If the SQL thread needs more events to be able to rotate the log (it - might need to finish the current group first), then it can ask for one - more at a time. Thus we don't outgrow the relay log indefinitely, + might need to finish the current group first), then it can ask for + one more at a time. Thus we don't outgrow the relay log indefinitely, but rather in a controlled manner, until the next rotate. When the SQL thread starts it sets ignore_log_space_limit to false. We should also reset ignore_log_space_limit to 0 when the user does - RESET SLAVE, but in fact, no need as RESET SLAVE requires that the slave - be stopped, and the SQL thread sets ignore_log_space_limit to 0 when + RESET SLAVE, but in fact, no need as RESET SLAVE requires that the + slave be stopped, and the SQL thread sets ignore_log_space_limit + to 0 when it stops. */ mysql_mutex_lock(&rli->log_space_lock); @@ -5197,10 +6282,10 @@ static Log_event* next_event(Relay_log_info* rli) mysql_mutex_unlock(&rli->log_space_lock); mysql_cond_broadcast(&rli->log_space_cond); // Note that wait_for_update_relay_log unlocks lock_log ! - rli->relay_log.wait_for_update_relay_log(rli->sql_thd); + rli->relay_log.wait_for_update_relay_log(rli->sql_driver_thd); // re-acquire data lock since we released it earlier mysql_mutex_lock(&rli->data_lock); - rli->last_master_timestamp= save_timestamp; + rli->sql_thread_caught_up= false; continue; } /* @@ -5269,11 +6354,6 @@ static Log_event* next_event(Relay_log_info* rli) mysql_mutex_lock(log_lock); if (rli->relay_log.is_active(rli->linfo.log_file_name)) { -#ifdef EXTRA_DEBUG - if (global_system_variables.log_warnings) - sql_print_information("next log '%s' is currently active", - rli->linfo.log_file_name); -#endif rli->cur_log= cur_log= rli->relay_log.get_log_file(); rli->cur_log_old_open_count= rli->relay_log.get_open_count(); DBUG_ASSERT(rli->cur_log_fd == -1); @@ -5356,11 +6436,6 @@ static Log_event* next_event(Relay_log_info* rli) ourselves. We are sure that the log is still not hot now (a log can get from hot to cold, but not from cold to hot). No need for LOCK_log. */ -#ifdef EXTRA_DEBUG - if (global_system_variables.log_warnings) - sql_print_information("next log '%s' is not active", - rli->linfo.log_file_name); -#endif // open_binlog() will check the magic header if ((rli->cur_log_fd=open_binlog(cur_log,rli->linfo.log_file_name, &errmsg)) <0) @@ -5540,20 +6615,14 @@ bool rpl_master_has_bug(const Relay_log_info *rli, uint bug_id, bool report, */ bool rpl_master_erroneous_autoinc(THD *thd) { - if (active_mi && active_mi->rli.sql_thd == thd) + if (thd->rgi_slave) { - Relay_log_info *rli= &active_mi->rli; DBUG_EXECUTE_IF("simulate_bug33029", return TRUE;); - return rpl_master_has_bug(rli, 33029, FALSE, NULL, NULL); + return rpl_master_has_bug(thd->rgi_slave->rli, 33029, FALSE, NULL, NULL); } return FALSE; } -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class I_List_iterator<i_string>; -template class I_List_iterator<i_string_pair>; -#endif - /** @} (end of group Replication) */ diff --git a/sql/slave.h b/sql/slave.h index 6b4bcffe109..3981a9d4f2c 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -45,9 +45,13 @@ #define MAX_SLAVE_ERROR 2000 +#define MAX_REPLICATION_THREAD 64 + // Forward declarations class Relay_log_info; class Master_info; +class Master_info_index; +struct rpl_parallel_thread; int init_intvar_from_file(int* var, IO_CACHE* f, int default_val); int init_strvar_from_file(char *var, int max_size, IO_CACHE *f, @@ -197,7 +201,8 @@ int mysql_table_dump(THD* thd, const char* db, int fetch_master_table(THD* thd, const char* db_name, const char* table_name, Master_info* mi, MYSQL* mysql, bool overwrite); -bool show_master_info(THD* thd, Master_info* mi); +bool show_master_info(THD* thd, Master_info* mi, bool full); +bool show_all_master_info(THD* thd); bool show_binlog_info(THD* thd); bool rpl_master_has_bug(const Relay_log_info *rli, uint bug_id, bool report, bool (*pred)(const void *), const void *param); @@ -223,14 +228,20 @@ int purge_relay_logs(Relay_log_info* rli, THD *thd, bool just_reset, void set_slave_thread_options(THD* thd); void set_slave_thread_default_charset(THD *thd, Relay_log_info const *rli); int rotate_relay_log(Master_info* mi); -int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli); +int apply_event_and_update_pos(Log_event* ev, THD* thd, + struct rpl_group_info *rgi, + rpl_parallel_thread *rpt); pthread_handler_t handle_slave_io(void *arg); +void slave_output_error_info(Relay_log_info *rli, THD *thd); pthread_handler_t handle_slave_sql(void *arg); bool net_request_file(NET* net, const char* fname); extern bool volatile abort_loop; extern Master_info main_mi, *active_mi; /* active_mi for multi-master */ +extern Master_info *default_master_info; /* To replace active_mi */ +extern Master_info_index *master_info_index; +extern LEX_STRING default_master_connection_name; extern LIST master_list; extern my_bool replicate_same_server_id; diff --git a/sql/sp.cc b/sql/sp.cc index 13f027493b4..9a8a22e1919 100644 --- a/sql/sp.cc +++ b/sql/sp.cc @@ -50,7 +50,8 @@ db_load_routine(THD *thd, stored_procedure_type type, sp_name *name, sp_head **sphp, ulonglong sql_mode, const char *params, const char *returns, const char *body, st_sp_chistics &chistics, - const char *definer, longlong created, longlong modified, + LEX_STRING *definer_user_name, LEX_STRING *definer_host_name, + longlong created, longlong modified, Stored_program_creation_ctx *creation_ctx); static const @@ -168,7 +169,7 @@ TABLE_FIELD_TYPE proc_table_fields[MYSQL_PROC_FIELD_COUNT] = }; static const TABLE_FIELD_DEF - proc_table_def= {MYSQL_PROC_FIELD_COUNT, proc_table_fields}; +proc_table_def= {MYSQL_PROC_FIELD_COUNT, proc_table_fields, 0, (uint*) 0 }; /*************************************************************************/ @@ -548,6 +549,10 @@ db_find_routine(THD *thd, stored_procedure_type type, sp_name *name, ulonglong sql_mode, saved_mode= thd->variables.sql_mode; Open_tables_backup open_tables_state_backup; Stored_program_creation_ctx *creation_ctx; + char definer_user_name_holder[USERNAME_LENGTH + 1]; + LEX_STRING definer_user_name= { definer_user_name_holder, USERNAME_LENGTH }; + char definer_host_name_holder[HOSTNAME_LENGTH + 1]; + LEX_STRING definer_host_name= { definer_host_name_holder, HOSTNAME_LENGTH }; DBUG_ENTER("db_find_routine"); DBUG_PRINT("enter", ("type: %d name: %.*s", @@ -657,9 +662,19 @@ db_find_routine(THD *thd, stored_procedure_type type, sp_name *name, close_system_tables(thd, &open_tables_state_backup); table= 0; + if (parse_user(definer, strlen(definer), + definer_user_name.str, &definer_user_name.length, + definer_host_name.str, &definer_host_name.length) && + definer_user_name.length && !definer_host_name.length) + { + // 'user@' -> 'user@%' + definer_host_name= host_not_specified; + } + ret= db_load_routine(thd, type, name, sphp, sql_mode, params, returns, body, chistics, - definer, created, modified, creation_ctx); + &definer_user_name, &definer_host_name, + created, modified, creation_ctx); done: /* Restore the time zone flag as the timezone usage in proc table @@ -804,7 +819,8 @@ db_load_routine(THD *thd, stored_procedure_type type, sp_name *name, sp_head **sphp, ulonglong sql_mode, const char *params, const char *returns, const char *body, st_sp_chistics &chistics, - const char *definer, longlong created, longlong modified, + LEX_STRING *definer_user_name, LEX_STRING *definer_host_name, + longlong created, longlong modified, Stored_program_creation_ctx *creation_ctx) { LEX *old_lex= thd->lex, newlex; @@ -814,22 +830,12 @@ db_load_routine(THD *thd, stored_procedure_type type, { saved_cur_db_name_buf, sizeof(saved_cur_db_name_buf) }; bool cur_db_changed; Bad_db_error_handler db_not_exists_handler; - char definer_user_name_holder[USERNAME_LENGTH + 1]; - LEX_STRING definer_user_name= { definer_user_name_holder, - USERNAME_LENGTH }; - - char definer_host_name_holder[HOSTNAME_LENGTH + 1]; - LEX_STRING definer_host_name= { definer_host_name_holder, HOSTNAME_LENGTH }; int ret= 0; thd->lex= &newlex; newlex.current_select= NULL; - parse_user(definer, strlen(definer), - definer_user_name.str, &definer_user_name.length, - definer_host_name.str, &definer_host_name.length); - defstr.set_charset(creation_ctx->get_client_cs()); /* @@ -845,7 +851,7 @@ db_load_routine(THD *thd, stored_procedure_type type, params, strlen(params), returns, strlen(returns), body, strlen(body), - &chistics, &definer_user_name, &definer_host_name, + &chistics, definer_user_name, definer_host_name, sql_mode)) { ret= SP_INTERNAL_ERROR; @@ -895,7 +901,7 @@ db_load_routine(THD *thd, stored_procedure_type type, goto end; } - (*sphp)->set_definer(&definer_user_name, &definer_host_name); + (*sphp)->set_definer(definer_user_name, definer_host_name); (*sphp)->set_info(created, modified, &chistics, sql_mode); (*sphp)->set_creation_ctx(creation_ctx); (*sphp)->optimize(); @@ -974,7 +980,8 @@ sp_create_routine(THD *thd, stored_procedure_type type, sp_head *sp) { int ret; TABLE *table; - char definer[USER_HOST_BUFF_SIZE]; + char definer_buf[USER_HOST_BUFF_SIZE]; + LEX_STRING definer; ulonglong saved_mode= thd->variables.sql_mode; MDL_key::enum_mdl_namespace mdl_type= type == TYPE_ENUM_FUNCTION ? MDL_key::FUNCTION : MDL_key::PROCEDURE; @@ -984,9 +991,6 @@ sp_create_routine(THD *thd, stored_procedure_type type, sp_head *sp) enum_check_fields saved_count_cuted_fields; bool store_failed= FALSE; - - bool save_binlog_row_based; - DBUG_ENTER("sp_create_routine"); DBUG_PRINT("enter", ("type: %d name: %.*s", (int) type, (int) sp->m_name.length, @@ -1004,14 +1008,6 @@ sp_create_routine(THD *thd, stored_procedure_type type, sp_head *sp) /* Reset sql_mode during data dictionary operations. */ thd->variables.sql_mode= 0; - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - saved_count_cuted_fields= thd->count_cuted_fields; thd->count_cuted_fields= CHECK_FIELD_WARN; @@ -1022,8 +1018,7 @@ sp_create_routine(THD *thd, stored_procedure_type type, sp_head *sp) restore_record(table, s->default_values); // Get default values for fields /* NOTE: all needed privilege checks have been already done. */ - strxnmov(definer, sizeof(definer)-1, thd->lex->definer->user.str, "@", - thd->lex->definer->host.str, NullS); + thd->lex->definer->set_lex_string(&definer, definer_buf); if (table->s->fields < MYSQL_PROC_FIELD_COUNT) { @@ -1098,7 +1093,7 @@ sp_create_routine(THD *thd, stored_procedure_type type, sp_head *sp) store_failed= store_failed || table->field[MYSQL_PROC_FIELD_DEFINER]-> - store(definer, (uint)strlen(definer), system_charset_info); + store(definer.str, definer.length, system_charset_info); ((Field_timestamp *)table->field[MYSQL_PROC_FIELD_CREATED])->set_time(); ((Field_timestamp *)table->field[MYSQL_PROC_FIELD_MODIFIED])->set_time(); @@ -1178,6 +1173,9 @@ sp_create_routine(THD *thd, stored_procedure_type type, sp_head *sp) ret= SP_OK; if (table->file->ha_write_row(table->record[0])) ret= SP_WRITE_ROW_FAILED; + /* Make change permanent and avoid 'table is marked as crashed' errors */ + table->file->extra(HA_EXTRA_FLUSH); + if (ret == SP_OK) sp_cache_invalidate(); @@ -1217,10 +1215,7 @@ sp_create_routine(THD *thd, stored_procedure_type type, sp_head *sp) done: thd->count_cuted_fields= saved_count_cuted_fields; thd->variables.sql_mode= saved_mode; - /* Restore the state of binlog format */ DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(ret); } @@ -1245,7 +1240,6 @@ sp_drop_routine(THD *thd, stored_procedure_type type, sp_name *name) { TABLE *table; int ret; - bool save_binlog_row_based; MDL_key::enum_mdl_namespace mdl_type= type == TYPE_ENUM_FUNCTION ? MDL_key::FUNCTION : MDL_key::PROCEDURE; DBUG_ENTER("sp_drop_routine"); @@ -1267,13 +1261,12 @@ sp_drop_routine(THD *thd, stored_procedure_type type, sp_name *name) row-based replication. The flag will be reset at the end of the statement. */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - if ((ret= db_find_routine_aux(thd, type, name, table)) == SP_OK) { if (table->file->ha_delete_row(table->record[0])) ret= SP_DELETE_ROW_FAILED; + /* Make change permanent and avoid 'table is marked as crashed' errors */ + table->file->extra(HA_EXTRA_FLUSH); } if (ret == SP_OK) @@ -1296,10 +1289,7 @@ sp_drop_routine(THD *thd, stored_procedure_type type, sp_name *name) sp_cache_flush_obsolete(spc, &sp); } } - /* Restore the state of binlog format */ DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(ret); } @@ -1327,7 +1317,6 @@ sp_update_routine(THD *thd, stored_procedure_type type, sp_name *name, { TABLE *table; int ret; - bool save_binlog_row_based; MDL_key::enum_mdl_namespace mdl_type= type == TYPE_ENUM_FUNCTION ? MDL_key::FUNCTION : MDL_key::PROCEDURE; DBUG_ENTER("sp_update_routine"); @@ -1345,14 +1334,6 @@ sp_update_routine(THD *thd, stored_procedure_type type, sp_name *name, if (!(table= open_proc_table_for_update(thd))) DBUG_RETURN(SP_OPEN_TABLE_FAILED); - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - if ((ret= db_find_routine_aux(thd, type, name, table)) == SP_OK) { if (type == TYPE_ENUM_FUNCTION && ! trust_function_creators && @@ -1380,7 +1361,6 @@ sp_update_routine(THD *thd, stored_procedure_type type, sp_name *name, } store_record(table,record[1]); - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; ((Field_timestamp *)table->field[MYSQL_PROC_FIELD_MODIFIED])->set_time(); if (chistics->suid != SP_IS_DEFAULT_SUID) table->field[MYSQL_PROC_FIELD_SECURITY_TYPE]-> @@ -1397,6 +1377,8 @@ sp_update_routine(THD *thd, stored_procedure_type type, sp_name *name, ret= SP_WRITE_ROW_FAILED; else ret= 0; + /* Make change permanent and avoid 'table is marked as crashed' errors */ + table->file->extra(HA_EXTRA_FLUSH); } if (ret == SP_OK) @@ -1406,10 +1388,7 @@ sp_update_routine(THD *thd, stored_procedure_type type, sp_name *name, sp_cache_invalidate(); } err: - /* Restore the state of binlog format */ DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(ret); } @@ -1574,7 +1553,11 @@ sp_drop_db_routines(THD *thd, char *db) if (nxtres != HA_ERR_END_OF_FILE) ret= SP_KEY_NOT_FOUND; if (deleted) + { sp_cache_invalidate(); + /* Make change permanent and avoid 'table is marked as crashed' errors */ + table->file->extra(HA_EXTRA_FLUSH); + } } table->file->ha_index_end(); @@ -1682,7 +1665,6 @@ sp_find_routine(THD *thd, stored_procedure_type type, sp_name *name, ulong level; sp_head *new_sp; const char *returns= ""; - char definer[USER_HOST_BUFF_SIZE]; /* String buffer for RETURNS data type must have system charset; @@ -1719,8 +1701,6 @@ sp_find_routine(THD *thd, stored_procedure_type type, sp_name *name, DBUG_RETURN(0); } - strxmov(definer, sp->m_definer_user.str, "@", - sp->m_definer_host.str, NullS); if (type == TYPE_ENUM_FUNCTION) { sp_returns_type(thd, retstr, sp); @@ -1728,7 +1708,8 @@ sp_find_routine(THD *thd, stored_procedure_type type, sp_name *name, } if (db_load_routine(thd, type, name, &new_sp, sp->m_sql_mode, sp->m_params.str, returns, - sp->m_body.str, *sp->m_chistics, definer, + sp->m_body.str, *sp->m_chistics, + &sp->m_definer_user, &sp->m_definer_host, sp->m_created, sp->m_modified, sp->get_creation_ctx()) == SP_OK) { diff --git a/sql/sp_head.cc b/sql/sp_head.cc index 99f9b172b16..80d4f4c8e9f 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -42,6 +42,7 @@ #include "sql_parse.h" // cleanup_items #include "sql_base.h" // close_thread_tables #include "transaction.h" // trans_commit_stmt +#include "sql_audit.h" /* Sufficient max length of printed destinations and frame offsets (all uints). @@ -219,6 +220,7 @@ sp_get_flags_for_command(LEX *lex) case SQLCOM_SHOW_CREATE_TRIGGER: case SQLCOM_SHOW_DATABASES: case SQLCOM_SHOW_ERRORS: + case SQLCOM_SHOW_EXPLAIN: case SQLCOM_SHOW_FIELDS: case SQLCOM_SHOW_FUNC_CODE: case SQLCOM_SHOW_GRANTS: @@ -281,9 +283,12 @@ sp_get_flags_for_command(LEX *lex) case SQLCOM_CREATE_VIEW: case SQLCOM_CREATE_TRIGGER: case SQLCOM_CREATE_USER: + case SQLCOM_CREATE_ROLE: case SQLCOM_ALTER_TABLE: case SQLCOM_GRANT: + case SQLCOM_GRANT_ROLE: case SQLCOM_REVOKE: + case SQLCOM_REVOKE_ROLE: case SQLCOM_BEGIN: case SQLCOM_RENAME_TABLE: case SQLCOM_RENAME_USER: @@ -291,6 +296,7 @@ sp_get_flags_for_command(LEX *lex) case SQLCOM_DROP_DB: case SQLCOM_REVOKE_ALL: case SQLCOM_DROP_USER: + case SQLCOM_DROP_ROLE: case SQLCOM_DROP_VIEW: case SQLCOM_DROP_TRIGGER: case SQLCOM_TRUNCATE: @@ -311,6 +317,33 @@ sp_get_flags_for_command(LEX *lex) case SQLCOM_UNINSTALL_PLUGIN: flags= sp_head::HAS_COMMIT_OR_ROLLBACK; break; + case SQLCOM_DELETE: + case SQLCOM_DELETE_MULTI: + { + /* + DELETE normally doesn't return resultset, but there are two exceptions: + - DELETE ... RETURNING + - EXPLAIN DELETE ... + */ + if (lex->select_lex.item_list.is_empty() && !lex->describe) + flags= 0; + else + flags= sp_head::MULTI_RESULTS; + break; + } + case SQLCOM_UPDATE: + case SQLCOM_UPDATE_MULTI: + case SQLCOM_INSERT: + case SQLCOM_REPLACE: + case SQLCOM_REPLACE_SELECT: + case SQLCOM_INSERT_SELECT: + { + if (!lex->describe) + flags= 0; + else + flags= sp_head::MULTI_RESULTS; + break; + } default: flags= 0; break; @@ -510,7 +543,7 @@ sp_head::operator new(size_t size) throw() MEM_ROOT own_root; sp_head *sp; - init_sql_alloc(&own_root, MEM_ROOT_BLOCK_SIZE, MEM_ROOT_PREALLOC); + init_sql_alloc(&own_root, MEM_ROOT_BLOCK_SIZE, MEM_ROOT_PREALLOC, MYF(0)); sp= (sp_head *) alloc_root(&own_root, size); if (sp == NULL) DBUG_RETURN(NULL); @@ -594,7 +627,7 @@ sp_head::init(LEX *lex) types of stored procedures to simplify reset_lex()/restore_lex() code. */ lex->trg_table_fields.empty(); - my_init_dynamic_array(&m_instr, sizeof(sp_instr *), 16, 8); + my_init_dynamic_array(&m_instr, sizeof(sp_instr *), 16, 8, MYF(0)); m_param_begin= NULL; m_param_end= NULL; @@ -1256,7 +1289,7 @@ sp_head::execute(THD *thd, bool merge_da_on_success) DBUG_RETURN(TRUE); /* init per-instruction memroot */ - init_sql_alloc(&execute_mem_root, MEM_ROOT_BLOCK_SIZE, 0); + init_sql_alloc(&execute_mem_root, MEM_ROOT_BLOCK_SIZE, 0, MYF(0)); DBUG_ASSERT(!(m_flags & IS_INVOKED)); m_flags|= IS_INVOKED; @@ -1714,7 +1747,7 @@ sp_head::execute_trigger(THD *thd, TODO: we should create sp_rcontext once per command and reuse it on subsequent executions of a trigger. */ - init_sql_alloc(&call_mem_root, MEM_ROOT_BLOCK_SIZE, 0); + init_sql_alloc(&call_mem_root, MEM_ROOT_BLOCK_SIZE, 0, MYF(0)); thd->set_n_backup_active_arena(&call_arena, &backup_arena); if (!(nctx= new sp_rcontext(m_pcont, 0, octx)) || @@ -1831,7 +1864,7 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, TODO: we should create sp_rcontext once per command and reuse it on subsequent executions of a function/trigger. */ - init_sql_alloc(&call_mem_root, MEM_ROOT_BLOCK_SIZE, 0); + init_sql_alloc(&call_mem_root, MEM_ROOT_BLOCK_SIZE, 0, MYF(0)); thd->set_n_backup_active_arena(&call_arena, &backup_arena); if (!(nctx= new sp_rcontext(m_pcont, return_value_fld, octx)) || @@ -2438,7 +2471,6 @@ sp_head::fill_field_definition(THD *thd, LEX *lex, { LEX_STRING cmt = { 0, 0 }; uint unused1= 0; - int unused2= 0; if (field_def->init(thd, (char*) "", field_type, lex->length, lex->dec, lex->type, (Item*) 0, (Item*) 0, &cmt, 0, @@ -2446,7 +2478,7 @@ sp_head::fill_field_definition(THD *thd, LEX *lex, lex->charset ? lex->charset : thd->variables.collation_database, lex->uint_geom_type, - lex->vcol_info, NULL)) + lex->vcol_info, NULL, FALSE)) return TRUE; if (field_def->interval_list.elements) @@ -2455,8 +2487,7 @@ sp_head::fill_field_definition(THD *thd, LEX *lex, sp_prepare_create_field(thd, field_def); - if (prepare_create_field(field_def, &unused1, &unused2, &unused2, - HA_CAN_GEOMETRY)) + if (prepare_create_field(field_def, &unused1, HA_CAN_GEOMETRY)) { return TRUE; } @@ -2527,8 +2558,13 @@ sp_head::set_definer(const char *definer, uint definerlen) char host_name_holder[HOSTNAME_LENGTH + 1]; LEX_STRING host_name= { host_name_holder, HOSTNAME_LENGTH }; - parse_user(definer, definerlen, user_name.str, &user_name.length, - host_name.str, &host_name.length); + if (parse_user(definer, definerlen, user_name.str, &user_name.length, + host_name.str, &host_name.length) && + user_name.length && !host_name.length) + { + // 'user@' -> 'user@%' + host_name= host_not_specified; + } set_definer(&user_name, &host_name); } @@ -3027,6 +3063,8 @@ sp_lex_keeper::reset_lex_and_exec_core(THD *thd, uint *nextp, thd->mdl_context.release_statement_locks(); } } + //TODO: why is this here if log_slow_query is in sp_instr_stmt_execute? + delete_explain_query(m_lex); if (m_lex->query_tables_own_last) { @@ -3150,6 +3188,10 @@ sp_instr_stmt::execute(THD *thd, uint *nextp) query_cache_end_of_result(thd); + mysql_audit_general(thd, MYSQL_AUDIT_GENERAL_STATUS, + thd->stmt_da->is_error() ? thd->stmt_da->sql_errno() : 0, + command_name[COM_QUERY].str); + if (!res && unlikely(thd->enable_slow_log)) log_slow_statement(thd); } @@ -3250,6 +3292,7 @@ sp_instr_set::exec_core(THD *thd, uint *nextp) my_error(ER_OUT_OF_RESOURCES, MYF(ME_FATALERROR)); } } + delete_explain_query(thd->lex); *nextp = m_ip+1; return res; diff --git a/sql/sp_pcontext.cc b/sql/sp_pcontext.cc index 4c5087eaf27..f11daeecb7b 100644 --- a/sql/sp_pcontext.cc +++ b/sql/sp_pcontext.cc @@ -61,20 +61,20 @@ sp_pcontext::sp_pcontext() m_label_scope(LABEL_DEFAULT_SCOPE) { (void) my_init_dynamic_array(&m_vars, sizeof(sp_variable_t *), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); (void) my_init_dynamic_array(&m_case_expr_id_lst, sizeof(int), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); (void) my_init_dynamic_array(&m_conds, sizeof(sp_cond_type_t *), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); (void) my_init_dynamic_array(&m_cursors, sizeof(LEX_STRING), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); (void) my_init_dynamic_array(&m_handlers, sizeof(sp_cond_type_t *), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); m_label.empty(); m_children.empty(); @@ -89,20 +89,20 @@ sp_pcontext::sp_pcontext(sp_pcontext *prev, label_scope_type label_scope) m_label_scope(label_scope) { (void) my_init_dynamic_array(&m_vars, sizeof(sp_variable_t *), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); (void) my_init_dynamic_array(&m_case_expr_id_lst, sizeof(int), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); (void) my_init_dynamic_array(&m_conds, sizeof(sp_cond_type_t *), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); (void) my_init_dynamic_array(&m_cursors, sizeof(LEX_STRING), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); (void) my_init_dynamic_array(&m_handlers, sizeof(sp_cond_type_t *), - PCONTEXT_ARRAY_INIT_ALLOC, - PCONTEXT_ARRAY_INCREMENT_ALLOC); + PCONTEXT_ARRAY_INIT_ALLOC, + PCONTEXT_ARRAY_INCREMENT_ALLOC, MYF(0)); m_label.empty(); m_children.empty(); diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 721d49a7f88..58bc440174f 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -44,13 +44,13 @@ #include "sp.h" #include "transaction.h" #include "lock.h" // MYSQL_LOCK_IGNORE_TIMEOUT -#include "records.h" // init_read_record, end_read_record #include <sql_common.h> #include <mysql/plugin_auth.h> #include "sql_connect.h" #include "hostname.h" #include "sql_db.h" #include "sql_array.h" +#include "sql_hset.h" #include "sql_plugin_compat.h" @@ -59,15 +59,15 @@ bool mysql_user_table_is_in_short_password_format= false; static const TABLE_FIELD_TYPE mysql_db_table_fields[MYSQL_DB_FIELD_COUNT] = { { - { C_STRING_WITH_LEN("Host") }, + { C_STRING_WITH_LEN("Host") }, { C_STRING_WITH_LEN("char(60)") }, {NULL, 0} - }, + }, { - { C_STRING_WITH_LEN("Db") }, + { C_STRING_WITH_LEN("Db") }, { C_STRING_WITH_LEN("char(64)") }, {NULL, 0} - }, + }, { { C_STRING_WITH_LEN("User") }, { C_STRING_WITH_LEN("char(") }, @@ -171,24 +171,45 @@ TABLE_FIELD_TYPE mysql_db_table_fields[MYSQL_DB_FIELD_COUNT] = { }; const TABLE_FIELD_DEF - mysql_db_table_def= {MYSQL_DB_FIELD_COUNT, mysql_db_table_fields}; +mysql_db_table_def= {MYSQL_DB_FIELD_COUNT, mysql_db_table_fields, 0, (uint*) 0 }; static LEX_STRING native_password_plugin_name= { C_STRING_WITH_LEN("mysql_native_password") }; - + static LEX_STRING old_password_plugin_name= { C_STRING_WITH_LEN("mysql_old_password") }; - + /// @todo make it configurable LEX_STRING *default_auth_plugin_name= &native_password_plugin_name; +/* + Wildcard host, matches any hostname +*/ +LEX_STRING host_not_specified= { C_STRING_WITH_LEN("%") }; + +/* + Constants, used in the SHOW GRANTS command. + Their actual string values are irrelevant, they're always compared + as pointers to these string constants. +*/ +LEX_STRING current_user= { C_STRING_WITH_LEN("*current_user") }; +LEX_STRING current_role= { C_STRING_WITH_LEN("*current_role") }; +LEX_STRING current_user_and_current_role= { C_STRING_WITH_LEN("*current_user_and_current_role") }; + + #ifndef NO_EMBEDDED_ACCESS_CHECKS static plugin_ref old_password_plugin; #endif static plugin_ref native_password_plugin; +static char *safe_str(char *str) +{ return str ? str : const_cast<char*>(""); } + +static const char *safe_str(const char *str) +{ return str ? str : ""; } + /* Classes */ struct acl_host_and_ip @@ -197,6 +218,12 @@ struct acl_host_and_ip long ip, ip_mask; // Used with masked ip:s }; +#ifndef NO_EMBEDDED_ACCESS_CHECKS +static bool compare_hostname(const acl_host_and_ip *, const char *, const char *); +#else +#define compare_hostname(X,Y,Z) 0 +#endif + class ACL_ACCESS { public: ulong sort; @@ -212,15 +239,27 @@ public: char *db; }; -class ACL_USER :public ACL_ACCESS +class ACL_USER_BASE :public ACL_ACCESS +{ + +public: + static void *operator new(size_t size, MEM_ROOT *mem_root) + { return (void*) alloc_root(mem_root, size); } + + uchar flags; // field used to store various state information + LEX_STRING user; + /* list to hold references to granted roles (ACL_ROLE instances) */ + DYNAMIC_ARRAY role_grants; +}; + +class ACL_USER :public ACL_USER_BASE { public: acl_host_and_ip host; uint hostname_length; USER_RESOURCES user_resource; - char *user; uint8 salt[SCRAMBLE_LENGTH + 1]; // scrambled password in binary form - uint8 salt_len; // 0 - no password, 4 - 3.20, 8 - 4.0, 20 - 4.1.1 + uint8 salt_len; // 0 - no password, 4 - 3.20, 8 - 4.0, 20 - 4.1.1 enum SSL_type ssl_type; const char *ssl_cipher, *x509_issuer, *x509_subject; LEX_STRING plugin; @@ -232,7 +271,8 @@ public: if (!dst) return 0; *dst= *this; - dst->user= safe_strdup_root(root, user); + dst->user.str= safe_strdup_root(root, user.str); + dst->user.length= user.length; dst->ssl_cipher= safe_strdup_root(root, ssl_cipher); dst->x509_issuer= safe_strdup_root(root, x509_issuer); dst->x509_subject= safe_strdup_root(root, x509_subject); @@ -243,8 +283,55 @@ public: dst->plugin.str= strmake_root(root, plugin.str, plugin.length); dst->auth_string.str= safe_strdup_root(root, auth_string.str); dst->host.hostname= safe_strdup_root(root, host.hostname); + bzero(&dst->role_grants, sizeof(role_grants)); return dst; } + + int cmp(const char *user2, const char *host2) + { + CHARSET_INFO *cs= system_charset_info; + int res; + res= strcmp(safe_str(user.str), safe_str(user2)); + if (!res) + res= my_strcasecmp(cs, host.hostname, host2); + return res; + } + + bool eq(const char *user2, const char *host2) { return !cmp(user2, host2); } + + bool wild_eq(const char *user2, const char *host2, const char *ip2 = 0) + { + if (strcmp(safe_str(user.str), safe_str(user2))) + return false; + + return compare_hostname(&host, host2, ip2 ? ip2 : host2); + } +}; + +class ACL_ROLE :public ACL_USER_BASE +{ +public: + /* + In case of granting a role to a role, the access bits are merged together + via a bit OR operation and placed in the ACL_USER::access field. + + When rebuilding role_grants via the rebuild_role_grant function, + the ACL_USER::access field needs to be reset first. The field + initial_role_access holds initial grants, as granted directly to the role + */ + ulong initial_role_access; + /* + In subgraph traversal, when we need to traverse only a part of the graph + (e.g. all direct and indirect grantees of a role X), the counter holds the + number of affected neighbour nodes. + See also propagate_role_grants() + */ + uint counter; + DYNAMIC_ARRAY parent_grantee; // array of backlinks to elements granted + + ACL_ROLE(ACL_USER * user, MEM_ROOT *mem); + ACL_ROLE(const char * rolename, ulong privileges, MEM_ROOT *mem); + }; class ACL_DB :public ACL_ACCESS @@ -252,16 +339,30 @@ class ACL_DB :public ACL_ACCESS public: acl_host_and_ip host; char *user,*db; + ulong initial_access; /* access bits present in the table */ }; +#ifndef DBUG_OFF +/* status variables, only visible in SHOW STATUS after -#d,role_merge_stats */ +ulong role_global_merges= 0, role_db_merges= 0, role_table_merges= 0, + role_column_merges= 0, role_routine_merges= 0; +#endif #ifndef NO_EMBEDDED_ACCESS_CHECKS static void update_hostname(acl_host_and_ip *host, const char *hostname); static ulong get_sort(uint count,...); -static bool compare_hostname(const acl_host_and_ip *host, const char *hostname, - const char *ip); -static bool show_proxy_grants (THD *thd, LEX_USER *user, - char *buff, size_t buffsize); +static bool show_proxy_grants (THD *, const char *, const char *, + char *, size_t); +static bool show_role_grants(THD *, const char *, const char *, + ACL_USER_BASE *, char *, size_t); +static bool show_global_privileges(THD *, ACL_USER_BASE *, + bool, char *, size_t); +static bool show_database_privileges(THD *, const char *, const char *, + char *, size_t); +static bool show_table_and_column_privileges(THD *, const char *, const char *, + char *, size_t); +static int show_routine_grants(THD *, const char *, const char *, HASH *, + const char *, int, char *, int); class ACL_PROXY_USER :public ACL_ACCESS { @@ -271,11 +372,11 @@ class ACL_PROXY_USER :public ACL_ACCESS const char *proxied_user; bool with_grant; - typedef enum { - MYSQL_PROXIES_PRIV_HOST, - MYSQL_PROXIES_PRIV_USER, + typedef enum { + MYSQL_PROXIES_PRIV_HOST, + MYSQL_PROXIES_PRIV_USER, MYSQL_PROXIES_PRIV_PROXIED_HOST, - MYSQL_PROXIES_PRIV_PROXIED_USER, + MYSQL_PROXIES_PRIV_PROXIED_USER, MYSQL_PROXIES_PRIV_WITH_GRANT, MYSQL_PROXIES_PRIV_GRANTOR, MYSQL_PROXIES_PRIV_TIMESTAMP } old_acl_proxy_users; @@ -287,16 +388,14 @@ public: bool with_grant_arg) { user= (user_arg && *user_arg) ? user_arg : NULL; - update_hostname (&host, - (host_arg && *host_arg) ? host_arg : NULL); - proxied_user= (proxied_user_arg && *proxied_user_arg) ? + update_hostname (&host, (host_arg && *host_arg) ? host_arg : NULL); + proxied_user= (proxied_user_arg && *proxied_user_arg) ? proxied_user_arg : NULL; - update_hostname (&proxied_host, + update_hostname (&proxied_host, (proxied_host_arg && *proxied_host_arg) ? proxied_host_arg : NULL); with_grant= with_grant_arg; - sort= get_sort(4, host.hostname, user, - proxied_host.hostname, proxied_user); + sort= get_sort(4, host.hostname, user, proxied_host.hostname, proxied_user); } void init(MEM_ROOT *mem, const char *host_arg, const char *user_arg, @@ -305,9 +404,9 @@ public: { init ((host_arg && *host_arg) ? strdup_root (mem, host_arg) : NULL, (user_arg && *user_arg) ? strdup_root (mem, user_arg) : NULL, - (proxied_host_arg && *proxied_host_arg) ? + (proxied_host_arg && *proxied_host_arg) ? strdup_root (mem, proxied_host_arg) : NULL, - (proxied_user_arg && *proxied_user_arg) ? + (proxied_user_arg && *proxied_user_arg) ? strdup_root (mem, proxied_user_arg) : NULL, with_grant_arg); } @@ -326,29 +425,27 @@ public: const char *get_host() { return host.hostname; } const char *get_proxied_user() { return proxied_user; } const char *get_proxied_host() { return proxied_host.hostname; } - void set_user(MEM_ROOT *mem, const char *user_arg) - { + void set_user(MEM_ROOT *mem, const char *user_arg) + { user= user_arg && *user_arg ? strdup_root(mem, user_arg) : NULL; } - void set_host(MEM_ROOT *mem, const char *host_arg) - { - update_hostname(&host, - (host_arg && *host_arg) ? - strdup_root(mem, host_arg) : NULL); + void set_host(MEM_ROOT *mem, const char *host_arg) + { + update_hostname(&host, safe_strdup_root(mem, host_arg)); } bool check_validity(bool check_no_resolve) { - if (check_no_resolve && + if (check_no_resolve && (hostname_requires_resolving(host.hostname) || hostname_requires_resolving(proxied_host.hostname))) { sql_print_warning("'proxies_priv' entry '%s@%s %s@%s' " "ignored in --skip-name-resolve mode.", - proxied_user ? proxied_user : "", - proxied_host.hostname ? proxied_host.hostname : "", - user ? user : "", - host.hostname ? host.hostname : ""); + safe_str(proxied_user), + safe_str(proxied_host.hostname), + safe_str(user), + safe_str(host.hostname)); return TRUE; } return FALSE; @@ -362,22 +459,15 @@ public: "compare_hostname(%s,%s,%s) &&" "wild_compare (%s,%s) &&" "wild_compare (%s,%s)", - host.hostname ? host.hostname : "<NULL>", - host_arg ? host_arg : "<NULL>", - ip_arg ? ip_arg : "<NULL>", - proxied_host.hostname ? proxied_host.hostname : "<NULL>", - host_arg ? host_arg : "<NULL>", - ip_arg ? ip_arg : "<NULL>", - user_arg ? user_arg : "<NULL>", - user ? user : "<NULL>", - proxied_user_arg ? proxied_user_arg : "<NULL>", - proxied_user ? proxied_user : "<NULL>")); + host.hostname, host_arg, ip_arg, proxied_host.hostname, + host_arg, ip_arg, user_arg, user, + proxied_user_arg, proxied_user)); DBUG_RETURN(compare_hostname(&host, host_arg, ip_arg) && compare_hostname(&proxied_host, host_arg, ip_arg) && (!user || (user_arg && !wild_compare(user_arg, user, TRUE))) && - (!proxied_user || - (proxied_user && !wild_compare(proxied_user_arg, + (!proxied_user || + (proxied_user && !wild_compare(proxied_user_arg, proxied_user, TRUE)))); } @@ -395,21 +485,16 @@ public: "strcmp(%s,%s) &&" "wild_compare (%s,%s) &&" "wild_compare (%s,%s)", - user ? user : "<NULL>", - grant->user ? grant->user : "<NULL>", - proxied_user ? proxied_user : "<NULL>", - grant->proxied_user ? grant->proxied_user : "<NULL>", - host.hostname ? host.hostname : "<NULL>", - grant->host.hostname ? grant->host.hostname : "<NULL>", - proxied_host.hostname ? proxied_host.hostname : "<NULL>", - grant->proxied_host.hostname ? - grant->proxied_host.hostname : "<NULL>")); + user, grant->user, proxied_user, grant->proxied_user, + host.hostname, grant->host.hostname, + proxied_host.hostname, grant->proxied_host.hostname)); - DBUG_RETURN(auth_element_equals(user, grant->user) && - auth_element_equals(proxied_user, grant->proxied_user) && - auth_element_equals(host.hostname, grant->host.hostname) && - auth_element_equals(proxied_host.hostname, - grant->proxied_host.hostname)); + bool res= auth_element_equals(user, grant->user) && + auth_element_equals(proxied_user, grant->proxied_user) && + auth_element_equals(host.hostname, grant->host.hostname) && + auth_element_equals(proxied_host.hostname, + grant->proxied_host.hostname); + DBUG_RETURN(res); } @@ -446,23 +531,21 @@ public: with_grant= grant->with_grant; } - static int store_pk(TABLE *table, - const LEX_STRING *host, + static int store_pk(TABLE *table, + const LEX_STRING *host, const LEX_STRING *user, - const LEX_STRING *proxied_host, + const LEX_STRING *proxied_host, const LEX_STRING *proxied_user) { DBUG_ENTER("ACL_PROXY_USER::store_pk"); DBUG_PRINT("info", ("host=%s, user=%s, proxied_host=%s, proxied_user=%s", - host->str ? host->str : "<NULL>", - user->str ? user->str : "<NULL>", - proxied_host->str ? proxied_host->str : "<NULL>", - proxied_user->str ? proxied_user->str : "<NULL>")); - if (table->field[MYSQL_PROXIES_PRIV_HOST]->store(host->str, + host->str, user->str, + proxied_host->str, proxied_user->str)); + if (table->field[MYSQL_PROXIES_PRIV_HOST]->store(host->str, host->length, system_charset_info)) DBUG_RETURN(TRUE); - if (table->field[MYSQL_PROXIES_PRIV_USER]->store(user->str, + if (table->field[MYSQL_PROXIES_PRIV_USER]->store(user->str, user->length, system_charset_info)) DBUG_RETURN(TRUE); @@ -490,10 +573,10 @@ public: if (store_pk(table, host, user, proxied_host, proxied_user)) DBUG_RETURN(TRUE); DBUG_PRINT("info", ("with_grant=%s", with_grant ? "TRUE" : "FALSE")); - if (table->field[MYSQL_PROXIES_PRIV_WITH_GRANT]->store(with_grant ? 1 : 0, - TRUE)) + if (table->field[MYSQL_PROXIES_PRIV_WITH_GRANT]->store(with_grant ? 1 : 0, + TRUE)) DBUG_RETURN(TRUE); - if (table->field[MYSQL_PROXIES_PRIV_GRANTOR]->store(grantor, + if (table->field[MYSQL_PROXIES_PRIV_GRANTOR]->store(grantor, strlen(grantor), system_charset_info)) DBUG_RETURN(TRUE); @@ -520,6 +603,81 @@ static uchar* acl_entry_get_key(acl_entry *entry, size_t *length, return (uchar*) entry->key; } +static uchar* acl_role_get_key(ACL_ROLE *entry, size_t *length, + my_bool not_used __attribute__((unused))) +{ + *length=(uint) entry->user.length; + return (uchar*) entry->user.str; +} + +struct ROLE_GRANT_PAIR : public Sql_alloc +{ + char *u_uname; + char *u_hname; + char *r_uname; + LEX_STRING hashkey; + bool with_admin; + + bool init(MEM_ROOT *mem, char *username, char *hostname, char *rolename, + bool with_admin_option); +}; + +static uchar* acl_role_map_get_key(ROLE_GRANT_PAIR *entry, size_t *length, + my_bool not_used __attribute__((unused))) +{ + *length=(uint) entry->hashkey.length; + return (uchar*) entry->hashkey.str; +} + +bool ROLE_GRANT_PAIR::init(MEM_ROOT *mem, char *username, + char *hostname, char *rolename, + bool with_admin_option) +{ + if (!this) + return true; + + size_t uname_l = username ? strlen(username) : 0; + size_t hname_l = hostname ? strlen(hostname) : 0; + size_t rname_l = rolename ? strlen(rolename) : 0; + /* + Create a buffer that holds all 3 NULL terminated strings in succession + To save memory space, the same buffer is used as the hashkey + */ + size_t bufflen = uname_l + hname_l + rname_l + 3; //add the '\0' aswell + char *buff= (char *)alloc_root(mem, bufflen); + if (!buff) + return true; + + /* + Offsets in the buffer for all 3 strings + */ + char *username_pos= buff; + char *hostname_pos= buff + uname_l + 1; + char *rolename_pos= buff + uname_l + hname_l + 2; + + if (username) //prevent undefined behaviour + memcpy(username_pos, username, uname_l); + username_pos[uname_l]= '\0'; //#1 string terminator + u_uname= username_pos; + + if (hostname) //prevent undefined behaviour + memcpy(hostname_pos, hostname, hname_l); + hostname_pos[hname_l]= '\0'; //#2 string terminator + u_hname= hostname_pos; + + if (rolename) //prevent undefined behaviour + memcpy(rolename_pos, rolename, rname_l); + rolename_pos[rname_l]= '\0'; //#3 string terminator + r_uname= rolename_pos; + + hashkey.str = buff; + hashkey.length = bufflen; + + with_admin= with_admin_option; + + return false; +} + #define IP_ADDR_STRLEN (3 + 1 + 3 + 1 + 3 + 1 + 3) #define ACL_KEY_LENGTH (IP_ADDR_STRLEN + 1 + NAME_LEN + \ 1 + USERNAME_LENGTH + 1) @@ -542,7 +700,28 @@ static uchar* acl_entry_get_key(acl_entry *entry, size_t *length, #endif /* HAVE_OPENSSL && !EMBEDDED_LIBRARY */ #define NORMAL_HANDSHAKE_SIZE 6 +#define ROLE_ASSIGN_COLUMN_IDX 42 +/* various flags valid for ACL_USER */ +#define IS_ROLE (1L << 0) +/* Flag to mark that a ROLE is on the recursive DEPTH_FIRST_SEARCH stack */ +#define ROLE_ON_STACK (1L << 1) +/* + Flag to mark that a ROLE and all it's neighbours have + been visited +*/ +#define ROLE_EXPLORED (1L << 2) +/* Flag to mark that on_node was already called for this role */ +#define ROLE_OPENED (1L << 3) + static DYNAMIC_ARRAY acl_hosts, acl_users, acl_dbs, acl_proxy_users; +static HASH acl_roles; +/* + An hash containing mappings user <--> role + + A hash is used so as to make updates quickly + The hashkey used represents all the entries combined +*/ +static HASH acl_roles_mappings; static MEM_ROOT mem, memex; static bool initialized=0; static bool allow_all_hosts=1; @@ -551,38 +730,101 @@ static DYNAMIC_ARRAY acl_wild_hosts; static hash_filo *acl_cache; static uint grant_version=0; /* Version of priv tables. incremented by acl_load */ static ulong get_access(TABLE *form,uint fieldnr, uint *next_field=0); +static bool check_is_role(TABLE *form); static int acl_compare(ACL_ACCESS *a,ACL_ACCESS *b); static ulong get_sort(uint count,...); static void init_check_host(void); static void rebuild_check_host(void); -static ACL_USER *find_acl_user(const char *host, const char *user, - my_bool exact); +static void rebuild_role_grants(void); +static ACL_USER *find_user_exact(const char *host, const char *user); +static ACL_USER *find_user_wild(const char *host, const char *user, const char *ip= 0); +static ACL_ROLE *find_acl_role(const char *user); +static ROLE_GRANT_PAIR *find_role_grant_pair(const LEX_STRING *u, const LEX_STRING *h, const LEX_STRING *r); +static ACL_USER_BASE *find_acl_user_base(const char *user, const char *host); static bool update_user_table(THD *thd, TABLE *table, const char *host, const char *user, const char *new_password, uint new_password_len); static my_bool acl_load(THD *thd, TABLE_LIST *tables); static my_bool grant_load(THD *thd, TABLE_LIST *tables); static inline void get_grantor(THD *thd, char* grantor); +static bool add_role_user_mapping(const char *uname, const char *hname, const char *rname); + +#define ROLE_CYCLE_FOUND 2 +static int traverse_role_graph_up(ACL_ROLE *, void *, + int (*) (ACL_ROLE *, void *), + int (*) (ACL_ROLE *, ACL_ROLE *, void *)); + +static int traverse_role_graph_down(ACL_USER_BASE *, void *, + int (*) (ACL_USER_BASE *, void *), + int (*) (ACL_USER_BASE *, ACL_ROLE *, void *)); + /* Enumeration of various ACL's and Hashes used in handle_grant_struct() */ enum enum_acl_lists { USER_ACL= 0, + ROLE_ACL, DB_ACL, COLUMN_PRIVILEGES_HASH, PROC_PRIVILEGES_HASH, FUNC_PRIVILEGES_HASH, - PROXY_USERS_ACL + PROXY_USERS_ACL, + ROLES_MAPPINGS_HASH }; +ACL_ROLE::ACL_ROLE(ACL_USER *user, MEM_ROOT *root) : counter(0) +{ + + access= user->access; + /* set initial role access the same as the table row privileges */ + initial_role_access= user->access; + this->user.str= safe_strdup_root(root, user->user.str); + this->user.length= user->user.length; + bzero(&role_grants, sizeof(role_grants)); + bzero(&parent_grantee, sizeof(parent_grantee)); + flags= IS_ROLE; +} + +ACL_ROLE::ACL_ROLE(const char * rolename, ulong privileges, MEM_ROOT *root) : + initial_role_access(privileges), counter(0) +{ + this->access= initial_role_access; + this->user.str= safe_strdup_root(root, rolename); + this->user.length= strlen(rolename); + bzero(&role_grants, sizeof(role_grants)); + bzero(&parent_grantee, sizeof(parent_grantee)); + flags= IS_ROLE; +} + + +static bool is_invalid_role_name(const char *str) +{ + if (strcasecmp(str, "PUBLIC") && strcasecmp(str, "NONE")) + return false; + + my_error(ER_INVALID_ROLE, MYF(0), str); + return true; +} + + +static void free_acl_user(ACL_USER *user) +{ + delete_dynamic(&(user->role_grants)); +} + +static void free_acl_role(ACL_ROLE *role) +{ + delete_dynamic(&(role->role_grants)); + delete_dynamic(&(role->parent_grantee)); +} + /* - Convert scrambled password to binary form, according to scramble type, + Convert scrambled password to binary form, according to scramble type, Binary form is stored in user.salt. */ -static -void +static void set_user_salt(ACL_USER *acl_user, const char *password, uint password_len) { if (password_len == SCRAMBLED_PASSWORD_CHAR_LENGTH) @@ -637,11 +879,20 @@ static bool fix_user_plugin_ptr(ACL_USER *user) user->plugin= old_password_plugin_name; else return true; - + set_user_salt(user, user->auth_string.str, user->auth_string.length); return false; } +static bool get_YN_as_bool(Field *field) +{ + char buff[2]; + String res(buff,sizeof(buff),&my_charset_latin1); + field->val_str(&res); + return res[0] == 'Y' || res[0] == 'y'; +} + + /* Initialize structures responsible for user/db-level privilege checking and load privilege information for them from tables in the 'mysql' database. @@ -703,7 +954,7 @@ my_bool acl_init(bool dont_read_acl_tables) return_val= acl_reload(thd); delete thd; /* Remember that we don't have a THD */ - my_pthread_setspecific_ptr(THR_THD, 0); + set_current_thd(0); DBUG_RETURN(return_val); } @@ -711,10 +962,9 @@ my_bool acl_init(bool dont_read_acl_tables) Choose from either native or old password plugins when assigning a password */ -static bool -set_user_plugin (ACL_USER *user, int password_len) +static bool set_user_plugin (ACL_USER *user, int password_len) { - switch (password_len) + switch (password_len) { case 0: /* no password */ case SCRAMBLED_PASSWORD_CHAR_LENGTH: @@ -725,8 +975,8 @@ set_user_plugin (ACL_USER *user, int password_len) return FALSE; default: sql_print_warning("Found invalid password for user: '%s@%s'; " - "Ignoring user", user->user ? user->user : "", - user->host.hostname ? user->host.hostname : ""); + "Ignoring user", safe_str(user->user.str), + safe_str(user->host.hostname)); return TRUE; } } @@ -739,8 +989,9 @@ set_user_plugin (ACL_USER *user, int password_len) SYNOPSIS acl_load() thd Current thread - tables List containing open "mysql.host", "mysql.user" and - "mysql.db" tables. + tables List containing open "mysql.host", "mysql.user", + "mysql.db", "mysql.proxies_priv" and "mysql.roles_mapping" + tables. RETURN VALUES FALSE Success @@ -762,15 +1013,13 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) grant_version++; /* Privileges updated */ - acl_cache->clear(1); // Clear locked hostname cache - - init_sql_alloc(&mem, ACL_ALLOC_BLOCK_SIZE, 0); + init_sql_alloc(&mem, ACL_ALLOC_BLOCK_SIZE, 0, MYF(0)); if (init_read_record(&read_record_info,thd,table= tables[0].table,NULL,1,0, FALSE)) goto end; table->use_all_columns(); - (void) my_init_dynamic_array(&acl_hosts,sizeof(ACL_HOST),20,50); + (void) my_init_dynamic_array(&acl_hosts,sizeof(ACL_HOST), 20, 50, MYF(0)); while (!(read_record_info.read_record(&read_record_info))) { ACL_HOST host; @@ -794,8 +1043,7 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) "case that has been forced to lowercase because " "lower_case_table_names is set. It will not be " "possible to remove this privilege using REVOKE.", - host.host.hostname ? host.host.hostname : "", - host.db ? host.db : ""); + host.host.hostname, host.db); } host.access= get_access(table,2); host.access= fix_rights_for_db(host.access); @@ -804,8 +1052,8 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) { sql_print_warning("'host' entry '%s|%s' " "ignored in --skip-name-resolve mode.", - host.host.hostname ? host.host.hostname : "", - host.db ? host.db : ""); + safe_str(host.host.hostname), + safe_str(host.db)); continue; } #ifndef TO_BE_REMOVED @@ -827,7 +1075,11 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) goto end; table->use_all_columns(); - (void) my_init_dynamic_array(&acl_users,sizeof(ACL_USER),50,100); + (void) my_init_dynamic_array(&acl_users,sizeof(ACL_USER), 50, 100, MYF(0)); + (void) my_hash_init2(&acl_roles,50, &my_charset_utf8_bin, + 0,0,0, (my_hash_get_key) acl_role_get_key, + (void (*)(void *))free_acl_role, 0); + username_char_length= min(table->field[1]->char_length(), USERNAME_CHAR_LENGTH); password_length= table->field[2]->field_length / table->field[2]->charset()->mbmaxlen; @@ -874,25 +1126,42 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) while (!(read_record_info.read_record(&read_record_info))) { ACL_USER user; + bool is_role= FALSE; bzero(&user, sizeof(user)); update_hostname(&user.host, get_field(&mem, table->field[0])); - user.user= get_field(&mem, table->field[1]); - if (check_no_resolve && hostname_requires_resolving(user.host.hostname)) + char *username= get_field(&mem, table->field[1]); + user.user.str= username; + user.user.length= username? strlen(username) : 0; + + /* + If the user entry is a role, skip password and hostname checks + A user can not log in with a role so some checks are not necessary + */ + is_role= check_is_role(table); + + if (is_role && is_invalid_role_name(username)) + { + thd->clear_error(); // the warning is still issued + continue; + } + + if (!is_role && check_no_resolve && + hostname_requires_resolving(user.host.hostname)) { sql_print_warning("'user' entry '%s@%s' " "ignored in --skip-name-resolve mode.", - user.user ? user.user : "", - user.host.hostname ? user.host.hostname : ""); + safe_str(user.user.str), + safe_str(user.host.hostname)); continue; } char *password= get_field(&mem, table->field[2]); uint password_len= password ? strlen(password) : 0; - user.auth_string.str= password ? password : const_cast<char*>(""); + user.auth_string.str= safe_str(password); user.auth_string.length= password_len; set_user_salt(&user, password, password_len); - if (set_user_plugin(&user, password_len)) + if (!is_role && set_user_plugin(&user, password_len)) continue; { @@ -934,7 +1203,7 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) if (table->s->fields <= 38 && (user.access & SUPER_ACL)) user.access|= TRIGGER_ACL; - user.sort= get_sort(2,user.host.hostname,user.user); + user.sort= get_sort(2, user.host.hostname, user.user.str); user.hostname_length= (user.host.hostname ? (uint) strlen(user.host.hostname) : 0); @@ -972,7 +1241,7 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) user.user_resource.user_conn= ptr ? atoi(ptr) : 0; } - if (table->s->fields >= 41) + if (!is_role && table->s->fields >= 41) { /* We may have plugin & auth_String fields */ char *tmpstr= get_field(&mem, table->field[next_field++]); @@ -985,12 +1254,11 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) sql_print_warning("'user' entry '%s@%s' has both a password " "and an authentication plugin specified. The " "password will be ignored.", - user.user ? user.user : "", - user.host.hostname ? user.host.hostname : ""); + safe_str(user.user.str), + safe_str(user.host.hostname)); } - user.auth_string.str= get_field(&mem, table->field[next_field++]); - if (!user.auth_string.str) - user.auth_string.str= const_cast<char*>(""); + user.auth_string.str= + safe_str(get_field(&mem, table->field[next_field++])); user.auth_string.length= strlen(user.auth_string.str); fix_user_plugin_ptr(&user); @@ -1014,7 +1282,26 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) user.access|= SUPER_ACL | EXECUTE_ACL; #endif } - (void) push_dynamic(&acl_users,(uchar*) &user); + + (void) my_init_dynamic_array(&user.role_grants,sizeof(ACL_ROLE *), + 8, 8, MYF(0)); + + if (is_role) + { + DBUG_PRINT("info", ("Found role %s", user.user.str)); + ACL_ROLE *entry= new (&mem) ACL_ROLE(&user, &mem); + entry->role_grants = user.role_grants; + (void) my_init_dynamic_array(&entry->parent_grantee, + sizeof(ACL_USER_BASE *), 8, 8, MYF(0)); + my_hash_insert(&acl_roles, (uchar *)entry); + + continue; + } + else + { + DBUG_PRINT("info", ("Found user %s", user.user.str)); + (void) push_dynamic(&acl_users,(uchar*) &user); + } if (!user.host.hostname || (user.host.hostname[0] == wild_many && !user.host.hostname[1])) allow_all_hosts=1; // Anyone can connect @@ -1030,29 +1317,31 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) goto end; table->use_all_columns(); - (void) my_init_dynamic_array(&acl_dbs,sizeof(ACL_DB),50,100); + (void) my_init_dynamic_array(&acl_dbs,sizeof(ACL_DB), 50, 100, MYF(0)); while (!(read_record_info.read_record(&read_record_info))) { ACL_DB db; - update_hostname(&db.host,get_field(&mem, table->field[MYSQL_DB_FIELD_HOST])); + db.user=get_field(&mem, table->field[MYSQL_DB_FIELD_USER]); + const char *hostname= get_field(&mem, table->field[MYSQL_DB_FIELD_HOST]); + if (!hostname && find_acl_role(db.user)) + hostname= ""; + update_hostname(&db.host, hostname); db.db=get_field(&mem, table->field[MYSQL_DB_FIELD_DB]); if (!db.db) { sql_print_warning("Found an entry in the 'db' table with empty database name; Skipped"); continue; } - db.user=get_field(&mem, table->field[MYSQL_DB_FIELD_USER]); if (check_no_resolve && hostname_requires_resolving(db.host.hostname)) { sql_print_warning("'db' entry '%s %s@%s' " "ignored in --skip-name-resolve mode.", - db.db, - db.user ? db.user : "", - db.host.hostname ? db.host.hostname : ""); + db.db, safe_str(db.user), safe_str(db.host.hostname)); continue; } db.access=get_access(table,3); db.access=fix_rights_for_db(db.access); + db.initial_access= db.access; if (lower_case_table_names) { /* @@ -1072,9 +1361,7 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) "case that has been forced to lowercase because " "lower_case_table_names is set. It will not be " "possible to remove this privilege using REVOKE.", - db.db, - db.user ? db.user : "", - db.host.hostname ? db.host.hostname : ""); + db.db, safe_str(db.user), safe_str(db.host.hostname)); } } db.sort=get_sort(3,db.host.hostname,db.db,db.user); @@ -1092,8 +1379,8 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) end_read_record(&read_record_info); freeze_size(&acl_dbs); - (void) my_init_dynamic_array(&acl_proxy_users, sizeof(ACL_PROXY_USER), - 50, 100); + (void) my_init_dynamic_array(&acl_proxy_users, sizeof(ACL_PROXY_USER), + 50, 100, MYF(0)); if (tables[3].table) { init_read_record(&read_record_info, thd, table= tables[3].table, NULL, 1, @@ -1123,6 +1410,48 @@ static my_bool acl_load(THD *thd, TABLE_LIST *tables) } freeze_size(&acl_proxy_users); + if (tables[4].table) + { + if (init_read_record(&read_record_info, thd, table= tables[4].table, + NULL, 1, 1, FALSE)) + goto end; + table->use_all_columns(); + /* account for every role mapping */ + + (void) my_hash_init2(&acl_roles_mappings, 50, system_charset_info, + 0,0,0, (my_hash_get_key) acl_role_map_get_key, 0,0); + MEM_ROOT temp_root; + init_alloc_root(&temp_root, ACL_ALLOC_BLOCK_SIZE, 0, MYF(0)); + while (!(read_record_info.read_record(&read_record_info))) + { + char *hostname= safe_str(get_field(&temp_root, table->field[0])); + char *username= safe_str(get_field(&temp_root, table->field[1])); + char *rolename= safe_str(get_field(&temp_root, table->field[2])); + bool with_grant_option= get_YN_as_bool(table->field[3]); + + if (add_role_user_mapping(username, hostname, rolename)) { + sql_print_error("Invalid roles_mapping table entry user:'%s@%s', rolename:'%s'", + username, hostname, rolename); + continue; + } + + ROLE_GRANT_PAIR *mapping= new (&mem) ROLE_GRANT_PAIR; + + if (mapping->init(&mem, username, hostname, rolename, with_grant_option)) + continue; + + my_hash_insert(&acl_roles_mappings, (uchar*) mapping); + } + + free_root(&temp_root, MYF(0)); + end_read_record(&read_record_info); + } + else + { + sql_print_error("Missing system table mysql.roles_mapping; " + "please run mysql_upgrade to create it"); + } + init_check_host(); initialized=1; @@ -1136,13 +1465,15 @@ end: void acl_free(bool end) { + my_hash_free(&acl_roles); free_root(&mem,MYF(0)); delete_dynamic(&acl_hosts); - delete_dynamic(&acl_users); + delete_dynamic_with_callback(&acl_users, (FREE_FUNC) free_acl_user); delete_dynamic(&acl_dbs); delete_dynamic(&acl_wild_hosts); delete_dynamic(&acl_proxy_users); my_hash_free(&acl_check_hosts); + my_hash_free(&acl_roles_mappings); plugin_unlock(0, native_password_plugin); plugin_unlock(0, old_password_plugin); if (!end) @@ -1176,10 +1507,10 @@ void acl_free(bool end) my_bool acl_reload(THD *thd) { - TABLE_LIST tables[4]; + TABLE_LIST tables[5]; DYNAMIC_ARRAY old_acl_hosts, old_acl_users, old_acl_dbs, old_acl_proxy_users; + HASH old_acl_roles, old_acl_roles_mappings; MEM_ROOT old_mem; - bool old_initialized; my_bool return_val= TRUE; DBUG_ENTER("acl_reload"); @@ -1196,12 +1527,18 @@ my_bool acl_reload(THD *thd) tables[3].init_one_table(C_STRING_WITH_LEN("mysql"), C_STRING_WITH_LEN("proxies_priv"), "proxies_priv", TL_READ); + tables[4].init_one_table(C_STRING_WITH_LEN("mysql"), + C_STRING_WITH_LEN("roles_mapping"), + "roles_mapping", TL_READ); tables[0].next_local= tables[0].next_global= tables + 1; tables[1].next_local= tables[1].next_global= tables + 2; tables[2].next_local= tables[2].next_global= tables + 3; + tables[3].next_local= tables[3].next_global= tables + 4; tables[0].open_type= tables[1].open_type= tables[2].open_type= - tables[3].open_type= OT_BASE_ONLY; - tables[3].open_strategy= TABLE_LIST::OPEN_IF_EXISTS; + tables[3].open_type= tables[4].open_type= OT_BASE_ONLY; + tables[0].open_strategy= tables[3].open_strategy= + tables[4].open_strategy= TABLE_LIST::OPEN_IF_EXISTS; + if (open_and_lock_tables(thd, tables, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT)) { @@ -1215,11 +1552,13 @@ my_bool acl_reload(THD *thd) goto end; } - if ((old_initialized=initialized)) - mysql_mutex_lock(&acl_cache->lock); + acl_cache->clear(0); + mysql_mutex_lock(&acl_cache->lock); old_acl_hosts= acl_hosts; old_acl_users= acl_users; + old_acl_roles= acl_roles; + old_acl_roles_mappings= acl_roles_mappings; old_acl_proxy_users= acl_proxy_users; old_acl_dbs= acl_dbs; old_mem= mem; @@ -1232,6 +1571,8 @@ my_bool acl_reload(THD *thd) acl_free(); /* purecov: inspected */ acl_hosts= old_acl_hosts; acl_users= old_acl_users; + acl_roles= old_acl_roles; + acl_roles_mappings= old_acl_roles_mappings; acl_proxy_users= old_acl_proxy_users; acl_dbs= old_acl_dbs; mem= old_mem; @@ -1239,20 +1580,20 @@ my_bool acl_reload(THD *thd) } else { + my_hash_free(&old_acl_roles); free_root(&old_mem,MYF(0)); delete_dynamic(&old_acl_hosts); - delete_dynamic(&old_acl_users); + delete_dynamic_with_callback(&old_acl_users, (FREE_FUNC) free_acl_user); delete_dynamic(&old_acl_proxy_users); delete_dynamic(&old_acl_dbs); + my_hash_free(&old_acl_roles_mappings); } - if (old_initialized) - mysql_mutex_unlock(&acl_cache->lock); + mysql_mutex_unlock(&acl_cache->lock); end: close_mysql_tables(thd); DBUG_RETURN(return_val); } - /* Get all access bits from table after fieldnr @@ -1284,8 +1625,7 @@ static ulong get_access(TABLE *form, uint fieldnr, uint *next_field) ((Field_enum*) (*pos))->typelib->count == 2 ; pos++, fieldnr++, bit<<=1) { - (*pos)->val_str(&res); - if (my_toupper(&my_charset_latin1, res[0]) == 'Y') + if (get_YN_as_bool(*pos)) access_bits|= bit; } if (next_field) @@ -1293,6 +1633,34 @@ static ulong get_access(TABLE *form, uint fieldnr, uint *next_field) return access_bits; } +/* + Check if a user entry in the user table is marked as being a role entry + + IMPLEMENTATION + Access the coresponding column and check the coresponding ENUM of the form + ENUM('N', 'Y') + + SYNOPSIS + check_is_role() + form an open table to read the entry from. + The record should be already read in table->record[0] + + RETURN VALUE + TRUE if the user is marked as a role + FALSE otherwise +*/ + +static bool check_is_role(TABLE *form) +{ + char buff[2]; + String res(buff, sizeof(buff), &my_charset_latin1); + /* Table version does not support roles */ + if (form->s->fields <= ROLE_ASSIGN_COLUMN_IDX) + return FALSE; + + return get_YN_as_bool(form->field[ROLE_ASSIGN_COLUMN_IDX]); +} + /* Return a number which, if sorted 'desc', puts strings in this order: @@ -1372,12 +1740,11 @@ bool acl_getroot(Security_context *sctx, char *user, char *host, DBUG_ENTER("acl_getroot"); DBUG_PRINT("enter", ("Host: '%s', Ip: '%s', User: '%s', db: '%s'", - (host ? host : "(NULL)"), (ip ? ip : "(NULL)"), - user, (db ? db : "(NULL)"))); + host, ip, user, db)); sctx->user= user; sctx->host= host; sctx->ip= ip; - sctx->host_or_ip= host ? host : (ip ? ip : ""); + sctx->host_or_ip= host ? host : (safe_str(ip)); if (!initialized) { @@ -1392,63 +1759,168 @@ bool acl_getroot(Security_context *sctx, char *user, char *host, sctx->master_access= 0; sctx->db_access= 0; - *sctx->priv_user= *sctx->priv_host= 0; + *sctx->priv_user= *sctx->priv_host= *sctx->priv_role= 0; - /* - Find acl entry in user database. - This is specially tailored to suit the check we do for CALL of - a stored procedure; user is set to what is actually a - priv_user, which can be ''. - */ - for (i=0 ; i < acl_users.elements ; i++) + if (host[0]) // User, not Role { - ACL_USER *acl_user_tmp= dynamic_element(&acl_users,i,ACL_USER*); - if ((!acl_user_tmp->user && !user[0]) || - (acl_user_tmp->user && strcmp(user, acl_user_tmp->user) == 0)) + acl_user= find_user_wild(host, user, ip); + + if (acl_user) { - if (compare_hostname(&acl_user_tmp->host, host, ip)) + res= 0; + for (i=0 ; i < acl_dbs.elements ; i++) { - acl_user= acl_user_tmp; - res= 0; - break; + ACL_DB *acl_db= dynamic_element(&acl_dbs, i, ACL_DB*); + if (!acl_db->user || + (user && user[0] && !strcmp(user, acl_db->user))) + { + if (compare_hostname(&acl_db->host, host, ip)) + { + if (!acl_db->db || (db && !wild_compare(db, acl_db->db, 0))) + { + sctx->db_access= acl_db->access; + break; + } + } + } } + sctx->master_access= acl_user->access; + + if (acl_user->user.str) + strmake_buf(sctx->priv_user, user); + + if (acl_user->host.hostname) + strmake_buf(sctx->priv_host, acl_user->host.hostname); } } - - if (acl_user) + else // Role, not User { - for (i=0 ; i < acl_dbs.elements ; i++) + ACL_ROLE *acl_role= find_acl_role(user); + if (acl_role) { - ACL_DB *acl_db= dynamic_element(&acl_dbs, i, ACL_DB*); - if (!acl_db->user || - (user && user[0] && !strcmp(user, acl_db->user))) + res= 0; + for (i=0 ; i < acl_dbs.elements ; i++) { - if (compare_hostname(&acl_db->host, host, ip)) - { - if (!acl_db->db || (db && !wild_compare(db, acl_db->db, 0))) - { - sctx->db_access= acl_db->access; - break; - } - } + ACL_DB *acl_db= dynamic_element(&acl_dbs, i, ACL_DB*); + if (!acl_db->user || + (user && user[0] && !strcmp(user, acl_db->user))) + { + if (compare_hostname(&acl_db->host, "", "")) + { + if (!acl_db->db || (db && !wild_compare(db, acl_db->db, 0))) + { + sctx->db_access= acl_db->access; + break; + } + } + } } + sctx->master_access= acl_role->access; + + if (acl_role->user.str) + strmake_buf(sctx->priv_user, user); + sctx->priv_host[0]= 0; } - sctx->master_access= acl_user->access; + } - if (acl_user->user) - strmake_buf(sctx->priv_user, user); - else - *sctx->priv_user= 0; + mysql_mutex_unlock(&acl_cache->lock); + DBUG_RETURN(res); +} - if (acl_user->host.hostname) - strmake_buf(sctx->priv_host, acl_user->host.hostname); - else - *sctx->priv_host= 0; +int acl_check_setrole(THD *thd, char *rolename, ulonglong *access) +{ + ACL_ROLE *role; + ACL_USER_BASE *acl_user_base; + ACL_USER *UNINIT_VAR(acl_user); + bool is_granted= FALSE; + int result= 0; + + /* clear role privileges */ + mysql_mutex_lock(&acl_cache->lock); + + if (!strcasecmp(rolename, "NONE")) + { + /* have to clear the privileges */ + /* get the current user */ + acl_user= find_user_exact(thd->security_ctx->priv_host, + thd->security_ctx->priv_user); + if (acl_user == NULL) + { + my_error(ER_INVALID_CURRENT_USER, MYF(0), rolename); + result= -1; + } + else if (access) + *access= acl_user->access; + + goto end; + } + + role= find_acl_role(rolename); + + /* According to SQL standard, the same error message must be presented */ + if (role == NULL) { + my_error(ER_INVALID_ROLE, MYF(0), rolename); + result= -1; + goto end; + } + + for (uint i=0 ; i < role->parent_grantee.elements ; i++) + { + acl_user_base= *(dynamic_element(&role->parent_grantee, i, ACL_USER_BASE**)); + if (acl_user_base->flags & IS_ROLE) + continue; + + acl_user= (ACL_USER *)acl_user_base; + /* Yes! priv_user@host. Don't ask why - that's what check_access() does. */ + if (acl_user->wild_eq(thd->security_ctx->priv_user, + thd->security_ctx->host)) + { + is_granted= TRUE; + break; + } + } + + /* According to SQL standard, the same error message must be presented */ + if (!is_granted) + { + my_error(ER_INVALID_ROLE, MYF(0), rolename); + result= 1; + goto end; } + + if (access) + { + *access = acl_user->access | role->access; + } +end: mysql_mutex_unlock(&acl_cache->lock); - DBUG_RETURN(res); + return result; } + +int acl_setrole(THD *thd, char *rolename, ulonglong access) +{ + /* merge the privileges */ + Security_context *sctx= thd->security_ctx; + sctx->master_access= static_cast<ulong>(access); + if (thd->db) + sctx->db_access= acl_get(sctx->host, sctx->ip, sctx->user, thd->db, FALSE); + + if (!strcasecmp(rolename, "NONE")) + { + thd->security_ctx->priv_role[0]= 0; + } + else + { + if (thd->db) + sctx->db_access|= acl_get("", "", rolename, thd->db, FALSE); + /* mark the current role */ + strmake_buf(thd->security_ctx->priv_role, rolename); + } + return 0; +} + + static uchar* check_get_key(ACL_USER *buff, size_t *length, my_bool not_used __attribute__((unused))) { @@ -1457,6 +1929,14 @@ static uchar* check_get_key(ACL_USER *buff, size_t *length, } +static void acl_update_role(const char *rolename, ulong privileges) +{ + ACL_ROLE *role= find_acl_role(rolename); + if (role) + role->initial_role_access= role->access= privileges; +} + + static void acl_update_user(const char *user, const char *host, const char *password, uint password_len, enum SSL_type ssl_type, @@ -1473,57 +1953,66 @@ static void acl_update_user(const char *user, const char *host, for (uint i=0 ; i < acl_users.elements ; i++) { ACL_USER *acl_user=dynamic_element(&acl_users,i,ACL_USER*); - if ((!acl_user->user && !user[0]) || - (acl_user->user && !strcmp(user,acl_user->user))) + if (acl_user->eq(user, host)) { - if ((!acl_user->host.hostname && !host[0]) || - (acl_user->host.hostname && - !my_strcasecmp(system_charset_info, host, acl_user->host.hostname))) + if (plugin->str[0]) { - if (plugin->str[0]) + acl_user->plugin= *plugin; + acl_user->auth_string.str= auth->str ? + strmake_root(&mem, auth->str, auth->length) : const_cast<char*>(""); + acl_user->auth_string.length= auth->length; + if (fix_user_plugin_ptr(acl_user)) + acl_user->plugin.str= strmake_root(&mem, plugin->str, plugin->length); + } + else + if (password[0]) { - acl_user->plugin= *plugin; - acl_user->auth_string.str= auth->str ? - strmake_root(&mem, auth->str, auth->length) : const_cast<char*>(""); - acl_user->auth_string.length= auth->length; - if (fix_user_plugin_ptr(acl_user)) - acl_user->plugin.str= strmake_root(&mem, plugin->str, plugin->length); + acl_user->auth_string.str= strmake_root(&mem, password, password_len); + acl_user->auth_string.length= password_len; + set_user_salt(acl_user, password, password_len); + set_user_plugin(acl_user, password_len); } - else - if (password[0]) - { - acl_user->auth_string.str= strmake_root(&mem, password, password_len); - acl_user->auth_string.length= password_len; - set_user_salt(acl_user, password, password_len); - set_user_plugin(acl_user, password_len); - } - acl_user->access=privileges; - if (mqh->specified_limits & USER_RESOURCES::QUERIES_PER_HOUR) - acl_user->user_resource.questions=mqh->questions; - if (mqh->specified_limits & USER_RESOURCES::UPDATES_PER_HOUR) - acl_user->user_resource.updates=mqh->updates; - if (mqh->specified_limits & USER_RESOURCES::CONNECTIONS_PER_HOUR) - acl_user->user_resource.conn_per_hour= mqh->conn_per_hour; - if (mqh->specified_limits & USER_RESOURCES::USER_CONNECTIONS) - acl_user->user_resource.user_conn= mqh->user_conn; - if (ssl_type != SSL_TYPE_NOT_SPECIFIED) - { - acl_user->ssl_type= ssl_type; - acl_user->ssl_cipher= (ssl_cipher ? strdup_root(&mem,ssl_cipher) : - 0); - acl_user->x509_issuer= (x509_issuer ? strdup_root(&mem,x509_issuer) : - 0); - acl_user->x509_subject= (x509_subject ? - strdup_root(&mem,x509_subject) : 0); - } - /* search complete: */ - break; + acl_user->access=privileges; + if (mqh->specified_limits & USER_RESOURCES::QUERIES_PER_HOUR) + acl_user->user_resource.questions=mqh->questions; + if (mqh->specified_limits & USER_RESOURCES::UPDATES_PER_HOUR) + acl_user->user_resource.updates=mqh->updates; + if (mqh->specified_limits & USER_RESOURCES::CONNECTIONS_PER_HOUR) + acl_user->user_resource.conn_per_hour= mqh->conn_per_hour; + if (mqh->specified_limits & USER_RESOURCES::USER_CONNECTIONS) + acl_user->user_resource.user_conn= mqh->user_conn; + if (ssl_type != SSL_TYPE_NOT_SPECIFIED) + { + acl_user->ssl_type= ssl_type; + acl_user->ssl_cipher= (ssl_cipher ? strdup_root(&mem,ssl_cipher) : + 0); + acl_user->x509_issuer= (x509_issuer ? strdup_root(&mem,x509_issuer) : + 0); + acl_user->x509_subject= (x509_subject ? + strdup_root(&mem,x509_subject) : 0); } + /* search complete: */ + break; } } } +static void acl_insert_role(const char *rolename, ulong privileges) +{ + ACL_ROLE *entry; + + mysql_mutex_assert_owner(&acl_cache->lock); + entry= new (&mem) ACL_ROLE(rolename, privileges, &mem); + (void) my_init_dynamic_array(&entry->parent_grantee, + sizeof(ACL_USER_BASE *), 8, 8, MYF(0)); + (void) my_init_dynamic_array(&entry->role_grants,sizeof(ACL_ROLE *), + 8, 8, MYF(0)); + + my_hash_insert(&acl_roles, (uchar *)entry); +} + + static void acl_insert_user(const char *user, const char *host, const char *password, uint password_len, enum SSL_type ssl_type, @@ -1539,8 +2028,9 @@ static void acl_insert_user(const char *user, const char *host, mysql_mutex_assert_owner(&acl_cache->lock); - acl_user.user=*user ? strdup_root(&mem,user) : 0; - update_hostname(&acl_user.host, *host ? strdup_root(&mem, host): 0); + acl_user.user.str=*user ? strdup_root(&mem,user) : 0; + acl_user.user.length= strlen(user); + update_hostname(&acl_user.host, safe_strdup_root(&mem, host)); if (plugin->str[0]) { acl_user.plugin= *plugin; @@ -1558,15 +2048,18 @@ static void acl_insert_user(const char *user, const char *host, set_user_plugin(&acl_user, password_len); } + acl_user.flags= 0; acl_user.access=privileges; acl_user.user_resource = *mqh; - acl_user.sort=get_sort(2,acl_user.host.hostname,acl_user.user); + acl_user.sort=get_sort(2, acl_user.host.hostname, acl_user.user.str); acl_user.hostname_length=(uint) strlen(host); acl_user.ssl_type= (ssl_type != SSL_TYPE_NOT_SPECIFIED ? ssl_type : SSL_TYPE_NONE); acl_user.ssl_cipher= ssl_cipher ? strdup_root(&mem,ssl_cipher) : 0; acl_user.x509_issuer= x509_issuer ? strdup_root(&mem,x509_issuer) : 0; acl_user.x509_subject=x509_subject ? strdup_root(&mem,x509_subject) : 0; + (void) my_init_dynamic_array(&acl_user.role_grants, sizeof(ACL_USER *), + 8, 8, MYF(0)); (void) push_dynamic(&acl_users,(uchar*) &acl_user); if (!acl_user.host.hostname || @@ -1577,11 +2070,17 @@ static void acl_insert_user(const char *user, const char *host, /* Rebuild 'acl_check_hosts' since 'acl_users' has been modified */ rebuild_check_host(); + + /* + Rebuild every user's role_grants since 'acl_users' has been sorted + and old pointers to ACL_USER elements are no longer valid + */ + rebuild_role_grants(); } static void acl_update_db(const char *user, const char *host, const char *db, - ulong privileges) + ulong privileges) { mysql_mutex_assert_owner(&acl_cache->lock); @@ -1601,7 +2100,10 @@ static void acl_update_db(const char *user, const char *host, const char *db, { if (privileges) - acl_db->access=privileges; + { + acl_db->access= privileges; + acl_db->initial_access= acl_db->access; + } else delete_dynamic_element(&acl_dbs,i); } @@ -1626,14 +2128,14 @@ static void acl_update_db(const char *user, const char *host, const char *db, */ static void acl_insert_db(const char *user, const char *host, const char *db, - ulong privileges) + ulong privileges) { ACL_DB acl_db; mysql_mutex_assert_owner(&acl_cache->lock); acl_db.user=strdup_root(&mem,user); - update_hostname(&acl_db.host, *host ? strdup_root(&mem,host) : 0); + update_hostname(&acl_db.host, safe_strdup_root(&mem, host)); acl_db.db=strdup_root(&mem,db); - acl_db.access=privileges; + acl_db.initial_access= acl_db.access= privileges; acl_db.sort=get_sort(3,acl_db.host.hostname,acl_db.db,acl_db.user); (void) push_dynamic(&acl_dbs,(uchar*) &acl_db); my_qsort((uchar*) dynamic_element(&acl_dbs,0,ACL_DB*),acl_dbs.elements, @@ -1641,7 +2143,6 @@ static void acl_insert_db(const char *user, const char *host, const char *db, } - /* Get privilege for a host, user and db combination @@ -1659,7 +2160,7 @@ ulong acl_get(const char *host, const char *ip, acl_entry *entry; DBUG_ENTER("acl_get"); - tmp_db= strmov(strmov(key, ip ? ip : "") + 1, user) + 1; + tmp_db= strmov(strmov(key, safe_str(ip)) + 1, user) + 1; end= strnmov(tmp_db, db, key + sizeof(key) - tmp_db); if (end >= key + sizeof(key)) // db name was truncated @@ -1692,12 +2193,15 @@ ulong acl_get(const char *host, const char *ip, { if (compare_hostname(&acl_db->host,host,ip)) { - if (!acl_db->db || !wild_compare(db,acl_db->db,db_is_pattern)) - { - db_access=acl_db->access; - if (acl_db->host.hostname) - goto exit; // Fully specified. Take it - break; /* purecov: tested */ + if (!acl_db->db || !wild_compare(db,acl_db->db,db_is_pattern)) + { + db_access=acl_db->access; + if (acl_db->host.hostname) + goto exit; // Fully specified. Take it + /* the host table is not used for roles */ + if ((!host || !host[0]) && !acl_db->host.hostname && find_acl_role(user)) + goto exit; + break; /* purecov: tested */ } } } @@ -1748,7 +2252,7 @@ static void init_check_host(void) { DBUG_ENTER("init_check_host"); (void) my_init_dynamic_array(&acl_wild_hosts,sizeof(struct acl_host_and_ip), - acl_users.elements,1); + acl_users.elements, 1, MYF(0)); (void) my_hash_init(&acl_check_hosts,system_charset_info, acl_users.elements, 0, 0, (my_hash_get_key) check_get_key, 0, 0); @@ -1806,9 +2310,143 @@ void rebuild_check_host(void) init_check_host(); } +/* + Reset a role role_grants dynamic array. + Also, the role's access bits are reset to the ones present in the table. +*/ +static my_bool acl_role_reset_role_arrays(void *ptr, + void * not_used __attribute__((unused))) +{ + ACL_ROLE *role= (ACL_ROLE *)ptr; + reset_dynamic(&role->role_grants); + reset_dynamic(&role->parent_grantee); + role->counter= 0; + return 0; +} -/* Return true if there is no users that can match the given host */ +/* + Add a the coresponding pointers present in the mapping to the entries in + acl_users and acl_roles +*/ +static bool add_role_user_mapping(ACL_USER_BASE *grantee, ACL_ROLE *role) +{ + return push_dynamic(&grantee->role_grants, (uchar*) &role) + || push_dynamic(&role->parent_grantee, (uchar*) &grantee); +} + +/* + Revert the last add_role_user_mapping() action +*/ +static void undo_add_role_user_mapping(ACL_USER_BASE *grantee, ACL_ROLE *role) +{ + void *pop __attribute__((unused)); + + pop= pop_dynamic(&grantee->role_grants); + DBUG_ASSERT(role == *(ACL_ROLE**)pop); + + pop= pop_dynamic(&role->parent_grantee); + DBUG_ASSERT(grantee == *(ACL_USER_BASE**)pop); +} + +/* + this helper is used when building role_grants and parent_grantee arrays + from scratch. + + this happens either on initial loading of data from tables, in acl_load(). + or in rebuild_role_grants after acl_role_reset_role_arrays(). +*/ +static bool add_role_user_mapping(const char *uname, const char *hname, + const char *rname) +{ + ACL_USER_BASE *grantee= find_acl_user_base(uname, hname); + ACL_ROLE *role= find_acl_role(rname); + + if (grantee == NULL || role == NULL) + return 1; + + /* + because all arrays are rebuilt completely, and counters were also reset, + we can increment them here, and after the rebuild all counters will + have correct values (equal to the number of roles granted). + */ + if (grantee->flags & IS_ROLE) + ((ACL_ROLE*)grantee)->counter++; + return add_role_user_mapping(grantee, role); +} + +/* + This helper function is used to removes roles and grantees + from the corresponding cross-reference arrays. see remove_role_user_mapping(). + as such, it asserts that an element to delete is present in the array, + and is present only once. +*/ +static void remove_ptr_from_dynarray(DYNAMIC_ARRAY *array, void *ptr) +{ + bool found __attribute__((unused))= false; + for (uint i= 0; i < array->elements; i++) + { + if (ptr == *dynamic_element(array, i, void**)) + { + DBUG_ASSERT(!found); + delete_dynamic_element(array, i); + IF_DBUG(found= true, break); + } + } + DBUG_ASSERT(found); +} + +static void remove_role_user_mapping(ACL_USER_BASE *grantee, ACL_ROLE *role, + int grantee_idx=-1, int role_idx=-1) +{ + remove_ptr_from_dynarray(&grantee->role_grants, role); + remove_ptr_from_dynarray(&role->parent_grantee, grantee); +} + + +static my_bool add_role_user_mapping_action(void *ptr, void *unused __attribute__((unused))) +{ + ROLE_GRANT_PAIR *pair= (ROLE_GRANT_PAIR*)ptr; + my_bool status __attribute__((unused)); + status= add_role_user_mapping(pair->u_uname, pair->u_hname, pair->r_uname); + /* + The invariant chosen is that acl_roles_mappings should _always_ + only contain valid entries, referencing correct user and role grants. + If add_role_user_mapping detects an invalid entry, it will not add + the mapping into the ACL_USER::role_grants array. + */ + DBUG_ASSERT(status >= 0); + return 0; +} + + +/* + Rebuild the role grants every time the acl_users is modified + + The role grants in the ACL_USER class need to be rebuilt, as they contain + pointers to elements of the acl_users array. +*/ + +static void rebuild_role_grants(void) +{ + DBUG_ENTER("rebuild_role_grants"); + /* + Reset every user's and role's role_grants array + */ + for (uint i=0; i < acl_users.elements; i++) { + ACL_USER *user= dynamic_element(&acl_users, i, ACL_USER *); + reset_dynamic(&user->role_grants); + } + my_hash_iterate(&acl_roles, acl_role_reset_role_arrays, NULL); + + /* Rebuild the direct links between users and roles in ACL_USER::role_grants */ + my_hash_iterate(&acl_roles_mappings, add_role_user_mapping_action, NULL); + + DBUG_VOID_RETURN; +} + + +/* Return true if there is no users that can match the given host */ bool acl_check_host(const char *host, const char *ip) { if (allow_all_hosts) @@ -1861,20 +2499,25 @@ int check_change_password(THD *thd, const char *host, const char *user, my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), "--skip-grant-tables"); return(1); } + if (!thd->slave_thread && !thd->security_ctx->priv_user[0]) + { + my_message(ER_PASSWORD_ANONYMOUS_USER, ER(ER_PASSWORD_ANONYMOUS_USER), + MYF(0)); + return(1); + } + if (!host) // Role + { + my_error(ER_PASSWORD_NO_MATCH, MYF(0)); + return 1; + } if (!thd->slave_thread && - (strcmp(thd->security_ctx->user, user) || + (strcmp(thd->security_ctx->priv_user, user) || my_strcasecmp(system_charset_info, host, thd->security_ctx->priv_host))) { if (check_access(thd, UPDATE_ACL, "mysql", NULL, NULL, 1, 0)) return(1); } - if (!thd->slave_thread && !thd->security_ctx->user[0]) - { - my_message(ER_PASSWORD_ANONYMOUS_USER, ER(ER_PASSWORD_ANONYMOUS_USER), - MYF(0)); - return(1); - } size_t len= strlen(new_password); if (len && len != SCRAMBLED_PASSWORD_CHAR_LENGTH && len != SCRAMBLED_PASSWORD_CHAR_LENGTH_323) @@ -1906,10 +2549,11 @@ bool change_password(THD *thd, const char *host, const char *user, { TABLE_LIST tables; TABLE *table; + Rpl_filter *rpl_filter= thd->rpl_filter; /* Buffer should be extended when password length is extended. */ char buff[512]; ulong query_length; - bool save_binlog_row_based; + enum_binlog_format save_binlog_format; uint new_password_len= (uint) strlen(new_password); bool result= 1; DBUG_ENTER("change_password"); @@ -1946,13 +2590,14 @@ bool change_password(THD *thd, const char *host, const char *user, This statement will be replicated as a statement, even when using row-based replication. The flag will be reset at the end of the statement. + This has to be handled here as it's called by set_var.cc, which is + not automaticly handled by sql_parse.cc */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); mysql_mutex_lock(&acl_cache->lock); ACL_USER *acl_user; - if (!(acl_user= find_acl_user(host, user, TRUE))) + if (!(acl_user= find_user_exact(host, user))) { mysql_mutex_unlock(&acl_cache->lock); my_message(ER_PASSWORD_NO_MATCH, ER(ER_PASSWORD_NO_MATCH), MYF(0)); @@ -1960,7 +2605,7 @@ bool change_password(THD *thd, const char *host, const char *user, } /* update loaded acl entry: */ - if (acl_user->plugin.str == native_password_plugin_name.str || + if (acl_user->plugin.str == native_password_plugin_name.str || acl_user->plugin.str == old_password_plugin_name.str) { acl_user->auth_string.str= strmake_root(&mem, new_password, new_password_len); @@ -1973,8 +2618,8 @@ bool change_password(THD *thd, const char *host, const char *user, ER_SET_PASSWORD_AUTH_PLUGIN, ER(ER_SET_PASSWORD_AUTH_PLUGIN)); if (update_user_table(thd, table, - acl_user->host.hostname ? acl_user->host.hostname : "", - acl_user->user ? acl_user->user : "", + safe_str(acl_user->host.hostname), + safe_str(acl_user->user.str), new_password, new_password_len)) { mysql_mutex_unlock(&acl_cache->lock); /* purecov: deadcode */ @@ -1988,8 +2633,8 @@ bool change_password(THD *thd, const char *host, const char *user, { query_length= sprintf(buff,"SET PASSWORD FOR '%-.120s'@'%-.120s'='%-.120s'", - acl_user->user ? acl_user->user : "", - acl_user->host.hostname ? acl_user->host.hostname : "", + safe_str(acl_user->user.str), + safe_str(acl_user->host.hostname), new_password); thd->clear_error(); result= thd->binlog_query(THD::STMT_QUERY_TYPE, buff, query_length, @@ -1997,11 +2642,7 @@ bool change_password(THD *thd, const char *host, const char *user, } end: close_mysql_tables(thd); - - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); + thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(result); } @@ -2017,7 +2658,7 @@ end: RETURN FALSE user not fond - TRUE there are such user + TRUE there is such user */ bool is_acl_user(const char *host, const char *user) @@ -2029,45 +2670,96 @@ bool is_acl_user(const char *host, const char *user) return TRUE; mysql_mutex_lock(&acl_cache->lock); - res= find_acl_user(host, user, TRUE) != NULL; + + if (*host) // User + res= find_user_exact(host, user) != NULL; + else // Role + res= find_acl_role(user) != NULL; + mysql_mutex_unlock(&acl_cache->lock); return res; } /* - Find first entry that matches the current user + unlike find_user_exact and find_user_wild, + this function finds anonymous users too, it's when a + user is not empty, but priv_user (acl_user->user) is empty. */ +static ACL_USER *find_user_or_anon(const char *host, const char *user, const char *ip) +{ + ACL_USER *result= NULL; + mysql_mutex_assert_owner(&acl_cache->lock); + for (uint i=0; i < acl_users.elements; i++) + { + ACL_USER *acl_user_tmp= dynamic_element(&acl_users, i, ACL_USER*); + if ((!acl_user_tmp->user.str || + !strcmp(user, acl_user_tmp->user.str)) && + compare_hostname(&acl_user_tmp->host, host, ip)) + { + result= acl_user_tmp; + break; + } + } + return result; +} + -static ACL_USER * -find_acl_user(const char *host, const char *user, my_bool exact) +/* + Find first entry that matches the specified user@host pair +*/ +static ACL_USER * find_user_exact(const char *host, const char *user) { - DBUG_ENTER("find_acl_user"); - DBUG_PRINT("enter",("host: '%s' user: '%s'",host,user)); + mysql_mutex_assert_owner(&acl_cache->lock); + for (uint i=0 ; i < acl_users.elements ; i++) + { + ACL_USER *acl_user=dynamic_element(&acl_users,i,ACL_USER*); + if (acl_user->eq(user, host)) + return acl_user; + } + return 0; +} + +/* + Find first entry that matches the specified user@host pair +*/ +static ACL_USER * find_user_wild(const char *host, const char *user, const char *ip) +{ mysql_mutex_assert_owner(&acl_cache->lock); for (uint i=0 ; i < acl_users.elements ; i++) { ACL_USER *acl_user=dynamic_element(&acl_users,i,ACL_USER*); - DBUG_PRINT("info",("strcmp('%s','%s'), compare_hostname('%s','%s'),", - user, acl_user->user ? acl_user->user : "", - host, - acl_user->host.hostname ? acl_user->host.hostname : - "")); - if ((!acl_user->user && !user[0]) || - (acl_user->user && !strcmp(user,acl_user->user))) - { - if (exact ? !my_strcasecmp(system_charset_info, host, - acl_user->host.hostname ? - acl_user->host.hostname : "") : - compare_hostname(&acl_user->host,host,host)) - { - DBUG_RETURN(acl_user); - } - } + if (acl_user->wild_eq(user, host, ip)) + return acl_user; } - DBUG_RETURN(0); + return 0; +} + +/* + Find a role with the specified name +*/ +static ACL_ROLE *find_acl_role(const char *role) +{ + DBUG_ENTER("find_acl_role"); + DBUG_PRINT("enter",("role: '%s'", role)); + DBUG_PRINT("info", ("Hash elements: %ld", acl_roles.records)); + + mysql_mutex_assert_owner(&acl_cache->lock); + + ACL_ROLE *r= (ACL_ROLE *)my_hash_search(&acl_roles, (uchar *)role, + role ? strlen(role) : 0); + DBUG_RETURN(r); +} + + +static ACL_USER_BASE *find_acl_user_base(const char *user, const char *host) +{ + if (*host) + return find_user_exact(host, user); + + return find_acl_role(user); } @@ -2104,10 +2796,11 @@ static const char *calc_ip(const char *ip, long *val, char end) static void update_hostname(acl_host_and_ip *host, const char *hostname) { + // fix historical undocumented convention that empty host is the same as '%' + hostname=const_cast<char*>(hostname ? hostname : host_not_specified.str); host->hostname=(char*) hostname; // This will not be modified! - if (!hostname || - (!(hostname=calc_ip(hostname,&host->ip,'/')) || - !(hostname=calc_ip(hostname+1,&host->ip_mask,'\0')))) + if (!(hostname= calc_ip(hostname,&host->ip,'/')) || + !(hostname= calc_ip(hostname+1,&host->ip_mask,'\0'))) { host->ip= host->ip_mask=0; // Not a masked ip } @@ -2279,6 +2972,8 @@ static bool test_if_create_new_users(THD *thd) db_access=acl_get(sctx->host, sctx->ip, sctx->priv_user, tl.db, 0); + if (sctx->priv_role[0]) + db_access|= acl_get("", "", sctx->priv_role, tl.db, 0); if (!(db_access & INSERT_ACL)) { if (check_grant(thd, INSERT_ACL, &tl, FALSE, UINT_MAX, TRUE)) @@ -2295,12 +2990,13 @@ static bool test_if_create_new_users(THD *thd) static int replace_user_table(THD *thd, TABLE *table, LEX_USER &combo, ulong rights, bool revoke_grant, - bool can_create_user, bool no_auto_create) + bool can_create_user, bool no_auto_create) { int error = -1; bool old_row_exists=0; char what= (revoke_grant) ? 'N' : 'Y'; uchar user_key[MAX_KEY_LENGTH]; + bool handle_as_role= combo.is_role(); LEX *lex= thd->lex; DBUG_ENTER("replace_user_table"); @@ -2318,6 +3014,15 @@ static int replace_user_table(THD *thd, TABLE *table, LEX_USER &combo, else combo.password= empty_lex_str; + /* if the user table is not up to date, we can't handle role updates */ + if (table->s->fields <= ROLE_ASSIGN_COLUMN_IDX && handle_as_role) + { + my_error(ER_COL_COUNT_DOESNT_MATCH_PLEASE_UPDATE, MYF(0), + table->alias.c_ptr(), ROLE_ASSIGN_COLUMN_IDX + 1, table->s->fields, + static_cast<int>(table->s->mysql_version), MYSQL_VERSION_ID); + DBUG_RETURN(-1); + } + table->use_all_columns(); table->field[0]->store(combo.host.str,combo.host.length, system_charset_info); @@ -2476,6 +3181,16 @@ static int replace_user_table(THD *thd, TABLE *table, LEX_USER &combo, table->field[next_field + 1]->reset(); } } + + /* table format checked earlier */ + if (handle_as_role) + { + if (old_row_exists && !check_is_role(table)) + { + goto end; + } + table->field[ROLE_ASSIGN_COLUMN_IDX]->store("Y", 1, system_charset_info); + } } if (old_row_exists) @@ -2489,10 +3204,10 @@ static int replace_user_table(THD *thd, TABLE *table, LEX_USER &combo, if ((error= table->file->ha_update_row(table->record[1],table->record[0])) && error != HA_ERR_RECORD_IS_THE_SAME) - { // This should never happen - table->file->print_error(error,MYF(0)); /* purecov: deadcode */ - error= -1; /* purecov: deadcode */ - goto end; /* purecov: deadcode */ + { // This should never happen + table->file->print_error(error,MYF(0)); /* purecov: deadcode */ + error= -1; /* purecov: deadcode */ + goto end; /* purecov: deadcode */ } else error= 0; @@ -2514,27 +3229,37 @@ end: { acl_cache->clear(1); // Clear privilege cache if (old_row_exists) - acl_update_user(combo.user.str, combo.host.str, - combo.password.str, combo.password.length, - lex->ssl_type, - lex->ssl_cipher, - lex->x509_issuer, - lex->x509_subject, - &lex->mqh, - rights, - &combo.plugin, - &combo.auth); + { + if (handle_as_role) + acl_update_role(combo.user.str, rights); + else + acl_update_user(combo.user.str, combo.host.str, + combo.password.str, combo.password.length, + lex->ssl_type, + lex->ssl_cipher, + lex->x509_issuer, + lex->x509_subject, + &lex->mqh, + rights, + &combo.plugin, + &combo.auth); + } else - acl_insert_user(combo.user.str, combo.host.str, - combo.password.str, combo.password.length, - lex->ssl_type, - lex->ssl_cipher, - lex->x509_issuer, - lex->x509_subject, - &lex->mqh, - rights, - &combo.plugin, - &combo.auth); + { + if (handle_as_role) + acl_insert_role(combo.user.str, rights); + else + acl_insert_user(combo.user.str, combo.host.str, + combo.password.str, combo.password.length, + lex->ssl_type, + lex->ssl_cipher, + lex->x509_issuer, + lex->x509_subject, + &lex->mqh, + rights, + &combo.plugin, + &combo.auth); + } } DBUG_RETURN(error); } @@ -2563,10 +3288,14 @@ static int replace_db_table(TABLE *table, const char *db, } /* Check if there is such a user in user table in memory? */ - if (!find_acl_user(combo.host.str,combo.user.str, FALSE)) + if (!find_user_wild(combo.host.str,combo.user.str)) { - my_message(ER_PASSWORD_NO_MATCH, ER(ER_PASSWORD_NO_MATCH), MYF(0)); - DBUG_RETURN(-1); + /* The user could be a role, check if the user is registered as a role */ + if (!combo.host.length && !find_acl_role(combo.user.str)) + { + my_message(ER_PASSWORD_NO_MATCH, ER(ER_PASSWORD_NO_MATCH), MYF(0)); + DBUG_RETURN(-1); + } } table->use_all_columns(); @@ -2648,8 +3377,107 @@ abort: DBUG_RETURN(-1); } +/** + Updates the mysql.roles_mapping table and the acl_roles_mappings hash. + + @param table TABLE to update + @param user user name of the grantee + @param host host name of the grantee + @param role role name to grant + @param with_admin WITH ADMIN OPTION flag + @param existing the entry in the acl_roles_mappings hash or NULL. + it is never NULL if revoke_grant is true. + it is NULL when a new pair is added, it's not NULL + when an existing pair is updated. + @param revoke_grant true for REVOKE, false for GRANT +*/ +static int +replace_roles_mapping_table(TABLE *table, LEX_STRING *user, LEX_STRING *host, + LEX_STRING *role, bool with_admin, + ROLE_GRANT_PAIR *existing, bool revoke_grant) +{ + DBUG_ENTER("replace_roles_mapping_table"); + + uchar row_key[MAX_KEY_LENGTH]; + int error; + table->use_all_columns(); + restore_record(table, s->default_values); + table->field[0]->store(host->str, host->length, system_charset_info); + table->field[1]->store(user->str, user->length, system_charset_info); + table->field[2]->store(role->str, role->length, system_charset_info); + + DBUG_ASSERT(!revoke_grant || existing); + + if (existing) // delete or update + { + key_copy(row_key, table->record[0], table->key_info, + table->key_info->key_length); + if (table->file->ha_index_read_idx_map(table->record[1], 0, row_key, + HA_WHOLE_KEY, HA_READ_KEY_EXACT)) + { + /* No match */ + DBUG_RETURN(1); + } + if (revoke_grant && !with_admin) + { + if ((error= table->file->ha_delete_row(table->record[1]))) + { + DBUG_PRINT("info", ("error deleting row '%s' '%s' '%s'", + host->str, user->str, role->str)); + goto table_error; + } + /* + This should always return something, as the check was performed + earlier + */ + my_hash_delete(&acl_roles_mappings, (uchar*)existing); + } + else + { + if (revoke_grant) + existing->with_admin= false; + else + existing->with_admin|= with_admin; + + table->field[3]->store(existing->with_admin + 1); + + if ((error= table->file->ha_update_row(table->record[1], table->record[0]))) + { + DBUG_PRINT("info", ("error updating row '%s' '%s' '%s'", + host->str, user->str, role->str)); + goto table_error; + } + } + DBUG_RETURN(0); + } + + table->field[3]->store(with_admin + 1); + + if ((error= table->file->ha_write_row(table->record[0]))) + { + DBUG_PRINT("info", ("error inserting row '%s' '%s' '%s'", + host->str, user->str, role->str)); + goto table_error; + } + else + { + /* allocate a new entry that will go in the hash */ + ROLE_GRANT_PAIR *hash_entry= new (&mem) ROLE_GRANT_PAIR; + if (hash_entry->init(&mem, user->str, host->str, role->str, with_admin)) + DBUG_RETURN(1); + my_hash_insert(&acl_roles_mappings, (uchar*) hash_entry); + } + + /* all ok */ + DBUG_RETURN(0); -static void +table_error: + DBUG_PRINT("info", ("table error")); + table->file->print_error(error, MYF(0)); + DBUG_RETURN(1); +} + +static void acl_update_proxy_user(ACL_PROXY_USER *new_value, bool is_revoke) { mysql_mutex_assert_owner(&acl_cache->lock); @@ -2657,7 +3485,7 @@ acl_update_proxy_user(ACL_PROXY_USER *new_value, bool is_revoke) DBUG_ENTER("acl_update_proxy_user"); for (uint i= 0; i < acl_proxy_users.elements; i++) { - ACL_PROXY_USER *acl_user= + ACL_PROXY_USER *acl_user= dynamic_element(&acl_proxy_users, i, ACL_PROXY_USER *); if (acl_user->pk_equals(new_value)) @@ -2679,7 +3507,7 @@ acl_update_proxy_user(ACL_PROXY_USER *new_value, bool is_revoke) } -static void +static void acl_insert_proxy_user(ACL_PROXY_USER *new_value) { DBUG_ENTER("acl_insert_proxy_user"); @@ -2692,9 +3520,9 @@ acl_insert_proxy_user(ACL_PROXY_USER *new_value) } -static int +static int replace_proxies_priv_table(THD *thd, TABLE *table, const LEX_USER *user, - const LEX_USER *proxied_user, bool with_grant_arg, + const LEX_USER *proxied_user, bool with_grant_arg, bool revoke_grant) { bool old_row_exists= 0; @@ -2712,14 +3540,14 @@ replace_proxies_priv_table(THD *thd, TABLE *table, const LEX_USER *user, } /* Check if there is such a user in user table in memory? */ - if (!find_acl_user(user->host.str,user->user.str, FALSE)) + if (!find_user_wild(user->host.str,user->user.str)) { my_message(ER_PASSWORD_NO_MATCH, ER(ER_PASSWORD_NO_MATCH), MYF(0)); DBUG_RETURN(-1); } table->use_all_columns(); - ACL_PROXY_USER::store_pk (table, &user->host, &user->user, + ACL_PROXY_USER::store_pk (table, &user->host, &user->user, &proxied_user->host, &proxied_user->user); key_copy(user_key, table->record[0], table->key_info, @@ -2818,11 +3646,16 @@ class GRANT_COLUMN :public Sql_alloc public: char *column; ulong rights; + ulong init_rights; uint key_length; - GRANT_COLUMN(String &c, ulong y) :rights (y) + GRANT_COLUMN(String &c, ulong y) :rights (y), init_rights(y) { column= (char*) memdup_root(&memex,c.ptr(), key_length=c.length()); } + + /* this constructor assumes thas source->column is allocated in memex */ + GRANT_COLUMN(GRANT_COLUMN *source) : column(source->column), + rights (source->rights), init_rights(0), key_length(source->key_length) { } }; @@ -2833,13 +3666,13 @@ static uchar* get_key_column(GRANT_COLUMN *buff, size_t *length, return (uchar*) buff->column; } - class GRANT_NAME :public Sql_alloc { public: acl_host_and_ip host; char *db, *user, *tname, *hash_key; ulong privs; + ulong init_privs; /* privileges found in physical table */ ulong sort; size_t key_length; GRANT_NAME(const char *h, const char *d,const char *u, @@ -2857,6 +3690,7 @@ class GRANT_TABLE :public GRANT_NAME { public: ulong cols; + ulong init_cols; /* privileges found in physical table */ HASH hash_columns; GRANT_TABLE(const char *h, const char *d,const char *u, @@ -2864,6 +3698,11 @@ public: GRANT_TABLE (TABLE *form, TABLE *col_privs); ~GRANT_TABLE(); bool ok() { return privs != 0 || cols != 0; } + void init_hash() + { + my_hash_init2(&hash_columns, 4, system_charset_info, + 0, 0, 0, (my_hash_get_key) get_key_column, 0, 0); + } }; @@ -2894,7 +3733,7 @@ void GRANT_NAME::set_user_details(const char *h, const char *d, GRANT_NAME::GRANT_NAME(const char *h, const char *d,const char *u, const char *t, ulong p, bool is_routine) - :db(0), tname(0), privs(p) + :db(0), tname(0), privs(p), init_privs(p) { set_user_details(h, d, u, t, is_routine); } @@ -2903,18 +3742,25 @@ GRANT_TABLE::GRANT_TABLE(const char *h, const char *d,const char *u, const char *t, ulong p, ulong c) :GRANT_NAME(h,d,u,t,p, FALSE), cols(c) { - (void) my_hash_init2(&hash_columns,4,system_charset_info, - 0,0,0, (my_hash_get_key) get_key_column,0,0); + init_hash(); } - +/* + create a new GRANT_TABLE entry for role inheritance. init_* fields are set + to 0 +*/ GRANT_NAME::GRANT_NAME(TABLE *form, bool is_routine) { - update_hostname(&host, get_field(&memex, form->field[0])); + user= safe_str(get_field(&memex,form->field[2])); + + const char *hostname= get_field(&memex, form->field[0]); + mysql_mutex_lock(&acl_cache->lock); + if (!hostname && find_acl_role(user)) + hostname= ""; + mysql_mutex_unlock(&acl_cache->lock); + update_hostname(&host, hostname); + db= get_field(&memex,form->field[1]); - user= get_field(&memex,form->field[2]); - if (!user) - user= (char*) ""; sort= get_sort(3, host.hostname, db, user); tname= get_field(&memex,form->field[3]); if (!db || !tname) @@ -2936,6 +3782,7 @@ GRANT_NAME::GRANT_NAME(TABLE *form, bool is_routine) strmov(strmov(strmov(hash_key,user)+1,db)+1,tname); privs = (ulong) form->field[6]->val_int(); privs = fix_rights_for_table(privs); + init_privs= privs; } @@ -2952,10 +3799,17 @@ GRANT_TABLE::GRANT_TABLE(TABLE *form, TABLE *col_privs) return; } cols= (ulong) form->field[7]->val_int(); - cols = fix_rights_for_column(cols); + cols= fix_rights_for_column(cols); + /* + Initial columns privileges are the same as column privileges on creation. + In case of roles, the cols privilege bits can get inherited and thus + cause the cols field to change. The init_cols field is always the same + as the physical table entry + */ + init_cols= cols; + + init_hash(); - (void) my_hash_init2(&hash_columns,4,system_charset_info, - 0,0,0, (my_hash_get_key) get_key_column,0,0); if (cols) { uint key_prefix_len; @@ -2978,6 +3832,7 @@ GRANT_TABLE::GRANT_TABLE(TABLE *form, TABLE *col_privs) if (col_privs->file->ha_index_init(0, 1)) { cols= 0; + init_cols= 0; return; } @@ -2985,7 +3840,8 @@ GRANT_TABLE::GRANT_TABLE(TABLE *form, TABLE *col_privs) (key_part_map)15, HA_READ_KEY_EXACT)) { - cols = 0; /* purecov: deadcode */ + cols= 0; /* purecov: deadcode */ + init_cols= 0; col_privs->file->ha_index_end(); return; } @@ -3000,13 +3856,13 @@ GRANT_TABLE::GRANT_TABLE(TABLE *form, TABLE *col_privs) fix_rights_for_column(priv)))) { /* Don't use this entry */ - privs = cols = 0; /* purecov: deadcode */ + privs= cols= init_privs= init_cols=0; /* purecov: deadcode */ return; /* purecov: deadcode */ } if (my_hash_insert(&hash_columns, (uchar *) mem_check)) { /* Invalidate this entry */ - privs= cols= 0; + privs= cols= init_privs= init_cols=0; return; } } while (!col_privs->file->ha_index_next(col_privs->record[0]) && @@ -3030,9 +3886,9 @@ static uchar* get_grant_table(GRANT_NAME *buff, size_t *length, } -void free_grant_table(GRANT_TABLE *grant_table) +static void free_grant_table(GRANT_TABLE *grant_table) { - my_hash_free(&grant_table->hash_columns); + grant_table->~GRANT_TABLE(); } @@ -3087,7 +3943,7 @@ static GRANT_NAME *name_hash_search(HASH *name_hash, } -inline GRANT_NAME * +static GRANT_NAME * routine_hash_search(const char *host, const char *ip, const char *db, const char *user, const char *tname, bool proc, bool exact) { @@ -3097,7 +3953,7 @@ routine_hash_search(const char *host, const char *ip, const char *db, } -inline GRANT_TABLE * +static GRANT_TABLE * table_hash_search(const char *host, const char *ip, const char *db, const char *user, const char *tname, bool exact) { @@ -3106,7 +3962,7 @@ table_hash_search(const char *host, const char *ip, const char *db, } -inline GRANT_COLUMN * +static GRANT_COLUMN * column_hash_search(GRANT_TABLE *t, const char *cname, uint length) { return (GRANT_COLUMN*) my_hash_search(&t->hash_columns, @@ -3288,7 +4144,10 @@ static int replace_column_table(GRANT_TABLE *g_t, goto end; /* purecov: deadcode */ } if (grant_column) - grant_column->rights = privileges; // Update hash + { + grant_column->rights = privileges; // Update hash + grant_column->init_rights = privileges; + } } else { @@ -3345,11 +4204,14 @@ static int replace_table_table(THD *thd, GRANT_TABLE *grant_table, The following should always succeed as new users are created before this function is called! */ - if (!find_acl_user(combo.host.str,combo.user.str, FALSE)) + if (!find_user_wild(combo.host.str,combo.user.str)) { - my_message(ER_PASSWORD_NO_MATCH, ER(ER_PASSWORD_NO_MATCH), - MYF(0)); /* purecov: deadcode */ - DBUG_RETURN(-1); /* purecov: deadcode */ + if (!combo.host.length && !find_acl_role(combo.user.str)) + { + my_message(ER_PASSWORD_NO_MATCH, ER(ER_PASSWORD_NO_MATCH), + MYF(0)); /* purecov: deadcode */ + DBUG_RETURN(-1); /* purecov: deadcode */ + } } table->use_all_columns(); @@ -3433,6 +4295,9 @@ static int replace_table_table(THD *thd, GRANT_TABLE *grant_table, if (rights | col_rights) { + grant_table->init_privs= rights; + grant_table->init_cols= col_rights; + grant_table->privs= rights; grant_table->cols= col_rights; } @@ -3462,6 +4327,7 @@ static int replace_routine_table(THD *thd, GRANT_NAME *grant_name, int old_row_exists= 1; int error=0; ulong store_proc_rights; + HASH *hash= is_proc ? &proc_priv_hash : &func_priv_hash; DBUG_ENTER("replace_routine_table"); if (!initialized) @@ -3470,6 +4336,12 @@ static int replace_routine_table(THD *thd, GRANT_NAME *grant_name, DBUG_RETURN(-1); } + if (revoke_grant && !grant_name->init_privs) // only inherited role privs + { + my_hash_delete(hash, (uchar*) grant_name); + DBUG_RETURN(0); + } + get_grantor(thd, grantor); /* New users are created before this function is called. @@ -3499,6 +4371,9 @@ static int replace_routine_table(THD *thd, GRANT_NAME *grant_name, The following should never happen as we first check the in memory grant tables for the user. There is however always a small change that the user has modified the grant tables directly. + + Also, there is also a second posibility that this routine entry + is created for a role by being inherited from a granted role. */ if (revoke_grant) { // no row, no revoke @@ -3553,12 +4428,12 @@ static int replace_routine_table(THD *thd, GRANT_NAME *grant_name, if (rights) { + grant_name->init_privs= rights; grant_name->privs= rights; } else { - my_hash_delete(is_proc ? &proc_priv_hash : &func_priv_hash,(uchar*) - grant_name); + my_hash_delete(hash, (uchar*) grant_name); } DBUG_RETURN(0); @@ -3569,6 +4444,886 @@ table_error: } +/***************************************************************** + Role privilege propagation and graph traversal functionality + + According to the SQL standard, a role can be granted to a role, + thus role grants can create an arbitrarily complex directed acyclic + graph (a standard explicitly specifies that cycles are not allowed). + + When a privilege is granted to a role, it becomes available to all grantees. + The code below recursively traverses a DAG of role grants, propagating + privilege changes. + + The traversal function can work both ways, from roles to grantees or + from grantees to roles. The first is used for privilege propagation, + the second - for SHOW GRANTS and I_S.APPLICABLE_ROLES + + The role propagation code is smart enough to propagate only privilege + changes to one specific database, table, or routine, if only they + were changed (like in GRANT ... ON ... TO ...) or it can propagate + everything (on startup or after FLUSH PRIVILEGES). + + It traverses only a subgraph that's accessible from the modified role, + only visiting roles that can be possibly affected by the GRANT statement. + + Additionally, it stops traversal early, if this particular GRANT statement + didn't result in any changes of privileges (e.g. both role1 and role2 + are granted to the role3, both role1 and role2 have SELECT privilege. + if SELECT is revoked from role1 it won't change role3 privileges, + so we won't traverse from role3 to its grantees). +******************************************************************/ +struct PRIVS_TO_MERGE +{ + enum what { ALL, GLOBAL, DB, TABLE_COLUMN, PROC, FUNC } what; + const char *db, *name; +}; + +static int init_role_for_merging(ACL_ROLE *role, void *context) +{ + role->counter= 0; + return 0; +} + +static int count_subgraph_nodes(ACL_ROLE *role, ACL_ROLE *grantee, void *context) +{ + grantee->counter++; + return 0; +} + +static int merge_role_privileges(ACL_ROLE *, ACL_ROLE *, void *); + +/** + rebuild privileges of all affected roles + + entry point into role privilege propagation. after privileges of the + 'role' were changed, this function rebuilds privileges of all affected roles + as necessary. +*/ +static void propagate_role_grants(ACL_ROLE *role, + enum PRIVS_TO_MERGE::what what, + const char *db, const char *name) +{ + + mysql_mutex_assert_owner(&acl_cache->lock); + PRIVS_TO_MERGE data= { what, db, name }; + + /* + Changing privileges of a role causes all other roles that had + this role granted to them to have their rights invalidated. + + We need to rebuild all roles' related access bits. + + This cannot be a simple depth-first search, instead we have to merge + privieges for all roles granted to a specific grantee, *before* + merging privileges for this grantee. In other words, we must visit all + parent nodes of a specific node, before descencing into this node. + + For example, if role1 is granted to role2 and role3, and role3 is + granted to role2, after "GRANT ... role1", we cannot merge privileges + for role2, until role3 is merged. The counter will be 0 for role1, 2 + for role2, 1 for role3. Traversal will start from role1, go to role2, + decrement the counter, backtrack, go to role3, merge it, go to role2 + again, merge it. + + And the counter is not just "all parent nodes", but only parent nodes + that are part of the subgraph we're interested in. For example, if + both roleA and roleB are granted to roleC, then roleC has two parent + nodes. But when granting a privilege to roleA, we're only looking at a + subgraph that includes roleA and roleC (roleB cannot be possibly + affected by that grant statement). In this subgraph roleC has only one + parent. + + (on the other hand, in acl_load we want to update all roles, and + the counter is exactly equal to the number of all parent nodes) + + Thus, we do two graph traversals here. First we only count parents + that are part of the subgraph. On the second traversal we decrement + the counter and actually merge privileges for a node when a counter + drops to zero. + */ + traverse_role_graph_up(role, &data, init_role_for_merging, count_subgraph_nodes); + traverse_role_graph_up(role, &data, NULL, merge_role_privileges); +} + + +// State of a node during a Depth First Search exploration +struct NODE_STATE +{ + ACL_USER_BASE *node_data; /* pointer to the node data */ + uint neigh_idx; /* the neighbour that needs to be evaluated next */ +}; + +/** + Traverse the role grant graph and invoke callbacks at the specified points. + + @param user user or role to start traversal from + @param context opaque parameter to pass to callbacks + @param offset offset to ACL_ROLE::parent_grantee or to + ACL_USER_BASE::role_grants. Depending on this value, + traversal will go from roles to grantees or from + grantees to roles. + @param on_node called when a node is visited for the first time. + Returning a value <0 will abort the traversal. + @param on_edge called for every edge in the graph, when traversal + goes from a node to a neighbour node. + Returning <0 will abort the traversal. Returning >0 + will make the traversal not to follow this edge. + + @note + The traverse method is a DEPTH FIRST SEARCH, but callbacks can influence + that (on_edge returning >0 value). + + @note + This function should not be called directly, use + traverse_role_graph_up() and traverse_role_graph_down() instead. + + @retval 0 traversal finished successfully + @retval ROLE_CYCLE_FOUND traversal aborted, cycle detected + @retval <0 traversal was aborted, because a callback returned + this error code +*/ +static int traverse_role_graph_impl(ACL_USER_BASE *user, void *context, + off_t offset, + int (*on_node) (ACL_USER_BASE *role, void *context), + int (*on_edge) (ACL_USER_BASE *current, ACL_ROLE *neighbour, void *context)) +{ + DBUG_ENTER("traverse_role_graph_impl"); + DBUG_ASSERT(user); + DBUG_PRINT("enter",("role: '%s'", user->user.str)); + /* + The search operation should always leave the ROLE_ON_STACK and + ROLE_EXPLORED flags clean for all nodes involved in the search + */ + DBUG_ASSERT(!(user->flags & ROLE_ON_STACK)); + DBUG_ASSERT(!(user->flags & ROLE_EXPLORED)); + mysql_mutex_assert_owner(&acl_cache->lock); + + /* + Stack used to simulate the recursive calls of DFS. + It uses a Dynamic_array to reduce the number of + malloc calls to a minimum + */ + Dynamic_array<NODE_STATE> stack(20,50); + Dynamic_array<ACL_USER_BASE *> to_clear(20,50); + NODE_STATE state; /* variable used to insert elements in the stack */ + int result= 0; + + state.neigh_idx= 0; + state.node_data= user; + user->flags|= ROLE_ON_STACK; + + stack.push(state); + to_clear.push(user); + + user->flags|= ROLE_OPENED; + if (on_node && ((result= on_node(user, context)) < 0)) + goto end; + + while (stack.elements()) + { + NODE_STATE *curr_state= stack.back() - 1; + + DBUG_ASSERT(curr_state->node_data->flags & ROLE_ON_STACK); + + ACL_USER_BASE *current= curr_state->node_data; + ACL_USER_BASE *neighbour= NULL; + DBUG_PRINT("info", ("Examining role %s", current->user.str)); + /* + Iterate through the neighbours until a first valid jump-to + neighbour is found + */ + my_bool found= FALSE; + uint i; + DYNAMIC_ARRAY *array= (DYNAMIC_ARRAY *)(((char*)current) + offset); + + DBUG_ASSERT(array == ¤t->role_grants || current->flags & IS_ROLE); + for (i= curr_state->neigh_idx; i < array->elements; i++) + { + neighbour= *(dynamic_element(array, i, ACL_ROLE**)); + if (!(neighbour->flags & IS_ROLE)) + continue; + + DBUG_PRINT("info", ("Examining neighbour role %s", neighbour->user.str)); + + /* check if it forms a cycle */ + if (neighbour->flags & ROLE_ON_STACK) + { + DBUG_PRINT("info", ("Found cycle")); + result= ROLE_CYCLE_FOUND; + goto end; + } + + if (!(neighbour->flags & ROLE_OPENED)) + { + neighbour->flags|= ROLE_OPENED; + to_clear.push(neighbour); + if (on_node && ((result= on_node(neighbour, context)) < 0)) + goto end; + } + + if (on_edge) + { + result= on_edge(current, (ACL_ROLE*)neighbour, context); + if (result < 0) + goto end; + if (result > 0) + continue; + } + + /* Check if it was already explored, in that case, move on */ + if (neighbour->flags & ROLE_EXPLORED) + continue; + + found= TRUE; + break; + } + + /* found states that we have found a node to jump next into */ + if (found) + { + curr_state->neigh_idx= i + 1; + + /* some sanity checks */ + DBUG_ASSERT(!(neighbour->flags & ROLE_ON_STACK)); + + /* add the neighbour on the stack */ + neighbour->flags|= ROLE_ON_STACK; + state.neigh_idx= 0; + state.node_data= neighbour; + stack.push(state); + } + else + { + /* Make sure we got a correct node */ + DBUG_ASSERT(curr_state->node_data->flags & ROLE_ON_STACK); + /* Finished with exploring the current node, pop it off the stack */ + curr_state= stack.pop(); + curr_state->node_data->flags&= ~ROLE_ON_STACK; /* clear the on-stack bit */ + curr_state->node_data->flags|= ROLE_EXPLORED; + } + } + +end: + /* Cleanup */ + for (uint i= 0; i < to_clear.elements(); i++) + { + ACL_USER_BASE *current= to_clear.at(i); + DBUG_ASSERT(current->flags & (ROLE_EXPLORED | ROLE_ON_STACK | ROLE_OPENED)); + current->flags&= ~(ROLE_EXPLORED | ROLE_ON_STACK | ROLE_OPENED); + } + DBUG_RETURN(result); +} + +/** + Traverse the role grant graph, going from a role to its grantees. + + This is used to propagate changes in privileges, for example, + when GRANT or REVOKE is issued for a role. +*/ + +static int traverse_role_graph_up(ACL_ROLE *role, void *context, + int (*on_node) (ACL_ROLE *role, void *context), + int (*on_edge) (ACL_ROLE *current, ACL_ROLE *neighbour, void *context)) +{ + return traverse_role_graph_impl(role, context, + my_offsetof(ACL_ROLE, parent_grantee), + (int (*)(ACL_USER_BASE *, void *))on_node, + (int (*)(ACL_USER_BASE *, ACL_ROLE *, void *))on_edge); +} + +/** + Traverse the role grant graph, going from a user or a role to granted roles. + + This is used, for example, to print all grants available to a user or a role + (as in SHOW GRANTS). +*/ + +static int traverse_role_graph_down(ACL_USER_BASE *user, void *context, + int (*on_node) (ACL_USER_BASE *role, void *context), + int (*on_edge) (ACL_USER_BASE *current, ACL_ROLE *neighbour, void *context)) +{ + return traverse_role_graph_impl(user, context, + my_offsetof(ACL_USER_BASE, role_grants), + on_node, on_edge); +} + +/* + To find all db/table/routine privilege for a specific role + we need to scan the array of privileges. It can be big. + But the set of privileges granted to a role in question (or + to roles directly granted to the role in question) is supposedly + much smaller. + + We put a role and all roles directly granted to it in a hash, and iterate + the (suposedly long) array of privileges, filtering out "interesting" + entries using the role hash. We put all these "interesting" + entries in a (suposedly small) dynamic array and them use it for merging. +*/ +static uchar* role_key(const ACL_ROLE *role, size_t *klen, my_bool) +{ + *klen= role->user.length; + return (uchar*) role->user.str; +} +typedef Hash_set<ACL_ROLE> role_hash_t; + +static bool merge_role_global_privileges(ACL_ROLE *grantee) +{ + ulong old= grantee->access; + grantee->access= grantee->initial_role_access; + + DBUG_EXECUTE_IF("role_merge_stats", role_global_merges++;); + + for (uint i= 0; i < grantee->role_grants.elements; i++) + { + ACL_ROLE *r= *dynamic_element(&grantee->role_grants, i, ACL_ROLE**); + grantee->access|= r->access; + } + return old != grantee->access; +} + +static int db_name_sort(ACL_DB * const *db1, ACL_DB * const *db2) +{ + return strcmp((*db1)->db, (*db2)->db); +} + +/** + update ACL_DB for given database and a given role with merged privileges + + @param merged ACL_DB of the role in question (or NULL if it wasn't found) + @param first first ACL_DB in an array for the database in question + @param access new privileges for the given role on the gived database + @param role the name of the given role + + @return a bitmap of + 1 - privileges were changed + 2 - ACL_DB was added + 4 - ACL_DB was deleted +*/ +static int update_role_db(ACL_DB *merged, ACL_DB **first, ulong access, char *role) +{ + if (!first) + return 0; + + DBUG_EXECUTE_IF("role_merge_stats", role_db_merges++;); + + if (merged == NULL) + { + /* + there's no ACL_DB for this role (all db grants come from granted roles) + we need to create it + + Note that we cannot use acl_insert_db() now: + 1. it'll sort elements in the acl_dbs, so the pointers will become invalid + 2. we may need many of them, no need to sort every time + */ + DBUG_ASSERT(access); + ACL_DB acl_db; + acl_db.user= role; + acl_db.host.hostname= const_cast<char*>(""); + acl_db.host.ip= acl_db.host.ip_mask= 0; + acl_db.db= first[0]->db; + acl_db.access= access; + acl_db.initial_access= 0; + acl_db.sort=get_sort(3, "", acl_db.db, role); + push_dynamic(&acl_dbs,(uchar*) &acl_db); + return 2; + } + else if (access == 0) + { + /* + there is ACL_DB but the role has no db privileges granted + (all privileges were coming from granted roles, and now those roles + were dropped or had their privileges revoked). + we need to remove this ACL_DB entry + + Note, that we cannot delete now: + 1. it'll shift elements in the acl_dbs, so the pointers will become invalid + 2. it's O(N) operation, and we may need many of them + so we only mark elements deleted and will delete later. + */ + merged->sort= 0; // lower than any valid ACL_DB sort value, will be sorted last + return 4; + } + else if (merged->access != access) + { + /* this is easy */ + merged->access= access; + return 1; + } + return 0; +} + +/** + merges db privileges from roles granted to the role 'grantee'. + + @return true if database privileges of the 'grantee' were changed + +*/ +static bool merge_role_db_privileges(ACL_ROLE *grantee, const char *dbname, + role_hash_t *rhash) +{ + Dynamic_array<ACL_DB *> dbs; + + /* + Supposedly acl_dbs can be huge, but only a handful of db grants + apply to grantee or roles directly granted to grantee. + + Collect these applicable db grants. + */ + for (uint i=0 ; i < acl_dbs.elements ; i++) + { + ACL_DB *db= dynamic_element(&acl_dbs,i,ACL_DB*); + if (db->host.hostname[0]) + continue; + if (dbname && strcmp(db->db, dbname)) + continue; + ACL_ROLE *r= rhash->find(db->user, strlen(db->user)); + if (!r) + continue; + dbs.append(db); + } + dbs.sort(db_name_sort); + + /* + Because dbs array is sorted by the db name, all grants for the same db + (that should be merged) are sorted together. The grantee's ACL_DB element + is not necessarily the first and may be not present at all. + */ + ACL_DB **first= NULL, *UNINIT_VAR(merged); + ulong UNINIT_VAR(access), update_flags= 0; + for (ACL_DB **cur= dbs.front(); cur < dbs.back(); cur++) + { + if (!first || (!dbname && strcmp(cur[0]->db, cur[-1]->db))) + { // new db name series + update_flags|= update_role_db(merged, first, access, grantee->user.str); + merged= NULL; + access= 0; + first= cur; + } + if (strcmp(cur[0]->user, grantee->user.str) == 0) + access|= (merged= cur[0])->initial_access; + else + access|= cur[0]->access; + } + update_flags|= update_role_db(merged, first, access, grantee->user.str); + + /* + to make this code a bit simpler, we sort on deletes, to move + deleted elements to the end of the array. strictly speaking it's + unnecessary, it'd be faster to remove them in one O(N) array scan. + + on the other hand, qsort on almost sorted array is pretty fast anyway... + */ + if (update_flags & (2|4)) + { // inserted or deleted, need to sort + my_qsort((uchar*) dynamic_element(&acl_dbs,0,ACL_DB*),acl_dbs.elements, + sizeof(ACL_DB),(qsort_cmp) acl_compare); + } + if (update_flags & 4) + { // deleted, trim the end + while (acl_dbs.elements && + dynamic_element(&acl_dbs, acl_dbs.elements-1, ACL_DB*)->sort == 0) + acl_dbs.elements--; + } + return update_flags; +} + +static int table_name_sort(GRANT_TABLE * const *tbl1, GRANT_TABLE * const *tbl2) +{ + int res = strcmp((*tbl1)->db, (*tbl2)->db); + if (res) return res; + return strcmp((*tbl1)->tname, (*tbl2)->tname); +} + +/** + merges column privileges for the entry 'merged' + + @param merged GRANT_TABLE to merge the privileges into + @param cur first entry in the array of GRANT_TABLE's for a given table + @param last last entry in the array of GRANT_TABLE's for a given table, + all entries between cur and last correspond to the *same* table + + @return 1 if the _set of columns_ in 'merged' was changed + (not if the _set of privileges_ was changed). +*/ +static int update_role_columns(GRANT_TABLE *merged, + GRANT_TABLE **cur, GRANT_TABLE **last) + +{ + ulong rights __attribute__((unused))= 0; + int changed= 0; + if (!merged->cols) + { + changed= merged->hash_columns.records > 0; + my_hash_reset(&merged->hash_columns); + return changed; + } + + DBUG_EXECUTE_IF("role_merge_stats", role_column_merges++;); + + HASH *mh= &merged->hash_columns; + for (uint i=0 ; i < mh->records ; i++) + { + GRANT_COLUMN *col = (GRANT_COLUMN *)my_hash_element(mh, i); + col->rights= col->init_rights; + } + + for (; cur < last; cur++) + { + if (*cur == merged) + continue; + HASH *ch= &cur[0]->hash_columns; + for (uint i=0 ; i < ch->records ; i++) + { + GRANT_COLUMN *ccol = (GRANT_COLUMN *)my_hash_element(ch, i); + GRANT_COLUMN *mcol = (GRANT_COLUMN *)my_hash_search(mh, + (uchar *)ccol->column, ccol->key_length); + if (mcol) + mcol->rights|= ccol->rights; + else + { + changed= 1; + my_hash_insert(mh, (uchar*)new (&memex) GRANT_COLUMN(ccol)); + } + } + } + + for (uint i=0 ; i < mh->records ; i++) + { + GRANT_COLUMN *col = (GRANT_COLUMN *)my_hash_element(mh, i); + rights|= col->rights; + if (!col->rights) + { + changed= 1; + my_hash_delete(mh, (uchar*)col); + } + } + DBUG_ASSERT(rights == merged->cols); + return changed; +} + +/** + update GRANT_TABLE for a given table and a given role with merged privileges + + @param merged GRANT_TABLE of the role in question (or NULL if it wasn't found) + @param first first GRANT_TABLE in an array for the table in question + @param last last entry in the array of GRANT_TABLE's for a given table, + all entries between first and last correspond to the *same* table + @param privs new table-level privileges for 'merged' + @param cols new OR-ed column-level privileges for 'merged' + @param role the name of the given role + + @return a bitmap of + 1 - privileges were changed + 2 - GRANT_TABLE was added + 4 - GRANT_TABLE was deleted +*/ +static int update_role_table_columns(GRANT_TABLE *merged, + GRANT_TABLE **first, GRANT_TABLE **last, + ulong privs, ulong cols, char *role) +{ + if (!first) + return 0; + + DBUG_EXECUTE_IF("role_merge_stats", role_table_merges++;); + + if (merged == NULL) + { + /* + there's no GRANT_TABLE for this role (all table grants come from granted + roles) we need to create it + */ + DBUG_ASSERT(privs | cols); + merged= new (&memex) GRANT_TABLE("", first[0]->db, role, first[0]->tname, + privs, cols); + merged->init_privs= merged->init_cols= 0; + update_role_columns(merged, first, last); + my_hash_insert(&column_priv_hash,(uchar*) merged); + return 2; + } + else if ((privs | cols) == 0) + { + /* + there is GRANT_TABLE object but the role has no table or column + privileges granted (all privileges were coming from granted roles, and + now those roles were dropped or had their privileges revoked). + we need to remove this GRANT_TABLE + */ + DBUG_EXECUTE_IF("role_merge_stats", role_column_merges+= test(merged->cols);); + my_hash_delete(&column_priv_hash,(uchar*) merged); + return 4; + } + else + { + bool changed= merged->cols != cols || merged->privs != privs; + merged->cols= cols; + merged->privs= privs; + if (update_role_columns(merged, first, last)) + changed= true; + return changed; + } +} + +/** + merges table privileges from roles granted to the role 'grantee'. + + @return true if table privileges of the 'grantee' were changed + +*/ +static bool merge_role_table_and_column_privileges(ACL_ROLE *grantee, + const char *db, const char *tname, role_hash_t *rhash) +{ + Dynamic_array<GRANT_TABLE *> grants; + DBUG_ASSERT(test(db) == test(tname)); // both must be set, or neither + + /* + first, collect table/column privileges granted to + roles in question. + */ + for (uint i=0 ; i < column_priv_hash.records ; i++) + { + GRANT_TABLE *grant= (GRANT_TABLE *) my_hash_element(&column_priv_hash, i); + if (grant->host.hostname[0]) + continue; + if (tname && (strcmp(grant->db, db) || strcmp(grant->tname, tname))) + continue; + ACL_ROLE *r= rhash->find(grant->user, strlen(grant->user)); + if (!r) + continue; + grants.append(grant); + } + grants.sort(table_name_sort); + + GRANT_TABLE **first= NULL, *UNINIT_VAR(merged), **cur; + ulong UNINIT_VAR(privs), UNINIT_VAR(cols), update_flags= 0; + for (cur= grants.front(); cur < grants.back(); cur++) + { + if (!first || + (!tname && (strcmp(cur[0]->db, cur[-1]->db) || + strcmp(cur[0]->tname, cur[-1]->tname)))) + { // new db.tname series + update_flags|= update_role_table_columns(merged, first, cur, + privs, cols, grantee->user.str); + merged= NULL; + privs= cols= 0; + first= cur; + } + if (strcmp(cur[0]->user, grantee->user.str) == 0) + { + merged= cur[0]; + cols|= cur[0]->init_cols; + privs|= cur[0]->init_privs; + } + else + { + cols|= cur[0]->cols; + privs|= cur[0]->privs; + } + } + update_flags|= update_role_table_columns(merged, first, cur, + privs, cols, grantee->user.str); + + return update_flags; +} + +static int routine_name_sort(GRANT_NAME * const *r1, GRANT_NAME * const *r2) +{ + int res= strcmp((*r1)->db, (*r2)->db); + if (res) return res; + return strcmp((*r1)->tname, (*r2)->tname); +} + +/** + update GRANT_NAME for a given routine and a given role with merged privileges + + @param merged GRANT_NAME of the role in question (or NULL if it wasn't found) + @param first first GRANT_NAME in an array for the routine in question + @param privs new routine-level privileges for 'merged' + @param role the name of the given role + @param hash proc_priv_hash or func_priv_hash + + @return a bitmap of + 1 - privileges were changed + 2 - GRANT_NAME was added + 4 - GRANT_NAME was deleted +*/ +static int update_role_routines(GRANT_NAME *merged, GRANT_NAME **first, + ulong privs, char *role, HASH *hash) +{ + if (!first) + return 0; + + if (merged == NULL) + { + /* + there's no GRANT_NAME for this role (all routine grants come from granted + roles) we need to create it + */ + DBUG_ASSERT(privs); + merged= new (&memex) GRANT_NAME("", first[0]->db, role, first[0]->tname, + privs, true); + merged->init_privs= 0; // all privs are inherited + my_hash_insert(hash, (uchar *)merged); + return 2; + } + else if (privs == 0) + { + /* + there is GRANT_NAME but the role has no privileges granted + (all privileges were coming from granted roles, and now those roles + were dropped or had their privileges revoked). + we need to remove this entry + */ + my_hash_delete(hash, (uchar*)merged); + return 4; + } + else if (merged->privs != privs) + { + /* this is easy */ + merged->privs= privs; + return 1; + } + return 0; +} + +/** + merges routine privileges from roles granted to the role 'grantee'. + + @return true if routine privileges of the 'grantee' were changed + +*/ +static bool merge_role_routine_grant_privileges(ACL_ROLE *grantee, + const char *db, const char *tname, role_hash_t *rhash, HASH *hash) +{ + ulong update_flags= 0; + + DBUG_ASSERT(test(db) == test(tname)); // both must be set, or neither + + DBUG_EXECUTE_IF("role_merge_stats", role_routine_merges++;); + + Dynamic_array<GRANT_NAME *> grants; + + /* first, collect routine privileges granted to roles in question */ + for (uint i=0 ; i < hash->records ; i++) + { + GRANT_NAME *grant= (GRANT_NAME *) my_hash_element(hash, i); + if (grant->host.hostname[0]) + continue; + if (tname && (strcmp(grant->db, db) || strcmp(grant->tname, tname))) + continue; + ACL_ROLE *r= rhash->find(grant->user, strlen(grant->user)); + if (!r) + continue; + grants.append(grant); + } + grants.sort(routine_name_sort); + + GRANT_NAME **first= NULL, *UNINIT_VAR(merged); + ulong UNINIT_VAR(privs); + for (GRANT_NAME **cur= grants.front(); cur < grants.back(); cur++) + { + if (!first || + (!tname && (strcmp(cur[0]->db, cur[-1]->db) || + strcmp(cur[0]->tname, cur[-1]->tname)))) + { // new db.tname series + update_flags|= update_role_routines(merged, first, privs, + grantee->user.str, hash); + merged= NULL; + privs= 0; + first= cur; + } + if (strcmp(cur[0]->user, grantee->user.str) == 0) + { + merged= cur[0]; + privs|= cur[0]->init_privs; + } + else + { + privs|= cur[0]->privs; + } + } + update_flags|= update_role_routines(merged, first, privs, + grantee->user.str, hash); + return update_flags; +} + +/** + update privileges of the 'grantee' from all roles, granted to it +*/ +static int merge_role_privileges(ACL_ROLE *role __attribute__((unused)), + ACL_ROLE *grantee, void *context) +{ + PRIVS_TO_MERGE *data= (PRIVS_TO_MERGE *)context; + + if (--grantee->counter) + return 1; // don't recurse into grantee just yet + + /* if we'll do db/table/routine privileges, create a hash of role names */ + role_hash_t role_hash(role_key); + if (data->what != PRIVS_TO_MERGE::GLOBAL) + { + role_hash.insert(grantee); + for (uint i= 0; i < grantee->role_grants.elements; i++) + role_hash.insert(*dynamic_element(&grantee->role_grants, i, ACL_ROLE**)); + } + + bool all= data->what == PRIVS_TO_MERGE::ALL; + bool changed= false; + if (all || data->what == PRIVS_TO_MERGE::GLOBAL) + changed|= merge_role_global_privileges(grantee); + if (all || data->what == PRIVS_TO_MERGE::DB) + changed|= merge_role_db_privileges(grantee, data->db, &role_hash); + if (all || data->what == PRIVS_TO_MERGE::TABLE_COLUMN) + changed|= merge_role_table_and_column_privileges(grantee, + data->db, data->name, &role_hash); + if (all || data->what == PRIVS_TO_MERGE::PROC) + changed|= merge_role_routine_grant_privileges(grantee, + data->db, data->name, &role_hash, &proc_priv_hash); + if (all || data->what == PRIVS_TO_MERGE::FUNC) + changed|= merge_role_routine_grant_privileges(grantee, + data->db, data->name, &role_hash, &func_priv_hash); + + return !changed; // don't recurse into the subgraph if privs didn't change +} + +/***************************************************************** + End of the role privilege propagation and graph traversal code +******************************************************************/ + +bool copy_and_check_auth(LEX_USER *to, LEX_USER *from, LEX *lex) +{ + if (to != from) + { + /* preserve authentication information, if LEX_USER was reallocated */ + to->password= from->password; + to->plugin= from->plugin; + to->auth= from->auth; + } + + /* + Note, that no password is null_lex_str, while no plugin is empty_lex_str. + See sql_yacc.yy + */ + bool has_auth= to->password.str || to->plugin.length || to->auth.length || + lex->ssl_type != SSL_TYPE_NOT_SPECIFIED || lex->ssl_cipher || + lex->x509_issuer || lex->x509_subject || + lex->mqh.specified_limits; + + /* + Specifying authentication clauses forces the name to be interpreted + as a user, not a role. See also check_change_password() + */ + if (to->is_role() && has_auth) + { + my_error(ER_PASSWORD_NO_MATCH, MYF(0)); + return true; + } + + return false; +} + + /* Store table level and column level grants in the privilege tables @@ -3597,7 +5352,7 @@ int mysql_table_grant(THD *thd, TABLE_LIST *table_list, TABLE_LIST tables[3]; bool create_new_users=0; char *db_name, *table_name; - bool save_binlog_row_based; + Rpl_filter *rpl_filter= thd->rpl_filter; DBUG_ENTER("mysql_table_grant"); if (!initialized) @@ -3687,14 +5442,6 @@ int mysql_table_grant(THD *thd, TABLE_LIST *table_list, if (column_priv || (revoke_grant && ((rights & COL_ACLS) || columns.elements))) tables[1].next_local= tables[1].next_global= tables+2; - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - #ifdef HAVE_REPLICATION /* GRANT and REVOKE are applied the slave in/exclusion rules as they are @@ -3708,17 +5455,11 @@ int mysql_table_grant(THD *thd, TABLE_LIST *table_list, */ tables[0].updating= tables[1].updating= tables[2].updating= 1; if (!(thd->spcont || rpl_filter->tables_ok(0, tables))) - { - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(FALSE); - } } #endif - /* + /* The lock api is depending on the thd->lex variable which needs to be re-initialized. */ @@ -3732,11 +5473,7 @@ int mysql_table_grant(THD *thd, TABLE_LIST *table_list, thd->lex->sql_command= backup.sql_command; if (open_and_lock_tables(thd, tables, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT)) { // Should never happen - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); thd->lex->restore_backup_query_tables_list(&backup); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(TRUE); /* purecov: deadcode */ } @@ -3753,16 +5490,19 @@ int mysql_table_grant(THD *thd, TABLE_LIST *table_list, { int error; GRANT_TABLE *grant_table; - if (!(Str= get_current_user(thd, tmp_Str))) + if (!(Str= get_current_user(thd, tmp_Str, false))) { result= TRUE; continue; - } + } /* Create user if needed */ - error=replace_user_table(thd, tables[0].table, *Str, - 0, revoke_grant, create_new_users, - test(thd->variables.sql_mode & - MODE_NO_AUTO_CREATE_USER)); + if (copy_and_check_auth(Str, tmp_Str, thd->lex)) + error= -1; + else + error=replace_user_table(thd, tables[0].table, *Str, + 0, revoke_grant, create_new_users, + test(thd->variables.sql_mode & + MODE_NO_AUTO_CREATE_USER)); if (error) { result= TRUE; // Remember error @@ -3839,15 +5579,17 @@ int mysql_table_grant(THD *thd, TABLE_LIST *table_list, } else if (tables[2].table) { - if ((replace_column_table(grant_table, tables[2].table, *Str, - columns, - db_name, table_name, - rights, revoke_grant))) + if (replace_column_table(grant_table, tables[2].table, *Str, columns, + db_name, table_name, rights, revoke_grant)) { result= TRUE; } } + if (Str->is_role()) + propagate_role_grants(find_acl_role(Str->user.str), + PRIVS_TO_MERGE::TABLE_COLUMN, db_name, table_name); } + thd->mem_root= old_root; mysql_mutex_unlock(&acl_cache->lock); @@ -3864,9 +5606,6 @@ int mysql_table_grant(THD *thd, TABLE_LIST *table_list, /* Tables are automatically closed */ thd->lex->restore_backup_query_tables_list(&backup); /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result); } @@ -3895,7 +5634,7 @@ bool mysql_routine_grant(THD *thd, TABLE_LIST *table_list, bool is_proc, TABLE_LIST tables[2]; bool create_new_users=0, result=0; char *db_name, *table_name; - bool save_binlog_row_based; + Rpl_filter *rpl_filter= thd->rpl_filter; DBUG_ENTER("mysql_routine_grant"); if (!initialized) @@ -3925,14 +5664,6 @@ bool mysql_routine_grant(THD *thd, TABLE_LIST *table_list, bool is_proc, C_STRING_WITH_LEN("procs_priv"), "procs_priv", TL_WRITE); tables[0].next_local= tables[0].next_global= tables+1; - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - #ifdef HAVE_REPLICATION /* GRANT and REVOKE are applied the slave in/exclusion rules as they are @@ -3947,23 +5678,15 @@ bool mysql_routine_grant(THD *thd, TABLE_LIST *table_list, bool is_proc, tables[0].updating= tables[1].updating= 1; if (!(thd->spcont || rpl_filter->tables_ok(0, tables))) { - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(FALSE); } } #endif if (open_and_lock_tables(thd, tables, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT)) - { // Should never happen - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(TRUE); - } + + DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); if (!revoke_grant) create_new_users= test_if_create_new_users(thd); @@ -3978,11 +5701,11 @@ bool mysql_routine_grant(THD *thd, TABLE_LIST *table_list, bool is_proc, { int error; GRANT_NAME *grant_name; - if (!(Str= get_current_user(thd, tmp_Str))) + if (!(Str= get_current_user(thd, tmp_Str, false))) { result= TRUE; continue; - } + } /* Create user if needed */ error=replace_user_table(thd, tables[0].table, *Str, 0, revoke_grant, create_new_users, @@ -3999,7 +5722,7 @@ bool mysql_routine_grant(THD *thd, TABLE_LIST *table_list, bool is_proc, grant_name= routine_hash_search(Str->host.str, NullS, db_name, Str->user.str, table_name, is_proc, 1); - if (!grant_name) + if (!grant_name || !grant_name->init_privs) { if (revoke_grant) { @@ -4021,12 +5744,16 @@ bool mysql_routine_grant(THD *thd, TABLE_LIST *table_list, bool is_proc, } if (replace_routine_table(thd, grant_name, tables[1].table, *Str, - db_name, table_name, is_proc, rights, + db_name, table_name, is_proc, rights, revoke_grant) != 0) { result= TRUE; continue; } + if (Str->is_role()) + propagate_role_grants(find_acl_role(Str->user.str), + is_proc ? PRIVS_TO_MERGE::PROC : PRIVS_TO_MERGE::FUNC, + db_name, table_name); } thd->mem_root= old_root; mysql_mutex_unlock(&acl_cache->lock); @@ -4038,15 +5765,282 @@ bool mysql_routine_grant(THD *thd, TABLE_LIST *table_list, bool is_proc, } mysql_rwlock_unlock(&LOCK_grant); - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); /* Tables are automatically closed */ DBUG_RETURN(result); } +static void append_user(String *str, const char *u, const char *h) +{ + if (str->length()) + str->append(','); + str->append('\''); + str->append(u); + /* hostname part is not relevant for roles, it is always empty */ + if (*h) + { + str->append(STRING_WITH_LEN("'@'")); + str->append(h); + } + str->append('\''); +} + +static int can_grant_role_callback(ACL_USER_BASE *grantee, + ACL_ROLE *role, void *data) +{ + ROLE_GRANT_PAIR *pair; + + if (role != (ACL_ROLE*)data) + return 0; // keep searching + + if (grantee->flags & IS_ROLE) + pair= find_role_grant_pair(&grantee->user, &empty_lex_str, &role->user); + else + { + ACL_USER *user= (ACL_USER *)grantee; + LEX_STRING host= { user->host.hostname, user->hostname_length }; + pair= find_role_grant_pair(&user->user, &host, &role->user); + } + if (!pair->with_admin) + return 0; // keep searching + + return -1; // abort the traversal +} + + +/* + One can only grant a role if SELECT * FROM I_S.APPLICABLE_ROLES shows this + role as grantable. + + What this really means - we need to traverse role graph for the current user + looking for our role being granted with the admin option. +*/ +static bool can_grant_role(THD *thd, ACL_ROLE *role) +{ + Security_context *sctx= thd->security_ctx; + + if (!sctx->user) // replication + return true; + + ACL_USER *grantee= find_user_exact(sctx->priv_host, sctx->priv_user); + if (!grantee) + return false; + + return traverse_role_graph_down(grantee, role, NULL, + can_grant_role_callback) == -1; +} + + +bool mysql_grant_role(THD *thd, List <LEX_USER> &list, bool revoke) +{ + DBUG_ENTER("mysql_grant_role"); + /* + The first entry in the list is the granted role. Need at least two + entries for the command to be valid + */ + DBUG_ASSERT(list.elements >= 2); + bool result= 0; + String wrong_users; + LEX_USER *user, *granted_role; + LEX_STRING rolename; + LEX_STRING username; + LEX_STRING hostname; + ACL_ROLE *role, *role_as_user; + + List_iterator <LEX_USER> user_list(list); + granted_role= user_list++; + if (!(granted_role= get_current_user(thd, granted_role))) + DBUG_RETURN(TRUE); + + DBUG_ASSERT(granted_role->is_role()); + rolename= granted_role->user; + + TABLE_LIST tables; + tables.init_one_table(C_STRING_WITH_LEN("mysql"), + C_STRING_WITH_LEN("roles_mapping"), + "roles_mapping", TL_WRITE); + + if (open_and_lock_tables(thd, &tables, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT)) + DBUG_RETURN(TRUE); /* purecov: deadcode */ + + mysql_rwlock_wrlock(&LOCK_grant); + mysql_mutex_lock(&acl_cache->lock); + if (!(role= find_acl_role(rolename.str))) + { + mysql_mutex_unlock(&acl_cache->lock); + mysql_rwlock_unlock(&LOCK_grant); + my_error(ER_INVALID_ROLE, MYF(0), rolename.str); + DBUG_RETURN(TRUE); + } + + if (!can_grant_role(thd, role)) + { + mysql_mutex_unlock(&acl_cache->lock); + mysql_rwlock_unlock(&LOCK_grant); + my_error(ER_ACCESS_DENIED_NO_PASSWORD_ERROR, MYF(0), + thd->security_ctx->priv_user, thd->security_ctx->priv_host); + DBUG_RETURN(TRUE); + } + + while ((user= user_list++)) + { + role_as_user= NULL; + /* current_role is treated slightly different */ + if (user->user.str == current_role.str) + { + /* current_role is NONE */ + if (!thd->security_ctx->priv_role[0]) + { + my_error(ER_INVALID_ROLE, MYF(0), "NONE"); + append_user(&wrong_users, "NONE", ""); + result= 1; + continue; + } + if (!(role_as_user= find_acl_role(thd->security_ctx->priv_role))) + { + append_user(&wrong_users, thd->security_ctx->priv_role, ""); + result= 1; + continue; + } + + /* can not grant current_role to current_role */ + if (granted_role->user.str == current_role.str) + { + append_user(&wrong_users, thd->security_ctx->priv_role, ""); + result= 1; + continue; + } + username.str= thd->security_ctx->priv_role; + username.length= strlen(username.str); + hostname= empty_lex_str; + } + else if (user->user.str == current_user.str) + { + username.str= thd->security_ctx->priv_user; + username.length= strlen(username.str); + hostname.str= thd->security_ctx->priv_host; + hostname.length= strlen(hostname.str); + } + else + { + username= user->user; + if (user->host.str) + hostname= user->host; + else + if ((role_as_user= find_acl_role(user->user.str))) + hostname= empty_lex_str; + else + { + if (is_invalid_role_name(username.str)) + { + append_user(&wrong_users, username.str, ""); + result= 1; + continue; + } + hostname= host_not_specified; + } + } + + ROLE_GRANT_PAIR *hash_entry= find_role_grant_pair(&username, &hostname, + &rolename); + ACL_USER_BASE *grantee= role_as_user; + + if (!grantee) + grantee= find_user_exact(hostname.str, username.str); + + if (!grantee) + { + append_user(&wrong_users, username.str, hostname.str); + result= 1; + continue; + } + + if (!revoke) + { + if (hash_entry) + { + // perhaps, updating an existing grant, adding WITH ADMIN OPTION + } + else + { + add_role_user_mapping(grantee, role); + + /* + Check if this grant would cause a cycle. It only needs to be run + if we're granting a role to a role + */ + if (role_as_user && + traverse_role_graph_down(role, 0, 0, 0) == ROLE_CYCLE_FOUND) + { + append_user(&wrong_users, username.str, ""); + result= 1; + undo_add_role_user_mapping(grantee, role); + continue; + } + } + } + else + { + /* grant was already removed or never existed */ + if (!hash_entry) + { + append_user(&wrong_users, username.str, hostname.str); + result= 1; + continue; + } + if (thd->lex->with_admin_option) + { + // only revoking an admin option, not the complete grant + } + else + { + /* revoke a role grant */ + remove_role_user_mapping(grantee, role); + } + } + + /* write into the roles_mapping table */ + if (replace_roles_mapping_table(tables.table, + &username, &hostname, &rolename, + thd->lex->with_admin_option, + hash_entry, revoke)) + { + append_user(&wrong_users, username.str, ""); + result= 1; + if (!revoke) + { + /* need to remove the mapping added previously */ + undo_add_role_user_mapping(grantee, role); + } + else + { + /* need to restore the mapping deleted previously */ + add_role_user_mapping(grantee, role); + } + continue; + } + + /* + Only need to propagate grants when granting/revoking a role to/from + a role + */ + if (role_as_user) + propagate_role_grants(role_as_user, PRIVS_TO_MERGE::ALL, 0, 0); + } + + mysql_mutex_unlock(&acl_cache->lock); + + if (result) + my_error(revoke ? ER_CANNOT_REVOKE_ROLE : ER_CANNOT_GRANT_ROLE, MYF(0), + rolename.str, wrong_users.c_ptr_safe()); + else + result= write_bin_log(thd, TRUE, thd->query(), thd->query_length()); + + mysql_rwlock_unlock(&LOCK_grant); + + DBUG_RETURN(result); +} + bool mysql_grant(THD *thd, const char *db, List <LEX_USER> &list, ulong rights, bool revoke_grant, bool is_proxy) @@ -4056,8 +6050,9 @@ bool mysql_grant(THD *thd, const char *db, List <LEX_USER> &list, char tmp_db[SAFE_NAME_LEN+1]; bool create_new_users=0; TABLE_LIST tables[2]; - bool save_binlog_row_based; + Rpl_filter *rpl_filter= thd->rpl_filter; DBUG_ENTER("mysql_grant"); + if (!initialized) { my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), @@ -4090,23 +6085,15 @@ bool mysql_grant(THD *thd, const char *db, List <LEX_USER> &list, tables[1].init_one_table(C_STRING_WITH_LEN("mysql"), C_STRING_WITH_LEN("proxies_priv"), - "proxies_priv", + "proxies_priv", TL_WRITE); else tables[1].init_one_table(C_STRING_WITH_LEN("mysql"), - C_STRING_WITH_LEN("db"), - "db", + C_STRING_WITH_LEN("db"), + "db", TL_WRITE); tables[0].next_local= tables[0].next_global= tables+1; - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - #ifdef HAVE_REPLICATION /* GRANT and REVOKE are applied the slave in/exclusion rules as they are @@ -4120,24 +6107,14 @@ bool mysql_grant(THD *thd, const char *db, List <LEX_USER> &list, */ tables[0].updating= tables[1].updating= 1; if (!(thd->spcont || rpl_filter->tables_ok(0, tables))) - { - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(FALSE); - } } #endif if (open_and_lock_tables(thd, tables, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT)) - { // This should never happen - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(TRUE); /* purecov: deadcode */ - } + + DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); if (!revoke_grant) create_new_users= test_if_create_new_users(thd); @@ -4147,21 +6124,25 @@ bool mysql_grant(THD *thd, const char *db, List <LEX_USER> &list, mysql_mutex_lock(&acl_cache->lock); grant_version++; + if (proxied_user) + { + if (!(proxied_user= get_current_user(thd, proxied_user, false))) + DBUG_RETURN(TRUE); + DBUG_ASSERT(proxied_user->host.length); // not a Role + } + int result=0; while ((tmp_Str = str_list++)) { - if (!(Str= get_current_user(thd, tmp_Str))) + if (!(Str= get_current_user(thd, tmp_Str, false))) { result= TRUE; continue; } - /* - No User, but a password? - They did GRANT ... TO CURRENT_USER() IDENTIFIED BY ... ! - Get the current user, and shallow-copy the new password to them! - */ - if (!tmp_Str->user.str && tmp_Str->password.str) - Str->password= tmp_Str->password; + + if (copy_and_check_auth(Str, tmp_Str, thd->lex)) + result= -1; + else if (replace_user_table(thd, tables[0].table, *Str, (!db ? rights : 0), revoke_grant, create_new_users, test(thd->variables.sql_mode & @@ -4185,10 +6166,14 @@ bool mysql_grant(THD *thd, const char *db, List <LEX_USER> &list, else if (is_proxy) { if (replace_proxies_priv_table (thd, tables[1].table, Str, proxied_user, - rights & GRANT_ACL ? TRUE : FALSE, + rights & GRANT_ACL ? TRUE : FALSE, revoke_grant)) result= -1; } + if (Str->is_role()) + propagate_role_grants(find_acl_role(Str->user.str), + db ? PRIVS_TO_MERGE::DB : PRIVS_TO_MERGE::GLOBAL, + db, 0); } mysql_mutex_unlock(&acl_cache->lock); @@ -4201,10 +6186,6 @@ bool mysql_grant(THD *thd, const char *db, List <LEX_USER> &list, if (!result) my_ok(thd); - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result); } @@ -4245,106 +6226,7 @@ my_bool grant_init() return_val= grant_reload(thd); delete thd; /* Remember that we don't have a THD */ - my_pthread_setspecific_ptr(THR_THD, 0); - DBUG_RETURN(return_val); -} - - -/** - @brief Helper function to grant_reload_procs_priv - - Reads the procs_priv table into memory hash. - - @param table A pointer to the procs_priv table structure. - - @see grant_reload - @see grant_reload_procs_priv - - @return Error state - @retval TRUE An error occurred - @retval FALSE Success -*/ - -static my_bool grant_load_procs_priv(TABLE *p_table) -{ - MEM_ROOT *memex_ptr; - my_bool return_val= 1; - bool check_no_resolve= specialflag & SPECIAL_NO_RESOLVE; - MEM_ROOT **save_mem_root_ptr= my_pthread_getspecific_ptr(MEM_ROOT**, - THR_MALLOC); - DBUG_ENTER("grant_load_procs_priv"); - (void) my_hash_init(&proc_priv_hash, &my_charset_utf8_bin, - 0,0,0, (my_hash_get_key) get_grant_table, - 0,0); - (void) my_hash_init(&func_priv_hash, &my_charset_utf8_bin, - 0,0,0, (my_hash_get_key) get_grant_table, - 0,0); - - if (p_table->file->ha_index_init(0, 1)) - DBUG_RETURN(TRUE); - - p_table->use_all_columns(); - - if (!p_table->file->ha_index_first(p_table->record[0])) - { - memex_ptr= &memex; - my_pthread_setspecific_ptr(THR_MALLOC, &memex_ptr); - do - { - GRANT_NAME *mem_check; - HASH *hash; - if (!(mem_check=new (memex_ptr) GRANT_NAME(p_table, TRUE))) - { - /* This could only happen if we are out memory */ - goto end_unlock; - } - - if (check_no_resolve) - { - if (hostname_requires_resolving(mem_check->host.hostname)) - { - sql_print_warning("'procs_priv' entry '%s %s@%s' " - "ignored in --skip-name-resolve mode.", - mem_check->tname, mem_check->user, - mem_check->host.hostname ? - mem_check->host.hostname : ""); - continue; - } - } - if (p_table->field[4]->val_int() == TYPE_ENUM_PROCEDURE) - { - hash= &proc_priv_hash; - } - else - if (p_table->field[4]->val_int() == TYPE_ENUM_FUNCTION) - { - hash= &func_priv_hash; - } - else - { - sql_print_warning("'procs_priv' entry '%s' " - "ignored, bad routine type", - mem_check->tname); - continue; - } - - mem_check->privs= fix_rights_for_procedure(mem_check->privs); - if (! mem_check->ok()) - delete mem_check; - else if (my_hash_insert(hash, (uchar*) mem_check)) - { - delete mem_check; - goto end_unlock; - } - } - while (!p_table->file->ha_index_next(p_table->record[0])); - } - /* Return ok */ - return_val= 0; - -end_unlock: - p_table->file->ha_index_end(); - my_pthread_setspecific_ptr(THR_MALLOC, save_mem_root_ptr); + set_current_thd(0); DBUG_RETURN(return_val); } @@ -4368,7 +6250,7 @@ static my_bool grant_load(THD *thd, TABLE_LIST *tables) { MEM_ROOT *memex_ptr; my_bool return_val= 1; - TABLE *t_table= 0, *c_table= 0; + TABLE *t_table, *c_table, *p_table; bool check_no_resolve= specialflag & SPECIAL_NO_RESOLVE; MEM_ROOT **save_mem_root_ptr= my_pthread_getspecific_ptr(MEM_ROOT**, THR_MALLOC); @@ -4380,9 +6262,15 @@ static my_bool grant_load(THD *thd, TABLE_LIST *tables) (void) my_hash_init(&column_priv_hash, &my_charset_utf8_bin, 0,0,0, (my_hash_get_key) get_grant_table, (my_hash_free_key) free_grant_table,0); + (void) my_hash_init(&proc_priv_hash, &my_charset_utf8_bin, + 0,0,0, (my_hash_get_key) get_grant_table, 0,0); + (void) my_hash_init(&func_priv_hash, &my_charset_utf8_bin, + 0,0,0, (my_hash_get_key) get_grant_table, 0,0); + init_sql_alloc(&memex, ACL_ALLOC_BLOCK_SIZE, 0, MYF(0)); - t_table = tables[0].table; - c_table = tables[1].table; + t_table= tables[0].table; + c_table= tables[1].table; + p_table= tables[2].table; // this can be NULL if (t_table->file->ha_index_init(0, 1)) goto end_index_init; @@ -4390,10 +6278,11 @@ static my_bool grant_load(THD *thd, TABLE_LIST *tables) t_table->use_all_columns(); c_table->use_all_columns(); + memex_ptr= &memex; + my_pthread_setspecific_ptr(THR_MALLOC, &memex_ptr); + if (!t_table->file->ha_index_first(t_table->record[0])) { - memex_ptr= &memex; - my_pthread_setspecific_ptr(THR_MALLOC, &memex_ptr); do { GRANT_TABLE *mem_check; @@ -4410,9 +6299,8 @@ static my_bool grant_load(THD *thd, TABLE_LIST *tables) sql_print_warning("'tables_priv' entry '%s %s@%s' " "ignored in --skip-name-resolve mode.", mem_check->tname, - mem_check->user ? mem_check->user : "", - mem_check->host.hostname ? - mem_check->host.hostname : ""); + safe_str(mem_check->user), + safe_str(mem_check->host.hostname)); continue; } } @@ -4428,8 +6316,72 @@ static my_bool grant_load(THD *thd, TABLE_LIST *tables) while (!t_table->file->ha_index_next(t_table->record[0])); } - return_val=0; // Return ok + return_val= 0; + + if (p_table) + { + if (p_table->file->ha_index_init(0, 1)) + goto end_unlock; + + p_table->use_all_columns(); + + if (!p_table->file->ha_index_first(p_table->record[0])) + { + do + { + GRANT_NAME *mem_check; + HASH *hash; + if (!(mem_check=new (memex_ptr) GRANT_NAME(p_table, TRUE))) + { + /* This could only happen if we are out memory */ + goto end_unlock_p; + } + + if (check_no_resolve) + { + if (hostname_requires_resolving(mem_check->host.hostname)) + { + sql_print_warning("'procs_priv' entry '%s %s@%s' " + "ignored in --skip-name-resolve mode.", + mem_check->tname, mem_check->user, + safe_str(mem_check->host.hostname)); + continue; + } + } + if (p_table->field[4]->val_int() == TYPE_ENUM_PROCEDURE) + { + hash= &proc_priv_hash; + } + else + if (p_table->field[4]->val_int() == TYPE_ENUM_FUNCTION) + { + hash= &func_priv_hash; + } + else + { + sql_print_warning("'procs_priv' entry '%s' " + "ignored, bad routine type", + mem_check->tname); + continue; + } + mem_check->privs= fix_rights_for_procedure(mem_check->privs); + mem_check->init_privs= mem_check->privs; + if (! mem_check->ok()) + delete mem_check; + else if (my_hash_insert(hash, (uchar*) mem_check)) + { + delete mem_check; + goto end_unlock_p; + } + } + while (!p_table->file->ha_index_next(p_table->record[0])); + } + } + +end_unlock_p: + if (p_table) + p_table->file->ha_index_end(); end_unlock: t_table->file->ha_index_end(); my_pthread_setspecific_ptr(THR_MALLOC, save_mem_root_ptr); @@ -4439,56 +6391,16 @@ end_index_init: } -/** - @brief Helper function to grant_reload. Reloads procs_priv table is it - exists. - - @param thd A pointer to the thread handler object. - - @see grant_reload - - @return Error state - @retval FALSE Success - @retval TRUE An error has occurred. -*/ - -static my_bool grant_reload_procs_priv(THD *thd) +my_bool role_propagate_grants_action(void *ptr, void *unused __attribute__((unused))) { - HASH old_proc_priv_hash, old_func_priv_hash; - TABLE_LIST table; - my_bool return_val= FALSE; - DBUG_ENTER("grant_reload_procs_priv"); - - table.init_one_table("mysql", 5, "procs_priv", - strlen("procs_priv"), "procs_priv", - TL_READ); - table.open_type= OT_BASE_ONLY; - - if (open_and_lock_tables(thd, &table, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT)) - DBUG_RETURN(TRUE); - - mysql_rwlock_wrlock(&LOCK_grant); - /* Save a copy of the current hash if we need to undo the grant load */ - old_proc_priv_hash= proc_priv_hash; - old_func_priv_hash= func_priv_hash; - - if ((return_val= grant_load_procs_priv(table.table))) - { - /* Error; Reverting to old hash */ - DBUG_PRINT("error",("Reverting to old privileges")); - grant_free(); - proc_priv_hash= old_proc_priv_hash; - func_priv_hash= old_func_priv_hash; - } - else - { - my_hash_free(&old_proc_priv_hash); - my_hash_free(&old_func_priv_hash); - } - mysql_rwlock_unlock(&LOCK_grant); + ACL_ROLE *role= (ACL_ROLE *)ptr; + if (role->counter) + return 0; - close_mysql_tables(thd); - DBUG_RETURN(return_val); + mysql_mutex_assert_owner(&acl_cache->lock); + PRIVS_TO_MERGE data= { PRIVS_TO_MERGE::ALL, 0, 0 }; + traverse_role_graph_up(role, &data, NULL, merge_role_privileges); + return 0; } @@ -4509,8 +6421,8 @@ static my_bool grant_reload_procs_priv(THD *thd) my_bool grant_reload(THD *thd) { - TABLE_LIST tables[2]; - HASH old_column_priv_hash; + TABLE_LIST tables[3]; + HASH old_column_priv_hash, old_proc_priv_hash, old_func_priv_hash; MEM_ROOT old_mem; my_bool return_val= 1; DBUG_ENTER("grant_reload"); @@ -4525,8 +6437,13 @@ my_bool grant_reload(THD *thd) tables[1].init_one_table(C_STRING_WITH_LEN("mysql"), C_STRING_WITH_LEN("columns_priv"), "columns_priv", TL_READ); + tables[2].init_one_table(C_STRING_WITH_LEN("mysql"), + C_STRING_WITH_LEN("procs_priv"), + "procs_priv", TL_READ); tables[0].next_local= tables[0].next_global= tables+1; - tables[0].open_type= tables[1].open_type= OT_BASE_ONLY; + tables[1].next_local= tables[1].next_global= tables+2; + tables[0].open_type= tables[1].open_type= tables[2].open_type= OT_BASE_ONLY; + tables[2].open_strategy= TABLE_LIST::OPEN_IF_EXISTS; /* To avoid deadlocks we should obtain table locks before @@ -4536,41 +6453,42 @@ my_bool grant_reload(THD *thd) goto end; mysql_rwlock_wrlock(&LOCK_grant); + grant_version++; old_column_priv_hash= column_priv_hash; + old_proc_priv_hash= proc_priv_hash; + old_func_priv_hash= func_priv_hash; /* Create a new memory pool but save the current memory pool to make an undo opertion possible in case of failure. */ old_mem= memex; - init_sql_alloc(&memex, ACL_ALLOC_BLOCK_SIZE, 0); if ((return_val= grant_load(thd, tables))) { // Error. Revert to old hash DBUG_PRINT("error",("Reverting to old privileges")); grant_free(); /* purecov: deadcode */ column_priv_hash= old_column_priv_hash; /* purecov: deadcode */ + proc_priv_hash= old_proc_priv_hash; + func_priv_hash= old_func_priv_hash; memex= old_mem; /* purecov: deadcode */ } else { my_hash_free(&old_column_priv_hash); + my_hash_free(&old_proc_priv_hash); + my_hash_free(&old_func_priv_hash); free_root(&old_mem,MYF(0)); } - mysql_rwlock_unlock(&LOCK_grant); - close_mysql_tables(thd); - /* - It is OK failing to load procs_priv table because we may be - working with 4.1 privilege tables. - */ - if (grant_reload_procs_priv(thd)) - return_val= 1; + mysql_mutex_lock(&acl_cache->lock); + my_hash_iterate(&acl_roles, role_propagate_grants_action, NULL); + mysql_mutex_unlock(&acl_cache->lock); - mysql_rwlock_wrlock(&LOCK_grant); - grant_version++; mysql_rwlock_unlock(&LOCK_grant); + close_mysql_tables(thd); + end: DBUG_RETURN(return_val); } @@ -4601,12 +6519,16 @@ end: @see check_access @see check_table_access - @note This functions assumes that either number of tables to be inspected + @note + This functions assumes that either number of tables to be inspected by it is limited explicitly (i.e. is is not UINT_MAX) or table list used and thd->lex->query_tables_own_last value correspond to each other (the latter should be either 0 or point to next_global member of one of elements of this table list). + We delay locking of LOCK_grant until we really need it as we assume that + most privileges be resolved with user or db level accesses. + @return Access status @retval FALSE Access granted; But column privileges might need to be checked. @@ -4623,6 +6545,9 @@ bool check_grant(THD *thd, ulong want_access, TABLE_LIST *tables, Security_context *sctx= thd->security_ctx; uint i; ulong orig_want_access= want_access; + my_bool locked= 0; + GRANT_TABLE *grant_table; + GRANT_TABLE *grant_table_role= NULL; DBUG_ENTER("check_grant"); DBUG_ASSERT(number > 0); @@ -4646,11 +6571,9 @@ bool check_grant(THD *thd, ulong want_access, TABLE_LIST *tables, */ tl->grant.orig_want_privilege= (want_access & ~SHOW_VIEW_ACL); } + number= i; - mysql_rwlock_rdlock(&LOCK_grant); - for (tl= tables; - tl && number-- && tl != first_not_own_table; - tl= tl->next_global) + for (tl= tables; number-- ; tl= tl->next_global) { sctx = test(tl->security_ctx) ? tl->security_ctx : thd->security_ctx; @@ -4703,16 +6626,28 @@ bool check_grant(THD *thd, ulong want_access, TABLE_LIST *tables, } continue; } - GRANT_TABLE *grant_table= table_hash_search(sctx->host, sctx->ip, - tl->get_db_name(), - sctx->priv_user, - tl->get_table_name(), - FALSE); - if (!grant_table) + if (!locked) + { + locked= 1; + mysql_rwlock_rdlock(&LOCK_grant); + } + + grant_table= table_hash_search(sctx->host, sctx->ip, + tl->get_db_name(), + sctx->priv_user, + tl->get_table_name(), + FALSE); + if (sctx->priv_role[0]) + grant_table_role= table_hash_search("", NULL, tl->get_db_name(), + sctx->priv_role, + tl->get_table_name(), + TRUE); + + if (!grant_table && !grant_table_role) { - want_access &= ~tl->grant.privilege; - goto err; // No grants + want_access&= ~tl->grant.privilege; + goto err; } /* @@ -4722,25 +6657,30 @@ bool check_grant(THD *thd, ulong want_access, TABLE_LIST *tables, if (any_combination_will_do) continue; - tl->grant.grant_table= grant_table; // Remember for column test + tl->grant.grant_table_user= grant_table; // Remember for column test + tl->grant.grant_table_role= grant_table_role; tl->grant.version= grant_version; - tl->grant.privilege|= grant_table->privs; + tl->grant.privilege|= grant_table ? grant_table->privs : 0; + tl->grant.privilege|= grant_table_role ? grant_table_role->privs : 0; tl->grant.want_privilege= ((want_access & COL_ACLS) & ~tl->grant.privilege); if (!(~tl->grant.privilege & want_access)) continue; - if (want_access & ~(grant_table->cols | tl->grant.privilege)) + if ((want_access&= ~((grant_table ? grant_table->cols : 0) | + (grant_table_role ? grant_table_role->cols : 0) | + tl->grant.privilege))) { - want_access &= ~(grant_table->cols | tl->grant.privilege); - goto err; // impossible + goto err; // impossible } } - mysql_rwlock_unlock(&LOCK_grant); + if (locked) + mysql_rwlock_unlock(&LOCK_grant); DBUG_RETURN(FALSE); err: - mysql_rwlock_unlock(&LOCK_grant); + if (locked) + mysql_rwlock_unlock(&LOCK_grant); if (!no_errors) // Not a silent skip of table { char command[128]; @@ -4780,6 +6720,7 @@ bool check_grant_column(THD *thd, GRANT_INFO *grant, const char *name, uint length, Security_context *sctx) { GRANT_TABLE *grant_table; + GRANT_TABLE *grant_table_role; GRANT_COLUMN *grant_column; ulong want_access= grant->want_privilege & ~grant->privilege; DBUG_ENTER("check_grant_column"); @@ -4794,17 +6735,40 @@ bool check_grant_column(THD *thd, GRANT_INFO *grant, if (grant->version != grant_version) { - grant->grant_table= + grant->grant_table_user= table_hash_search(sctx->host, sctx->ip, db_name, sctx->priv_user, table_name, 0); /* purecov: inspected */ + grant->grant_table_role= + sctx->priv_role[0] ? table_hash_search("", NULL, db_name, + sctx->priv_role, + table_name, TRUE) : NULL; grant->version= grant_version; /* purecov: inspected */ } - if (!(grant_table= grant->grant_table)) - goto err; /* purecov: deadcode */ - grant_column=column_hash_search(grant_table, name, length); - if (grant_column && !(~grant_column->rights & want_access)) + grant_table= grant->grant_table_user; + grant_table_role= grant->grant_table_role; + + if (!grant_table && !grant_table_role) + goto err; + + if (grant_table) + { + grant_column= column_hash_search(grant_table, name, length); + if (grant_column) + { + want_access&= ~grant_column->rights; + } + } + if (grant_table_role) + { + grant_column= column_hash_search(grant_table_role, name, length); + if (grant_column) + { + want_access&= ~grant_column->rights; + } + } + if (!want_access) { mysql_rwlock_unlock(&LOCK_grant); DBUG_RETURN(0); @@ -4814,6 +6778,7 @@ err: mysql_rwlock_unlock(&LOCK_grant); char command[128]; get_privilege_desc(command, sizeof(command), want_access); + /* TODO perhaps error should print current rolename aswell */ my_error(ER_COLUMNACCESS_DENIED_ERROR, MYF(0), command, sctx->priv_user, @@ -4862,7 +6827,7 @@ bool check_column_grant_in_table_ref(THD *thd, TABLE_LIST * table_ref, grant= &(table_ref->grant); db_name= table_ref->view_db.str; table_name= table_ref->view_name.str; - if (table_ref->belong_to_view && + if (table_ref->belong_to_view && thd->lex->sql_command == SQLCOM_SHOW_FIELDS) { view_privs= get_column_grant(thd, grant, db_name, table_name, name); @@ -4894,7 +6859,7 @@ bool check_column_grant_in_table_ref(THD *thd, TABLE_LIST * table_ref, } -/** +/** @brief check if a query can access a set of columns @param thd the current thread @@ -4903,24 +6868,23 @@ bool check_column_grant_in_table_ref(THD *thd, TABLE_LIST * table_ref, @return Operation status @retval 0 Success @retval 1 Falure - @details This function walks over the columns of a table reference + @details This function walks over the columns of a table reference The columns may originate from different tables, depending on the kind of table reference, e.g. join, view. For each table it will retrieve the grant information and will use it to check the required access privileges for the fields requested from it. -*/ -bool check_grant_all_columns(THD *thd, ulong want_access_arg, +*/ +bool check_grant_all_columns(THD *thd, ulong want_access_arg, Field_iterator_table_ref *fields) { Security_context *sctx= thd->security_ctx; - ulong want_access= want_access_arg; + ulong UNINIT_VAR(want_access); const char *table_name= NULL; - - const char* db_name; + const char* db_name; GRANT_INFO *grant; - /* Initialized only to make gcc happy */ - GRANT_TABLE *grant_table= NULL; - /* + GRANT_TABLE *UNINIT_VAR(grant_table); + GRANT_TABLE *UNINIT_VAR(grant_table_role); + /* Flag that gets set if privilege checking has to be performed on column level. */ @@ -4944,26 +6908,46 @@ bool check_grant_all_columns(THD *thd, ulong want_access_arg, /* reload table if someone has modified any grants */ if (grant->version != grant_version) { - grant->grant_table= + grant->grant_table_user= table_hash_search(sctx->host, sctx->ip, db_name, sctx->priv_user, table_name, 0); /* purecov: inspected */ + grant->grant_table_role= + sctx->priv_role[0] ? table_hash_search("", NULL, db_name, + sctx->priv_role, + table_name, TRUE) : NULL; grant->version= grant_version; /* purecov: inspected */ } - grant_table= grant->grant_table; - DBUG_ASSERT (grant_table); + grant_table= grant->grant_table_user; + grant_table_role= grant->grant_table_role; + DBUG_ASSERT (grant_table || grant_table_role); } } if (want_access) { - GRANT_COLUMN *grant_column= - column_hash_search(grant_table, field_name, - (uint) strlen(field_name)); - if (grant_column) + ulong have_access= 0; + if (grant_table) + { + GRANT_COLUMN *grant_column= + column_hash_search(grant_table, field_name, + (uint) strlen(field_name)); + if (grant_column) + have_access= grant_column->rights; + } + if (grant_table_role) + { + GRANT_COLUMN *grant_column= + column_hash_search(grant_table_role, field_name, + (uint) strlen(field_name)); + if (grant_column) + have_access|= grant_column->rights; + } + + if (have_access) using_column_privileges= TRUE; - if (!grant_column || (~grant_column->rights & want_access)) + if (want_access & ~have_access) goto err; } } @@ -4982,7 +6966,7 @@ err: if (using_column_privileges) my_error(ER_TABLEACCESS_DENIED_ERROR, MYF(0), command, sctx->priv_user, - sctx->host_or_ip, table_name); + sctx->host_or_ip, table_name); else my_error(ER_COLUMNACCESS_DENIED_ERROR, MYF(0), command, @@ -5008,6 +6992,12 @@ static bool check_grant_db_routine(THD *thd, const char *db, HASH *hash) { return FALSE; } + if (sctx->priv_role[0] && strcmp(item->user, sctx->priv_role) == 0 && + strcmp(item->db, db) == 0 && + (!item->host.hostname || !item->host.hostname[0])) + { + return FALSE; /* Found current role match */ + } } return TRUE; @@ -5020,11 +7010,12 @@ static bool check_grant_db_routine(THD *thd, const char *db, HASH *hash) Return 1 if access is denied */ -bool check_grant_db(THD *thd,const char *db) +bool check_grant_db(THD *thd, const char *db) { Security_context *sctx= thd->security_ctx; char helping [SAFE_NAME_LEN + USERNAME_LENGTH+2], *end; - uint len; + char helping2 [SAFE_NAME_LEN + USERNAME_LENGTH+2]; + uint len, UNINIT_VAR(len2); bool error= TRUE; end= strmov(helping, sctx->priv_user) + 1; @@ -5035,6 +7026,18 @@ bool check_grant_db(THD *thd,const char *db) len= (uint) (end - helping) + 1; + /* + If a role is set, we need to check for privileges + here aswell + */ + if (sctx->priv_role[0]) + { + end= strmov(helping2, sctx->priv_role) + 1; + end= strnmov(end, db, helping2 + sizeof(helping2) - end); + len2= (uint) (end - helping2) + 1; + } + + mysql_rwlock_rdlock(&LOCK_grant); for (uint idx=0 ; idx < column_priv_hash.records ; idx++) @@ -5049,6 +7052,14 @@ bool check_grant_db(THD *thd,const char *db) error= FALSE; /* Found match. */ break; } + if (sctx->priv_role[0] && + len2 < grant_table->key_length && + !memcmp(grant_table->hash_key,helping2,len) && + (!grant_table->host.hostname || !grant_table->host.hostname[0])) + { + error= FALSE; /* Found role match */ + break; + } } if (error) @@ -5085,6 +7096,7 @@ bool check_grant_routine(THD *thd, ulong want_access, Security_context *sctx= thd->security_ctx; char *user= sctx->priv_user; char *host= sctx->priv_host; + char *role= sctx->priv_role; DBUG_ENTER("check_grant_routine"); want_access&= ~sctx->master_access; @@ -5098,6 +7110,12 @@ bool check_grant_routine(THD *thd, ulong want_access, if ((grant_proc= routine_hash_search(host, sctx->ip, table->db, user, table->table_name, is_proc, 0))) table->grant.privilege|= grant_proc->privs; + if (role[0]) /* current role set check */ + { + if ((grant_proc= routine_hash_search("", NULL, table->db, role, + table->table_name, is_proc, 0))) + table->grant.privilege|= grant_proc->privs; + } if (want_access & ~table->grant.privilege) { @@ -5129,9 +7147,9 @@ err: /* - Check if routine has any of the + Check if routine has any of the routine level grants - + SYNPOSIS bool check_routine_level_acl() thd Thread handler @@ -5139,11 +7157,11 @@ err: name Routine name RETURN - 0 Ok + 0 Ok 1 error */ -bool check_routine_level_acl(THD *thd, const char *db, const char *name, +bool check_routine_level_acl(THD *thd, const char *db, const char *name, bool is_proc) { bool no_routine_acl= 1; @@ -5155,6 +7173,15 @@ bool check_routine_level_acl(THD *thd, const char *db, const char *name, sctx->priv_user, name, is_proc, 0))) no_routine_acl= !(grant_proc->privs & SHOW_PROC_ACLS); + + if (no_routine_acl && sctx->priv_role[0]) /* current set role check */ + { + if ((grant_proc= routine_hash_search("", + NULL, db, + sctx->priv_role, + name, is_proc, 0))) + no_routine_acl= !(grant_proc->privs & SHOW_PROC_ACLS); + } mysql_rwlock_unlock(&LOCK_grant); return no_routine_acl; } @@ -5170,18 +7197,26 @@ ulong get_table_grant(THD *thd, TABLE_LIST *table) Security_context *sctx= thd->security_ctx; const char *db = table->db ? table->db : thd->db; GRANT_TABLE *grant_table; + GRANT_TABLE *grant_table_role= NULL; mysql_rwlock_rdlock(&LOCK_grant); #ifdef EMBEDDED_LIBRARY grant_table= NULL; + grant_table_role= NULL; #else grant_table= table_hash_search(sctx->host, sctx->ip, db, sctx->priv_user, table->table_name, 0); + if (sctx->priv_role[0]) + grant_table_role= table_hash_search("", "", db, sctx->priv_role, + table->table_name, 0); #endif - table->grant.grant_table=grant_table; // Remember for column test + table->grant.grant_table_user= grant_table; // Remember for column test + table->grant.grant_table_role= grant_table_role; table->grant.version=grant_version; if (grant_table) table->grant.privilege|= grant_table->privs; + if (grant_table_role) + table->grant.privilege|= grant_table_role->privs; privilege= table->grant.privilege; mysql_rwlock_unlock(&LOCK_grant); return privilege; @@ -5211,31 +7246,52 @@ ulong get_column_grant(THD *thd, GRANT_INFO *grant, const char *field_name) { GRANT_TABLE *grant_table; + GRANT_TABLE *grant_table_role; GRANT_COLUMN *grant_column; - ulong priv; + ulong priv= 0; mysql_rwlock_rdlock(&LOCK_grant); /* reload table if someone has modified any grants */ if (grant->version != grant_version) { Security_context *sctx= thd->security_ctx; - grant->grant_table= + grant->grant_table_user= table_hash_search(sctx->host, sctx->ip, db_name, sctx->priv_user, - table_name, 0); /* purecov: inspected */ + table_name, 0); /* purecov: inspected */ + grant->grant_table_role= + sctx->priv_role[0] ? table_hash_search("", "", db_name, + sctx->priv_role, + table_name, TRUE) : NULL; grant->version= grant_version; /* purecov: inspected */ } - if (!(grant_table= grant->grant_table)) + grant_table= grant->grant_table_user; + grant_table_role= grant->grant_table_role; + + if (!grant_table && !grant_table_role) priv= grant->privilege; else { - grant_column= column_hash_search(grant_table, field_name, - (uint) strlen(field_name)); - if (!grant_column) - priv= (grant->privilege | grant_table->privs); - else - priv= (grant->privilege | grant_table->privs | grant_column->rights); + if (grant_table) + { + grant_column= column_hash_search(grant_table, field_name, + (uint) strlen(field_name)); + if (!grant_column) + priv= (grant->privilege | grant_table->privs); + else + priv= (grant->privilege | grant_table->privs | grant_column->rights); + } + + if (grant_table_role) + { + grant_column= column_hash_search(grant_table_role, field_name, + (uint) strlen(field_name)); + if (!grant_column) + priv|= (grant->privilege | grant_table_role->privs); + else + priv|= (grant->privilege | grant_table->privs | grant_column->rights); + } } mysql_rwlock_unlock(&LOCK_grant); return priv; @@ -5275,9 +7331,43 @@ static uint command_lengths[]= }; -static int show_routine_grants(THD *thd, LEX_USER *lex_user, HASH *hash, - const char *type, int typelen, - char *buff, int buffsize); +static bool print_grants_for_role(THD *thd, ACL_ROLE * role) +{ + char buff[1024]; + + if (show_role_grants(thd, role->user.str, "", role, buff, sizeof(buff))) + return TRUE; + + if (show_global_privileges(thd, role, TRUE, buff, sizeof(buff))) + return TRUE; + + if (show_database_privileges(thd, role->user.str, "", buff, sizeof(buff))) + return TRUE; + + if (show_table_and_column_privileges(thd, role->user.str, "", buff, sizeof(buff))) + return TRUE; + + if (show_routine_grants(thd, role->user.str, "", &proc_priv_hash, + STRING_WITH_LEN("PROCEDURE"), buff, sizeof(buff))) + return TRUE; + + if (show_routine_grants(thd, role->user.str, "", &func_priv_hash, + STRING_WITH_LEN("FUNCTION"), buff, sizeof(buff))) + return TRUE; + + return FALSE; + +} + + +static int show_grants_callback(ACL_USER_BASE *role, void *data) +{ + THD *thd= (THD *)data; + DBUG_ASSERT(role->flags & IS_ROLE); + if (print_grants_for_role(thd, (ACL_ROLE *)role)) + return -1; + return 0; +} /* @@ -5287,18 +7377,18 @@ static int show_routine_grants(THD *thd, LEX_USER *lex_user, HASH *hash, Send to client grant-like strings depicting user@host privileges */ -bool mysql_show_grants(THD *thd,LEX_USER *lex_user) +bool mysql_show_grants(THD *thd, LEX_USER *lex_user) { - ulong want_access; - uint counter,index; - int error = 0; - ACL_USER *acl_user; - ACL_DB *acl_db; + int error = -1; + ACL_USER *UNINIT_VAR(acl_user); + ACL_ROLE *acl_role= NULL; char buff[1024]; Protocol *protocol= thd->protocol; + char *username= NULL; + char *hostname= NULL; + char *rolename= NULL; DBUG_ENTER("mysql_show_grants"); - LINT_INIT(acl_user); if (!initialized) { my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), "--skip-grant-tables"); @@ -5308,26 +7398,54 @@ bool mysql_show_grants(THD *thd,LEX_USER *lex_user) mysql_rwlock_rdlock(&LOCK_grant); mysql_mutex_lock(&acl_cache->lock); - acl_user= find_acl_user(lex_user->host.str, lex_user->user.str, TRUE); - if (!acl_user) + if (lex_user->user.str == current_user.str) { - mysql_mutex_unlock(&acl_cache->lock); - mysql_rwlock_unlock(&LOCK_grant); + username= thd->security_ctx->priv_user; + hostname= thd->security_ctx->priv_host; + } + else if (lex_user->user.str == current_role.str) + { + rolename= thd->security_ctx->priv_role; + } + else if (lex_user->user.str == current_user_and_current_role.str) + { + username= thd->security_ctx->priv_user; + hostname= thd->security_ctx->priv_host; + rolename= thd->security_ctx->priv_role; + } + else + { + lex_user= get_current_user(thd, lex_user, false); + if (!lex_user) + { + mysql_mutex_unlock(&acl_cache->lock); + mysql_rwlock_unlock(&LOCK_grant); + DBUG_RETURN(TRUE); + } - my_error(ER_NONEXISTING_GRANT, MYF(0), - lex_user->user.str, lex_user->host.str); - DBUG_RETURN(TRUE); + if (lex_user->is_role()) + { + rolename= lex_user->user.str; + } + else + { + username= lex_user->user.str; + hostname= lex_user->host.str; + } } + DBUG_ASSERT(rolename || username); Item_string *field=new Item_string("",0,&my_charset_latin1); List<Item> field_list; field->name=buff; field->max_length=1024; - strxmov(buff,"Grants for ",lex_user->user.str,"@", - lex_user->host.str,NullS); + if (!username) + strxmov(buff,"Grants for ",rolename, NullS); + else + strxmov(buff,"Grants for ",username,"@",hostname, NullS); field_list.push_back(field); if (protocol->send_result_set_metadata(&field_list, - Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF)) + Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF)) { mysql_mutex_unlock(&acl_cache->lock); mysql_rwlock_unlock(&LOCK_grant); @@ -5335,39 +7453,188 @@ bool mysql_show_grants(THD *thd,LEX_USER *lex_user) DBUG_RETURN(TRUE); } - /* Add first global access grants */ + if (username) { - String global(buff,sizeof(buff),system_charset_info); - global.length(0); - global.append(STRING_WITH_LEN("GRANT ")); + acl_user= find_user_exact(hostname, username); + if (!acl_user) + { + mysql_mutex_unlock(&acl_cache->lock); + mysql_rwlock_unlock(&LOCK_grant); - want_access= acl_user->access; - if (test_all_bits(want_access, (GLOBAL_ACLS & ~ GRANT_ACL))) - global.append(STRING_WITH_LEN("ALL PRIVILEGES")); - else if (!(want_access & ~GRANT_ACL)) - global.append(STRING_WITH_LEN("USAGE")); + my_error(ER_NONEXISTING_GRANT, MYF(0), + username, hostname); + DBUG_RETURN(TRUE); + } + + /* Show granted roles to acl_user */ + if (show_role_grants(thd, username, hostname, acl_user, buff, sizeof(buff))) + goto end; + + /* Add first global access grants */ + if (show_global_privileges(thd, acl_user, FALSE, buff, sizeof(buff))) + goto end; + + /* Add database access */ + if (show_database_privileges(thd, username, hostname, buff, sizeof(buff))) + goto end; + + /* Add table & column access */ + if (show_table_and_column_privileges(thd, username, hostname, buff, sizeof(buff))) + goto end; + + if (show_routine_grants(thd, username, hostname, &proc_priv_hash, + STRING_WITH_LEN("PROCEDURE"), buff, sizeof(buff))) + goto end; + + if (show_routine_grants(thd, username, hostname, &func_priv_hash, + STRING_WITH_LEN("FUNCTION"), buff, sizeof(buff))) + goto end; + + if (show_proxy_grants(thd, username, hostname, buff, sizeof(buff))) + goto end; + } + + if (rolename) + { + acl_role= find_acl_role(rolename); + if (acl_role) + { + /* get a list of all inherited roles */ + traverse_role_graph_down(acl_role, thd, show_grants_callback, NULL); + } else { - bool found=0; - ulong j,test_access= want_access & ~GRANT_ACL; - for (counter=0, j = SELECT_ACL;j <= GLOBAL_ACLS;counter++,j <<= 1) + if (lex_user->user.str == current_role.str) { - if (test_access & j) - { - if (found) - global.append(STRING_WITH_LEN(", ")); - found=1; - global.append(command_array[counter],command_lengths[counter]); - } + mysql_mutex_unlock(&acl_cache->lock); + mysql_rwlock_unlock(&LOCK_grant); + my_error(ER_NONEXISTING_GRANT, MYF(0), + thd->security_ctx->priv_user, + thd->security_ctx->priv_host); + DBUG_RETURN(TRUE); + } + } + } + + error= 0; +end: + mysql_mutex_unlock(&acl_cache->lock); + mysql_rwlock_unlock(&LOCK_grant); + + my_eof(thd); + DBUG_RETURN(error); +} + +static ROLE_GRANT_PAIR *find_role_grant_pair(const LEX_STRING *u, + const LEX_STRING *h, + const LEX_STRING *r) +{ + char buf[1024]; + String pair_key(buf, sizeof(buf), &my_charset_bin); + + size_t key_length= u->length + h->length + r->length + 3; + pair_key.alloc(key_length); + + strmov(strmov(strmov(const_cast<char*>(pair_key.ptr()), + u->str) + 1, h->str) + 1, r->str); + + return (ROLE_GRANT_PAIR *) + my_hash_search(&acl_roles_mappings, (uchar*)pair_key.ptr(), key_length); +} + +static bool show_role_grants(THD *thd, const char *username, + const char *hostname, ACL_USER_BASE *acl_entry, + char *buff, size_t buffsize) +{ + uint counter; + Protocol *protocol= thd->protocol; + LEX_STRING host= {const_cast<char*>(hostname), strlen(hostname)}; + + String grant(buff,sizeof(buff),system_charset_info); + for (counter= 0; counter < acl_entry->role_grants.elements; counter++) + { + grant.length(0); + grant.append(STRING_WITH_LEN("GRANT ")); + ACL_ROLE *acl_role= *(dynamic_element(&acl_entry->role_grants, counter, + ACL_ROLE**)); + grant.append(acl_role->user.str, acl_role->user.length, + system_charset_info); + grant.append(STRING_WITH_LEN(" TO '")); + grant.append(acl_entry->user.str, acl_entry->user.length, + system_charset_info); + if (!(acl_entry->flags & IS_ROLE)) + { + grant.append(STRING_WITH_LEN("'@'")); + grant.append(&host); + } + grant.append('\''); + + ROLE_GRANT_PAIR *pair= + find_role_grant_pair(&acl_entry->user, &host, &acl_role->user); + DBUG_ASSERT(pair); + + if (pair->with_admin) + grant.append(STRING_WITH_LEN(" WITH ADMIN OPTION")); + + protocol->prepare_for_resend(); + protocol->store(grant.ptr(),grant.length(),grant.charset()); + if (protocol->write()) + { + return TRUE; + } + } + return FALSE; +} + +static bool show_global_privileges(THD *thd, ACL_USER_BASE *acl_entry, + bool handle_as_role, + char *buff, size_t buffsize) +{ + uint counter; + ulong want_access; + Protocol *protocol= thd->protocol; + + String global(buff,sizeof(buff),system_charset_info); + global.length(0); + global.append(STRING_WITH_LEN("GRANT ")); + + if (handle_as_role) + want_access= ((ACL_ROLE *)acl_entry)->initial_role_access; + else + want_access= acl_entry->access; + if (test_all_bits(want_access, (GLOBAL_ACLS & ~ GRANT_ACL))) + global.append(STRING_WITH_LEN("ALL PRIVILEGES")); + else if (!(want_access & ~GRANT_ACL)) + global.append(STRING_WITH_LEN("USAGE")); + else + { + bool found=0; + ulong j,test_access= want_access & ~GRANT_ACL; + for (counter=0, j = SELECT_ACL;j <= GLOBAL_ACLS;counter++,j <<= 1) + { + if (test_access & j) + { + if (found) + global.append(STRING_WITH_LEN(", ")); + found=1; + global.append(command_array[counter],command_lengths[counter]); } } - global.append (STRING_WITH_LEN(" ON *.* TO '")); - global.append(lex_user->user.str, lex_user->user.length, - system_charset_info); - global.append (STRING_WITH_LEN("'@'")); - global.append(lex_user->host.str,lex_user->host.length, - system_charset_info); + } + global.append (STRING_WITH_LEN(" ON *.* TO '")); + global.append(acl_entry->user.str, acl_entry->user.length, + system_charset_info); + global.append('\''); + + if (!handle_as_role) + { + ACL_USER *acl_user= (ACL_USER *)acl_entry; + + global.append (STRING_WITH_LEN("@'")); + global.append(acl_user->host.hostname, acl_user->hostname_length, + system_charset_info); global.append ('\''); + if (acl_user->plugin.str == native_password_plugin_name.str || acl_user->plugin.str == old_password_plugin_name.str) { @@ -5381,14 +7648,14 @@ bool mysql_show_grants(THD *thd,LEX_USER *lex_user) } else { - global.append(STRING_WITH_LEN(" IDENTIFIED VIA ")); - global.append(acl_user->plugin.str, acl_user->plugin.length); - if (acl_user->auth_string.length) - { - global.append(STRING_WITH_LEN(" USING '")); - global.append(acl_user->auth_string.str, acl_user->auth_string.length); - global.append('\''); - } + global.append(STRING_WITH_LEN(" IDENTIFIED VIA ")); + global.append(acl_user->plugin.str, acl_user->plugin.length); + if (acl_user->auth_string.length) + { + global.append(STRING_WITH_LEN(" USING '")); + global.append(acl_user->auth_string.str, acl_user->auth_string.length); + global.append('\''); + } } /* "show grants" SSL related stuff */ if (acl_user->ssl_type == SSL_TYPE_ANY) @@ -5401,67 +7668,75 @@ bool mysql_show_grants(THD *thd,LEX_USER *lex_user) global.append(STRING_WITH_LEN(" REQUIRE ")); if (acl_user->x509_issuer) { - ssl_options++; - global.append(STRING_WITH_LEN("ISSUER \'")); - global.append(acl_user->x509_issuer,strlen(acl_user->x509_issuer)); - global.append('\''); + ssl_options++; + global.append(STRING_WITH_LEN("ISSUER \'")); + global.append(acl_user->x509_issuer,strlen(acl_user->x509_issuer)); + global.append('\''); } if (acl_user->x509_subject) { - if (ssl_options++) - global.append(' '); - global.append(STRING_WITH_LEN("SUBJECT \'")); - global.append(acl_user->x509_subject,strlen(acl_user->x509_subject), + if (ssl_options++) + global.append(' '); + global.append(STRING_WITH_LEN("SUBJECT \'")); + global.append(acl_user->x509_subject,strlen(acl_user->x509_subject), system_charset_info); - global.append('\''); + global.append('\''); } if (acl_user->ssl_cipher) { - if (ssl_options++) - global.append(' '); - global.append(STRING_WITH_LEN("CIPHER '")); - global.append(acl_user->ssl_cipher,strlen(acl_user->ssl_cipher), + if (ssl_options++) + global.append(' '); + global.append(STRING_WITH_LEN("CIPHER '")); + global.append(acl_user->ssl_cipher,strlen(acl_user->ssl_cipher), system_charset_info); - global.append('\''); + global.append('\''); } } if ((want_access & GRANT_ACL) || - (acl_user->user_resource.questions || + (acl_user->user_resource.questions || acl_user->user_resource.updates || acl_user->user_resource.conn_per_hour || acl_user->user_resource.user_conn)) { global.append(STRING_WITH_LEN(" WITH")); if (want_access & GRANT_ACL) - global.append(STRING_WITH_LEN(" GRANT OPTION")); + global.append(STRING_WITH_LEN(" GRANT OPTION")); add_user_option(&global, acl_user->user_resource.questions, - "MAX_QUERIES_PER_HOUR", 0); + "MAX_QUERIES_PER_HOUR", 0); add_user_option(&global, acl_user->user_resource.updates, - "MAX_UPDATES_PER_HOUR", 0); + "MAX_UPDATES_PER_HOUR", 0); add_user_option(&global, acl_user->user_resource.conn_per_hour, - "MAX_CONNECTIONS_PER_HOUR", 0); + "MAX_CONNECTIONS_PER_HOUR", 0); add_user_option(&global, acl_user->user_resource.user_conn, - "MAX_USER_CONNECTIONS", 1); - } - protocol->prepare_for_resend(); - protocol->store(global.ptr(),global.length(),global.charset()); - if (protocol->write()) - { - error= -1; - goto end; + "MAX_USER_CONNECTIONS", 1); } } - /* Add database access */ + protocol->prepare_for_resend(); + protocol->store(global.ptr(),global.length(),global.charset()); + if (protocol->write()) + return TRUE; + + return FALSE; + +} + +static bool show_database_privileges(THD *thd, const char *username, + const char *hostname, + char *buff, size_t buffsize) +{ + ACL_DB *acl_db; + ulong want_access; + uint counter; + Protocol *protocol= thd->protocol; + for (counter=0 ; counter < acl_dbs.elements ; counter++) { const char *user, *host; acl_db=dynamic_element(&acl_dbs,counter,ACL_DB*); - if (!(user=acl_db->user)) - user= ""; - if (!(host=acl_db->host.hostname)) - host= ""; + user= safe_str(acl_db->user); + host=acl_db->host.hostname; /* We do not make SHOW GRANTS case-sensitive here (like REVOKE), @@ -5470,68 +7745,84 @@ bool mysql_show_grants(THD *thd,LEX_USER *lex_user) would be wrong from a security point of view. */ - if (!strcmp(lex_user->user.str,user) && - !my_strcasecmp(system_charset_info, lex_user->host.str, host)) + if (!strcmp(username, user) && + !my_strcasecmp(system_charset_info, hostname, host)) { - want_access=acl_db->access; + /* + do not print inherited access bits for roles, + the role bits present in the table are what matters + */ + if (*hostname) // User + want_access=acl_db->access; + else // Role + want_access=acl_db->initial_access; if (want_access) { - String db(buff,sizeof(buff),system_charset_info); - db.length(0); - db.append(STRING_WITH_LEN("GRANT ")); - - if (test_all_bits(want_access,(DB_ACLS & ~GRANT_ACL))) - db.append(STRING_WITH_LEN("ALL PRIVILEGES")); - else if (!(want_access & ~GRANT_ACL)) - db.append(STRING_WITH_LEN("USAGE")); - else - { - int found=0, cnt; - ulong j,test_access= want_access & ~GRANT_ACL; - for (cnt=0, j = SELECT_ACL; j <= DB_ACLS; cnt++,j <<= 1) - { - if (test_access & j) - { - if (found) - db.append(STRING_WITH_LEN(", ")); - found = 1; - db.append(command_array[cnt],command_lengths[cnt]); - } - } - } - db.append (STRING_WITH_LEN(" ON ")); - append_identifier(thd, &db, acl_db->db, strlen(acl_db->db)); - db.append (STRING_WITH_LEN(".* TO '")); - db.append(lex_user->user.str, lex_user->user.length, - system_charset_info); - db.append (STRING_WITH_LEN("'@'")); - // host and lex_user->host are equal except for case - db.append(host, strlen(host), system_charset_info); - db.append ('\''); - if (want_access & GRANT_ACL) - db.append(STRING_WITH_LEN(" WITH GRANT OPTION")); - protocol->prepare_for_resend(); - protocol->store(db.ptr(),db.length(),db.charset()); - if (protocol->write()) - { - error= -1; - goto end; - } + String db(buff,sizeof(buff),system_charset_info); + db.length(0); + db.append(STRING_WITH_LEN("GRANT ")); + + if (test_all_bits(want_access,(DB_ACLS & ~GRANT_ACL))) + db.append(STRING_WITH_LEN("ALL PRIVILEGES")); + else if (!(want_access & ~GRANT_ACL)) + db.append(STRING_WITH_LEN("USAGE")); + else + { + int found=0, cnt; + ulong j,test_access= want_access & ~GRANT_ACL; + for (cnt=0, j = SELECT_ACL; j <= DB_ACLS; cnt++,j <<= 1) + { + if (test_access & j) + { + if (found) + db.append(STRING_WITH_LEN(", ")); + found = 1; + db.append(command_array[cnt],command_lengths[cnt]); + } + } + } + db.append (STRING_WITH_LEN(" ON ")); + append_identifier(thd, &db, acl_db->db, strlen(acl_db->db)); + db.append (STRING_WITH_LEN(".* TO '")); + db.append(username, strlen(username), + system_charset_info); + if (*hostname) + { + db.append (STRING_WITH_LEN("'@'")); + // host and lex_user->host are equal except for case + db.append(host, strlen(host), system_charset_info); + } + db.append ('\''); + if (want_access & GRANT_ACL) + db.append(STRING_WITH_LEN(" WITH GRANT OPTION")); + protocol->prepare_for_resend(); + protocol->store(db.ptr(),db.length(),db.charset()); + if (protocol->write()) + { + return TRUE; + } } } } + return FALSE; + +} + +static bool show_table_and_column_privileges(THD *thd, const char *username, + const char *hostname, + char *buff, size_t buffsize) +{ + uint counter, index; + Protocol *protocol= thd->protocol; - /* Add table & column access */ for (index=0 ; index < column_priv_hash.records ; index++) { const char *user, *host; GRANT_TABLE *grant_table= (GRANT_TABLE*) my_hash_element(&column_priv_hash, index); - if (!(user=grant_table->user)) - user= ""; - if (!(host= grant_table->host.hostname)) - host= ""; + user= safe_str(grant_table->user); + host= grant_table->host.hostname; /* We do not make SHOW GRANTS case-sensitive here (like REVOKE), @@ -5540,132 +7831,126 @@ bool mysql_show_grants(THD *thd,LEX_USER *lex_user) would be wrong from a security point of view. */ - if (!strcmp(lex_user->user.str,user) && - !my_strcasecmp(system_charset_info, lex_user->host.str, host)) + if (!strcmp(username,user) && + !my_strcasecmp(system_charset_info, hostname, host)) { - ulong table_access= grant_table->privs; - if ((table_access | grant_table->cols) != 0) + ulong table_access; + ulong cols_access; + if (*hostname) // User + { + table_access= grant_table->privs; + cols_access= grant_table->cols; + } + else // Role { - String global(buff, sizeof(buff), system_charset_info); - ulong test_access= (table_access | grant_table->cols) & ~GRANT_ACL; + table_access= grant_table->init_privs; + cols_access= grant_table->init_cols; + } - global.length(0); - global.append(STRING_WITH_LEN("GRANT ")); + if ((table_access | cols_access) != 0) + { + String global(buff, sizeof(buff), system_charset_info); + ulong test_access= (table_access | cols_access) & ~GRANT_ACL; - if (test_all_bits(table_access, (TABLE_ACLS & ~GRANT_ACL))) - global.append(STRING_WITH_LEN("ALL PRIVILEGES")); - else if (!test_access) - global.append(STRING_WITH_LEN("USAGE")); - else - { + global.length(0); + global.append(STRING_WITH_LEN("GRANT ")); + + if (test_all_bits(table_access, (TABLE_ACLS & ~GRANT_ACL))) + global.append(STRING_WITH_LEN("ALL PRIVILEGES")); + else if (!test_access) + global.append(STRING_WITH_LEN("USAGE")); + else + { /* Add specific column access */ - int found= 0; - ulong j; + int found= 0; + ulong j; - for (counter= 0, j= SELECT_ACL; j <= TABLE_ACLS; counter++, j<<= 1) - { - if (test_access & j) - { - if (found) - global.append(STRING_WITH_LEN(", ")); - found= 1; - global.append(command_array[counter],command_lengths[counter]); + for (counter= 0, j= SELECT_ACL; j <= TABLE_ACLS; counter++, j<<= 1) + { + if (test_access & j) + { + if (found) + global.append(STRING_WITH_LEN(", ")); + found= 1; + global.append(command_array[counter],command_lengths[counter]); - if (grant_table->cols) - { - uint found_col= 0; - for (uint col_index=0 ; - col_index < grant_table->hash_columns.records ; - col_index++) - { - GRANT_COLUMN *grant_column = (GRANT_COLUMN*) - my_hash_element(&grant_table->hash_columns,col_index); - if (grant_column->rights & j) - { - if (!found_col) - { - found_col= 1; - /* - If we have a duplicated table level privilege, we - must write the access privilege name again. - */ - if (table_access & j) - { - global.append(STRING_WITH_LEN(", ")); - global.append(command_array[counter], - command_lengths[counter]); - } - global.append(STRING_WITH_LEN(" (")); - } - else - global.append(STRING_WITH_LEN(", ")); - global.append(grant_column->column, - grant_column->key_length, - system_charset_info); - } - } - if (found_col) - global.append(')'); - } - } - } - } - global.append(STRING_WITH_LEN(" ON ")); - append_identifier(thd, &global, grant_table->db, - strlen(grant_table->db)); - global.append('.'); - append_identifier(thd, &global, grant_table->tname, - strlen(grant_table->tname)); - global.append(STRING_WITH_LEN(" TO '")); - global.append(lex_user->user.str, lex_user->user.length, - system_charset_info); - global.append(STRING_WITH_LEN("'@'")); - // host and lex_user->host are equal except for case - global.append(host, strlen(host), system_charset_info); - global.append('\''); - if (table_access & GRANT_ACL) - global.append(STRING_WITH_LEN(" WITH GRANT OPTION")); - protocol->prepare_for_resend(); - protocol->store(global.ptr(),global.length(),global.charset()); - if (protocol->write()) - { - error= -1; - break; - } + if (grant_table->cols) + { + uint found_col= 0; + HASH *hash_columns; + hash_columns= &grant_table->hash_columns; + + for (uint col_index=0 ; + col_index < hash_columns->records ; + col_index++) + { + GRANT_COLUMN *grant_column = (GRANT_COLUMN*) + my_hash_element(hash_columns,col_index); + if (j & (*hostname ? grant_column->rights // User + : grant_column->init_rights)) // Role + { + if (!found_col) + { + found_col= 1; + /* + If we have a duplicated table level privilege, we + must write the access privilege name again. + */ + if (table_access & j) + { + global.append(STRING_WITH_LEN(", ")); + global.append(command_array[counter], + command_lengths[counter]); + } + global.append(STRING_WITH_LEN(" (")); + } + else + global.append(STRING_WITH_LEN(", ")); + global.append(grant_column->column, + grant_column->key_length, + system_charset_info); + } + } + if (found_col) + global.append(')'); + } + } + } + } + global.append(STRING_WITH_LEN(" ON ")); + append_identifier(thd, &global, grant_table->db, + strlen(grant_table->db)); + global.append('.'); + append_identifier(thd, &global, grant_table->tname, + strlen(grant_table->tname)); + global.append(STRING_WITH_LEN(" TO '")); + global.append(username, strlen(username), + system_charset_info); + if (*hostname) + { + global.append(STRING_WITH_LEN("'@'")); + // host and lex_user->host are equal except for case + global.append(host, strlen(host), system_charset_info); + } + global.append('\''); + if (table_access & GRANT_ACL) + global.append(STRING_WITH_LEN(" WITH GRANT OPTION")); + protocol->prepare_for_resend(); + protocol->store(global.ptr(),global.length(),global.charset()); + if (protocol->write()) + { + return TRUE; + } } } } + return FALSE; - if (show_routine_grants(thd, lex_user, &proc_priv_hash, - STRING_WITH_LEN("PROCEDURE"), buff, sizeof(buff))) - { - error= -1; - goto end; - } - - if (show_routine_grants(thd, lex_user, &func_priv_hash, - STRING_WITH_LEN("FUNCTION"), buff, sizeof(buff))) - { - error= -1; - goto end; - } - - if (show_proxy_grants(thd, lex_user, buff, sizeof(buff))) - { - error= -1; - goto end; - } - -end: - mysql_mutex_unlock(&acl_cache->lock); - mysql_rwlock_unlock(&LOCK_grant); - - my_eof(thd); - DBUG_RETURN(error); } -static int show_routine_grants(THD* thd, LEX_USER *lex_user, HASH *hash, - const char *type, int typelen, +static int show_routine_grants(THD* thd, + const char *username, const char *hostname, + HASH *hash, const char *type, int typelen, char *buff, int buffsize) { uint counter, index; @@ -5677,10 +7962,8 @@ static int show_routine_grants(THD* thd, LEX_USER *lex_user, HASH *hash, const char *user, *host; GRANT_NAME *grant_proc= (GRANT_NAME*) my_hash_element(hash, index); - if (!(user=grant_proc->user)) - user= ""; - if (!(host= grant_proc->host.hostname)) - host= ""; + user= safe_str(grant_proc->user); + host= grant_proc->host.hostname; /* We do not make SHOW GRANTS case-sensitive here (like REVOKE), @@ -5689,10 +7972,15 @@ static int show_routine_grants(THD* thd, LEX_USER *lex_user, HASH *hash, would be wrong from a security point of view. */ - if (!strcmp(lex_user->user.str,user) && - !my_strcasecmp(system_charset_info, lex_user->host.str, host)) + if (!strcmp(username, user) && + !my_strcasecmp(system_charset_info, hostname, host)) { - ulong proc_access= grant_proc->privs; + ulong proc_access; + if (*hostname) // User + proc_access= grant_proc->privs; + else // Role + proc_access= grant_proc->init_privs; + if (proc_access != 0) { String global(buff, buffsize, system_charset_info); @@ -5729,11 +8017,14 @@ static int show_routine_grants(THD* thd, LEX_USER *lex_user, HASH *hash, append_identifier(thd, &global, grant_proc->tname, strlen(grant_proc->tname)); global.append(STRING_WITH_LEN(" TO '")); - global.append(lex_user->user.str, lex_user->user.length, + global.append(username, strlen(username), system_charset_info); - global.append(STRING_WITH_LEN("'@'")); - // host and lex_user->host are equal except for case - global.append(host, strlen(host), system_charset_info); + if (*hostname) + { + global.append(STRING_WITH_LEN("'@'")); + // host and lex_user->host are equal except for case + global.append(host, strlen(host), system_charset_info); + } global.append('\''); if (proc_access & GRANT_ACL) global.append(STRING_WITH_LEN(" WITH GRANT OPTION")); @@ -5750,6 +8041,7 @@ static int show_routine_grants(THD* thd, LEX_USER *lex_user, HASH *hash, return error; } + /* Make a clear-text version of the requested privilege. */ @@ -5784,7 +8076,7 @@ void get_mqh(const char *user, const char *host, USER_CONN *uc) mysql_mutex_lock(&acl_cache->lock); - if (initialized && (acl_user= find_acl_user(host,user, FALSE))) + if (initialized && (acl_user= find_user_wild(host,user))) uc->user_resources= acl_user->user_resource; else bzero((char*) &uc->user_resources, sizeof(uc->user_resources)); @@ -5798,7 +8090,7 @@ void get_mqh(const char *user, const char *host, USER_CONN *uc) SYNOPSIS open_grant_tables() thd The current thread. - tables (out) The 4 elements array for the opened tables. + tables (out) The 7 elements array for the opened tables. DESCRIPTION Tables are numbered as follows: @@ -5806,6 +8098,9 @@ void get_mqh(const char *user, const char *host, USER_CONN *uc) 1 db 2 tables_priv 3 columns_priv + 4 procs_priv + 5 proxies_priv + 6 roles_mapping RETURN 1 Skip GRANT handling during replication. @@ -5813,9 +8108,10 @@ void get_mqh(const char *user, const char *host, USER_CONN *uc) < 0 Error. */ -#define GRANT_TABLES 6 -int open_grant_tables(THD *thd, TABLE_LIST *tables) +#define GRANT_TABLES 7 +static int open_grant_tables(THD *thd, TABLE_LIST *tables) { + Rpl_filter *rpl_filter= thd->rpl_filter; DBUG_ENTER("open_grant_tables"); if (!initialized) @@ -5840,13 +8136,19 @@ int open_grant_tables(THD *thd, TABLE_LIST *tables) (tables+5)->init_one_table(C_STRING_WITH_LEN("mysql"), C_STRING_WITH_LEN("proxies_priv"), "proxies_priv", TL_WRITE); - tables[5].open_strategy= TABLE_LIST::OPEN_IF_EXISTS; + (tables+5)->open_strategy= TABLE_LIST::OPEN_IF_EXISTS; + (tables+6)->init_one_table(C_STRING_WITH_LEN("mysql"), + C_STRING_WITH_LEN("roles_mapping"), + "roles_mapping", TL_WRITE); + (tables+6)->open_strategy= TABLE_LIST::OPEN_IF_EXISTS; + tables->next_local= tables->next_global= tables + 1; (tables+1)->next_local= (tables+1)->next_global= tables + 2; (tables+2)->next_local= (tables+2)->next_global= tables + 3; (tables+3)->next_local= (tables+3)->next_global= tables + 4; (tables+4)->next_local= (tables+4)->next_global= tables + 5; + (tables+5)->next_local= (tables+5)->next_global= tables + 6; #ifdef HAVE_REPLICATION /* @@ -5860,11 +8162,13 @@ int open_grant_tables(THD *thd, TABLE_LIST *tables) account in tests. */ tables[0].updating= tables[1].updating= tables[2].updating= - tables[3].updating= tables[4].updating= tables[5].updating= 1; + tables[3].updating= tables[4].updating= tables[5].updating= + tables[6].updating= 1; if (!(thd->spcont || rpl_filter->tables_ok(0, tables))) DBUG_RETURN(1); tables[0].updating= tables[1].updating= tables[2].updating= - tables[3].updating= tables[4].updating= tables[5].updating= 0; + tables[3].updating= tables[4].updating= tables[5].updating= + tables[6].updating= 0; } #endif @@ -5876,8 +8180,7 @@ int open_grant_tables(THD *thd, TABLE_LIST *tables) DBUG_RETURN(0); } -ACL_USER *check_acl_user(LEX_USER *user_name, - uint *acl_acl_userdx) +ACL_USER *check_acl_user(LEX_USER *user_name, uint *acl_acl_userdx) { ACL_USER *acl_user= 0; uint counter; @@ -5886,14 +8189,8 @@ ACL_USER *check_acl_user(LEX_USER *user_name, for (counter= 0 ; counter < acl_users.elements ; counter++) { - const char *user,*host; acl_user= dynamic_element(&acl_users, counter, ACL_USER*); - if (!(user=acl_user->user)) - user= ""; - if (!(host=acl_user->host.hostname)) - host= ""; - if (!strcmp(user_name->user.str,user) && - !my_strcasecmp(system_charset_info, user_name->host.str, host)) + if(acl_user->eq(user_name->user.str, user_name->host.str)) break; } if (counter == acl_users.elements) @@ -5937,7 +8234,7 @@ static int modify_grant_table(TABLE *table, Field *host_field, system_charset_info); user_field->store(user_to->user.str, user_to->user.length, system_charset_info); - if ((error= table->file->ha_update_row(table->record[1], + if ((error= table->file->ha_update_row(table->record[1], table->record[0])) && error != HA_ERR_RECORD_IS_THE_SAME) table->file->print_error(error, MYF(0)); @@ -5955,6 +8252,89 @@ static int modify_grant_table(TABLE *table, Field *host_field, } /* + Handle the roles_mappings privilege table +*/ +static int handle_roles_mappings_table(TABLE *table, bool drop, + LEX_USER *user_from, LEX_USER *user_to) +{ + /* + All entries (Host, User) that match user_from will be renamed, + as well as all Role entries that match if user_from.host.str == "" + + Otherwise, only matching (Host, User) will be renamed. + */ + DBUG_ENTER("handle_roles_mappings_table"); + + int error; + int result= 0; + THD *thd= current_thd; + const char *host, *user, *role; + Field *host_field= table->field[0]; + Field *user_field= table->field[1]; + Field *role_field= table->field[2]; + + DBUG_PRINT("info", ("Rewriting entry in roles_mappings table: %s@%s", + user_from->user.str, user_from->host.str)); + table->use_all_columns(); + if ((error= table->file->ha_rnd_init(1))) + { + table->file->print_error(error, MYF(0)); + result= -1; + } + else + { + while((error= table->file->ha_rnd_next(table->record[0])) != + HA_ERR_END_OF_FILE) + { + if (error) + { + DBUG_PRINT("info", ("scan error: %d", error)); + continue; + } + + host= safe_str(get_field(thd->mem_root, host_field)); + user= safe_str(get_field(thd->mem_root, user_field)); + + if (!(strcmp(user_from->user.str, user) || + my_strcasecmp(system_charset_info, user_from->host.str, host))) + result= ((drop || user_to) && + modify_grant_table(table, host_field, user_field, user_to)) ? + -1 : result ? result : 1; /* Error or keep result or found. */ + else + { + role= safe_str(get_field(thd->mem_root, role_field)); + + if (strcmp(user_from->user.str, role)) + continue; + + error= 0; + + if (drop) /* drop if requested */ + { + if ((error= table->file->ha_delete_row(table->record[0]))) + table->file->print_error(error, MYF(0)); + } + else if (user_to) + { + store_record(table, record[1]); + role_field->store(user_to->user.str, user_to->user.length, + system_charset_info); + if ((error= table->file->ha_update_row(table->record[1], + table->record[0])) && + error != HA_ERR_RECORD_IS_THE_SAME) + table->file->print_error(error, MYF(0)); + } + + /* Error or keep result or found. */ + result= error ? -1 : result ? result : 1; + } + } + table->file->ha_rnd_end(); + } + DBUG_RETURN(result); +} + +/* Handle a privilege table. SYNOPSIS @@ -5979,6 +8359,8 @@ static int modify_grant_table(TABLE *table, Field *host_field, 2 tables_priv 3 columns_priv 4 procs_priv + 5 proxies_priv + 6 roles_mapping RETURN > 0 At least one record matched. @@ -5994,8 +8376,8 @@ static int handle_grant_table(TABLE_LIST *tables, uint table_no, bool drop, TABLE *table= tables[table_no].table; Field *host_field= table->field[0]; Field *user_field= table->field[table_no && table_no != 5 ? 2 : 1]; - char *host_str= user_from->host.str; - char *user_str= user_from->user.str; + const char *host_str= user_from->host.str; + const char *user_str= user_from->user.str; const char *host; const char *user; uchar user_key[MAX_KEY_LENGTH]; @@ -6003,6 +8385,12 @@ static int handle_grant_table(TABLE_LIST *tables, uint table_no, bool drop, DBUG_ENTER("handle_grant_table"); THD *thd= current_thd; + if (table_no == 6) + { + result= handle_roles_mappings_table(table, drop, user_from, user_to); + DBUG_RETURN(result); + } + table->use_all_columns(); if (! table_no) // mysql.user table { @@ -6024,9 +8412,15 @@ static int handle_grant_table(TABLE_LIST *tables, uint table_no, bool drop, table->key_info->key_part[1].store_length); key_copy(user_key, table->record[0], table->key_info, key_prefix_length); - if ((error= table->file->ha_index_read_idx_map(table->record[0], 0, - user_key, (key_part_map)3, - HA_READ_KEY_EXACT))) + error= table->file->ha_index_read_idx_map(table->record[0], 0, + user_key, (key_part_map)3, + HA_READ_KEY_EXACT); + if (!error && !*host_str) + { // verify that we got a role or a user, as needed + if (check_is_role(table) != user_from->is_role()) + error= HA_ERR_KEY_NOT_FOUND; + } + if (error) { if (error != HA_ERR_KEY_NOT_FOUND && error != HA_ERR_END_OF_FILE) { @@ -6061,7 +8455,7 @@ static int handle_grant_table(TABLE_LIST *tables, uint table_no, bool drop, DBUG_PRINT("info",("scan table: '%s' search: '%s'@'%s'", table->s->table_name.str, user_str, host_str)); #endif - while ((error= table->file->ha_rnd_next(table->record[0])) != + while ((error= table->file->ha_rnd_next(table->record[0])) != HA_ERR_END_OF_FILE) { if (error) @@ -6070,10 +8464,8 @@ static int handle_grant_table(TABLE_LIST *tables, uint table_no, bool drop, DBUG_PRINT("info",("scan error: %d", error)); continue; } - if (! (host= get_field(thd->mem_root, host_field))) - host= ""; - if (! (user= get_field(thd->mem_root, user_field))) - user= ""; + host= safe_str(get_field(thd->mem_root, host_field)); + user= safe_str(get_field(thd->mem_root, user_field)); #ifdef EXTRA_DEBUG if (table_no != 5) @@ -6110,7 +8502,7 @@ static int handle_grant_table(TABLE_LIST *tables, uint table_no, bool drop, /** Handle an in-memory privilege structure. - @param struct_no The number of the structure to handle (0..5). + @param struct_no The number of the structure to handle (0..6). @param drop If user_from is to be dropped. @param user_from The the user to be searched/dropped/renamed. @param user_to The new name for the user if to be renamed, NULL otherwise. @@ -6121,13 +8513,6 @@ static int handle_grant_table(TABLE_LIST *tables, uint table_no, bool drop, Delete from grant structure if drop is true. Update in grant structure if drop is false and user_to is not NULL. Search in grant structure if drop is false and user_to is NULL. - Structures are enumerated as follows: - 0 ACL_USER - 1 ACL_DB - 2 COLUMN_PRIVILEGES_HASH - 3 PROC_PRIVILEGES_HASH - 4 FUNC_PRIVILEGES_HASH - 5 PROXY_USERS_ACL @retval > 0 At least one element matched. @retval 0 OK, but no element matched. @@ -6139,22 +8524,76 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, int result= 0; int idx; int elements; - const char *user; - const char *host; + const char *UNINIT_VAR(user); + const char *UNINIT_VAR(host); + const char *UNINIT_VAR(role); + uint role_not_matched= 1; ACL_USER *acl_user= NULL; + ACL_ROLE *acl_role= NULL; ACL_DB *acl_db= NULL; ACL_PROXY_USER *acl_proxy_user= NULL; GRANT_NAME *grant_name= NULL; + ROLE_GRANT_PAIR *UNINIT_VAR(role_grant_pair); HASH *grant_name_hash= NULL; + HASH *roles_mappings_hash= NULL; DBUG_ENTER("handle_grant_struct"); DBUG_PRINT("info",("scan struct: %u search: '%s'@'%s'", struct_no, user_from->user.str, user_from->host.str)); - LINT_INIT(user); - LINT_INIT(host); - mysql_mutex_assert_owner(&acl_cache->lock); + /* No point in querying ROLE ACL if user_from is not a role */ + if (struct_no == ROLE_ACL && user_from->host.length) + DBUG_RETURN(0); + + /* same. no roles in PROXY_USERS_ACL */ + if (struct_no == PROXY_USERS_ACL && user_from->is_role()) + DBUG_RETURN(0); + + if (struct_no == ROLE_ACL) //no need to scan the structures in this case + { + acl_role= find_acl_role(user_from->user.str); + if (!acl_role) + DBUG_RETURN(0); + + if (!drop && !user_to) //role was found + DBUG_RETURN(1); + + /* this calls for a role update */ + char *old_key= acl_role->user.str; + size_t old_key_length= acl_role->user.length; + if (drop) + { + /* all grants must be revoked from this role by now. propagate this */ + propagate_role_grants(acl_role, PRIVS_TO_MERGE::ALL, 0, 0); + + // delete the role from cross-reference arrays + for (uint i=0; i < acl_role->role_grants.elements; i++) + { + ACL_ROLE *grant= *dynamic_element(&acl_role->role_grants, + i, ACL_ROLE**); + remove_ptr_from_dynarray(&grant->parent_grantee, acl_role); + } + + for (uint i=0; i < acl_role->parent_grantee.elements; i++) + { + ACL_USER_BASE *grantee= *dynamic_element(&acl_role->parent_grantee, + i, ACL_USER_BASE**); + remove_ptr_from_dynarray(&grantee->role_grants, acl_role); + } + + my_hash_delete(&acl_roles, (uchar*) acl_role); + DBUG_RETURN(1); + } + acl_role->user.str= strdup_root(&mem, user_to->user.str); + acl_role->user.length= user_to->user.length; + + my_hash_update(&acl_roles, (uchar*) acl_role, (uchar*) old_key, + old_key_length); + DBUG_RETURN(1); + + } + /* Get the number of elements in the in-memory structure. */ switch (struct_no) { case USER_ACL: @@ -6178,9 +8617,13 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, case PROXY_USERS_ACL: elements= acl_proxy_users.elements; break; + case ROLES_MAPPINGS_HASH: + roles_mappings_hash= &acl_roles_mappings; + elements= roles_mappings_hash->records; + break; default: DBUG_ASSERT(0); - return -1; + DBUG_RETURN(-1); } #ifdef EXTRA_DEBUG @@ -6196,7 +8639,7 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, switch (struct_no) { case USER_ACL: acl_user= dynamic_element(&acl_users, idx, ACL_USER*); - user= acl_user->user; + user= acl_user->user.str; host= acl_user->host.hostname; break; @@ -6220,6 +8663,13 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, host= acl_proxy_user->get_host(); break; + case ROLES_MAPPINGS_HASH: + role_grant_pair= (ROLE_GRANT_PAIR *) my_hash_element(roles_mappings_hash, idx); + user= role_grant_pair->u_uname; + host= role_grant_pair->u_hname; + role= role_grant_pair->r_uname; + break; + default: DBUG_ASSERT(0); } @@ -6227,14 +8677,28 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, user= ""; if (! host) host= ""; + if (! role) + role= ""; #ifdef EXTRA_DEBUG DBUG_PRINT("loop",("scan struct: %u index: %u user: '%s' host: '%s'", struct_no, idx, user, host)); #endif - if (strcmp(user_from->user.str, user) || - my_strcasecmp(system_charset_info, user_from->host.str, host)) + + if (struct_no == ROLES_MAPPINGS_HASH) + { + role_not_matched= strcmp(user_from->user.str, role); + if (role_not_matched && + (strcmp(user_from->user.str, user) || + my_strcasecmp(system_charset_info, user_from->host.str, host))) continue; + } + else + { + if (strcmp(user_from->user.str, user) || + my_strcasecmp(system_charset_info, user_from->host.str, host)) + continue; + } result= 1; /* At least one element found. */ if ( drop ) @@ -6242,6 +8706,7 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, elements--; switch ( struct_no ) { case USER_ACL: + free_acl_user(dynamic_element(&acl_users, idx, ACL_USER*)); delete_dynamic_element(&acl_users, idx); break; @@ -6268,13 +8733,21 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, delete_dynamic_element(&acl_proxy_users, idx); break; + case ROLES_MAPPINGS_HASH: + my_hash_delete(roles_mappings_hash, (uchar*) role_grant_pair); + break; + + default: + DBUG_ASSERT(0); + break; } } else if ( user_to ) { switch ( struct_no ) { case USER_ACL: - acl_user->user= strdup_root(&mem, user_to->user.str); + acl_user->user.str= strdup_root(&mem, user_to->user.str); + acl_user->user.length= user_to->user.length; acl_user->host.hostname= strdup_root(&mem, user_to->host.str); break; @@ -6288,7 +8761,7 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, case FUNC_PRIVILEGES_HASH: { /* - Save old hash key and its length to be able properly update + Save old hash key and its length to be able to properly update element position in hash. */ char *old_key= grant_name->hash_key; @@ -6325,7 +8798,38 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, acl_proxy_user->set_user (&mem, user_to->user.str); acl_proxy_user->set_host (&mem, user_to->host.str); break; + + case ROLES_MAPPINGS_HASH: + { + /* + Save old hash key and its length to be able to properly update + element position in hash. + */ + char *old_key= role_grant_pair->hashkey.str; + size_t old_key_length= role_grant_pair->hashkey.length; + bool oom; + + if (role_not_matched) + oom= role_grant_pair->init(&mem, user_to->user.str, + user_to->host.str, + role_grant_pair->r_uname, false); + else + oom= role_grant_pair->init(&mem, role_grant_pair->u_uname, + role_grant_pair->u_hname, + user_to->user.str, false); + if (oom) + DBUG_RETURN(-1); + + my_hash_update(roles_mappings_hash, (uchar*) role_grant_pair, + (uchar*) old_key, old_key_length); + break; + } + + default: + DBUG_ASSERT(0); + break; } + } else { @@ -6370,24 +8874,21 @@ static int handle_grant_data(TABLE_LIST *tables, bool drop, { int result= 0; int found; + bool handle_as_role= user_from->is_role(); + bool search_only= !drop && !user_to; DBUG_ENTER("handle_grant_data"); - /* Handle user table. */ - if ((found= handle_grant_table(tables, 0, drop, user_from, user_to)) < 0) - { - /* Handle of table failed, don't touch the in-memory array. */ - result= -1; - } - else + if (user_to) + DBUG_ASSERT(handle_as_role == user_to->is_role()); + + if (search_only) { - /* Handle user array. */ - if ((handle_grant_struct(USER_ACL, drop, user_from, user_to)) || found) - { - result= 1; /* At least one record/element found. */ - /* If search is requested, we do not need to search further. */ - if (! drop && ! user_to) - goto end; - } + /* quickly search in-memory structures first */ + if (handle_as_role && find_acl_role(user_from->user.str)) + DBUG_RETURN(1); // found + + if (!handle_as_role && find_user_exact(user_from->host.str, user_from->user.str)) + DBUG_RETURN(1); // found } /* Handle db table. */ @@ -6399,13 +8900,14 @@ static int handle_grant_data(TABLE_LIST *tables, bool drop, else { /* Handle db array. */ - if (((handle_grant_struct(DB_ACL, drop, user_from, user_to) && ! result) || - found) && ! result) + if ((handle_grant_struct(DB_ACL, drop, user_from, user_to) || found) + && ! result) { result= 1; /* At least one record/element found. */ /* If search is requested, we do not need to search further. */ - if (! drop && ! user_to) + if (search_only) goto end; + acl_cache->clear(1); } } @@ -6418,21 +8920,21 @@ static int handle_grant_data(TABLE_LIST *tables, bool drop, else { /* Handle procs array. */ - if (((handle_grant_struct(PROC_PRIVILEGES_HASH, drop, user_from, user_to) && ! result) || - found) && ! result) + if ((handle_grant_struct(PROC_PRIVILEGES_HASH, drop, user_from, user_to) || found) + && ! result) { result= 1; /* At least one record/element found. */ /* If search is requested, we do not need to search further. */ - if (! drop && ! user_to) + if (search_only) goto end; } /* Handle funcs array. */ - if (((handle_grant_struct(FUNC_PRIVILEGES_HASH, drop, user_from, user_to) && ! result) || - found) && ! result) + if ((handle_grant_struct(FUNC_PRIVILEGES_HASH, drop, user_from, user_to) || found) + && ! result) { result= 1; /* At least one record/element found. */ /* If search is requested, we do not need to search further. */ - if (! drop && ! user_to) + if (search_only) goto end; } } @@ -6449,7 +8951,7 @@ static int handle_grant_data(TABLE_LIST *tables, bool drop, { result= 1; /* At least one record found. */ /* If search is requested, we do not need to search further. */ - if (! drop && ! user_to) + if (search_only) goto end; } @@ -6462,9 +8964,11 @@ static int handle_grant_data(TABLE_LIST *tables, bool drop, else { /* Handle columns hash. */ - if (((handle_grant_struct(COLUMN_PRIVILEGES_HASH, drop, user_from, user_to) && ! result) || - found) && ! result) + if ((handle_grant_struct(COLUMN_PRIVILEGES_HASH, drop, user_from, user_to) || found) + && ! result) result= 1; /* At least one record/element found. */ + if (search_only) + goto end; } } @@ -6479,27 +8983,74 @@ static int handle_grant_data(TABLE_LIST *tables, bool drop, else { /* Handle proxies_priv array. */ - if ((handle_grant_struct(PROXY_USERS_ACL, drop, user_from, user_to) && !result) || - found) + if ((handle_grant_struct(PROXY_USERS_ACL, drop, user_from, user_to) || found) + && ! result) result= 1; /* At least one record/element found. */ + if (search_only) + goto end; + } + } + + /* Handle roles_mappings table. */ + if (tables[6].table) + { + if ((found= handle_grant_table(tables, 6, drop, user_from, user_to)) < 0) + { + /* Handle of table failed, don't touch the in-memory array. */ + result= -1; + } + else + { + /* Handle acl_roles_mappings array */ + if ((handle_grant_struct(ROLES_MAPPINGS_HASH, drop, user_from, user_to) || found) + && ! result) + result= 1; /* At least one record/element found */ + if (search_only) + goto end; } } - end: + + /* Handle user table. */ + if ((found= handle_grant_table(tables, 0, drop, user_from, user_to)) < 0) + { + /* Handle of table failed, don't touch the in-memory array. */ + result= -1; + } + else + { + enum enum_acl_lists what= handle_as_role ? ROLE_ACL : USER_ACL; + if (((handle_grant_struct(what, drop, user_from, user_to)) || found) && !result) + { + result= 1; /* At least one record/element found. */ + DBUG_ASSERT(! search_only); + } + } + +end: DBUG_RETURN(result); } - static void append_user(String *str, LEX_USER *user) { if (str->length()) str->append(','); str->append('\''); str->append(user->user.str); - str->append(STRING_WITH_LEN("'@'")); - str->append(user->host.str); + /* hostname part is not relevant for roles, it is always empty */ + if (!user->is_role()) + { + str->append(STRING_WITH_LEN("'@'")); + str->append(user->host.str); + } str->append('\''); } +static void append_str(String *str, const char *s, size_t l) +{ + if (str->length()) + str->append(','); + str->append(s, l); +} /* Create a list of users. @@ -6508,59 +9059,68 @@ static void append_user(String *str, LEX_USER *user) mysql_create_user() thd The current thread. list The users to create. + handle_as_role Handle the user list as roles if true RETURN FALSE OK. TRUE Error. */ -bool mysql_create_user(THD *thd, List <LEX_USER> &list) +bool mysql_create_user(THD *thd, List <LEX_USER> &list, bool handle_as_role) { int result; String wrong_users; - LEX_USER *user_name, *tmp_user_name; + LEX_USER *user_name; List_iterator <LEX_USER> user_list(list); TABLE_LIST tables[GRANT_TABLES]; bool some_users_created= FALSE; - bool save_binlog_row_based; DBUG_ENTER("mysql_create_user"); + DBUG_PRINT("entry", ("Handle as %s", handle_as_role ? "role" : "user")); - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); + if (handle_as_role && sp_process_definer(thd)) + DBUG_RETURN(TRUE); /* CREATE USER may be skipped on replication client. */ if ((result= open_grant_tables(thd, tables))) - { - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result != 1); - } mysql_rwlock_wrlock(&LOCK_grant); mysql_mutex_lock(&acl_cache->lock); - while ((tmp_user_name= user_list++)) + while ((user_name= user_list++)) { - if (!(user_name= get_current_user(thd, tmp_user_name))) + if (user_name->user.str == current_user.str) { + append_str(&wrong_users, STRING_WITH_LEN("CURRENT_USER")); + result= TRUE; + continue; + } + + if (user_name->user.str == current_role.str) + { + append_str(&wrong_users, STRING_WITH_LEN("CURRENT_ROLE")); + result= TRUE; + continue; + } + + if (handle_as_role && is_invalid_role_name(user_name->user.str)) + { + append_user(&wrong_users, user_name); result= TRUE; continue; } + if (!user_name->host.str) + user_name->host= host_not_specified; + /* Search all in-memory structures and grant tables - for a mention of the new user name. + for a mention of the new user/role name. */ if (handle_grant_data(tables, 0, user_name, NULL)) { append_user(&wrong_users, user_name); + result= TRUE; continue; } @@ -6570,26 +9130,51 @@ bool mysql_create_user(THD *thd, List <LEX_USER> &list) { append_user(&wrong_users, user_name); result= TRUE; + continue; + } + + // every created role is automatically granted to its creator-admin + if (handle_as_role) + { + ACL_USER_BASE *grantee= find_acl_user_base(thd->lex->definer->user.str, + thd->lex->definer->host.str); + ACL_ROLE *role= find_acl_role(user_name->user.str); + + /* + just like with routines, views, triggers, and events we allow + non-existant definers here with a warning (see sp_process_definer()) + */ + if (grantee) + add_role_user_mapping(grantee, role); + + if (replace_roles_mapping_table(tables[6].table, + &thd->lex->definer->user, + &thd->lex->definer->host, + &user_name->user, true, + NULL, false)) + { + append_user(&wrong_users, user_name); + if (grantee) + undo_add_role_user_mapping(grantee, role); + result= TRUE; + } } } mysql_mutex_unlock(&acl_cache->lock); if (result) - my_error(ER_CANNOT_USER, MYF(0), "CREATE USER", wrong_users.c_ptr_safe()); + my_error(ER_CANNOT_USER, MYF(0), + (handle_as_role) ? "CREATE ROLE" : "CREATE USER", + wrong_users.c_ptr_safe()); if (some_users_created) result |= write_bin_log(thd, FALSE, thd->query(), thd->query_length()); mysql_rwlock_unlock(&LOCK_grant); - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result); } - /* Drop a list of users and all their privileges. @@ -6603,7 +9188,7 @@ bool mysql_create_user(THD *thd, List <LEX_USER> &list) TRUE Error. */ -bool mysql_drop_user(THD *thd, List <LEX_USER> &list) +bool mysql_drop_user(THD *thd, List <LEX_USER> &list, bool handle_as_role) { int result; String wrong_users; @@ -6612,26 +9197,12 @@ bool mysql_drop_user(THD *thd, List <LEX_USER> &list) TABLE_LIST tables[GRANT_TABLES]; bool some_users_deleted= FALSE; ulonglong old_sql_mode= thd->variables.sql_mode; - bool save_binlog_row_based; DBUG_ENTER("mysql_drop_user"); - - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); + DBUG_PRINT("entry", ("Handle as %s", handle_as_role ? "role" : "user")); /* DROP USER may be skipped on replication client. */ if ((result= open_grant_tables(thd, tables))) - { - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result != 1); - } thd->variables.sql_mode&= ~MODE_PAD_CHAR_TO_FULL_LENGTH; @@ -6640,41 +9211,59 @@ bool mysql_drop_user(THD *thd, List <LEX_USER> &list) while ((tmp_user_name= user_list++)) { - if (!(user_name= get_current_user(thd, tmp_user_name))) + user_name= get_current_user(thd, tmp_user_name, false); + if (!user_name) { + thd->clear_error(); + append_str(&wrong_users, STRING_WITH_LEN("CURRENT_ROLE")); result= TRUE; continue; - } + } + + if (handle_as_role != user_name->is_role()) + { + append_user(&wrong_users, tmp_user_name); + result= TRUE; + continue; + } + if (handle_grant_data(tables, 1, user_name, NULL) <= 0) { append_user(&wrong_users, user_name); result= TRUE; continue; } + some_users_deleted= TRUE; } - /* Rebuild 'acl_check_hosts' since 'acl_users' has been modified */ - rebuild_check_host(); + if (!handle_as_role) + { + /* Rebuild 'acl_check_hosts' since 'acl_users' has been modified */ + rebuild_check_host(); + + /* + Rebuild every user's role_grants since 'acl_users' has been sorted + and old pointers to ACL_USER elements are no longer valid + */ + rebuild_role_grants(); + } mysql_mutex_unlock(&acl_cache->lock); if (result) - my_error(ER_CANNOT_USER, MYF(0), "DROP USER", wrong_users.c_ptr_safe()); + my_error(ER_CANNOT_USER, MYF(0), + (handle_as_role) ? "DROP ROLE" : "DROP USER", + wrong_users.c_ptr_safe()); if (some_users_deleted) result |= write_bin_log(thd, FALSE, thd->query(), thd->query_length()); mysql_rwlock_unlock(&LOCK_grant); thd->variables.sql_mode= old_sql_mode; - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result); } - /* Rename a user. @@ -6697,44 +9286,34 @@ bool mysql_rename_user(THD *thd, List <LEX_USER> &list) List_iterator <LEX_USER> user_list(list); TABLE_LIST tables[GRANT_TABLES]; bool some_users_renamed= FALSE; - bool save_binlog_row_based; DBUG_ENTER("mysql_rename_user"); - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - /* RENAME USER may be skipped on replication client. */ if ((result= open_grant_tables(thd, tables))) - { - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result != 1); - } + + DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); mysql_rwlock_wrlock(&LOCK_grant); mysql_mutex_lock(&acl_cache->lock); while ((tmp_user_from= user_list++)) { - if (!(user_from= get_current_user(thd, tmp_user_from))) + tmp_user_to= user_list++; + if (!(user_from= get_current_user(thd, tmp_user_from, false)) || + user_from->is_role()) { + append_user(&wrong_users, user_from); result= TRUE; continue; - } - tmp_user_to= user_list++; - if (!(user_to= get_current_user(thd, tmp_user_to))) + } + if (!(user_to= get_current_user(thd, tmp_user_to, false)) || + user_to->is_role()) { + append_user(&wrong_users, user_to); result= TRUE; continue; - } - DBUG_ASSERT(user_to != 0); /* Syntax enforces pairs of users. */ + } /* Search all in-memory structures and grant tables @@ -6743,29 +9322,32 @@ bool mysql_rename_user(THD *thd, List <LEX_USER> &list) if (handle_grant_data(tables, 0, user_to, NULL) || handle_grant_data(tables, 0, user_from, user_to) <= 0) { + /* NOTE TODO renaming roles is not yet implemented */ append_user(&wrong_users, user_from); result= TRUE; continue; } some_users_renamed= TRUE; } - + /* Rebuild 'acl_check_hosts' since 'acl_users' has been modified */ rebuild_check_host(); + /* + Rebuild every user's role_grants since 'acl_users' has been sorted + and old pointers to ACL_USER elements are no longer valid + */ + rebuild_role_grants(); + mysql_mutex_unlock(&acl_cache->lock); if (result) my_error(ER_CANNOT_USER, MYF(0), "RENAME USER", wrong_users.c_ptr_safe()); - + if (some_users_renamed && mysql_bin_log.is_open()) result |= write_bin_log(thd, FALSE, thd->query(), thd->query_length()); mysql_rwlock_unlock(&LOCK_grant); - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result); } @@ -6790,25 +9372,12 @@ bool mysql_revoke_all(THD *thd, List <LEX_USER> &list) int result; ACL_DB *acl_db; TABLE_LIST tables[GRANT_TABLES]; - bool save_binlog_row_based; DBUG_ENTER("mysql_revoke_all"); - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - if ((result= open_grant_tables(thd, tables))) - { - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result != 1); - } + + DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); mysql_rwlock_wrlock(&LOCK_grant); mysql_mutex_lock(&acl_cache->lock); @@ -6817,19 +9386,21 @@ bool mysql_revoke_all(THD *thd, List <LEX_USER> &list) List_iterator <LEX_USER> user_list(list); while ((tmp_lex_user= user_list++)) { - if (!(lex_user= get_current_user(thd, tmp_lex_user))) + if (!(lex_user= get_current_user(thd, tmp_lex_user, false))) { result= -1; continue; - } - if (!find_acl_user(lex_user->host.str, lex_user->user.str, TRUE)) + } + + /* This is not a role and the user could not be found */ + if (!lex_user->is_role() && + !find_user_exact(lex_user->host.str, lex_user->user.str)) { result= -1; continue; } - if (replace_user_table(thd, tables[0].table, - *lex_user, ~(ulong)0, 1, 0, 0)) + if (replace_user_table(thd, tables[0].table, *lex_user, ~(ulong)0, 1, 0, 0)) { result= -1; continue; @@ -6848,12 +9419,11 @@ bool mysql_revoke_all(THD *thd, List <LEX_USER> &list) const char *user,*host; acl_db=dynamic_element(&acl_dbs,counter,ACL_DB*); - if (!(user=acl_db->user)) - user= ""; - if (!(host=acl_db->host.hostname)) - host= ""; - if (!strcmp(lex_user->user.str,user) && + user= safe_str(acl_db->user); + host= safe_str(acl_db->host.hostname); + + if (!strcmp(lex_user->user.str, user) && !strcmp(lex_user->host.str, host)) { if (!replace_db_table(tables[1].table, acl_db->db, *lex_user, @@ -6880,10 +9450,8 @@ bool mysql_revoke_all(THD *thd, List <LEX_USER> &list) const char *user,*host; GRANT_TABLE *grant_table= (GRANT_TABLE*) my_hash_element(&column_priv_hash, counter); - if (!(user=grant_table->user)) - user= ""; - if (!(host=grant_table->host.hostname)) - host= ""; + user= safe_str(grant_table->user); + host= safe_str(grant_table->host.hostname); if (!strcmp(lex_user->user.str,user) && !strcmp(lex_user->host.str, host)) @@ -6926,10 +9494,8 @@ bool mysql_revoke_all(THD *thd, List <LEX_USER> &list) { const char *user,*host; GRANT_NAME *grant_proc= (GRANT_NAME*) my_hash_element(hash, counter); - if (!(user=grant_proc->user)) - user= ""; - if (!(host=grant_proc->host.hostname)) - host= ""; + user= safe_str(grant_proc->user); + host= safe_str(grant_proc->host.hostname); if (!strcmp(lex_user->user.str,user) && !strcmp(lex_user->host.str, host)) @@ -6948,6 +9514,59 @@ bool mysql_revoke_all(THD *thd, List <LEX_USER> &list) counter++; } } while (revoked); + + ACL_USER_BASE *user_or_role; + /* remove role grants */ + if (lex_user->is_role()) + { + /* this can not fail due to get_current_user already having searched for it */ + user_or_role= find_acl_role(lex_user->user.str); + } + else + { + user_or_role= find_user_exact(lex_user->host.str, lex_user->user.str); + } + /* + Find every role grant pair matching the role_grants array and remove it, + both from the acl_roles_mappings and the roles_mapping table + */ + for (counter= 0; counter < user_or_role->role_grants.elements; counter++) + { + ACL_ROLE *role_grant= *dynamic_element(&user_or_role->role_grants, + counter, ACL_ROLE**); + ROLE_GRANT_PAIR *pair = find_role_grant_pair(&lex_user->user, + &lex_user->host, + &role_grant->user); + if (replace_roles_mapping_table(tables[6].table, + &lex_user->user, + &lex_user->host, + &role_grant->user, false, pair, true)) + { + result= -1; //Something went wrong + } + /* + Delete from the parent_grantee array of the roles granted, + the entry pointing to this user_or_role + */ + remove_ptr_from_dynarray(&role_grant->parent_grantee, user_or_role); + } + /* TODO + How to handle an error in the replace_roles_mapping_table, in + regards to the privileges held in memory + */ + + /* Finally, clear the role_grants array */ + if (counter == user_or_role->role_grants.elements) + { + reset_dynamic(&user_or_role->role_grants); + } + /* + If we are revoking from a role, we need to update all the parent grantees + */ + if (lex_user->is_role()) + { + propagate_role_grants((ACL_ROLE *)user_or_role, PRIVS_TO_MERGE::ALL, 0, 0); + } } mysql_mutex_unlock(&acl_cache->lock); @@ -6959,10 +9578,6 @@ bool mysql_revoke_all(THD *thd, List <LEX_USER> &list) write_bin_log(thd, FALSE, thd->query(), thd->query_length()); mysql_rwlock_unlock(&LOCK_grant); - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(result); } @@ -7054,26 +9669,19 @@ bool sp_revoke_privileges(THD *thd, const char *sp_db, const char *sp_name, TABLE_LIST tables[GRANT_TABLES]; HASH *hash= is_proc ? &proc_priv_hash : &func_priv_hash; Silence_routine_definer_errors error_handler; - bool save_binlog_row_based; DBUG_ENTER("sp_revoke_privileges"); if ((result= open_grant_tables(thd, tables))) DBUG_RETURN(result != 1); + DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); + /* Be sure to pop this before exiting this scope! */ thd->push_internal_handler(&error_handler); mysql_rwlock_wrlock(&LOCK_grant); mysql_mutex_lock(&acl_cache->lock); - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - /* Remove procedure access */ do { @@ -7086,11 +9694,8 @@ bool sp_revoke_privileges(THD *thd, const char *sp_db, const char *sp_name, LEX_USER lex_user; lex_user.user.str= grant_proc->user; lex_user.user.length= strlen(grant_proc->user); - lex_user.host.str= grant_proc->host.hostname ? - grant_proc->host.hostname : (char*)""; - lex_user.host.length= grant_proc->host.hostname ? - strlen(grant_proc->host.hostname) : 0; - + lex_user.host.str= safe_str(grant_proc->host.hostname); + lex_user.host.length= strlen(lex_user.host.str); if (replace_routine_table(thd,grant_proc,tables[4].table,lex_user, grant_proc->db, grant_proc->tname, is_proc, ~(ulong)0, 1) == 0) @@ -7107,10 +9712,6 @@ bool sp_revoke_privileges(THD *thd, const char *sp_db, const char *sp_name, mysql_rwlock_unlock(&LOCK_grant); thd->pop_internal_handler(); - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(error_handler.has_errors()); } @@ -7149,13 +9750,13 @@ bool sp_grant_privileges(THD *thd, const char *sp_db, const char *sp_name, mysql_mutex_lock(&acl_cache->lock); - if ((au= find_acl_user(combo->host.str=(char*)sctx->host_or_ip,combo->user.str,FALSE))) + if ((au= find_user_wild(combo->host.str=(char*)sctx->host_or_ip, combo->user.str))) goto found_acl; - if ((au= find_acl_user(combo->host.str=(char*)sctx->host, combo->user.str,FALSE))) + if ((au= find_user_wild(combo->host.str=(char*)sctx->host, combo->user.str))) goto found_acl; - if ((au= find_acl_user(combo->host.str=(char*)sctx->ip, combo->user.str,FALSE))) + if ((au= find_user_wild(combo->host.str=(char*)sctx->ip, combo->user.str))) goto found_acl; - if((au= find_acl_user(combo->host.str=(char*)"%", combo->user.str, FALSE))) + if ((au= find_user_wild(combo->host.str=(char*)"%", combo->user.str))) goto found_acl; mysql_mutex_unlock(&acl_cache->lock); @@ -7170,10 +9771,8 @@ bool sp_grant_privileges(THD *thd, const char *sp_db, const char *sp_name, tables->db= (char*)sp_db; tables->table_name= tables->alias= (char*)sp_name; - thd->make_lex_string(&combo->user, - combo->user.str, strlen(combo->user.str), 0); - thd->make_lex_string(&combo->host, - combo->host.str, strlen(combo->host.str), 0); + thd->make_lex_string(&combo->user, combo->user.str, strlen(combo->user.str)); + thd->make_lex_string(&combo->host, combo->host.str, strlen(combo->host.str)); combo->password= empty_lex_str; combo->plugin= empty_lex_str; @@ -7229,23 +9828,12 @@ bool sp_grant_privileges(THD *thd, const char *sp_db, const char *sp_name, } -/***************************************************************************** - Instantiate used templates -*****************************************************************************/ - -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class List_iterator<LEX_COLUMN>; -template class List_iterator<LEX_USER>; -template class List<LEX_COLUMN>; -template class List<LEX_USER>; -#endif - /** Validate if a user can proxy as another user @thd current thread @param user the logged in user (proxy user) - @param authenticated_as the effective user a plugin is trying to + @param authenticated_as the effective user a plugin is trying to impersonate as (proxied user) @return proxy user definition @retval NULL proxy user definition not found or not applicable @@ -7253,7 +9841,7 @@ template class List<LEX_USER>; */ static ACL_PROXY_USER * -acl_find_proxy_user(const char *user, const char *host, const char *ip, +acl_find_proxy_user(const char *user, const char *host, const char *ip, const char *authenticated_as, bool *proxy_used) { uint i; @@ -7268,10 +9856,10 @@ acl_find_proxy_user(const char *user, const char *host, const char *ip, DBUG_RETURN (NULL); } - *proxy_used= TRUE; + *proxy_used= TRUE; for (i=0; i < acl_proxy_users.elements; i++) { - ACL_PROXY_USER *proxy= dynamic_element(&acl_proxy_users, i, + ACL_PROXY_USER *proxy= dynamic_element(&acl_proxy_users, i, ACL_PROXY_USER *); if (proxy->matches(host, user, ip, authenticated_as)) DBUG_RETURN(proxy); @@ -7286,7 +9874,7 @@ acl_check_proxy_grant_access(THD *thd, const char *host, const char *user, bool with_grant) { DBUG_ENTER("acl_check_proxy_grant_access"); - DBUG_PRINT("info", ("user=%s host=%s with_grant=%d", user, host, + DBUG_PRINT("info", ("user=%s host=%s with_grant=%d", user, host, (int) with_grant)); if (!initialized) { @@ -7317,7 +9905,7 @@ acl_check_proxy_grant_access(THD *thd, const char *host, const char *user, !my_strcasecmp(system_charset_info, host, thd->security_ctx->priv_host)) { - DBUG_PRINT("info", ("strcmp (%s, %s) my_casestrcmp (%s, %s) equal", + DBUG_PRINT("info", ("strcmp (%s, %s) my_casestrcmp (%s, %s) equal", thd->security_ctx->priv_user, user, host, thd->security_ctx->priv_host)); DBUG_RETURN(FALSE); @@ -7326,7 +9914,7 @@ acl_check_proxy_grant_access(THD *thd, const char *host, const char *user, /* check for matching WITH PROXY rights */ for (uint i=0; i < acl_proxy_users.elements; i++) { - ACL_PROXY_USER *proxy= dynamic_element(&acl_proxy_users, i, + ACL_PROXY_USER *proxy= dynamic_element(&acl_proxy_users, i, ACL_PROXY_USER *); if (proxy->matches(thd->security_ctx->host, thd->security_ctx->user, @@ -7347,7 +9935,8 @@ acl_check_proxy_grant_access(THD *thd, const char *host, const char *user, static bool -show_proxy_grants(THD *thd, LEX_USER *user, char *buff, size_t buffsize) +show_proxy_grants(THD *thd, const char *username, const char *hostname, + char *buff, size_t buffsize) { Protocol *protocol= thd->protocol; int error= 0; @@ -7356,7 +9945,7 @@ show_proxy_grants(THD *thd, LEX_USER *user, char *buff, size_t buffsize) { ACL_PROXY_USER *proxy= dynamic_element(&acl_proxy_users, i, ACL_PROXY_USER *); - if (proxy->granted_on(user->host.str, user->user.str)) + if (proxy->granted_on(hostname, username)) { String global(buff, buffsize, system_charset_info); global.length(0); @@ -7373,9 +9962,119 @@ show_proxy_grants(THD *thd, LEX_USER *user, char *buff, size_t buffsize) return error; } +static int enabled_roles_insert(ACL_USER_BASE *role, void *context_data) +{ + TABLE *table= (TABLE*) context_data; + DBUG_ASSERT(role->flags & IS_ROLE); + + restore_record(table, s->default_values); + table->field[0]->set_notnull(); + table->field[0]->store(role->user.str, role->user.length, + system_charset_info); + if (schema_table_store_record(table->in_use, table)) + return -1; + return 0; +} + +struct APPLICABLE_ROLES_DATA +{ + TABLE *table; + const LEX_STRING host; + const LEX_STRING user_and_host; + ACL_USER_BASE *user; +}; + +static int +applicable_roles_insert(ACL_USER_BASE *grantee, ACL_ROLE *role, void *ptr) +{ + APPLICABLE_ROLES_DATA *data= (APPLICABLE_ROLES_DATA *)ptr; + CHARSET_INFO *cs= system_charset_info; + TABLE *table= data->table; + bool is_role= grantee != data->user; + const LEX_STRING *user_and_host= is_role ? &grantee->user + : &data->user_and_host; + const LEX_STRING *host= is_role ? &empty_lex_str : &data->host; + + restore_record(table, s->default_values); + table->field[0]->store(user_and_host->str, user_and_host->length, cs); + table->field[1]->store(role->user.str, role->user.length, cs); + + ROLE_GRANT_PAIR *pair= + find_role_grant_pair(&grantee->user, host, &role->user); + DBUG_ASSERT(pair); + + if (pair->with_admin) + table->field[2]->store(STRING_WITH_LEN("YES"), cs); + else + table->field[2]->store(STRING_WITH_LEN("NO"), cs); + + if (schema_table_store_record(table->in_use, table)) + return -1; + return 0; +} #endif /*NO_EMBEDDED_ACCESS_CHECKS */ +int fill_schema_enabled_roles(THD *thd, TABLE_LIST *tables, COND *cond) +{ + TABLE *table= tables->table; +#ifndef NO_EMBEDDED_ACCESS_CHECKS + if (thd->security_ctx->priv_role[0]) + { + mysql_rwlock_rdlock(&LOCK_grant); + mysql_mutex_lock(&acl_cache->lock); + ACL_ROLE *acl_role= find_acl_role(thd->security_ctx->priv_role); + if (acl_role) + traverse_role_graph_down(acl_role, table, enabled_roles_insert, NULL); + mysql_mutex_unlock(&acl_cache->lock); + mysql_rwlock_unlock(&LOCK_grant); + if (acl_role) + return 0; + } +#endif + + restore_record(table, s->default_values); + table->field[0]->set_null(); + return schema_table_store_record(table->in_use, table); +} + + +/* + This shows all roles granted to current user + and recursively all roles granted to those roles +*/ +int fill_schema_applicable_roles(THD *thd, TABLE_LIST *tables, COND *cond) +{ + int res= 0; +#ifndef NO_EMBEDDED_ACCESS_CHECKS + if (initialized) + { + TABLE *table= tables->table; + Security_context *sctx= thd->security_ctx; + mysql_rwlock_rdlock(&LOCK_grant); + mysql_mutex_lock(&acl_cache->lock); + ACL_USER *user= find_user_exact(sctx->priv_host, sctx->priv_user); + if (user) + { + char buff[USER_HOST_BUFF_SIZE+10]; + DBUG_ASSERT(user->user.length + user->hostname_length +2 < sizeof(buff)); + char *end= strxmov(buff, user->user.str, "@", user->host.hostname, NULL); + APPLICABLE_ROLES_DATA data= { table, + { user->host.hostname, user->hostname_length }, + { buff, (size_t)(end - buff) }, user + }; + + res= traverse_role_graph_down(user, &data, 0, applicable_roles_insert); + } + + mysql_mutex_unlock(&acl_cache->lock); + mysql_rwlock_unlock(&LOCK_grant); + } +#endif + + return res; +} + int wild_case_compare(CHARSET_INFO *cs, const char *str,const char *wildstr) { @@ -7468,16 +10167,14 @@ int fill_schema_user_privileges(THD *thd, TABLE_LIST *tables, COND *cond) { const char *user,*host, *is_grantable="YES"; acl_user=dynamic_element(&acl_users,counter,ACL_USER*); - if (!(user=acl_user->user)) - user= ""; - if (!(host=acl_user->host.hostname)) - host= ""; + user= safe_str(acl_user->user.str); + host= safe_str(acl_user->host.hostname); if (no_global_access && (strcmp(thd->security_ctx->priv_user, user) || my_strcasecmp(system_charset_info, curr_host, host))) continue; - + want_access= acl_user->access; if (!(want_access & GRANT_ACL)) is_grantable= "NO"; @@ -7500,7 +10197,7 @@ int fill_schema_user_privileges(THD *thd, TABLE_LIST *tables, COND *cond) { if (test_access & j) { - if (update_schema_privilege(thd, table, buff, 0, 0, 0, 0, + if (update_schema_privilege(thd, table, buff, 0, 0, 0, 0, command_array[priv_id], command_lengths[priv_id], is_grantable)) { @@ -7544,10 +10241,8 @@ int fill_schema_schema_privileges(THD *thd, TABLE_LIST *tables, COND *cond) const char *user, *host, *is_grantable="YES"; acl_db=dynamic_element(&acl_dbs,counter,ACL_DB*); - if (!(user=acl_db->user)) - user= ""; - if (!(host=acl_db->host.hostname)) - host= ""; + user= safe_str(acl_db->user); + host= safe_str(acl_db->host.hostname); if (no_global_access && (strcmp(thd->security_ctx->priv_user, user) || @@ -7617,11 +10312,9 @@ int fill_schema_table_privileges(THD *thd, TABLE_LIST *tables, COND *cond) { const char *user, *host, *is_grantable= "YES"; GRANT_TABLE *grant_table= (GRANT_TABLE*) my_hash_element(&column_priv_hash, - index); - if (!(user=grant_table->user)) - user= ""; - if (!(host= grant_table->host.hostname)) - host= ""; + index); + user= safe_str(grant_table->user); + host= safe_str(grant_table->host.hostname); if (no_global_access && (strcmp(thd->security_ctx->priv_user, user) || @@ -7671,7 +10364,7 @@ int fill_schema_table_privileges(THD *thd, TABLE_LIST *tables, COND *cond) } } } - } + } } err: mysql_rwlock_unlock(&LOCK_grant); @@ -7701,11 +10394,9 @@ int fill_schema_column_privileges(THD *thd, TABLE_LIST *tables, COND *cond) { const char *user, *host, *is_grantable= "YES"; GRANT_TABLE *grant_table= (GRANT_TABLE*) my_hash_element(&column_priv_hash, - index); - if (!(user=grant_table->user)) - user= ""; - if (!(host= grant_table->host.hostname)) - host= ""; + index); + user= safe_str(grant_table->user); + host= safe_str(grant_table->host.hostname); if (no_global_access && (strcmp(thd->security_ctx->priv_user, user) || @@ -7783,9 +10474,7 @@ void fill_effective_table_privileges(THD *thd, GRANT_INFO *grant, Security_context *sctx= thd->security_ctx; DBUG_ENTER("fill_effective_table_privileges"); DBUG_PRINT("enter", ("Host: '%s', Ip: '%s', User: '%s', table: `%s`.`%s`", - sctx->priv_host, (sctx->ip ? sctx->ip : "(NULL)"), - (sctx->priv_user ? sctx->priv_user : "(NULL)"), - db, table)); + sctx->priv_host, sctx->ip, sctx->priv_user, db, table)); /* --skip-grants */ if (!initialized) { @@ -7804,22 +10493,40 @@ void fill_effective_table_privileges(THD *thd, GRANT_INFO *grant, DBUG_VOID_RETURN; // it is slave } - /* db privileges */ - grant->privilege|= acl_get(sctx->host, sctx->ip, sctx->priv_user, db, 0); + if (!thd->db || strcmp(db, thd->db)) + { + /* db privileges */ + grant->privilege|= acl_get(sctx->host, sctx->ip, sctx->priv_user, db, 0); + /* db privileges for role */ + if (sctx->priv_role[0]) + grant->privilege|= acl_get("", "", sctx->priv_role, db, 0); + } + else + { + grant->privilege|= sctx->db_access; + } /* table privileges */ mysql_rwlock_rdlock(&LOCK_grant); if (grant->version != grant_version) { - grant->grant_table= + grant->grant_table_user= table_hash_search(sctx->host, sctx->ip, db, - sctx->priv_user, - table, 0); /* purecov: inspected */ + sctx->priv_user, + table, 0); /* purecov: inspected */ + grant->grant_table_role= + sctx->priv_role[0] ? table_hash_search("", "", db, + sctx->priv_role, + table, TRUE) : NULL; grant->version= grant_version; /* purecov: inspected */ } - if (grant->grant_table != 0) + if (grant->grant_table_user != 0) + { + grant->privilege|= grant->grant_table_user->privs; + } + if (grant->grant_table_role != 0) { - grant->privilege|= grant->grant_table->privs; + grant->privilege|= grant->grant_table_role->privs; } mysql_rwlock_unlock(&LOCK_grant); @@ -7841,6 +10548,54 @@ bool check_routine_level_acl(THD *thd, const char *db, const char *name, #endif +/** + Return information about user or current user. + + @param[in] thd thread handler + @param[in] user user + @param[in] lock whether &acl_cache->lock mutex needs to be locked + + @return + - On success, return a valid pointer to initialized + LEX_USER, which contains user information. + - On error, return 0. +*/ + +LEX_USER *get_current_user(THD *thd, LEX_USER *user, bool lock) +{ + if (user->user.str == current_user.str) // current_user + return create_default_definer(thd, false); + + if (user->user.str == current_role.str) // current_role + return create_default_definer(thd, true); + + if (user->host.str == NULL) // Possibly a role + { + // to be reexecution friendly we have to make a copy + LEX_USER *dup= (LEX_USER*) thd->memdup(user, sizeof(*user)); + if (!dup) + return 0; + +#ifndef NO_EMBEDDED_ACCESS_CHECKS + if (is_invalid_role_name(user->user.str)) + return 0; + + if (lock) + mysql_mutex_lock(&acl_cache->lock); + if (find_acl_role(dup->user.str)) + dup->host= empty_lex_str; + else + dup->host= host_not_specified; + if (lock) + mysql_mutex_unlock(&acl_cache->lock); +#endif + + return dup; + } + + return user; +} + struct ACL_internal_schema_registry_entry { const LEX_STRING *m_name; @@ -8005,9 +10760,9 @@ static void login_failed_error(THD *thd) thd->main_security_ctx.host_or_ip, thd->password ? ER(ER_YES) : ER(ER_NO)); status_var_increment(thd->status_var.access_denied_errors); - /* + /* Log access denied messages to the error log when log-warnings = 2 - so that the overhead of the general query log is not required to track + so that the overhead of the general query log is not required to track failed connections. */ if (global_system_variables.log_warnings > 1) @@ -8015,7 +10770,7 @@ static void login_failed_error(THD *thd) sql_print_warning(ER(access_denied_error_code(thd->password)), thd->main_security_ctx.user, thd->main_security_ctx.host_or_ip, - thd->password ? ER(ER_YES) : ER(ER_NO)); + thd->password ? ER(ER_YES) : ER(ER_NO)); } } @@ -8024,7 +10779,7 @@ static void login_failed_error(THD *thd) after the connection was established Packet format: - + Bytes Content ----- ---- 1 protocol version (always 10) @@ -8106,7 +10861,7 @@ static bool send_server_handshake_packet(MPVIO_EXT *mpvio, data_len= SCRAMBLE_LENGTH; } - end= strnmov(end, server_version, SERVER_VERSION_LENGTH) + 1; + end= strxnmov(end, SERVER_VERSION_LENGTH, RPL_VERSION_HACK, server_version, NullS) + 1; int4store((uchar*) end, mpvio->thd->thread_id); end+= 4; @@ -8118,7 +10873,7 @@ static bool send_server_handshake_packet(MPVIO_EXT *mpvio, end= (char*) memcpy(end, data, SCRAMBLE_LENGTH_323); end+= SCRAMBLE_LENGTH_323; *end++= 0; - + int2store(end, thd->client_capabilities); /* write server characteristics: up to 16 bytes allowed */ end[2]= (char) default_charset_info->number; @@ -8148,7 +10903,7 @@ static bool secure_auth(THD *thd) return 0; /* - If the server is running in secure auth mode, short scrambles are + If the server is running in secure auth mode, short scrambles are forbidden. Extra juggling to report the same error as the old code. */ if (thd->client_capabilities & CLIENT_PROTOCOL_41) @@ -8173,7 +10928,7 @@ static bool secure_auth(THD *thd) using a different authentication plugin Packet format: - + Bytes Content ----- ---- 1 byte with the value 254 @@ -8239,7 +10994,7 @@ static bool send_plugin_request_packet(MPVIO_EXT *mpvio, DBUG_RETURN (1); } - DBUG_PRINT("info", ("requesting client to use the %s plugin", + DBUG_PRINT("info", ("requesting client to use the %s plugin", client_auth_plugin)); DBUG_RETURN(net_write_command(net, switch_plugin_request_buf[0], (uchar*) client_auth_plugin, @@ -8250,13 +11005,10 @@ static bool send_plugin_request_packet(MPVIO_EXT *mpvio, #ifndef NO_EMBEDDED_ACCESS_CHECKS /** Finds acl entry in user database for authentication purposes. - + Finds a user and copies it into mpvio. Creates a fake user if no matching user account is found. - @note find_acl_user is not the same, because it doesn't take into - account the case when user is not empty, but acl_user->user is empty - @retval 0 found @retval 1 error */ @@ -8267,16 +11019,11 @@ static bool find_mpvio_user(MPVIO_EXT *mpvio) DBUG_ASSERT(mpvio->acl_user == 0); mysql_mutex_lock(&acl_cache->lock); - for (uint i=0; i < acl_users.elements; i++) - { - ACL_USER *acl_user_tmp= dynamic_element(&acl_users, i, ACL_USER*); - if ((!acl_user_tmp->user || !strcmp(sctx->user, acl_user_tmp->user)) && - compare_hostname(&acl_user_tmp->host, sctx->host, sctx->ip)) - { - mpvio->acl_user= acl_user_tmp->copy(mpvio->thd->mem_root); - break; - } - } + + ACL_USER *user= find_user_or_anon(sctx->host, sctx->user, sctx->ip); + if (user) + mpvio->acl_user= user->copy(&mem); + mysql_mutex_unlock(&acl_cache->lock); if (!mpvio->acl_user) @@ -8328,8 +11075,7 @@ static bool find_mpvio_user(MPVIO_EXT *mpvio) mpvio->auth_info.user_name_length= strlen(sctx->user); mpvio->auth_info.auth_string= mpvio->acl_user->auth_string.str; mpvio->auth_info.auth_string_length= (unsigned long) mpvio->acl_user->auth_string.length; - strmake_buf(mpvio->auth_info.authenticated_as, mpvio->acl_user->user ? - mpvio->acl_user->user : ""); + strmake_buf(mpvio->auth_info.authenticated_as, safe_str(mpvio->acl_user->user.str)); DBUG_PRINT("info", ("exit: user=%s, auth_string=%s, authenticated as=%s" "plugin=%s", @@ -8415,7 +11161,7 @@ static bool parse_com_change_user_packet(MPVIO_EXT *mpvio, uint packet_length) thd->user_connect= 0; strmake_buf(sctx->priv_user, sctx->user); - if (thd->make_lex_string(&mpvio->db, db_buff, db_len, 0) == 0) + if (thd->make_lex_string(&mpvio->db, db_buff, db_len) == 0) DBUG_RETURN(1); /* The error is set by make_lex_string(). */ /* @@ -8458,7 +11204,7 @@ static bool parse_com_change_user_packet(MPVIO_EXT *mpvio, uint packet_length) /* For a passwordless accounts we use native_password_plugin. But when an old 4.0 client connects to it, we change it to - old_password_plugin, otherwise MySQL will think that server + old_password_plugin, otherwise MySQL will think that server and client plugins don't match. */ if (mpvio->acl_user->auth_string.length == 0) @@ -8467,9 +11213,9 @@ static bool parse_com_change_user_packet(MPVIO_EXT *mpvio, uint packet_length) } DBUG_PRINT("info", ("client_plugin=%s, restart", client_plugin)); - /* - Remember the data part of the packet, to present it to plugin in - read_packet() + /* + Remember the data part of the packet, to present it to plugin in + read_packet() */ mpvio->cached_client_reply.pkt= passwd; mpvio->cached_client_reply.pkt_len= passwd_len; @@ -8642,7 +11388,7 @@ static ulong parse_client_handshake_packet(MPVIO_EXT *mpvio, Security_context *sctx= thd->security_ctx; - if (thd->make_lex_string(&mpvio->db, db, db_len, 0) == 0) + if (thd->make_lex_string(&mpvio->db, db, db_len) == 0) return packet_error; /* The error is set by make_lex_string(). */ my_free(sctx->user); if (!(sctx->user= my_strndup(user, user_len, MYF(MY_WME)))) @@ -8685,14 +11431,14 @@ static ulong parse_client_handshake_packet(MPVIO_EXT *mpvio, /* For a passwordless accounts we use native_password_plugin. But when an old 4.0 client connects to it, we change it to - old_password_plugin, otherwise MySQL will think that server + old_password_plugin, otherwise MySQL will think that server and client plugins don't match. */ if (mpvio->acl_user->auth_string.length == 0) mpvio->acl_user->plugin= old_password_plugin_name; } } - + /* if the acl_user needs a different plugin to authenticate (specified in GRANT ... AUTHENTICATED VIA plugin_name ..) @@ -8998,7 +11744,7 @@ static bool acl_check_ssl(THD *thd, const ACL_USER *acl_user) #else /* HAVE_OPENSSL */ default: /* - If we don't have SSL but SSL is required for this user the + If we don't have SSL but SSL is required for this user the authentication should fail. */ return 1; @@ -9104,7 +11850,7 @@ bool acl_authenticate(THD *thd, uint connect_errors, mpvio.status= MPVIO_EXT::FAILURE; mpvio.make_it_fail= false; mpvio.auth_info.host_or_ip= thd->security_ctx->host_or_ip; - mpvio.auth_info.host_or_ip_length= + mpvio.auth_info.host_or_ip_length= (unsigned int) strlen(thd->security_ctx->host_or_ip); DBUG_PRINT("info", ("com_change_user_pkt_len=%u", com_change_user_pkt_len)); @@ -9132,7 +11878,7 @@ bool acl_authenticate(THD *thd, uint connect_errors, the correct plugin. */ - res= do_auth_once(thd, auth_plugin_name, &mpvio); + res= do_auth_once(thd, auth_plugin_name, &mpvio); } /* @@ -9152,7 +11898,7 @@ bool acl_authenticate(THD *thd, uint connect_errors, Security_context *sctx= thd->security_ctx; const ACL_USER *acl_user= mpvio.acl_user; - thd->password= mpvio.auth_info.password_used; // remember for error messages + thd->password= mpvio.auth_info.password_used; // remember for error messages /* Log the command here so that the user can check the log @@ -9167,12 +11913,12 @@ bool acl_authenticate(THD *thd, uint connect_errors, general_log_print(thd, command, "%s@%s as %s on %s", sctx->user, sctx->host_or_ip, sctx->priv_user[0] ? sctx->priv_user : "anonymous", - mpvio.db.str ? mpvio.db.str : (char*) ""); + safe_str(mpvio.db.str)); } else general_log_print(thd, command, (char*) "%s@%s on %s", sctx->user, sctx->host_or_ip, - mpvio.db.str ? mpvio.db.str : (char*) ""); + safe_str(mpvio.db.str)); } if (res > CR_OK && mpvio.status != MPVIO_EXT::SUCCESS) @@ -9190,7 +11936,7 @@ bool acl_authenticate(THD *thd, uint connect_errors, { #ifndef NO_EMBEDDED_ACCESS_CHECKS bool is_proxy_user= FALSE; - const char *auth_user = acl_user->user ? acl_user->user : ""; + const char *auth_user = safe_str(acl_user->user.str); ACL_PROXY_USER *proxy_user; /* check if the user is allowed to proxy as another user */ proxy_user= acl_find_proxy_user(auth_user, sctx->host, sctx->ip, @@ -9210,13 +11956,12 @@ bool acl_authenticate(THD *thd, uint connect_errors, my_snprintf(sctx->proxy_user, sizeof(sctx->proxy_user) - 1, "'%s'@'%s'", auth_user, - acl_user->host.hostname ? acl_user->host.hostname : ""); + safe_str(acl_user->host.hostname)); /* we're proxying : find the proxy user definition */ mysql_mutex_lock(&acl_cache->lock); - acl_proxy_user= find_acl_user(proxy_user->get_proxied_host() ? - proxy_user->get_proxied_host() : "", - mpvio.auth_info.authenticated_as, TRUE); + acl_proxy_user= find_user_exact(safe_str(proxy_user->get_proxied_host()), + mpvio.auth_info.authenticated_as); if (!acl_proxy_user) { if (!thd->is_error()) @@ -9230,8 +11975,8 @@ bool acl_authenticate(THD *thd, uint connect_errors, #endif sctx->master_access= acl_user->access; - if (acl_user->user) - strmake_buf(sctx->priv_user, acl_user->user); + if (acl_user->user.str) + strmake_buf(sctx->priv_user, acl_user->user.str); else *sctx->priv_user= 0; @@ -9261,10 +12006,10 @@ bool acl_authenticate(THD *thd, uint connect_errors, acl_user->user_resource.updates || acl_user->user_resource.conn_per_hour || acl_user->user_resource.user_conn || max_user_connections_checking) && - get_or_create_user_conn(thd, - (opt_old_style_user_limits ? sctx->user : sctx->priv_user), - (opt_old_style_user_limits ? sctx->host_or_ip : sctx->priv_host), - &acl_user->user_resource)) + get_or_create_user_conn(thd, + (opt_old_style_user_limits ? sctx->user : sctx->priv_user), + (opt_old_style_user_limits ? sctx->host_or_ip : sctx->priv_host), + &acl_user->user_resource)) DBUG_RETURN(1); // The error is set by get_or_create_user_conn() } else @@ -9274,7 +12019,7 @@ bool acl_authenticate(THD *thd, uint connect_errors, (thd->user_connect->user_resources.conn_per_hour || thd->user_connect->user_resources.user_conn || max_user_connections_checking) && - check_for_max_user_connections(thd, thd->user_connect)) + check_for_max_user_connections(thd, thd->user_connect)) { /* Ensure we don't decrement thd->user_connections->connections twice */ thd->user_connect= 0; @@ -9429,7 +12174,7 @@ static int native_password_authenticate(MYSQL_PLUGIN_VIO *vio, DBUG_RETURN(CR_ERROR); } -static int old_password_authenticate(MYSQL_PLUGIN_VIO *vio, +static int old_password_authenticate(MYSQL_PLUGIN_VIO *vio, MYSQL_SERVER_AUTH_INFO *info) { uchar *pkt; @@ -9476,7 +12221,7 @@ static int old_password_authenticate(MYSQL_PLUGIN_VIO *vio, return CR_ERROR; return check_scramble_323(pkt, thd->scramble, - (ulong *) mpvio->acl_user->salt) ? + (ulong *) mpvio->acl_user->salt) ? CR_ERROR : CR_OK; } diff --git a/sql/sql_acl.h b/sql/sql_acl.h index 3169746419c..1b09b4bbdd4 100644 --- a/sql/sql_acl.h +++ b/sql/sql_acl.h @@ -20,30 +20,30 @@ #include "violite.h" /* SSL_type */ #include "sql_class.h" /* LEX_COLUMN */ -#define SELECT_ACL (1L << 0) -#define INSERT_ACL (1L << 1) -#define UPDATE_ACL (1L << 2) -#define DELETE_ACL (1L << 3) -#define CREATE_ACL (1L << 4) -#define DROP_ACL (1L << 5) -#define RELOAD_ACL (1L << 6) -#define SHUTDOWN_ACL (1L << 7) -#define PROCESS_ACL (1L << 8) -#define FILE_ACL (1L << 9) -#define GRANT_ACL (1L << 10) -#define REFERENCES_ACL (1L << 11) -#define INDEX_ACL (1L << 12) -#define ALTER_ACL (1L << 13) -#define SHOW_DB_ACL (1L << 14) -#define SUPER_ACL (1L << 15) -#define CREATE_TMP_ACL (1L << 16) -#define LOCK_TABLES_ACL (1L << 17) -#define EXECUTE_ACL (1L << 18) -#define REPL_SLAVE_ACL (1L << 19) -#define REPL_CLIENT_ACL (1L << 20) -#define CREATE_VIEW_ACL (1L << 21) -#define SHOW_VIEW_ACL (1L << 22) -#define CREATE_PROC_ACL (1L << 23) +#define SELECT_ACL (1L << 0) +#define INSERT_ACL (1L << 1) +#define UPDATE_ACL (1L << 2) +#define DELETE_ACL (1L << 3) +#define CREATE_ACL (1L << 4) +#define DROP_ACL (1L << 5) +#define RELOAD_ACL (1L << 6) +#define SHUTDOWN_ACL (1L << 7) +#define PROCESS_ACL (1L << 8) +#define FILE_ACL (1L << 9) +#define GRANT_ACL (1L << 10) +#define REFERENCES_ACL (1L << 11) +#define INDEX_ACL (1L << 12) +#define ALTER_ACL (1L << 13) +#define SHOW_DB_ACL (1L << 14) +#define SUPER_ACL (1L << 15) +#define CREATE_TMP_ACL (1L << 16) +#define LOCK_TABLES_ACL (1L << 17) +#define EXECUTE_ACL (1L << 18) +#define REPL_SLAVE_ACL (1L << 19) +#define REPL_CLIENT_ACL (1L << 20) +#define CREATE_VIEW_ACL (1L << 21) +#define SHOW_VIEW_ACL (1L << 22) +#define CREATE_PROC_ACL (1L << 23) #define ALTER_PROC_ACL (1L << 24) #define CREATE_USER_ACL (1L << 25) #define EVENT_ACL (1L << 26) @@ -57,7 +57,7 @@ 4. acl_init() or whatever - to define behaviour for old privilege tables 5. sql_yacc.yy - for GRANT/REVOKE to work */ -#define NO_ACCESS (1L << 30) +#define NO_ACCESS (1L << 30) #define DB_ACLS \ (UPDATE_ACL | SELECT_ACL | INSERT_ACL | DELETE_ACL | CREATE_ACL | DROP_ACL | \ GRANT_ACL | REFERENCES_ACL | INDEX_ACL | ALTER_ACL | CREATE_TMP_ACL | \ @@ -106,21 +106,21 @@ #define DB_CHUNK1 (GRANT_ACL | REFERENCES_ACL | INDEX_ACL | ALTER_ACL) #define DB_CHUNK2 (CREATE_TMP_ACL | LOCK_TABLES_ACL) #define DB_CHUNK3 (CREATE_VIEW_ACL | SHOW_VIEW_ACL | \ - CREATE_PROC_ACL | ALTER_PROC_ACL ) + CREATE_PROC_ACL | ALTER_PROC_ACL ) #define DB_CHUNK4 (EXECUTE_ACL) #define DB_CHUNK5 (EVENT_ACL | TRIGGER_ACL) #define fix_rights_for_db(A) (((A) & DB_CHUNK0) | \ - (((A) << 4) & DB_CHUNK1) | \ - (((A) << 6) & DB_CHUNK2) | \ - (((A) << 9) & DB_CHUNK3) | \ - (((A) << 2) & DB_CHUNK4))| \ + (((A) << 4) & DB_CHUNK1) | \ + (((A) << 6) & DB_CHUNK2) | \ + (((A) << 9) & DB_CHUNK3) | \ + (((A) << 2) & DB_CHUNK4))| \ (((A) << 9) & DB_CHUNK5) #define get_rights_for_db(A) (((A) & DB_CHUNK0) | \ - (((A) & DB_CHUNK1) >> 4) | \ - (((A) & DB_CHUNK2) >> 6) | \ - (((A) & DB_CHUNK3) >> 9) | \ - (((A) & DB_CHUNK4) >> 2))| \ + (((A) & DB_CHUNK1) >> 4) | \ + (((A) & DB_CHUNK2) >> 6) | \ + (((A) & DB_CHUNK3) >> 9) | \ + (((A) & DB_CHUNK4) >> 2))| \ (((A) & DB_CHUNK5) >> 9) #define TBL_CHUNK0 DB_CHUNK0 #define TBL_CHUNK1 DB_CHUNK1 @@ -137,11 +137,11 @@ #define fix_rights_for_column(A) (((A) & 7) | (((A) & ~7) << 8)) #define get_rights_for_column(A) (((A) & 7) | ((A) >> 8)) #define fix_rights_for_procedure(A) ((((A) << 18) & EXECUTE_ACL) | \ - (((A) << 23) & ALTER_PROC_ACL) | \ - (((A) << 8) & GRANT_ACL)) + (((A) << 23) & ALTER_PROC_ACL) | \ + (((A) << 8) & GRANT_ACL)) #define get_rights_for_procedure(A) ((((A) & EXECUTE_ACL) >> 18) | \ - (((A) & ALTER_PROC_ACL) >> 23) | \ - (((A) & GRANT_ACL) >> 8)) + (((A) & ALTER_PROC_ACL) >> 23) | \ + (((A) & GRANT_ACL) >> 8)) enum mysql_db_table_field { @@ -173,6 +173,11 @@ enum mysql_db_table_field extern const TABLE_FIELD_DEF mysql_db_table_def; extern bool mysql_user_table_is_in_short_password_format; +extern LEX_STRING host_not_specified; +extern LEX_STRING current_user; +extern LEX_STRING current_role; +extern LEX_STRING current_user_and_current_role; + static inline int access_denied_error_code(int passwd_used) { @@ -188,7 +193,7 @@ my_bool acl_init(bool dont_read_acl_tables); my_bool acl_reload(THD *thd); void acl_free(bool end=0); ulong acl_get(const char *host, const char *ip, - const char *user, const char *db, my_bool db_is_pattern); + const char *user, const char *db, my_bool db_is_pattern); bool acl_authenticate(THD *thd, uint connect_errors, uint com_change_user_pkt_len); bool acl_getroot(Security_context *sctx, char *user, char *host, char *ip, char *db); @@ -196,39 +201,43 @@ bool acl_check_host(const char *host, const char *ip); int check_change_password(THD *thd, const char *host, const char *user, char *password, uint password_len); bool change_password(THD *thd, const char *host, const char *user, - char *password); + char *password); + +bool mysql_grant_role(THD *thd, List<LEX_USER> &user_list, bool revoke); bool mysql_grant(THD *thd, const char *db, List <LEX_USER> &user_list, ulong rights, bool revoke, bool is_proxy); int mysql_table_grant(THD *thd, TABLE_LIST *table, List <LEX_USER> &user_list, List <LEX_COLUMN> &column_list, ulong rights, bool revoke); bool mysql_routine_grant(THD *thd, TABLE_LIST *table, bool is_proc, - List <LEX_USER> &user_list, ulong rights, - bool revoke, bool write_to_binlog); + List <LEX_USER> &user_list, ulong rights, + bool revoke, bool write_to_binlog); my_bool grant_init(); void grant_free(void); my_bool grant_reload(THD *thd); bool check_grant(THD *thd, ulong want_access, TABLE_LIST *tables, bool any_combination_will_do, uint number, bool no_errors); bool check_grant_column (THD *thd, GRANT_INFO *grant, - const char *db_name, const char *table_name, - const char *name, uint length, Security_context *sctx); + const char *db_name, const char *table_name, + const char *name, uint length, Security_context *sctx); bool check_column_grant_in_table_ref(THD *thd, TABLE_LIST * table_ref, const char *name, uint length); -bool check_grant_all_columns(THD *thd, ulong want_access, +bool check_grant_all_columns(THD *thd, ulong want_access, Field_iterator_table_ref *fields); bool check_grant_routine(THD *thd, ulong want_access, - TABLE_LIST *procs, bool is_proc, bool no_error); + TABLE_LIST *procs, bool is_proc, bool no_error); bool check_grant_db(THD *thd,const char *db); ulong get_table_grant(THD *thd, TABLE_LIST *table); ulong get_column_grant(THD *thd, GRANT_INFO *grant, const char *db_name, const char *table_name, const char *field_name); bool mysql_show_grants(THD *thd, LEX_USER *user); +int fill_schema_enabled_roles(THD *thd, TABLE_LIST *tables, COND *cond); +int fill_schema_applicable_roles(THD *thd, TABLE_LIST *tables, COND *cond); void get_privilege_desc(char *to, uint max_length, ulong access); void get_mqh(const char *user, const char *host, USER_CONN *uc); -bool mysql_create_user(THD *thd, List <LEX_USER> &list); -bool mysql_drop_user(THD *thd, List <LEX_USER> &list); +bool mysql_create_user(THD *thd, List <LEX_USER> &list, bool handle_as_role); +bool mysql_drop_user(THD *thd, List <LEX_USER> &list, bool handle_as_role); bool mysql_rename_user(THD *thd, List <LEX_USER> &list); bool mysql_revoke_all(THD *thd, List <LEX_USER> &list); void fill_effective_table_privileges(THD *thd, GRANT_INFO *grant, @@ -382,4 +391,11 @@ get_cached_table_access(GRANT_INTERNAL_INFO *grant_internal_info, bool acl_check_proxy_grant_access (THD *thd, const char *host, const char *user, bool with_grant); +int acl_setrole(THD *thd, char *rolename, ulonglong access); +int acl_check_setrole(THD *thd, char *rolename, ulonglong *access); + +#ifndef DBUG_OFF +extern ulong role_global_merges, role_db_merges, role_table_merges, + role_column_merges, role_routine_merges; +#endif #endif /* SQL_ACL_INCLUDED */ diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index 9beeca06945..48ac853310d 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -27,7 +27,9 @@ #include "sql_acl.h" // *_ACL #include "sp.h" // Sroutine_hash_entry #include "sql_parse.h" // check_table_access +#include "strfunc.h" #include "sql_admin.h" +#include "sql_statistics.h" /* Prepare, run and cleanup for mysql_recreate_table() */ @@ -97,8 +99,6 @@ static int prepare_for_repair(THD *thd, TABLE_LIST *table_list, if (!(table= table_list->table)) { - char key[MAX_DBKEY_LENGTH]; - uint key_length; /* If the table didn't exist, we have a shared metadata lock on it that is left from mysql_admin_table()'s attempt to @@ -111,9 +111,7 @@ static int prepare_for_repair(THD *thd, TABLE_LIST *table_list, Attempt to do full-blown table open in mysql_admin_table() has failed. Let us try to open at least a .FRM for this table. */ - my_hash_value_type hash_value; - key_length= create_table_def_key(thd, key, table_list, 0); table_list->mdl_request.init(MDL_key::TABLE, table_list->db, table_list->table_name, MDL_EXCLUSIVE, MDL_TRANSACTION); @@ -124,11 +122,8 @@ static int prepare_for_repair(THD *thd, TABLE_LIST *table_list, DBUG_RETURN(0); has_mdl_lock= TRUE; - hash_value= my_calc_hash(&table_def_cache, (uchar*) key, key_length); - mysql_mutex_lock(&LOCK_open); - share= get_table_share(thd, table_list, key, key_length, 0, - &error, hash_value); - mysql_mutex_unlock(&LOCK_open); + share= get_table_share(thd, table_list->db, table_list->table_name, + GTS_TABLE); if (share == NULL) DBUG_RETURN(0); // Can't open frm file @@ -323,7 +318,9 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, Protocol *protocol= thd->protocol; LEX *lex= thd->lex; int result_code; + int compl_result_code; bool need_repair_or_alter= 0; + DBUG_ENTER("mysql_admin_table"); DBUG_PRINT("enter", ("extra_open_options: %u", extra_open_options)); @@ -646,9 +643,92 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, } } - DBUG_PRINT("admin", ("calling operator_func '%s'", operator_name)); - result_code = (table->table->file->*operator_func)(thd, check_opt); - DBUG_PRINT("admin", ("operator_func returned: %d", result_code)); + result_code= compl_result_code= HA_ADMIN_OK; + + if (operator_func == &handler::ha_analyze) + { + TABLE *tab= table->table; + Field **field_ptr= tab->field; + + if (lex->with_persistent_for_clause && + tab->s->table_category != TABLE_CATEGORY_USER) + { + compl_result_code= result_code= HA_ADMIN_INVALID; + } + + if (!lex->column_list) + { + uint fields= 0; + for ( ; *field_ptr; field_ptr++, fields++) ; + bitmap_set_prefix(tab->read_set, fields); + } + else + { + int pos; + LEX_STRING *column_name; + List_iterator_fast<LEX_STRING> it(*lex->column_list); + + bitmap_clear_all(tab->read_set); + while ((column_name= it++)) + { + if (tab->s->fieldnames.type_names == 0 || + (pos= find_type(&tab->s->fieldnames, column_name->str, + column_name->length, 1)) <= 0) + { + compl_result_code= result_code= HA_ADMIN_INVALID; + break; + } + bitmap_set_bit(tab->read_set, pos-1); + } + tab->file->column_bitmaps_signal(); + } + + if (!lex->index_list) + { + tab->keys_in_use_for_query.init(tab->s->keys); + } + else + { + int pos; + LEX_STRING *index_name; + List_iterator_fast<LEX_STRING> it(*lex->index_list); + + tab->keys_in_use_for_query.clear_all(); + while ((index_name= it++)) + { + if (tab->s->keynames.type_names == 0 || + (pos= find_type(&tab->s->keynames, index_name->str, + index_name->length, 1)) <= 0) + { + compl_result_code= result_code= HA_ADMIN_INVALID; + break; + } + tab->keys_in_use_for_query.set_bit(--pos); + } + } + } + + if (result_code == HA_ADMIN_OK) + { + DBUG_PRINT("admin", ("calling operator_func '%s'", operator_name)); + result_code = (table->table->file->*operator_func)(thd, check_opt); + DBUG_PRINT("admin", ("operator_func returned: %d", result_code)); + } + + if (compl_result_code == HA_ADMIN_OK && + operator_func == &handler::ha_analyze && + table->table->s->table_category == TABLE_CATEGORY_USER && + (get_use_stat_tables_mode(thd) > NEVER || + lex->with_persistent_for_clause)) + { + if (!(compl_result_code= + alloc_statistics_for_table(thd, table->table)) && + !(compl_result_code= + collect_statistics_for_table(thd, table->table))) + compl_result_code= update_statistics_for_table(thd, table->table); + if (compl_result_code) + result_code= HA_ADMIN_FAILED; + } if (result_code == HA_ADMIN_NOT_IMPLEMENTED && need_repair_or_alter) { diff --git a/sql/sql_analyse.h b/sql/sql_analyse.h index 34c8e2da3d4..8bac29de5a3 100644 --- a/sql/sql_analyse.h +++ b/sql/sql_analyse.h @@ -121,7 +121,8 @@ public: must_be_blob(0), was_zero_fill(0), was_maybe_zerofill(0), can_be_still_num(1) { init_tree(&tree, 0, 0, sizeof(String), (qsort_cmp2) sortcmp2, - 0, (tree_element_free) free_string, NULL); }; + (tree_element_free) free_string, NULL, + MYF(MY_THREAD_SPECIFIC)); }; void add(); void get_opt_type(String*, ha_rows); @@ -162,7 +163,7 @@ public: { bin_size= my_decimal_get_binary_size(a->max_length, a->decimals); init_tree(&tree, 0, 0, bin_size, (qsort_cmp2)compare_decimal2, - 0, 0, (void *)&bin_size); + 0, (void *)&bin_size, MYF(MY_THREAD_SPECIFIC)); }; void add(); @@ -190,7 +191,8 @@ public: field_real(Item* a, analyse* b) :field_info(a,b), min_arg(0), max_arg(0), sum(0), sum_sqr(0), max_notzero_dec_len(0) { init_tree(&tree, 0, 0, sizeof(double), - (qsort_cmp2) compare_double2, 0, NULL, NULL); } + (qsort_cmp2) compare_double2, NULL, NULL, + MYF(MY_THREAD_SPECIFIC)); } void add(); void get_opt_type(String*, ha_rows); @@ -244,7 +246,8 @@ public: field_longlong(Item* a, analyse* b) :field_info(a,b), min_arg(0), max_arg(0), sum(0), sum_sqr(0) { init_tree(&tree, 0, 0, sizeof(longlong), - (qsort_cmp2) compare_longlong2, 0, NULL, NULL); } + (qsort_cmp2) compare_longlong2, NULL, NULL, + MYF(MY_THREAD_SPECIFIC)); } void add(); void get_opt_type(String*, ha_rows); @@ -289,7 +292,8 @@ public: field_ulonglong(Item* a, analyse * b) :field_info(a,b), min_arg(0), max_arg(0), sum(0),sum_sqr(0) { init_tree(&tree, 0, 0, sizeof(ulonglong), - (qsort_cmp2) compare_ulonglong2, 0, NULL, NULL); } + (qsort_cmp2) compare_ulonglong2, NULL, NULL, + MYF(MY_THREAD_SPECIFIC)); } void add(); void get_opt_type(String*, ha_rows); String *get_min_arg(String *s) { s->set(min_arg,my_thd_charset); return s; } diff --git a/sql/sql_array.h b/sql/sql_array.h index 67f1f1c2ad7..245cfe954f8 100644 --- a/sql/sql_array.h +++ b/sql/sql_array.h @@ -19,6 +19,77 @@ #include <my_sys.h> +/** + A wrapper class which provides array bounds checking. + We do *not* own the array, we simply have a pointer to the first element, + and a length. + + @remark + We want the compiler-generated versions of: + - the copy CTOR (memberwise initialization) + - the assignment operator (memberwise assignment) + + @param Element_type The type of the elements of the container. + */ +template <typename Element_type> class Bounds_checked_array +{ +public: + Bounds_checked_array() : m_array(NULL), m_size(0) {} + + Bounds_checked_array(Element_type *el, size_t size) + : m_array(el), m_size(size) + {} + + void reset() { m_array= NULL; m_size= 0; } + + void reset(Element_type *array, size_t size) + { + m_array= array; + m_size= size; + } + + /** + Set a new bound on the array. Does not resize the underlying + array, so the new size must be smaller than or equal to the + current size. + */ + void resize(size_t new_size) + { + DBUG_ASSERT(new_size <= m_size); + m_size= new_size; + } + + Element_type &operator[](size_t n) + { + DBUG_ASSERT(n < m_size); + return m_array[n]; + } + + const Element_type &operator[](size_t n) const + { + DBUG_ASSERT(n < m_size); + return m_array[n]; + } + + size_t element_size() const { return sizeof(Element_type); } + size_t size() const { return m_size; } + + bool is_null() const { return m_array == NULL; } + + void pop_front() + { + DBUG_ASSERT(m_size > 0); + m_array+= 1; + m_size-= 1; + } + + Element_type *array() const { return m_array; } + +private: + Element_type *m_array; + size_t m_size; +}; + /* A typesafe wrapper around DYNAMIC_ARRAY */ @@ -29,11 +100,13 @@ template <class Elem> class Dynamic_array public: Dynamic_array(uint prealloc=16, uint increment=16) { - my_init_dynamic_array(&array, sizeof(Elem), prealloc, increment); + my_init_dynamic_array(&array, sizeof(Elem), prealloc, increment, + MYF(MY_THREAD_SPECIFIC)); } - Elem& at(int idx) + Elem& at(size_t idx) { + DBUG_ASSERT(idx < array.elements); return *(((Elem*)array.buffer) + idx); } @@ -52,11 +125,48 @@ public: return (insert_dynamic(&array, (uchar*)&el)); } - int elements() + bool append_val(Elem el) + { + return (insert_dynamic(&array, (uchar*)&el)); + } + + bool push(Elem &el) + { + return append(el); + } + + Elem *pop() + { + return (Elem*)pop_dynamic(&array); + } + + size_t elements() { return array.elements; } + void set_elements(size_t n) + { + array.elements= n; + } + + bool resize(size_t new_size, Elem default_val) + { + size_t old_size= elements(); + if (allocate_dynamic(&array, new_size)) + return true; + + if (new_size > old_size) + { + set_dynamic(&array, (uchar*)&default_val, new_size - 1); + /*for (size_t i= old_size; i != new_size; i++) + { + at(i)= default_val; + }*/ + } + return false; + } + ~Dynamic_array() { delete_dynamic(&array); @@ -68,6 +178,12 @@ public: { my_qsort(array.buffer, array.elements, sizeof(Elem), (qsort_cmp)cmp_func); } + + typedef int (*CMP_FUNC2)(const Elem *el1, const Elem *el2, void *); + void sort(CMP_FUNC2 cmp_func, void *data) + { + my_qsort2(array.buffer, array.elements, sizeof(Elem), (qsort2_cmp)cmp_func, data); + } }; /* diff --git a/sql/sql_audit.cc b/sql/sql_audit.cc index 1dab45800d9..a92f69f3da4 100644 --- a/sql/sql_audit.cc +++ b/sql/sql_audit.cc @@ -183,7 +183,7 @@ static my_bool acquire_plugins(THD *thd, plugin_ref plugin, void *arg) { /* specify some reasonable initialization defaults */ my_init_dynamic_array(&thd->audit_class_plugins, - sizeof(plugin_ref), 16, 16); + sizeof(plugin_ref), 16, 16, MYF(0)); } /* lock the plugin and add it to the list */ diff --git a/sql/sql_audit.h b/sql/sql_audit.h index 9acd4abbdca..8172610607a 100644 --- a/sql/sql_audit.h +++ b/sql/sql_audit.h @@ -43,6 +43,11 @@ static inline bool mysql_audit_general_enabled() return mysql_global_audit_mask[0] & MYSQL_AUDIT_GENERAL_CLASSMASK; } +static inline bool mysql_audit_connection_enabled() +{ + return mysql_global_audit_mask[0] & MYSQL_AUDIT_CONNECTION_CLASSMASK; +} + static inline bool mysql_audit_table_enabled() { return mysql_global_audit_mask[0] & MYSQL_AUDIT_TABLE_CLASSMASK; @@ -52,6 +57,7 @@ static inline bool mysql_audit_table_enabled() static inline void mysql_audit_notify(THD *thd, uint event_class, uint event_subtype, ...) { } #define mysql_audit_general_enabled() 0 +#define mysql_audit_connection_enabled() 0 #define mysql_audit_table_enabled() 0 #endif extern void mysql_audit_release(THD *thd); @@ -145,46 +151,67 @@ void mysql_audit_general(THD *thd, uint event_subtype, } } -#define MYSQL_AUDIT_NOTIFY_CONNECTION_CONNECT(thd) mysql_audit_notify(\ - (thd), MYSQL_AUDIT_CONNECTION_CLASS, MYSQL_AUDIT_CONNECTION_CONNECT,\ - (thd)->stmt_da->is_error() ? (thd)->stmt_da->sql_errno() : 0,\ - (thd)->thread_id, (thd)->security_ctx->user,\ - (thd)->security_ctx->user ? strlen((thd)->security_ctx->user) : 0,\ - (thd)->security_ctx->priv_user, strlen((thd)->security_ctx->priv_user),\ - (thd)->security_ctx->external_user,\ - (thd)->security_ctx->external_user ?\ - strlen((thd)->security_ctx->external_user) : 0,\ - (thd)->security_ctx->proxy_user, strlen((thd)->security_ctx->proxy_user),\ - (thd)->security_ctx->host,\ - (thd)->security_ctx->host ? strlen((thd)->security_ctx->host) : 0,\ - (thd)->security_ctx->ip,\ - (thd)->security_ctx->ip ? strlen((thd)->security_ctx->ip) : 0,\ - (thd)->db, (thd)->db ? strlen((thd)->db) : 0) - -#define MYSQL_AUDIT_NOTIFY_CONNECTION_DISCONNECT(thd, errcode)\ - mysql_audit_notify(\ - (thd), MYSQL_AUDIT_CONNECTION_CLASS, MYSQL_AUDIT_CONNECTION_DISCONNECT,\ - (errcode), (thd)->thread_id, (thd)->security_ctx->user,\ - (thd)->security_ctx->user ? strlen((thd)->security_ctx->user) : 0,\ - 0, 0, 0, 0, 0, 0, (thd)->security_ctx->host,\ - (thd)->security_ctx->host ? strlen((thd)->security_ctx->host) : 0,\ - 0, 0, 0, 0) - -#define MYSQL_AUDIT_NOTIFY_CONNECTION_CHANGE_USER(thd) mysql_audit_notify(\ - (thd), MYSQL_AUDIT_CONNECTION_CLASS, MYSQL_AUDIT_CONNECTION_CHANGE_USER,\ - (thd)->stmt_da->is_error() ? (thd)->stmt_da->sql_errno() : 0,\ - (thd)->thread_id, (thd)->security_ctx->user,\ - (thd)->security_ctx->user ? strlen((thd)->security_ctx->user) : 0,\ - (thd)->security_ctx->priv_user, strlen((thd)->security_ctx->priv_user),\ - (thd)->security_ctx->external_user,\ - (thd)->security_ctx->external_user ?\ - strlen((thd)->security_ctx->external_user) : 0,\ - (thd)->security_ctx->proxy_user, strlen((thd)->security_ctx->proxy_user),\ - (thd)->security_ctx->host,\ - (thd)->security_ctx->host ? strlen((thd)->security_ctx->host) : 0,\ - (thd)->security_ctx->ip,\ - (thd)->security_ctx->ip ? strlen((thd)->security_ctx->ip) : 0,\ - (thd)->db, (thd)->db ? strlen((thd)->db) : 0) +static inline +void mysql_audit_notify_connection_connect(THD *thd) +{ + if (mysql_audit_connection_enabled()) + { + const Security_context *sctx= thd->security_ctx; + mysql_audit_notify(thd, MYSQL_AUDIT_CONNECTION_CLASS, + MYSQL_AUDIT_CONNECTION_CONNECT, + thd->stmt_da->is_error() ? thd->stmt_da->sql_errno() : 0, + thd->thread_id, + sctx->user, sctx->user ? strlen(sctx->user) : 0, + sctx->priv_user, strlen(sctx->priv_user), + sctx->external_user, + sctx->external_user ? strlen(sctx->external_user) : 0, + sctx->proxy_user, strlen(sctx->proxy_user), + sctx->host, sctx->host ? strlen(sctx->host) : 0, + sctx->ip, sctx->ip ? strlen(sctx->ip) : 0, + thd->db, thd->db ? strlen(thd->db) : 0); + } +} + +static inline +void mysql_audit_notify_connection_disconnect(THD *thd, int errcode) +{ + if (mysql_audit_connection_enabled()) + { + const Security_context *sctx= thd->security_ctx; + mysql_audit_notify(thd, MYSQL_AUDIT_CONNECTION_CLASS, + MYSQL_AUDIT_CONNECTION_DISCONNECT, + errcode, thd->thread_id, + sctx->user, sctx->user ? strlen(sctx->user) : 0, + sctx->priv_user, strlen(sctx->priv_user), + sctx->external_user, + sctx->external_user ? strlen(sctx->external_user) : 0, + sctx->proxy_user, strlen(sctx->proxy_user), + sctx->host, sctx->host ? strlen(sctx->host) : 0, + sctx->ip, sctx->ip ? strlen(sctx->ip) : 0, + thd->db, thd->db ? strlen(thd->db) : 0); + } +} + +static inline +void mysql_audit_notify_connection_change_user(THD *thd) +{ + if (mysql_audit_connection_enabled()) + { + const Security_context *sctx= thd->security_ctx; + mysql_audit_notify(thd, MYSQL_AUDIT_CONNECTION_CLASS, + MYSQL_AUDIT_CONNECTION_CHANGE_USER, + thd->stmt_da->is_error() ? thd->stmt_da->sql_errno() : 0, + thd->thread_id, + sctx->user, sctx->user ? strlen(sctx->user) : 0, + sctx->priv_user, strlen(sctx->priv_user), + sctx->external_user, + sctx->external_user ? strlen(sctx->external_user) : 0, + sctx->proxy_user, strlen(sctx->proxy_user), + sctx->host, sctx->host ? strlen(sctx->host) : 0, + sctx->ip, sctx->ip ? strlen(sctx->ip) : 0, + thd->db, thd->db ? strlen(thd->db) : 0); + } +} static inline void mysql_audit_external_lock(THD *thd, TABLE_SHARE *share, int lock) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 723c4f2c50d..948b2ee7285 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -49,13 +49,15 @@ #include "sql_trigger.h" #include "transaction.h" #include "sql_prepare.h" +#include "sql_statistics.h" #include <m_ctype.h> #include <my_dir.h> #include <hash.h> #include "rpl_filter.h" #include "sql_table.h" // build_table_filename -#include "datadict.h" // dd_frm_type() +#include "datadict.h" // dd_frm_is_view() #include "sql_hset.h" // Hash_set +#include "rpl_rli.h" // rpl_group_info #ifdef __WIN__ #include <io.h> #endif @@ -166,6 +168,24 @@ Repair_mrg_table_error_handler::handle_condition(THD *, Protects table_def_hash, used and unused lists in the TABLE_SHARE object, LRU lists of used TABLEs and used TABLE_SHAREs, refresh_version and the table id counter. + In particular: + + end_of_unused_share + last_table_id + oldest_unused_share + refresh_version + table_cache_count + table_def_cache + table_def_shutdown_in_progress + unused_tables + TABLE::next + TABLE::prev + TABLE_SHARE::free_tables + TABLE_SHARE::m_flush_tickets + TABLE_SHARE::next + TABLE_SHARE::prev + TABLE_SHARE::ref_count + TABLE_SHARE::used_tables */ mysql_mutex_t LOCK_open; @@ -303,18 +323,18 @@ static void check_unused(THD *thd) Create a table cache key SYNOPSIS - create_table_def_key() + create_tmp_table_def_key() thd Thread handler key Create key here (must be of size MAX_DBKEY_LENGTH) - table_list Table definition - tmp_table Set if table is a tmp table + db Database name. + table_name Table name. IMPLEMENTATION The table cache_key is created from: db_name + \0 table_name + \0 - if the table is a tmp table, we add the following to make each tmp table + additionally we add the following to make each tmp table unique on the slave: 4 bytes for master thread id @@ -324,19 +344,13 @@ static void check_unused(THD *thd) Length of key */ -uint create_table_def_key(THD *thd, char *key, - const TABLE_LIST *table_list, - bool tmp_table) +uint create_tmp_table_def_key(THD *thd, char *key, + const char *db, const char *table_name) { - uint key_length= create_table_def_key(key, table_list->db, - table_list->table_name); - - if (tmp_table) - { - int4store(key + key_length, thd->server_id); - int4store(key + key_length + 4, thd->variables.pseudo_thread_id); - key_length+= TMP_TABLE_KEY_EXTRA; - } + uint key_length= create_table_def_key(key, db, table_name); + int4store(key + key_length, thd->variables.server_id); + int4store(key + key_length + 4, thd->variables.pseudo_thread_id); + key_length+= TMP_TABLE_KEY_EXTRA; return key_length; } @@ -377,6 +391,14 @@ bool table_def_init(void) init_tdc_psi_keys(); #endif mysql_mutex_init(key_LOCK_open, &LOCK_open, MY_MUTEX_INIT_FAST); + mysql_mutex_record_order(&LOCK_active_mi, &LOCK_open); + /* + When we delete from the table_def_cache(), the free function + table_def_free_entry() is invoked from my_hash_delete(), which calls + free_table_share(), which may unload plugins, which can remove status + variables and hence takes LOCK_status. Record this locking order here. + */ + mysql_mutex_record_order(&LOCK_open, &LOCK_status); oldest_unused_share= &end_of_unused_share; end_of_unused_share.prev= &oldest_unused_share; @@ -395,6 +417,7 @@ bool table_def_init(void) void table_def_start_shutdown(void) { + DBUG_ENTER("table_def_start_shutdown"); if (table_def_inited) { mysql_mutex_lock(&LOCK_open); @@ -409,6 +432,7 @@ void table_def_start_shutdown(void) /* Free all cached but unused TABLEs and TABLE_SHAREs. */ close_cached_tables(NULL, NULL, FALSE, LONG_TIMEOUT); } + DBUG_VOID_RETURN; } @@ -556,97 +580,105 @@ static void table_def_unuse_table(TABLE *table) table_list Table that should be opened key Table cache key key_length Length of key - db_flags Flags to open_table_def(): - OPEN_VIEW - error out: Error code from open_table_def() + flags operation: what to open table or view + hash_value = my_calc_hash(&table_def_cache, key, key_length) IMPLEMENTATION Get a table definition from the table definition cache. If it doesn't exist, create a new from the table definition file. - NOTES - We must have wrlock on LOCK_open when we come here - (To be changed later) - RETURN 0 Error # Share for table */ -TABLE_SHARE *get_table_share(THD *thd, TABLE_LIST *table_list, char *key, - uint key_length, uint db_flags, int *error, +TABLE_SHARE *get_table_share(THD *thd, const char *db, const char *table_name, + char *key, uint key_length, uint flags, my_hash_value_type hash_value) { TABLE_SHARE *share; DBUG_ENTER("get_table_share"); - *error= 0; - - /* - To be able perform any operation on table we should own - some kind of metadata lock on it. - */ - DBUG_ASSERT(thd->mdl_context.is_lock_owner(MDL_key::TABLE, - table_list->db, - table_list->table_name, - MDL_SHARED)); + mysql_mutex_lock(&LOCK_open); /* Read table definition from cache */ - if ((share= (TABLE_SHARE*) my_hash_search_using_hash_value(&table_def_cache, - hash_value, (uchar*) key, key_length))) - goto found; + share= (TABLE_SHARE*) my_hash_search_using_hash_value(&table_def_cache, + hash_value, (uchar*) key, key_length); - if (!(share= alloc_table_share(table_list, key, key_length))) + if (!share) { - DBUG_RETURN(0); - } + if (!(share= alloc_table_share(db, table_name, key, key_length))) + goto err; - /* - We assign a new table id under the protection of LOCK_open. - We do this instead of creating a new mutex - and using it for the sole purpose of serializing accesses to a - static variable, we assign the table id here. We assign it to the - share before inserting it into the table_def_cache to be really - sure that it cannot be read from the cache without having a table - id assigned. - - CAVEAT. This means that the table cannot be used for - binlogging/replication purposes, unless get_table_share() has been - called directly or indirectly. - */ - assign_new_table_id(share); - - if (my_hash_insert(&table_def_cache, (uchar*) share)) - { - free_table_share(share); - DBUG_RETURN(0); // return error - } - if (open_table_def(thd, share, db_flags)) - { - *error= share->error; - (void) my_hash_delete(&table_def_cache, (uchar*) share); - DBUG_RETURN(0); + /* + We assign a new table id under the protection of LOCK_open. + We do this instead of creating a new mutex + and using it for the sole purpose of serializing accesses to a + static variable, we assign the table id here. We assign it to the + share before inserting it into the table_def_cache to be really + sure that it cannot be read from the cache without having a table + id assigned. + + CAVEAT. This means that the table cannot be used for + binlogging/replication purposes, unless get_table_share() has been + called directly or indirectly. + */ + assign_new_table_id(share); + + if (my_hash_insert(&table_def_cache, (uchar*) share)) + { + free_table_share(share); + goto err; + } + share->ref_count++; // Mark in use + share->error= OPEN_FRM_OPEN_ERROR; + mysql_mutex_lock(&share->LOCK_ha_data); + mysql_mutex_unlock(&LOCK_open); + + /* note that get_table_share() *always* uses discovery */ + open_table_def(thd, share, flags | GTS_USE_DISCOVERY); + + mysql_mutex_unlock(&share->LOCK_ha_data); + mysql_mutex_lock(&LOCK_open); + + if (share->error) + { + share->ref_count--; + (void) my_hash_delete(&table_def_cache, (uchar*) share); + goto err; + } + DBUG_PRINT("exit", ("share: 0x%lx ref_count: %u", + (ulong) share, share->ref_count)); + + goto end; } - share->ref_count++; // Mark in use - DBUG_PRINT("exit", ("share: 0x%lx ref_count: %u", - (ulong) share, share->ref_count)); - DBUG_RETURN(share); -found: + /* cannot force discovery of a cached share */ + DBUG_ASSERT(!(flags & GTS_FORCE_DISCOVERY)); + + /* make sure that open_table_def() for this share is not running */ + mysql_mutex_lock(&share->LOCK_ha_data); + mysql_mutex_unlock(&share->LOCK_ha_data); + /* We found an existing table definition. Return it if we didn't get an error when reading the table definition from file. */ if (share->error) { - /* Table definition contained an error */ - open_table_error(share, share->error, share->open_errno, share->errarg); - DBUG_RETURN(0); + open_table_error(share, share->error, share->open_errno); + goto err; } - if (share->is_view && !(db_flags & OPEN_VIEW)) + + if (share->is_view && !(flags & GTS_VIEW)) { - open_table_error(share, 1, ENOENT, 0); - DBUG_RETURN(0); + open_table_error(share, OPEN_FRM_NOT_A_TABLE, ENOENT); + goto err; + } + if (!share->is_view && !(flags & GTS_TABLE)) + { + open_table_error(share, OPEN_FRM_NOT_A_VIEW, ENOENT); + goto err; } ++share->ref_count; @@ -671,99 +703,28 @@ found: DBUG_PRINT("exit", ("share: 0x%lx ref_count: %u", (ulong) share, share->ref_count)); - DBUG_RETURN(share); -} - + goto end; -/** - Get a table share. If it didn't exist, try creating it from engine - - For arguments and return values, see get_table_share() -*/ - -static TABLE_SHARE * -get_table_share_with_discover(THD *thd, TABLE_LIST *table_list, - char *key, uint key_length, - uint db_flags, int *error, - my_hash_value_type hash_value) - -{ - TABLE_SHARE *share; - bool exists; - DBUG_ENTER("get_table_share_with_discover"); - - share= get_table_share(thd, table_list, key, key_length, db_flags, error, - hash_value); - /* - If share is not NULL, we found an existing share. - - If share is NULL, and there is no error, we're inside - pre-locking, which silences 'ER_NO_SUCH_TABLE' errors - with the intention to silently drop non-existing tables - from the pre-locking list. In this case we still need to try - auto-discover before returning a NULL share. - - Or, we're inside SHOW CREATE VIEW, which - also installs a silencer for ER_NO_SUCH_TABLE error. - - If share is NULL and the error is ER_NO_SUCH_TABLE, this is - the same as above, only that the error was not silenced by - pre-locking or SHOW CREATE VIEW. - - In both these cases it won't harm to try to discover the - table. - - Finally, if share is still NULL, it's a real error and we need - to abort. - - @todo Rework alternative ways to deal with ER_NO_SUCH TABLE. - */ - if (share || - (thd->is_error() && thd->stmt_da->sql_errno() != ER_NO_SUCH_TABLE && - thd->stmt_da->sql_errno() != ER_NO_SUCH_TABLE_IN_ENGINE)) - DBUG_RETURN(share); - - *error= 0; +err: + mysql_mutex_unlock(&LOCK_open); + DBUG_RETURN(0); - /* Table didn't exist. Check if some engine can provide it */ - if (ha_check_if_table_exists(thd, table_list->db, table_list->table_name, - &exists)) - { - thd->clear_error(); - /* Conventionally, the storage engine API does not report errors. */ - my_error(ER_OUT_OF_RESOURCES, MYF(0)); - } - else if (! exists) +end: + if (flags & GTS_NOLOCK) { + release_table_share(share); /* - No such table in any engine. - Hide "Table doesn't exist" errors if the table belongs to a view. - The check for thd->is_error() is necessary to not push an - unwanted error in case the error was already silenced. - @todo Rework the alternative ways to deal with ER_NO_SUCH TABLE. + if GTS_NOLOCK is requested, the returned share pointer cannot be used, + the share it points to may go away any moment. + But perhaps the caller is only interested to know whether a share or + table existed? + Let's return an invalid pointer here to catch dereferencing attempts. */ - if (thd->is_error()) - { - if (table_list->parent_l) - { - thd->clear_error(); - my_error(ER_WRONG_MRG_TABLE, MYF(0)); - } - else if (table_list->belong_to_view) - { - TABLE_LIST *view= table_list->belong_to_view; - thd->clear_error(); - my_error(ER_VIEW_INVALID, MYF(0), - view->view_db.str, view->view_name.str); - } - } + share= (TABLE_SHARE*) 1; } - else - { - thd->clear_error(); - *error= 7; /* Run auto-discover. */ - } - DBUG_RETURN(NULL); + + mysql_mutex_unlock(&LOCK_open); + DBUG_RETURN(share); } @@ -835,8 +796,9 @@ TABLE_SHARE *get_cached_table_share(const char *db, const char *table_name) mysql_mutex_assert_owner(&LOCK_open); key_length= create_table_def_key(key, db, table_name); - return (TABLE_SHARE*) my_hash_search(&table_def_cache, - (uchar*) key, key_length); + TABLE_SHARE* share= (TABLE_SHARE*)my_hash_search(&table_def_cache, + (uchar*) key, key_length); + return !share || share->error ? 0 : share; } @@ -1082,6 +1044,9 @@ bool close_cached_tables(THD *thd, TABLE_LIST *tables, mysql_mutex_unlock(&LOCK_open); + DBUG_PRINT("info", ("open table definitions: %d", + (int) table_def_cache.records)); + if (!wait_for_refresh) DBUG_RETURN(result); @@ -1266,11 +1231,24 @@ bool close_cached_connection_tables(THD *thd, LEX_STRING *connection) static void mark_temp_tables_as_free_for_reuse(THD *thd) { + DBUG_ENTER("mark_temp_tables_as_free_for_reuse"); + + thd->lock_temporary_tables(); for (TABLE *table= thd->temporary_tables ; table ; table= table->next) { if ((table->query_id == thd->query_id) && ! table->open_by_handler) mark_tmp_table_for_reuse(table); } + thd->unlock_temporary_tables(); + if (thd->rgi_slave) + { + /* + Temporary tables are shared with other by sql execution threads. + As a safety messure, clear the pointer to the common area. + */ + thd->temporary_tables= 0; + } + DBUG_VOID_RETURN; } @@ -1284,6 +1262,7 @@ static void mark_temp_tables_as_free_for_reuse(THD *thd) void mark_tmp_table_for_reuse(TABLE *table) { + DBUG_ENTER("mark_tmp_table_for_reuse"); DBUG_ASSERT(table->s->tmp_table); table->query_id= 0; @@ -1314,6 +1293,7 @@ void mark_tmp_table_for_reuse(TABLE *table) LOCK TABLES is allowed (but ignored) for a temporary table. */ table->reginfo.lock_type= TL_WRITE; + DBUG_VOID_RETURN; } @@ -1664,6 +1644,10 @@ static inline uint tmpkeyval(THD *thd, TABLE *table) /* Close all temporary tables created by 'CREATE TEMPORARY TABLE' for thread creates one DROP TEMPORARY TABLE binlog event for each pseudo-thread + + Temporary tables created in a sql slave is closed by + Relay_log_info::close_temporary_tables() + */ bool close_temporary_tables(THD *thd) @@ -1678,6 +1662,7 @@ bool close_temporary_tables(THD *thd) if (!thd->temporary_tables) DBUG_RETURN(FALSE); + DBUG_ASSERT(!thd->rgi_slave); if (!mysql_bin_log.is_open()) { @@ -2116,7 +2101,7 @@ TABLE *find_temporary_table(THD *thd, const char *db, const char *table_name) TABLE *find_temporary_table(THD *thd, const TABLE_LIST *tl) { char key[MAX_DBKEY_LENGTH]; - uint key_length= create_table_def_key(thd, key, tl, 1); + uint key_length= create_tmp_table_def_key(thd, key, tl->db, tl->table_name); return find_temporary_table(thd, key, key_length); } @@ -2132,16 +2117,42 @@ TABLE *find_temporary_table(THD *thd, const char *table_key, uint table_key_length) { + TABLE *result= 0; + if (!thd->have_temporary_tables()) + return NULL; + + thd->lock_temporary_tables(); for (TABLE *table= thd->temporary_tables; table; table= table->next) { if (table->s->table_cache_key.length == table_key_length && !memcmp(table->s->table_cache_key.str, table_key, table_key_length)) { - return table; + /* + We need to set the THD as it may be different in case of + parallel replication + */ + if (table->in_use != thd) + { + table->in_use= thd; +#ifdef REMOVE_AFTER_MERGE_WITH_10 + if (thd->rgi_slave) + { + /* + We may be stealing an opened temporary tables from one slave + thread to another, we need to let the performance schema know that, + for aggregates per thread to work properly. + */ + table->file->unbind_psi(); + table->file->rebind_psi(); + } +#endif + } + result= table; + break; } } - - return NULL; + thd->unlock_temporary_tables(); + return result; } @@ -2189,6 +2200,9 @@ int drop_temporary_table(THD *thd, TABLE_LIST *table_list, bool *is_trans) /* Table might be in use by some outer statement. */ if (table->query_id && table->query_id != thd->query_id) { + DBUG_PRINT("info", ("table->query_id: %lu thd->query_id: %lu", + (ulong) table->query_id, (ulong) thd->query_id)); + my_error(ER_CANT_REOPEN_TABLE, MYF(0), table->alias.c_ptr()); DBUG_RETURN(-1); } @@ -2217,6 +2231,7 @@ void close_temporary_table(THD *thd, TABLE *table, table->s->db.str, table->s->table_name.str, (long) table, table->alias.c_ptr())); + thd->lock_temporary_tables(); if (table->prev) { table->prev->next= table->next; @@ -2236,12 +2251,14 @@ void close_temporary_table(THD *thd, TABLE *table, if (thd->temporary_tables) table->next->prev= 0; } - if (thd->slave_thread) + if (thd->rgi_slave) { /* natural invariant of temporary_tables */ DBUG_ASSERT(slave_open_temp_tables || !thd->temporary_tables); - slave_open_temp_tables--; + thread_safe_decrement32(&slave_open_temp_tables, &thread_running_lock); + table->in_use= 0; // No statistics } + thd->unlock_temporary_tables(); close_temporary(table, free_share, delete_table); DBUG_VOID_RETURN; } @@ -2296,15 +2313,12 @@ bool rename_temporary_table(THD* thd, TABLE *table, const char *db, char *key; uint key_length; TABLE_SHARE *share= table->s; - TABLE_LIST table_list; DBUG_ENTER("rename_temporary_table"); if (!(key=(char*) alloc_root(&share->mem_root, MAX_DBKEY_LENGTH))) DBUG_RETURN(1); /* purecov: inspected */ - table_list.db= (char*) db; - table_list.table_name= (char*) table_name; - key_length= create_table_def_key(thd, key, &table_list, 1); + key_length= create_tmp_table_def_key(thd, key, db, table_name); share->set_table_cache_key(key, key_length); DBUG_RETURN(0); } @@ -2394,71 +2408,6 @@ void drop_open_table(THD *thd, TABLE *table, const char *db_name, /** - Check that table exists in table definition cache, on disk - or in some storage engine. - - @param thd Thread context - @param table Table list element - @param fast_check Check only if share or .frm file exists - @param[out] exists Out parameter which is set to TRUE if table - exists and to FALSE otherwise. - - @note This function acquires LOCK_open internally. - - @note If there is no .FRM file for the table but it exists in one - of engines (e.g. it was created on another node of NDB cluster) - this function will fetch and create proper .FRM file for it. - - @retval TRUE Some error occurred - @retval FALSE No error. 'exists' out parameter set accordingly. -*/ - -bool check_if_table_exists(THD *thd, TABLE_LIST *table, bool fast_check, - bool *exists) -{ - char path[FN_REFLEN + 1]; - TABLE_SHARE *share; - DBUG_ENTER("check_if_table_exists"); - - *exists= TRUE; - - DBUG_ASSERT(fast_check || - thd->mdl_context. - is_lock_owner(MDL_key::TABLE, table->db, - table->table_name, MDL_SHARED)); - - mysql_mutex_lock(&LOCK_open); - share= get_cached_table_share(table->db, table->table_name); - mysql_mutex_unlock(&LOCK_open); - - if (share) - goto end; - - build_table_filename(path, sizeof(path) - 1, table->db, table->table_name, - reg_ext, 0); - - if (!access(path, F_OK)) - goto end; - - if (fast_check) - { - *exists= FALSE; - goto end; - } - - /* .FRM file doesn't exist. Check if some engine can provide it. */ - if (ha_check_if_table_exists(thd, table->db, table->table_name, exists)) - { - my_printf_error(ER_OUT_OF_RESOURCES, "Failed to open '%-.64s', error while " - "unpacking from engine", MYF(0), table->table_name); - DBUG_RETURN(TRUE); - } -end: - DBUG_RETURN(FALSE); -} - - -/** An error handler which converts, if possible, ER_LOCK_DEADLOCK error that can occur when we are trying to acquire a metadata lock to a request for back-off and re-start of open_tables() process. @@ -2731,9 +2680,9 @@ bool open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root, char *alias= table_list->alias; uint flags= ot_ctx->get_flags(); MDL_ticket *mdl_ticket; - int error; TABLE_SHARE *share; my_hash_value_type hash_value; + uint gts_flags; DBUG_ENTER("open_table"); /* an open table operation needs a lot of the stack space */ @@ -2743,8 +2692,9 @@ bool open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root, if (thd->killed) DBUG_RETURN(TRUE); - key_length= (create_table_def_key(thd, key, table_list, 1) - - TMP_TABLE_KEY_EXTRA); + key_length= create_tmp_table_def_key(thd, key, table_list->db, + table_list->table_name) - + TMP_TABLE_KEY_EXTRA; /* Unless requested otherwise, try to resolve this table in the list @@ -2754,35 +2704,30 @@ bool open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root, TODO: move this block into a separate function. */ if (table_list->open_type != OT_BASE_ONLY && - ! (flags & MYSQL_OPEN_SKIP_TEMPORARY)) + ! (flags & MYSQL_OPEN_SKIP_TEMPORARY) && thd->have_temporary_tables()) { - for (table= thd->temporary_tables; table ; table=table->next) - { - if (table->s->table_cache_key.length == key_length + - TMP_TABLE_KEY_EXTRA && - !memcmp(table->s->table_cache_key.str, key, - key_length + TMP_TABLE_KEY_EXTRA)) + if ((table= find_temporary_table(thd, key, + key_length + TMP_TABLE_KEY_EXTRA))) + { + /* + Check if we're trying to use the same temporary table twice in a query. + Right now we don't support this because a temporary table + is always represented by only one TABLE object in THD, and + it can not be cloned. Emit an error for an unsupported behaviour. + */ + if (table->query_id) { - /* - We're trying to use the same temporary table twice in a query. - Right now we don't support this because a temporary table - is always represented by only one TABLE object in THD, and - it can not be cloned. Emit an error for an unsupported behaviour. - */ - if (table->query_id) - { - DBUG_PRINT("error", - ("query_id: %lu server_id: %u pseudo_thread_id: %lu", - (ulong) table->query_id, (uint) thd->server_id, - (ulong) thd->variables.pseudo_thread_id)); - my_error(ER_CANT_REOPEN_TABLE, MYF(0), table->alias.c_ptr()); - DBUG_RETURN(TRUE); - } - table->query_id= thd->query_id; - thd->thread_specific_used= TRUE; - DBUG_PRINT("info",("Using temporary table")); - goto reset; + DBUG_PRINT("error", + ("query_id: %lu server_id: %u pseudo_thread_id: %lu", + (ulong) table->query_id, (uint) thd->variables.server_id, + (ulong) thd->variables.pseudo_thread_id)); + my_error(ER_CANT_REOPEN_TABLE, MYF(0), table->alias.c_ptr()); + DBUG_RETURN(TRUE); } + table->query_id= thd->query_id; + thd->thread_specific_used= TRUE; + DBUG_PRINT("info",("Using temporary table")); + goto reset; } } @@ -2876,7 +2821,6 @@ bool open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root, MDL_SHARED)) { char path[FN_REFLEN + 1]; - enum legacy_db_type not_used; build_table_filename(path, sizeof(path) - 1, table_list->db, table_list->table_name, reg_ext, 0); /* @@ -2886,7 +2830,7 @@ bool open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root, during prelocking process (in this case in theory we still should hold shared metadata lock on it). */ - if (dd_frm_type(thd, path, ¬_used) == FRMTYPE_VIEW) + if (dd_frm_is_view(thd, path)) { if (!tdc_open_view(thd, table_list, alias, key, key_length, mem_root, 0)) @@ -2923,12 +2867,12 @@ bool open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root, global read lock until end of this statement in order to have this statement blocked by active FLUSH TABLES WITH READ LOCK. - We don't block acquire this protection under LOCK TABLES as + We don't need to acquire this protection under LOCK TABLES as such protection already acquired at LOCK TABLES time and not released until UNLOCK TABLES. We don't block statements which modify only temporary tables - as these tables are not preserved by backup by any form of + as these tables are not preserved by any form of backup which uses FLUSH TABLES WITH READ LOCK. TODO: The fact that we sometimes acquire protection against @@ -2993,12 +2937,7 @@ bool open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root, if (table_list->open_strategy == TABLE_LIST::OPEN_IF_EXISTS) { - bool exists; - - if (check_if_table_exists(thd, table_list, 0, &exists)) - DBUG_RETURN(TRUE); - - if (!exists) + if (!ha_table_exists(thd, table_list->db, table_list->table_name)) DBUG_RETURN(FALSE); /* Table exists. Let us try to open it. */ @@ -3006,26 +2945,40 @@ bool open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root, else if (table_list->open_strategy == TABLE_LIST::OPEN_STUB) DBUG_RETURN(FALSE); + if (table_list->i_s_requested_object & OPEN_TABLE_ONLY) + gts_flags= GTS_TABLE; + else if (table_list->i_s_requested_object & OPEN_VIEW_ONLY) + gts_flags= GTS_VIEW; + else + gts_flags= GTS_TABLE | GTS_VIEW; + retry_share: - mysql_mutex_lock(&LOCK_open); + share= get_table_share(thd, table_list->db, table_list->table_name, + key, key_length, gts_flags, hash_value); - if (!(share= get_table_share_with_discover(thd, table_list, key, - key_length, OPEN_VIEW, - &error, - hash_value))) + if (!share) { - mysql_mutex_unlock(&LOCK_open); /* - If thd->is_error() is not set, we either need discover - (error == 7), or the error was silenced by the prelocking - handler (error == 0), in which case we should skip this - table. + Hide "Table doesn't exist" errors if the table belongs to a view. + The check for thd->is_error() is necessary to not push an + unwanted error in case the error was already silenced. + @todo Rework the alternative ways to deal with ER_NO_SUCH TABLE. */ - if (error == 7 && !thd->is_error()) + if (thd->is_error()) { - (void) ot_ctx->request_backoff_action(Open_table_context::OT_DISCOVER, - table_list); + if (table_list->parent_l) + { + thd->clear_error(); + my_error(ER_WRONG_MRG_TABLE, MYF(0)); + } + else if (table_list->belong_to_view) + { + TABLE_LIST *view= table_list->belong_to_view; + thd->clear_error(); + my_error(ER_VIEW_INVALID, MYF(0), + view->view_db.str, view->view_name.str); + } } DBUG_RETURN(TRUE); } @@ -3039,7 +2992,7 @@ retry_share: if (table_list->parent_l) { my_error(ER_WRONG_MRG_TABLE, MYF(0)); - goto err_unlock; + goto err_lock; } /* @@ -3047,13 +3000,7 @@ retry_share: that it was a view when the statement was prepared. */ if (check_and_update_table_version(thd, table_list, share)) - goto err_unlock; - if (table_list->i_s_requested_object & OPEN_TABLE_ONLY) - { - my_error(ER_NO_SUCH_TABLE, MYF(0), table_list->db, - table_list->table_name); - goto err_unlock; - } + goto err_lock; /* Open view */ if (open_new_frm(thd, share, alias, @@ -3062,7 +3009,9 @@ retry_share: READ_KEYINFO | COMPUTE_TYPES | EXTRA_RECORD, thd->open_options, 0, table_list, mem_root)) - goto err_unlock; + goto err_lock; + + mysql_mutex_lock(&LOCK_open); /* TODO: Don't free this */ release_table_share(share); @@ -3073,23 +3022,9 @@ retry_share: DBUG_RETURN(FALSE); } - /* - Note that situation when we are trying to open a table for what - was a view during previous execution of PS will be handled in by - the caller. Here we should simply open our table even if - TABLE_LIST::view is true. - */ - - if (table_list->i_s_requested_object & OPEN_VIEW_ONLY) - { - my_error(ER_NO_SUCH_TABLE, MYF(0), table_list->db, - table_list->table_name); - goto err_unlock; - } - + mysql_mutex_lock(&LOCK_open); if (!(flags & MYSQL_OPEN_IGNORE_FLUSH) || - (share->protected_against_usage() && - !(flags & MYSQL_OPEN_FOR_REPAIR))) + (share->protected_against_usage() && !(flags & MYSQL_OPEN_FOR_REPAIR))) { if (share->has_old_version()) { @@ -3143,17 +3078,15 @@ retry_share: { table= share->free_tables.front(); table_def_use_table(thd, table); - /* - We need to release share as we have EXTRA reference to it in our hands. - */ + /* Release the share as we hold an extra reference to it */ DBUG_PRINT("info", ("release temporarily acquired table share")); release_table_share(share); } else { - /* - We have too many TABLE instances around let us try to get rid of them. - */ + enum open_frm_error error; + + /* If we have too many TABLE instances around, try to get rid of them */ while (table_cache_count > table_cache_size && unused_tables) free_cache_entry(unused_tables); @@ -3176,7 +3109,7 @@ retry_share: { my_free(table); - if (error == 7) + if (error == OPEN_FRM_DISCOVER) (void) ot_ctx->request_backoff_action(Open_table_context::OT_DISCOVER, table_list); else if (share->crashed) @@ -3222,7 +3155,6 @@ retry_share: err_lock: mysql_mutex_lock(&LOCK_open); -err_unlock: release_table_share(share); mysql_mutex_unlock(&LOCK_open); @@ -3852,38 +3784,25 @@ bool tdc_open_view(THD *thd, TABLE_LIST *table_list, const char *alias, MEM_ROOT *mem_root, uint flags) { TABLE not_used; - int error; - my_hash_value_type hash_value; TABLE_SHARE *share; - hash_value= my_calc_hash(&table_def_cache, (uchar*) cache_key, - cache_key_length); - mysql_mutex_lock(&LOCK_open); + if (!(share= get_table_share(thd, table_list->db, table_list->table_name, + cache_key, cache_key_length, GTS_VIEW))) + return TRUE; - if (!(share= get_table_share(thd, table_list, cache_key, - cache_key_length, - OPEN_VIEW, &error, - hash_value))) - goto err; + DBUG_ASSERT(share->is_view); - if (share->is_view && - !open_new_frm(thd, share, alias, - (uint) (HA_OPEN_KEYFILE | HA_OPEN_RNDFILE | - HA_GET_INDEX | HA_TRY_READ_ONLY), - READ_KEYINFO | COMPUTE_TYPES | EXTRA_RECORD | - flags, thd->open_options, ¬_used, table_list, - mem_root)) - { - release_table_share(share); - mysql_mutex_unlock(&LOCK_open); - return FALSE; - } + bool err= open_new_frm(thd, share, alias, + (HA_OPEN_KEYFILE | HA_OPEN_RNDFILE | + HA_GET_INDEX | HA_TRY_READ_ONLY), + READ_KEYINFO | COMPUTE_TYPES | EXTRA_RECORD | flags, + thd->open_options, ¬_used, table_list, mem_root); - my_error(ER_WRONG_OBJECT, MYF(0), share->db.str, share->table_name.str, "VIEW"); + mysql_mutex_lock(&LOCK_open); release_table_share(share); -err: mysql_mutex_unlock(&LOCK_open); - return TRUE; + + return err; } @@ -3937,40 +3856,20 @@ static bool open_table_entry_fini(THD *thd, TABLE_SHARE *share, TABLE *entry) static bool auto_repair_table(THD *thd, TABLE_LIST *table_list) { - char cache_key[MAX_DBKEY_LENGTH]; - uint cache_key_length; TABLE_SHARE *share; TABLE *entry; - int not_used; bool result= TRUE; - my_hash_value_type hash_value; - - cache_key_length= create_table_def_key(thd, cache_key, table_list, 0); thd->clear_error(); - hash_value= my_calc_hash(&table_def_cache, (uchar*) cache_key, - cache_key_length); - mysql_mutex_lock(&LOCK_open); + if (!(entry= (TABLE*)my_malloc(sizeof(TABLE), MYF(MY_WME)))) + return result; - if (!(share= get_table_share(thd, table_list, cache_key, - cache_key_length, - OPEN_VIEW, ¬_used, - hash_value))) - goto end_unlock; + if (!(share= get_table_share(thd, table_list->db, table_list->table_name, + GTS_TABLE))) + goto end_free; - if (share->is_view) - { - release_table_share(share); - goto end_unlock; - } - - if (!(entry= (TABLE*)my_malloc(sizeof(TABLE), MYF(MY_WME)))) - { - release_table_share(share); - goto end_unlock; - } - mysql_mutex_unlock(&LOCK_open); + DBUG_ASSERT(! share->is_view); if (open_table_from_share(thd, share, table_list->alias, (uint) (HA_OPEN_KEYFILE | HA_OPEN_RNDFILE | @@ -3995,7 +3894,6 @@ static bool auto_repair_table(THD *thd, TABLE_LIST *table_list) closefrm(entry, 0); result= FALSE; } - my_free(entry); mysql_mutex_lock(&LOCK_open); release_table_share(share); @@ -4003,8 +3901,9 @@ static bool auto_repair_table(THD *thd, TABLE_LIST *table_list) tdc_remove_table(thd, TDC_RT_REMOVE_ALL, table_list->db, table_list->table_name, TRUE); -end_unlock: mysql_mutex_unlock(&LOCK_open); +end_free: + my_free(entry); return result; } @@ -4145,11 +4044,16 @@ recover_from_failed_open() tdc_remove_table(m_thd, TDC_RT_REMOVE_ALL, m_failed_table->db, m_failed_table->table_name, FALSE); - ha_create_table_from_engine(m_thd, m_failed_table->db, - m_failed_table->table_name); m_thd->warning_info->clear_warning_info(m_thd->query_id); m_thd->clear_error(); // Clear error message + + if ((result= + !get_table_share(m_thd, m_failed_table->db, + m_failed_table->table_name, + GTS_TABLE | GTS_FORCE_DISCOVERY | GTS_NOLOCK))) + break; + m_thd->mdl_context.release_transactional_locks(); break; } @@ -4640,6 +4544,32 @@ open_and_process_table(THD *thd, LEX *lex, TABLE_LIST *tables, goto end; } + if (get_use_stat_tables_mode(thd) > NEVER && tables->table) + { + TABLE_SHARE *table_share= tables->table->s; + if (table_share && table_share->table_category == TABLE_CATEGORY_USER && + table_share->tmp_table == NO_TMP_TABLE) + { + if (table_share->stats_cb.stats_can_be_read || + !alloc_statistics_for_table_share(thd, table_share, FALSE)) + { + if (table_share->stats_cb.stats_can_be_read) + { + KEY *key_info= table_share->key_info; + KEY *key_info_end= key_info + table_share->keys; + KEY *table_key_info= tables->table->key_info; + for ( ; key_info < key_info_end; key_info++, table_key_info++) + table_key_info->read_stats= key_info->read_stats; + Field **field_ptr= table_share->field; + Field **table_field_ptr= tables->table->field; + for ( ; *field_ptr; field_ptr++, table_field_ptr++) + (*table_field_ptr)->read_stats= (*field_ptr)->read_stats; + tables->table->stats_is_read= table_share->stats_cb.stats_is_read; + } + } + } + } + process_view_routines: /* Again we may need cache all routines used by this view and add @@ -4666,10 +4596,9 @@ end: DBUG_RETURN(error); } -extern "C" uchar *schema_set_get_key(const uchar *record, size_t *length, +extern "C" uchar *schema_set_get_key(const TABLE_LIST *table, size_t *length, my_bool not_used __attribute__((unused))) { - TABLE_LIST *table=(TABLE_LIST*) record; *length= table->db_length; return (uchar*) table->db; } @@ -4710,7 +4639,7 @@ lock_table_names(THD *thd, MDL_request_list mdl_requests; TABLE_LIST *table; MDL_request global_request; - Hash_set<TABLE_LIST, schema_set_get_key> schema_set; + Hash_set<TABLE_LIST> schema_set(schema_set_get_key); ulong org_lock_wait_timeout= lock_wait_timeout; /* Check if we are using CREATE TABLE ... IF NOT EXISTS */ bool create_table; @@ -4739,7 +4668,7 @@ lock_table_names(THD *thd, if (mdl_requests.is_empty()) DBUG_RETURN(FALSE); - /* Check if CREATE TABLE IF NOT EXISTS was used */ + /* Check if CREATE TABLE was used */ create_table= (tables_start && tables_start->open_strategy == TABLE_LIST::OPEN_IF_EXISTS); @@ -4749,7 +4678,7 @@ lock_table_names(THD *thd, Scoped locks: Take intention exclusive locks on all involved schemas. */ - Hash_set<TABLE_LIST, schema_set_get_key>::Iterator it(schema_set); + Hash_set<TABLE_LIST>::Iterator it(schema_set); while ((table= it++)) { MDL_request *schema_request= new (thd->mem_root) MDL_request; @@ -4778,12 +4707,9 @@ lock_table_names(THD *thd, for (;;) { - bool exists= TRUE; - bool res; - if (create_table) thd->push_internal_handler(&error_handler); // Avoid warnings & errors - res= thd->mdl_context.acquire_locks(&mdl_requests, lock_wait_timeout); + bool res= thd->mdl_context.acquire_locks(&mdl_requests, lock_wait_timeout); if (create_table) thd->pop_internal_handler(); if (!res) @@ -4793,13 +4719,10 @@ lock_table_names(THD *thd, DBUG_RETURN(TRUE); // Return original error /* - We come here in the case of lock timeout when executing - CREATE TABLE IF NOT EXISTS. - Verify that table really exists (it should as we got a lock conflict) + We come here in the case of lock timeout when executing CREATE TABLE. + Verify that table does exist (it usually does, as we got a lock conflict) */ - if (check_if_table_exists(thd, tables_start, 1, &exists)) - DBUG_RETURN(TRUE); // Should never happen - if (exists) + if (ha_table_exists(thd, tables_start->db, tables_start->table_name)) { if (thd->lex->create_info.options & HA_LEX_CREATE_IF_NOT_EXISTS) { @@ -4811,17 +4734,16 @@ lock_table_names(THD *thd, my_error(ER_TABLE_EXISTS_ERROR, MYF(0), tables_start->table_name); DBUG_RETURN(TRUE); } - /* purecov: begin inspected */ /* - We got error from acquire_locks but table didn't exists. - In theory this should never happen, except maybe in - CREATE or DROP DATABASE scenario. + We got error from acquire_locks, but the table didn't exists. + This could happen if another connection runs a statement + involving this non-existent table, and this statement took the mdl, + but didn't error out with ER_NO_SUCH_TABLE yet (yes, a race condition). We play safe and restart the original acquire_locks with the - original timeout + original timeout. */ create_table= 0; lock_wait_timeout= org_lock_wait_timeout; - /* purecov: end */ } } @@ -4944,11 +4866,11 @@ bool open_tables(THD *thd, TABLE_LIST **start, uint *counter, uint flags, } /* - Initialize temporary MEM_ROOT for new .FRM parsing. Do not allocate + Initialize temporary MEM_ROOT for new .FRM parsing. Do not alloctaate anything yet, to avoid penalty for statements which don't use views and thus new .FRM format. */ - init_sql_alloc(&new_frm_mem, 8024, 0); + init_sql_alloc(&new_frm_mem, 8024, 0, MYF(0)); thd->current_tablenr= 0; restart: @@ -5654,6 +5576,8 @@ bool open_and_lock_tables(THD *thd, TABLE_LIST *tables, if (lock_tables(thd, tables, counter, flags)) goto err; + (void) read_statistics_for_tables_if_needed(thd, tables); + if (derived) { if (mysql_handle_derived(thd->lex, DT_INIT)) @@ -6044,6 +5968,8 @@ void close_tables_for_reopen(THD *thd, TABLE_LIST **tables, the opened TABLE instance will be addded to THD::temporary_tables list. @param thd Thread context. + @param hton Storage engine of the table, if known, + or NULL otherwise. @param path Path (without .frm) @param db Database name. @param table_name Table name. @@ -6059,7 +5985,8 @@ void close_tables_for_reopen(THD *thd, TABLE_LIST **tables, @retval NULL on error. */ -TABLE *open_table_uncached(THD *thd, const char *path, const char *db, +TABLE *open_table_uncached(THD *thd, handlerton *hton, + const char *path, const char *db, const char *table_name, bool add_to_temporary_tables_list) { @@ -6067,18 +5994,16 @@ TABLE *open_table_uncached(THD *thd, const char *path, const char *db, TABLE_SHARE *share; char cache_key[MAX_DBKEY_LENGTH], *saved_cache_key, *tmp_path; uint key_length; - TABLE_LIST table_list; DBUG_ENTER("open_table_uncached"); DBUG_PRINT("enter", ("table: '%s'.'%s' path: '%s' server_id: %u " "pseudo_thread_id: %lu", db, table_name, path, - (uint) thd->server_id, (ulong) thd->variables.pseudo_thread_id)); + (uint) thd->variables.server_id, + (ulong) thd->variables.pseudo_thread_id)); - table_list.db= (char*) db; - table_list.table_name= (char*) table_name; /* Create the cache_key for temporary tables */ - key_length= create_table_def_key(thd, cache_key, &table_list, 1); + key_length= create_tmp_table_def_key(thd, cache_key, db, table_name); if (!(tmp_table= (TABLE*) my_malloc(sizeof(*tmp_table) + sizeof(*share) + strlen(path)+1 + key_length, @@ -6092,8 +6017,9 @@ TABLE *open_table_uncached(THD *thd, const char *path, const char *db, init_tmp_table_share(thd, share, saved_cache_key, key_length, strend(saved_cache_key)+1, tmp_path); + share->db_plugin= ha_lock_engine(thd, hton); - if (open_table_def(thd, share, 0) || + if (open_table_def(thd, share, GTS_TABLE | GTS_USE_DISCOVERY) || open_table_from_share(thd, share, table_name, (uint) (HA_OPEN_KEYFILE | HA_OPEN_RNDFILE | HA_GET_INDEX), @@ -6113,14 +6039,18 @@ TABLE *open_table_uncached(THD *thd, const char *path, const char *db, if (add_to_temporary_tables_list) { + thd->lock_temporary_tables(); /* growing temp list at the head */ tmp_table->next= thd->temporary_tables; if (tmp_table->next) tmp_table->next->prev= tmp_table; thd->temporary_tables= tmp_table; thd->temporary_tables->prev= 0; - if (thd->slave_thread) - slave_open_temp_tables++; + if (thd->rgi_slave) + { + thread_safe_increment32(&slave_open_temp_tables, &thread_running_lock); + } + thd->unlock_temporary_tables(); } tmp_table->pos_in_table_list= 0; DBUG_PRINT("tmptable", ("opened table: '%s'.'%s' 0x%lx", tmp_table->s->db.str, @@ -8888,34 +8818,33 @@ err_no_arena: ******************************************************************************/ -/* - Fill fields with given items. +/** + Fill the fields of a table with the values of an Item list - SYNOPSIS - fill_record() - thd thread handler - fields Item_fields list to be filled - values values to fill with - ignore_errors TRUE if we should ignore errors + @param thd thread handler + @param table_arg the table that is being modified + @param fields Item_fields list to be filled + @param values values to fill with + @param ignore_errors TRUE if we should ignore errors - NOTE + @details fill_record() may set table->auto_increment_field_not_null and a caller should make sure that it is reset after their last call to this function. - RETURN - FALSE OK - TRUE error occured + @return Status + @retval true An error occured. + @retval false OK. */ static bool -fill_record(THD * thd, List<Item> &fields, List<Item> &values, +fill_record(THD * thd, TABLE *table_arg, List<Item> &fields, List<Item> &values, bool ignore_errors) { List_iterator_fast<Item> f(fields),v(values); Item *value, *fld; Item_field *field; - TABLE *table= 0, *vcol_table= 0; + TABLE *vcol_table= 0; bool save_abort_on_warning= thd->abort_on_warning; bool save_no_errors= thd->no_errors; DBUG_ENTER("fill_record"); @@ -8937,12 +8866,13 @@ fill_record(THD * thd, List<Item> &fields, List<Item> &values, my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), fld->name); goto err; } - table= field->field->table; - table->auto_increment_field_not_null= FALSE; + DBUG_ASSERT(field->field->table == table_arg); + table_arg->auto_increment_field_not_null= FALSE; f.rewind(); } else if (thd->lex->unit.insert_table_with_stored_vcol) vcol_table= thd->lex->unit.insert_table_with_stored_vcol; + while ((fld= f++)) { if (!(field= fld->filed_for_view_update())) @@ -8952,7 +8882,7 @@ fill_record(THD * thd, List<Item> &fields, List<Item> &values, } value=v++; Field *rfield= field->field; - table= rfield->table; + TABLE* table= rfield->table; if (rfield == table->next_number_field) table->auto_increment_field_not_null= TRUE; if (rfield->vcol_info && @@ -8971,6 +8901,7 @@ fill_record(THD * thd, List<Item> &fields, List<Item> &values, my_message(ER_UNKNOWN_ERROR, ER(ER_UNKNOWN_ERROR), MYF(0)); goto err; } + rfield->set_explicit_default(value); DBUG_ASSERT(vcol_table == 0 || vcol_table == table); vcol_table= table; } @@ -8987,8 +8918,8 @@ fill_record(THD * thd, List<Item> &fields, List<Item> &values, err: thd->abort_on_warning= save_abort_on_warning; thd->no_errors= save_no_errors; - if (table) - table->auto_increment_field_not_null= FALSE; + if (fields.elements) + table_arg->auto_increment_field_not_null= FALSE; DBUG_RETURN(TRUE); } @@ -8997,42 +8928,39 @@ err: Fill fields in list with values from the list of items and invoke before triggers. - SYNOPSIS - fill_record_n_invoke_before_triggers() - thd thread context - fields Item_fields list to be filled - values values to fill with - ignore_errors TRUE if we should ignore errors - triggers object holding list of triggers to be invoked - event event type for triggers to be invoked + @param thd thread context + @param table the table that is being modified + @param fields Item_fields list to be filled + @param values values to fill with + @param ignore_errors TRUE if we should ignore errors + @param event event type for triggers to be invoked - NOTE + @detail This function assumes that fields which values will be set and triggers to be invoked belong to the same table, and that TABLE::record[0] and record[1] buffers correspond to new and old versions of row respectively. - RETURN - FALSE OK - TRUE error occured + @return Status + @retval true An error occured. + @retval false OK. */ bool -fill_record_n_invoke_before_triggers(THD *thd, List<Item> &fields, +fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, List<Item> &fields, List<Item> &values, bool ignore_errors, - Table_triggers_list *triggers, enum trg_event_type event) { bool result; - result= (fill_record(thd, fields, values, ignore_errors) || + Table_triggers_list *triggers= table->triggers; + result= (fill_record(thd, table, fields, values, ignore_errors) || (triggers && triggers->process_triggers(thd, event, TRG_ACTION_BEFORE, TRUE))); /* Re-calculate virtual fields to cater for cases when base columns are updated by the triggers. */ - if (!result && triggers) + if (!result && triggers && table) { - TABLE *table= 0; List_iterator_fast<Item> f(fields); Item *fld; Item_field *item_field; @@ -9040,47 +8968,46 @@ fill_record_n_invoke_before_triggers(THD *thd, List<Item> &fields, { fld= (Item_field*)f++; item_field= fld->filed_for_view_update(); - if (item_field && item_field->field && - (table= item_field->field->table) && - table->vfield) + if (item_field && item_field->field && table && table->vfield) + { + DBUG_ASSERT(table == item_field->field->table); result= update_virtual_fields(thd, table, table->triggers ? VCOL_UPDATE_ALL : VCOL_UPDATE_FOR_WRITE); + } } } return result; } -/* - Fill field buffer with values from Field list +/** + Fill the field buffer of a table with the values of an Item list - SYNOPSIS - fill_record() - thd thread handler - ptr pointer on pointer to record - values list of fields - ignore_errors TRUE if we should ignore errors - use_value forces usage of value of the items instead of result + @param thd thread handler + @param table_arg the table that is being modified + @param ptr pointer on pointer to record of fields + @param values values to fill with + @param ignore_errors TRUE if we should ignore errors + @param use_value forces usage of value of the items instead of result - NOTE + @details fill_record() may set table->auto_increment_field_not_null and a caller should make sure that it is reset after their last call to this function. - RETURN - FALSE OK - TRUE error occured + @return Status + @retval true An error occured. + @retval false OK. */ bool -fill_record(THD *thd, Field **ptr, List<Item> &values, bool ignore_errors, - bool use_value) +fill_record(THD *thd, TABLE *table, Field **ptr, List<Item> &values, + bool ignore_errors, bool use_value) { List_iterator_fast<Item> v(values); List<TABLE> tbl_list; Item *value; - TABLE *table= 0; Field *field; bool abort_on_warning_saved= thd->abort_on_warning; DBUG_ENTER("fill_record"); @@ -9095,7 +9022,7 @@ fill_record(THD *thd, Field **ptr, List<Item> &values, bool ignore_errors, On INSERT or UPDATE fields are checked to be from the same table, thus we safely can take table from the first field. */ - table= (*ptr)->table; + DBUG_ASSERT((*ptr)->table == table); /* Reset the table->auto_increment_field_not_null as it is valid for @@ -9126,6 +9053,7 @@ fill_record(THD *thd, Field **ptr, List<Item> &values, bool ignore_errors, else if (value->save_in_field(field, 0) < 0) goto err; + field->set_explicit_default(value); } /* Update virtual fields*/ thd->abort_on_warning= FALSE; @@ -9145,36 +9073,34 @@ err: /* - Fill fields in array with values from the list of items and invoke + Fill fields in an array with values from the list of items and invoke before triggers. - SYNOPSIS - fill_record_n_invoke_before_triggers() - thd thread context - ptr NULL-ended array of fields to be filled - values values to fill with - ignore_errors TRUE if we should ignore errors - triggers object holding list of triggers to be invoked - event event type for triggers to be invoked + @param thd thread context + @param table the table that is being modified + @param ptr the fields to be filled + @param values values to fill with + @param ignore_errors TRUE if we should ignore errors + @param event event type for triggers to be invoked - NOTE + @detail This function assumes that fields which values will be set and triggers to be invoked belong to the same table, and that TABLE::record[0] and record[1] buffers correspond to new and old versions of row respectively. - RETURN - FALSE OK - TRUE error occured + @return Status + @retval true An error occured. + @retval false OK. */ bool -fill_record_n_invoke_before_triggers(THD *thd, Field **ptr, +fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, Field **ptr, List<Item> &values, bool ignore_errors, - Table_triggers_list *triggers, enum trg_event_type event) { bool result; - result= (fill_record(thd, ptr, values, ignore_errors, FALSE) || + Table_triggers_list *triggers= table->triggers; + result= (fill_record(thd, table, ptr, values, ignore_errors, FALSE) || (triggers && triggers->process_triggers(thd, event, TRG_ACTION_BEFORE, TRUE))); /* @@ -9183,7 +9109,7 @@ fill_record_n_invoke_before_triggers(THD *thd, Field **ptr, */ if (!result && triggers && *ptr) { - TABLE *table= (*ptr)->table; + DBUG_ASSERT(table == (*ptr)->table); if (table->vfield) result= update_virtual_fields(thd, table, table->triggers ? VCOL_UPDATE_ALL : @@ -9218,15 +9144,10 @@ my_bool mysql_rm_tmp_tables(void) /* Remove all SQLxxx tables from directory */ - for (idx=0 ; idx < (uint) dirp->number_off_files ; idx++) + for (idx=0 ; idx < (uint) dirp->number_of_files ; idx++) { file=dirp->dir_entry+idx; - /* skiping . and .. */ - if (file->name[0] == '.' && (!file->name[1] || - (file->name[1] == '.' && !file->name[2]))) - continue; - if (!memcmp(file->name, tmp_file_prefix, tmp_file_prefix_length)) { @@ -9242,7 +9163,7 @@ my_bool mysql_rm_tmp_tables(void) memcpy(filePathCopy, filePath, filePath_len - ext_len); filePathCopy[filePath_len - ext_len]= 0; init_tmp_table_share(thd, &share, "", 0, "", filePathCopy); - if (!open_table_def(thd, &share, 0) && + if (!open_table_def(thd, &share) && ((handler_file= get_new_handler(&share, thd->mem_root, share.db_type())))) { @@ -9262,7 +9183,7 @@ my_bool mysql_rm_tmp_tables(void) my_dirend(dirp); } delete thd; - my_pthread_setspecific_ptr(THR_THD, 0); + set_current_thd(0); DBUG_RETURN(0); } @@ -9685,6 +9606,12 @@ has_write_table_auto_increment_not_first_in_pk(TABLE_LIST *tables) must call close_system_tables() to close systems tables opened with this call. + NOTES + In some situations we use this function to open system tables for + writing. It happens, for examples, with statistical tables when + they are updated by an ANALYZE command. In these cases we should + guarantee that system tables will not be deadlocked. + RETURN FALSE Success TRUE Error @@ -9838,11 +9765,6 @@ open_log_table(THD *thd, TABLE_LIST *one_table, Open_tables_backup *backup) /* Make sure all columns get assigned to a default value */ table->use_all_columns(); table->no_replicate= 1; - /* - Don't set automatic timestamps as we may want to use time of logging, - not from query start - */ - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; } else thd->restore_backup_open_tables_state(backup); @@ -9898,6 +9820,7 @@ int dynamic_column_error_message(enum_dyncol_func_result rc) switch (rc) { case ER_DYNCOL_YES: case ER_DYNCOL_OK: + case ER_DYNCOL_TRUNCATED: break; // it is not an error case ER_DYNCOL_FORMAT: my_error(ER_DYN_COL_WRONG_FORMAT, MYF(0)); diff --git a/sql/sql_base.h b/sql/sql_base.h index d49554d5473..7e9992c8994 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -69,6 +69,8 @@ enum enum_tdc_remove_table_type {TDC_RT_REMOVE_ALL, TDC_RT_REMOVE_NOT_OWN, #define RTFC_WAIT_OTHER_THREAD_FLAG 0x0002 #define RTFC_CHECK_KILLED_FLAG 0x0004 +extern HASH table_def_cache; + bool check_dup(const char *db, const char *name, TABLE_LIST *tables); extern mysql_mutex_t LOCK_open; bool table_cache_init(void); @@ -79,9 +81,6 @@ void table_def_start_shutdown(void); void assign_new_table_id(TABLE_SHARE *share); uint cached_open_tables(void); uint cached_table_definitions(void); -uint create_table_def_key(THD *thd, char *key, - const TABLE_LIST *table_list, - bool tmp_table); /** Create a table cache key for non-temporary table. @@ -107,12 +106,34 @@ create_table_def_key(char *key, const char *db, const char *table_name) NAME_LEN) - key + 1); } -TABLE_SHARE *get_table_share(THD *thd, TABLE_LIST *table_list, char *key, - uint key_length, uint db_flags, int *error, +uint create_tmp_table_def_key(THD *thd, char *key, const char *db, + const char *table_name); +TABLE_SHARE *get_table_share(THD *thd, const char *db, const char *table_name, + char *key, uint key_length, uint flags, my_hash_value_type hash_value); void release_table_share(TABLE_SHARE *share); TABLE_SHARE *get_cached_table_share(const char *db, const char *table_name); +// convenience helper: call get_table_share() without precomputed hash_value +static inline TABLE_SHARE *get_table_share(THD *thd, const char *db, + const char *table_name, + char *key, uint key_length, + uint flags) +{ + return get_table_share(thd, db, table_name, key, key_length, flags, + my_calc_hash(&table_def_cache, (uchar*) key, key_length)); +} + +// convenience helper: call get_table_share() without precomputed cache key +static inline TABLE_SHARE *get_table_share(THD *thd, const char *db, + const char *table_name, uint flags) +{ + char key[MAX_DBKEY_LENGTH]; + uint key_length; + key_length= create_table_def_key(key, db, table_name); + return get_table_share(thd, db, table_name, key, key_length, flags); +} + TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type update, uint lock_flags); @@ -174,8 +195,8 @@ bool open_new_frm(THD *thd, TABLE_SHARE *share, const char *alias, bool get_key_map_from_key_list(key_map *map, TABLE *table, List<String> *index_list); -TABLE *open_table_uncached(THD *thd, const char *path, const char *db, - const char *table_name, +TABLE *open_table_uncached(THD *thd, handlerton *hton, const char *path, + const char *db, const char *table_name, bool add_to_temporary_tables_list); TABLE *find_locked_table(TABLE *list, const char *db, const char *table_name); TABLE *find_write_locked_table(TABLE *list, const char *db, @@ -197,15 +218,15 @@ TABLE *find_temporary_table(THD *thd, const TABLE_LIST *tl); TABLE *find_temporary_table(THD *thd, const char *table_key, uint table_key_length); void close_thread_tables(THD *thd); -bool fill_record_n_invoke_before_triggers(THD *thd, List<Item> &fields, +bool fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, + List<Item> &fields, List<Item> &values, bool ignore_errors, - Table_triggers_list *triggers, enum trg_event_type event); -bool fill_record_n_invoke_before_triggers(THD *thd, Field **field, +bool fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, + Field **field, List<Item> &values, bool ignore_errors, - Table_triggers_list *triggers, enum trg_event_type event); bool insert_fields(THD *thd, Name_resolution_context *context, const char *db_name, const char *table_name, @@ -218,7 +239,7 @@ bool setup_fields(THD *thd, Item** ref_pointer_array, List<Item> &item, enum_mark_columns mark_used_columns, List<Item> *sum_func_list, bool allow_sum_func); void unfix_fields(List<Item> &items); -bool fill_record(THD *thd, Field **field, List<Item> &values, +bool fill_record(THD *thd, TABLE *table, Field **field, List<Item> &values, bool ignore_errors, bool use_value); Field * @@ -301,6 +322,7 @@ bool rename_temporary_table(THD* thd, TABLE *table, const char *new_db, const char *table_name); bool is_equal(const LEX_STRING *a, const LEX_STRING *b); +class Open_tables_backup; /* Functions to work with system tables. */ bool open_system_tables_for_read(THD *thd, TABLE_LIST *table_list, Open_tables_backup *backup); @@ -326,22 +348,39 @@ void tdc_remove_table(THD *thd, enum_tdc_remove_table_type remove_type, bool tdc_open_view(THD *thd, TABLE_LIST *table_list, const char *alias, char *cache_key, uint cache_key_length, MEM_ROOT *mem_root, uint flags); + +static inline bool tdc_open_view(THD *thd, TABLE_LIST *table_list, + const char *alias, MEM_ROOT *mem_root, + uint flags) +{ + char key[MAX_DBKEY_LENGTH]; + uint key_length; + key_length= create_table_def_key(key, table_list->db, table_list->table_name); + return tdc_open_view(thd, table_list, alias, key, key_length, mem_root, flags); +} + void tdc_flush_unused_tables(); TABLE *find_table_for_mdl_upgrade(THD *thd, const char *db, const char *table_name, bool no_error); void mark_tmp_table_for_reuse(TABLE *table); -bool check_if_table_exists(THD *thd, TABLE_LIST *table, bool fast_check, - bool *exists); + int update_virtual_fields(THD *thd, TABLE *table, enum enum_vcol_update_mode vcol_update_mode= VCOL_UPDATE_FOR_READ); int dynamic_column_error_message(enum_dyncol_func_result rc); +/* open_and_lock_tables with optional derived handling */ +int open_and_lock_tables_derived(THD *thd, TABLE_LIST *tables, bool derived); + +extern "C" int simple_raw_key_cmp(void* arg, const void* key1, + const void* key2); +extern "C" int count_distinct_walk(void *elem, element_count count, void *arg); +int simple_str_key_cmp(void* arg, uchar* key1, uchar* key2); + extern TABLE *unused_tables; extern Item **not_found_item; extern Field *not_found_field; extern Field *view_ref_found; -extern HASH table_def_cache; /** clean/setup table fields and map. @@ -503,7 +542,6 @@ open_tables(THD *thd, TABLE_LIST **tables, uint *counter, uint flags) return open_tables(thd, tables, counter, flags, &prelocking_strategy); } - inline TABLE *open_n_lock_single_table(THD *thd, TABLE_LIST *table_l, thr_lock_type lock_type, uint flags) { diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc index 420bd0eb2f0..99ebf59235a 100644 --- a/sql/sql_binlog.cc +++ b/sql/sql_binlog.cc @@ -80,6 +80,8 @@ void mysql_client_binlog_statement(THD* thd) my_bool have_fd_event= TRUE; int err; Relay_log_info *rli; + rpl_group_info *rgi; + rli= thd->rli_fake; if (!rli) { @@ -95,6 +97,9 @@ void mysql_client_binlog_statement(THD* thd) new Format_description_log_event(4); have_fd_event= FALSE; } + if (!(rgi= thd->rgi_fake)) + rgi= thd->rgi_fake= new rpl_group_info(rli); + rgi->thd= thd; const char *error= 0; char *buf= (char *) my_malloc(decoded_len, MYF(MY_WME)); @@ -111,7 +116,7 @@ void mysql_client_binlog_statement(THD* thd) goto end; } - rli->sql_thd= thd; + rli->sql_driver_thd= thd; rli->no_storage= TRUE; for (char const *strptr= thd->lex->comment.str ; @@ -232,7 +237,7 @@ void mysql_client_binlog_statement(THD* thd) (ev->flags & LOG_EVENT_SKIP_REPLICATION_F ? OPTION_SKIP_REPLICATION : 0); - err= ev->apply_event(rli); + err= ev->apply_event(rgi); thd->variables.option_bits= (thd->variables.option_bits & ~OPTION_SKIP_REPLICATION) | @@ -267,7 +272,7 @@ void mysql_client_binlog_statement(THD* thd) end: thd->variables.option_bits= thd_options; - rli->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); my_free(buf); DBUG_VOID_RETURN; } diff --git a/sql/sql_bitmap.h b/sql/sql_bitmap.h index d286d8e1ef0..f9ef9b6c63c 100644 --- a/sql/sql_bitmap.h +++ b/sql/sql_bitmap.h @@ -1,4 +1,5 @@ -/* Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2003, 2013, Oracle and/or its affiliates + Copyright (c) 2009, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -60,7 +61,7 @@ public: intersect(map2buff); if (map.n_bits > sizeof(ulonglong) * 8) bitmap_set_above(&map, sizeof(ulonglong), - test(map2buff & (LL(1) << (sizeof(ulonglong) * 8 - 1)))); + test(map2buff & (1LL << (sizeof(ulonglong) * 8 - 1)))); } void subtract(Bitmap& map2) { bitmap_subtract(&map, &map2.map); } void merge(Bitmap& map2) { bitmap_union(&map, &map2.map); } diff --git a/sql/sql_cache.cc b/sql/sql_cache.cc index d558572f3a2..cbb8bbc0fd4 100644 --- a/sql/sql_cache.cc +++ b/sql/sql_cache.cc @@ -1038,8 +1038,8 @@ void query_cache_insert(const char *packet, ulong length, /* Current_thd can be NULL when a new connection is immediately ended due to "Too many connections". thd->store_globals() has not been - called at this time and hence my_pthread_setspecific_ptr(THR_THD, - this) has not been called for this thread. + called at this time and hence set_current_thd(this) has not been + called for this thread. */ if (!thd) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 2ead1b533d9..09ec7361fe3 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -73,23 +73,6 @@ char empty_c_string[1]= {0}; /* used for not defined db */ const char * const THD::DEFAULT_WHERE= "field list"; - -/***************************************************************************** -** Instansiate templates -*****************************************************************************/ - -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -/* Used templates */ -template class List<Key>; -template class List_iterator<Key>; -template class List<Key_part_spec>; -template class List_iterator<Key_part_spec>; -template class List<Alter_drop>; -template class List_iterator<Alter_drop>; -template class List<Alter_column>; -template class List_iterator<Alter_column>; -#endif - /**************************************************************************** ** User variables ****************************************************************************/ @@ -129,7 +112,8 @@ Key::Key(const Key &rhs, MEM_ROOT *mem_root) columns(rhs.columns, mem_root), name(rhs.name), option_list(rhs.option_list), - generated(rhs.generated) + generated(rhs.generated), + create_if_not_exists(rhs.create_if_not_exists) { list_copy_and_replace_each_value(columns, mem_root); } @@ -367,29 +351,6 @@ void thd_set_thread_stack(THD *thd, char *stack_start) } /** - Lock connection data for the set of connections this connection - belongs to - - @param thd THD object -*/ -void thd_lock_thread_count(THD *) -{ - mysql_mutex_lock(&LOCK_thread_count); -} - -/** - Lock connection data for the set of connections this connection - belongs to - - @param thd THD object -*/ -void thd_unlock_thread_count(THD *) -{ - mysql_cond_broadcast(&COND_thread_count); - mysql_mutex_unlock(&LOCK_thread_count); -} - -/** Close the socket used by this connection @param thd THD object @@ -644,6 +605,17 @@ void thd_set_ha_data(THD *thd, const struct handlerton *hton, } +/** + Allow storage engine to wakeup commits waiting in THD::wait_for_prior_commit. + @see thd_wakeup_subsequent_commits() definition in plugin.h +*/ +extern "C" +void thd_wakeup_subsequent_commits(THD *thd, int wakeup_error) +{ + thd->wakeup_subsequent_commits(wakeup_error); +} + + extern "C" long long thd_test_options(const THD *thd, long long test_options) { @@ -797,7 +769,7 @@ bool Drop_table_error_handler::handle_condition(THD *thd, THD::THD() :Statement(&main_lex, &main_mem_root, STMT_CONVENTIONAL_EXECUTION, /* statement id */ 0), - rli_fake(0), rli_slave(NULL), + rli_fake(0), rgi_fake(0), rgi_slave(NULL), in_sub_stmt(0), log_all_errors(0), binlog_unsafe_warning_flags(0), binlog_table_maps(0), @@ -811,6 +783,7 @@ THD::THD() accessed_rows_and_keys(0), warning_info(&main_warning_info), stmt_da(&main_da), + thread_id(0), global_disable_checkpoint(0), failed_com_change_user(0), is_fatal_error(0), @@ -826,17 +799,29 @@ THD::THD() #if defined(ENABLED_DEBUG_SYNC) debug_sync_control(0), #endif /* defined(ENABLED_DEBUG_SYNC) */ - main_warning_info(0, false) + wait_for_commit_ptr(0), + main_warning_info(0, false, false) { ulong tmp; mdl_context.init(this); /* + We set THR_THD to temporally point to this THD to register all the + variables that allocates memory for this THD + */ + THD *old_THR_THD= current_thd; + set_current_thd(this); + status_var.memory_used= 0; + + main_warning_info.init(); + /* Pass nominal parameters to init_alloc_root only to ensure that the destructor works OK in case of an error. The main_mem_root will be re-initialized in init_for_queries(). */ - init_sql_alloc(&main_mem_root, ALLOC_ROOT_MIN_BLOCK_SIZE, 0); + init_sql_alloc(&main_mem_root, ALLOC_ROOT_MIN_BLOCK_SIZE, 0, + MYF(MY_THREAD_SPECIFIC)); + stmt_arena= this; thread_stack= 0; scheduler= thread_scheduler; // Will be fixed later @@ -855,6 +840,7 @@ THD::THD() col_access=0; is_slave_error= thread_specific_used= FALSE; my_hash_clear(&handler_tables_hash); + my_hash_clear(&ull_hash); tmp_table=0; cuted_fields= 0L; sent_row_count= 0L; @@ -871,8 +857,10 @@ THD::THD() progress.max_counter= 0; current_linfo = 0; slave_thread = 0; + connection_name.str= 0; + connection_name.length= 0; + bzero(&variables, sizeof(variables)); - thread_id= 0; one_shot_set= 0; file_id = 0; query_id= 0; @@ -890,8 +878,8 @@ THD::THD() mysql_audit_init_thd(this); #endif net.vio=0; + net.buff= 0; client_capabilities= 0; // minimalistic client - ull=0; system_thread= NON_SYSTEM_THREAD; cleanup_done= abort_on_warning= 0; peer_port= 0; // For SHOW PROCESSLIST @@ -916,7 +904,7 @@ THD::THD() /* Variables with default values */ proc_info="login"; where= THD::DEFAULT_WHERE; - server_id = ::server_id; + variables.server_id = global_system_variables.server_id; slave_net = 0; command=COM_CONNECT; *scramble= '\0'; @@ -931,7 +919,7 @@ THD::THD() user_connect=(USER_CONN *)0; my_hash_init(&user_vars, system_charset_info, USER_VARS_HASH_SIZE, 0, 0, (my_hash_get_key) get_var_key, - (my_hash_free_key) free_user_var, 0); + (my_hash_free_key) free_user_var, HASH_THREAD_SPECIFIC); sp_proc_cache= NULL; sp_func_cache= NULL; @@ -939,7 +927,7 @@ THD::THD() /* For user vars replication*/ if (opt_bin_log) my_init_dynamic_array(&user_var_events, - sizeof(BINLOG_USER_VAR_EVENT *), 16, 16); + sizeof(BINLOG_USER_VAR_EVENT *), 16, 16, MYF(0)); else bzero((char*) &user_var_events, sizeof(user_var_events)); @@ -949,19 +937,28 @@ THD::THD() protocol_binary.init(this); tablespace_op=FALSE; - tmp= sql_rnd_with_mutex(); + + /* + Initialize the random generator. We call my_rnd() without a lock as + it's not really critical if two threads modifies the structure at the + same time. We ensure that we have an unique number foreach thread + by adding the address of the stack. + */ + tmp= (ulong) (my_rnd(&sql_rand) * 0xffffffff); my_rnd_init(&rand, tmp + (ulong) &rand, tmp + (ulong) ::global_query_id); substitute_null_with_insert_id = FALSE; thr_lock_info_init(&lock_info); /* safety: will be reset after start */ m_internal_handler= NULL; - m_binlog_invoker= FALSE; + m_binlog_invoker= INVOKER_NONE; arena_for_cached_items= 0; memset(&invoker_user, 0, sizeof(invoker_user)); memset(&invoker_host, 0, sizeof(invoker_host)); prepare_derived_at_open= FALSE; create_tmp_table_for_derived= FALSE; save_prep_leaf_list= FALSE; + /* Restore THR_THD */ + set_current_thd(old_THR_THD); } @@ -1209,8 +1206,8 @@ LEX_STRING *thd_make_lex_string(THD *thd, LEX_STRING *lex_str, const char *str, unsigned int size, int allocate_lex_string) { - return thd->make_lex_string(lex_str, str, size, - (bool) allocate_lex_string); + return allocate_lex_string ? thd->make_lex_string(str, size) + : thd->make_lex_string(lex_str, str, size); } extern "C" @@ -1237,6 +1234,7 @@ extern "C" THD *_current_thd_noinline(void) void THD::init(void) { + DBUG_ENTER("thd::init"); mysql_mutex_lock(&LOCK_global_system_variables); plugin_thdvar_init(this); /* @@ -1245,7 +1243,14 @@ void THD::init(void) avoid temporary tables replication failure. */ variables.pseudo_thread_id= thread_id; + + variables.default_master_connection.str= default_master_connection_buff; + ::strmake(variables.default_master_connection.str, + global_system_variables.default_master_connection.str, + variables.default_master_connection.length); + mysql_mutex_unlock(&LOCK_global_system_variables); + server_status= SERVER_STATUS_AUTOCOMMIT; if (variables.sql_mode & MODE_NO_BACKSLASH_ESCAPES) server_status|= SERVER_STATUS_NO_BACKSLASH_ESCAPES; @@ -1259,9 +1264,10 @@ void THD::init(void) tx_isolation= (enum_tx_isolation) variables.tx_isolation; update_charset(); reset_current_stmt_binlog_format_row(); - bzero((char *) &status_var, sizeof(status_var)); + set_status_var_init(); bzero((char *) &org_status_var, sizeof(org_status_var)); start_bytes_received= 0; + last_commit_gtid.seq_no= 0; if (variables.sql_log_bin) variables.option_bits|= OPTION_BIN_LOG; @@ -1276,6 +1282,8 @@ void THD::init(void) /* Initialize the Debug Sync Facility. See debug_sync.cc. */ debug_sync_init_thread(this); #endif /* defined(ENABLED_DEBUG_SYNC) */ + apc_target.init(&LOCK_thd_data); + DBUG_VOID_RETURN; } @@ -1415,8 +1423,6 @@ void THD::cleanup(void) if (global_read_lock.is_acquired()) global_read_lock.unlock_global_read_lock(this); - /* All metadata locks must have been released by now. */ - DBUG_ASSERT(!mdl_context.has_locks()); if (user_connect) { decrease_user_connections(user_connect); @@ -1434,14 +1440,11 @@ void THD::cleanup(void) sp_cache_clear(&sp_proc_cache); sp_cache_clear(&sp_func_cache); - if (ull) - { - mysql_mutex_lock(&LOCK_user_locks); - item_user_lock_release(ull); - mysql_mutex_unlock(&LOCK_user_locks); - ull= NULL; - } + mysql_ull_cleanup(this); + /* All metadata locks must have been released by now. */ + DBUG_ASSERT(!mdl_context.has_locks()); + apc_target.destroy(); cleanup_done=1; DBUG_VOID_RETURN; } @@ -1449,8 +1452,16 @@ void THD::cleanup(void) THD::~THD() { + THD *orig_thd= current_thd; THD_CHECK_SENTRY(this); DBUG_ENTER("~THD()"); + + /* + In error cases, thd may not be current thd. We have to fix this so + that memory allocation counting is done correctly + */ + set_current_thd(this); + /* Ensure that no one is using THD */ mysql_mutex_lock(&LOCK_thd_data); mysql_mutex_unlock(&LOCK_thd_data); @@ -1458,10 +1469,8 @@ THD::~THD() /* Close connection */ #ifndef EMBEDDED_LIBRARY if (net.vio) - { vio_delete(net.vio); - net_end(&net); - } + net_end(&net); #endif stmt_map.reset(); /* close all prepared statements */ if (!cleanup_done) @@ -1484,6 +1493,11 @@ THD::~THD() dbug_sentry= THD_SENTRY_GONE; #endif #ifndef EMBEDDED_LIBRARY + if (rgi_fake) + { + delete rgi_fake; + rgi_fake= NULL; + } if (rli_fake) { delete rli_fake; @@ -1491,11 +1505,20 @@ THD::~THD() } mysql_audit_free_thd(this); - if (rli_slave) - rli_slave->cleanup_after_session(); + if (rgi_slave) + rgi_slave->cleanup_after_session(); #endif free_root(&main_mem_root, MYF(0)); + main_warning_info.free_memory(); + if (status_var.memory_used != 0) + { + DBUG_PRINT("error", ("memory_used: %lld", status_var.memory_used)); + SAFEMALLOC_REPORT_MEMORY(my_thread_dbug_id()); + DBUG_ASSERT(status_var.memory_used == 0); // Ensure everything is freed + } + + set_current_thd(orig_thd); DBUG_VOID_RETURN; } @@ -1771,7 +1794,7 @@ bool THD::store_globals() */ DBUG_ASSERT(thread_stack); - if (my_pthread_setspecific_ptr(THR_THD, this) || + if (set_current_thd(this) || my_pthread_setspecific_ptr(THR_MALLOC, &mem_root)) return 1; /* @@ -1815,7 +1838,7 @@ void THD::reset_globals() mysql_mutex_unlock(&LOCK_thd_data); /* Undocking the thread specific data. */ - my_pthread_setspecific_ptr(THR_THD, NULL); + set_current_thd(0); my_pthread_setspecific_ptr(THR_MALLOC, NULL); } @@ -1868,7 +1891,7 @@ void THD::cleanup_after_query() which is intended to consume its event (there can be other SET statements between them). */ - if ((rli_slave || rli_fake) && is_update_query(lex->sql_command)) + if ((rgi_slave || rli_fake) && is_update_query(lex->sql_command)) auto_inc_intervals_forced.empty(); #endif } @@ -1887,41 +1910,17 @@ void THD::cleanup_after_query() where= THD::DEFAULT_WHERE; /* reset table map for multi-table update */ table_map_for_update= 0; - m_binlog_invoker= FALSE; + m_binlog_invoker= INVOKER_NONE; #ifndef EMBEDDED_LIBRARY - if (rli_slave) - rli_slave->cleanup_after_query(); + if (rgi_slave) + rgi_slave->cleanup_after_query(); #endif DBUG_VOID_RETURN; } -/** - Create a LEX_STRING in this connection. - - @param lex_str pointer to LEX_STRING object to be initialized - @param str initializer to be copied into lex_str - @param length length of str, in bytes - @param allocate_lex_string if TRUE, allocate new LEX_STRING object, - instead of using lex_str value - @return NULL on failure, or pointer to the LEX_STRING object -*/ -LEX_STRING *THD::make_lex_string(LEX_STRING *lex_str, - const char* str, uint length, - bool allocate_lex_string) -{ - if (allocate_lex_string) - if (!(lex_str= (LEX_STRING *)alloc_root(mem_root, sizeof(LEX_STRING)))) - return 0; - if (!(lex_str->str= strmake_root(mem_root, str, length))) - return 0; - lex_str->length= length; - return lex_str; -} - - /* Convert a string to another character set @@ -2109,6 +2108,21 @@ CHANGED_TABLE_LIST* THD::changed_table_dup(const char *key, long key_length) int THD::send_explain_fields(select_result *result) { List<Item> field_list; + make_explain_field_list(field_list); + result->prepare(field_list, NULL); + return (result->send_result_set_metadata(field_list, + Protocol::SEND_NUM_ROWS | + Protocol::SEND_EOF)); +} + + +/* + Populate the provided field_list with EXPLAIN output columns. + this->lex->describe has the EXPLAIN flags +*/ + +void THD::make_explain_field_list(List<Item> &field_list) +{ Item *item; CHARSET_INFO *cs= system_charset_info; field_list.push_back(item= new Item_return_int("id",3, MYSQL_TYPE_LONGLONG)); @@ -2147,10 +2161,9 @@ int THD::send_explain_fields(select_result *result) } item->maybe_null= 1; field_list.push_back(new Item_empty_string("Extra", 255, cs)); - return (result->send_result_set_metadata(field_list, - Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF)); } + #ifdef SIGNAL_WITH_VIO_CLOSE void THD::close_active_vio() { @@ -2353,7 +2366,8 @@ int select_send::send_data(List<Item> &items) Protocol *protocol= thd->protocol; DBUG_ENTER("select_send::send_data"); - if (unit->offset_limit_cnt) + /* unit is not set when using 'delete ... returning' */ + if (unit && unit->offset_limit_cnt) { // using limit offset,count unit->offset_limit_cnt--; DBUG_RETURN(FALSE); @@ -2383,6 +2397,7 @@ int select_send::send_data(List<Item> &items) DBUG_RETURN(0); } + bool select_send::send_eof() { /* @@ -3247,6 +3262,10 @@ void THD::end_statement() } +/* + Start using arena specified by @set. Current arena data will be saved to + *backup. +*/ void THD::set_n_backup_active_arena(Query_arena *set, Query_arena *backup) { DBUG_ENTER("THD::set_n_backup_active_arena"); @@ -3261,6 +3280,12 @@ void THD::set_n_backup_active_arena(Query_arena *set, Query_arena *backup) } +/* + Stop using the temporary arena, and start again using the arena that is + specified in *backup. + The temporary arena is returned back into *set. +*/ + void THD::restore_active_arena(Query_arena *set, Query_arena *backup) { DBUG_ENTER("THD::restore_active_arena"); @@ -3634,7 +3659,8 @@ void thd_increment_net_big_packet_count(ulong length) void THD::set_status_var_init() { - bzero((char*) &status_var, sizeof(status_var)); + bzero((char*) &status_var, offsetof(STATUS_VAR, + last_cleared_system_status_var)); } @@ -3642,7 +3668,7 @@ void Security_context::init() { host= user= ip= external_user= 0; host_or_ip= "connecting host"; - priv_user[0]= priv_host[0]= proxy_user[0]= '\0'; + priv_user[0]= priv_host[0]= proxy_user[0]= priv_role[0]= '\0'; master_access= 0; #ifndef NO_EMBEDDED_ACCESS_CHECKS db_access= NO_ACCESS; @@ -3839,12 +3865,7 @@ void THD::restore_backup_open_tables_state(Open_tables_backup *backup) #undef thd_killed extern "C" int thd_killed(const MYSQL_THD thd) { - if (!thd) - thd= current_thd; - - if (!(thd->killed & KILL_HARD_BIT)) - return 0; - return thd->killed != 0; + return thd_kill_level(thd) > THD_ABORT_SOFTLY; } #else #error now thd_killed() function can go away @@ -3856,8 +3877,17 @@ extern "C" int thd_killed(const MYSQL_THD thd) */ extern "C" enum thd_kill_levels thd_kill_level(const MYSQL_THD thd) { + THD* current= current_thd; + if (!thd) - thd= current_thd; + thd= current; + + if (thd == current) + { + Apc_target *apc_target= (Apc_target*)&thd->apc_target; + if (apc_target->have_apc_requests()) + apc_target->process_apc_requests(); + } if (likely(thd->killed == NOT_KILLED)) return THD_IS_NOT_KILLED; @@ -4001,6 +4031,15 @@ extern "C" enum_tx_isolation thd_get_trx_isolation(const MYSQL_THD thd) return thd->tx_isolation; } +/** + Check if THD socket is still connected. + */ +extern "C" int thd_is_connected(MYSQL_THD thd) +{ + return thd->is_connected(); +} + + #ifdef INNODB_COMPATIBILITY_HOOKS extern "C" const struct charset_info_st *thd_charset(MYSQL_THD thd) { @@ -4286,17 +4325,8 @@ void THD::set_query_and_id(char *query_arg, uint32 query_length_arg, { mysql_mutex_lock(&LOCK_thd_data); set_query_inner(query_arg, query_length_arg, cs); - query_id= new_query_id; mysql_mutex_unlock(&LOCK_thd_data); -} - -/** Assign a new value to thd->query_id. */ - -void THD::set_query_id(query_id_t new_query_id) -{ - mysql_mutex_lock(&LOCK_thd_data); query_id= new_query_id; - mysql_mutex_unlock(&LOCK_thd_data); } /** Assign a new value to thd->mysys_var. */ @@ -4330,13 +4360,15 @@ void THD::leave_locked_tables_mode() /* Also ensure that we don't release metadata locks for open HANDLERs. */ if (handler_tables_hash.records) mysql_ha_set_explicit_lock_duration(this); + if (ull_hash.records) + mysql_ull_set_explicit_lock_duration(this); } locked_tables_mode= LTM_NONE; } -void THD::get_definer(LEX_USER *definer) +void THD::get_definer(LEX_USER *definer, bool role) { - binlog_invoker(); + binlog_invoker(role); #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) if (slave_thread && has_invoker()) { @@ -4348,7 +4380,7 @@ void THD::get_definer(LEX_USER *definer) } else #endif - get_default_definer(this, definer); + get_default_definer(this, definer, role); } @@ -5004,27 +5036,6 @@ THD::binlog_prepare_pending_rows_event(TABLE* table, uint32 serv_id, DBUG_RETURN(pending); /* This is the current pending event */ } -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -/* - Instantiate the versions we need, we have -fno-implicit-template as - compiling option. -*/ -template Rows_log_event* -THD::binlog_prepare_pending_rows_event(TABLE*, uint32, MY_BITMAP const*, - size_t, size_t, bool, - Write_rows_log_event*); - -template Rows_log_event* -THD::binlog_prepare_pending_rows_event(TABLE*, uint32, MY_BITMAP const*, - size_t colcnt, size_t, bool, - Delete_rows_log_event *); - -template Rows_log_event* -THD::binlog_prepare_pending_rows_event(TABLE*, uint32, MY_BITMAP const*, - size_t colcnt, size_t, bool, - Update_rows_log_event *); -#endif - /* Declare in unnamed namespace. */ CPP_UNNAMED_NS_START /** @@ -5167,7 +5178,7 @@ int THD::binlog_write_row(TABLE* table, bool is_trans, size_t const len= pack_row(table, cols, row_data, record); Rows_log_event* const ev= - binlog_prepare_pending_rows_event(table, server_id, cols, colcnt, + binlog_prepare_pending_rows_event(table, variables.server_id, cols, colcnt, len, is_trans, static_cast<Write_rows_log_event*>(0)); @@ -5211,7 +5222,7 @@ int THD::binlog_update_row(TABLE* table, bool is_trans, #endif Rows_log_event* const ev= - binlog_prepare_pending_rows_event(table, server_id, cols, colcnt, + binlog_prepare_pending_rows_event(table, variables.server_id, cols, colcnt, before_size + after_size, is_trans, static_cast<Update_rows_log_event*>(0)); @@ -5242,7 +5253,7 @@ int THD::binlog_delete_row(TABLE* table, bool is_trans, size_t const len= pack_row(table, cols, row_data, record); Rows_log_event* const ev= - binlog_prepare_pending_rows_event(table, server_id, cols, colcnt, + binlog_prepare_pending_rows_event(table, variables.server_id, cols, colcnt, len, is_trans, static_cast<Delete_rows_log_event*>(0)); @@ -5638,6 +5649,303 @@ THD::signal_wakeup_ready() } +void THD::rgi_lock_temporary_tables() +{ + mysql_mutex_lock(&rgi_slave->rli->data_lock); + temporary_tables= rgi_slave->rli->save_temporary_tables; +} + +void THD::rgi_unlock_temporary_tables() +{ + rgi_slave->rli->save_temporary_tables= temporary_tables; + mysql_mutex_unlock(&rgi_slave->rli->data_lock); +} + +bool THD::rgi_have_temporary_tables() +{ + return rgi_slave->rli->save_temporary_tables != 0; +} + + +wait_for_commit::wait_for_commit() + : subsequent_commits_list(0), next_subsequent_commit(0), waitee(0), + opaque_pointer(0), + waiting_for_commit(false), wakeup_error(0), + wakeup_subsequent_commits_running(false) +{ + mysql_mutex_init(key_LOCK_wait_commit, &LOCK_wait_commit, MY_MUTEX_INIT_FAST); + mysql_cond_init(key_COND_wait_commit, &COND_wait_commit, 0); +} + + +wait_for_commit::~wait_for_commit() +{ + /* + Since we do a dirty read of the waiting_for_commit flag in + wait_for_prior_commit() and in unregister_wait_for_prior_commit(), we need + to take extra care before freeing the wait_for_commit object. + + It is possible for the waitee to be pre-empted inside wakeup(), just after + it has cleared the waiting_for_commit flag and before it has released the + LOCK_wait_commit mutex. And then it is possible for the waiter to find the + flag cleared in wait_for_prior_commit() and go finish up things and + de-allocate the LOCK_wait_commit and COND_wait_commit objects before the + waitee has time to be re-scheduled and finish unlocking the mutex and + signalling the condition. This would lead to the waitee accessing no + longer valid memory. + + To prevent this, we do an extra lock/unlock of the mutex here before + deallocation; this makes certain that any waitee has completed wakeup() + first. + */ + mysql_mutex_lock(&LOCK_wait_commit); + mysql_mutex_unlock(&LOCK_wait_commit); + + mysql_mutex_destroy(&LOCK_wait_commit); + mysql_cond_destroy(&COND_wait_commit); +} + + +void +wait_for_commit::wakeup(int wakeup_error) +{ + /* + We signal each waiter on their own condition and mutex (rather than using + pthread_cond_broadcast() or something like that). + + Otherwise we would need to somehow ensure that they were done + waking up before we could allow this THD to be destroyed, which would + be annoying and unnecessary. + + Note that wakeup_subsequent_commits2() depends on this function being a + full memory barrier (it is, because it takes a mutex lock). + + */ + mysql_mutex_lock(&LOCK_wait_commit); + waiting_for_commit= false; + this->wakeup_error= wakeup_error; + /* + Note that it is critical that the mysql_cond_signal() here is done while + still holding the mutex. As soon as we release the mutex, the waiter might + deallocate the condition object. + */ + mysql_cond_signal(&COND_wait_commit); + mysql_mutex_unlock(&LOCK_wait_commit); +} + + +/* + Register that the next commit of this THD should wait to complete until + commit in another THD (the waitee) has completed. + + The wait may occur explicitly, with the waiter sitting in + wait_for_prior_commit() until the waitee calls wakeup_subsequent_commits(). + + Alternatively, the TC (eg. binlog) may do the commits of both waitee and + waiter at once during group commit, resolving both of them in the right + order. + + Only one waitee can be registered for a waiter; it must be removed by + wait_for_prior_commit() or unregister_wait_for_prior_commit() before a new + one is registered. But it is ok for several waiters to register a wait for + the same waitee. It is also permissible for one THD to be both a waiter and + a waitee at the same time. +*/ +void +wait_for_commit::register_wait_for_prior_commit(wait_for_commit *waitee) +{ + waiting_for_commit= true; + wakeup_error= 0; + DBUG_ASSERT(!this->waitee /* No prior registration allowed */); + this->waitee= waitee; + + mysql_mutex_lock(&waitee->LOCK_wait_commit); + /* + If waitee is in the middle of wakeup, then there is nothing to wait for, + so we need not register. This is necessary to avoid a race in unregister, + see comments on wakeup_subsequent_commits2() for details. + */ + if (waitee->wakeup_subsequent_commits_running) + waiting_for_commit= false; + else + { + /* + Put ourself at the head of the waitee's list of transactions that must + wait for it to commit first. + */ + this->next_subsequent_commit= waitee->subsequent_commits_list; + waitee->subsequent_commits_list= this; + } + mysql_mutex_unlock(&waitee->LOCK_wait_commit); +} + + +/* + Wait for commit of another transaction to complete, as already registered + with register_wait_for_prior_commit(). If the commit already completed, + returns immediately. +*/ +int +wait_for_commit::wait_for_prior_commit2(THD *thd) +{ + const char *old_msg; + wait_for_commit *loc_waitee; + + mysql_mutex_lock(&LOCK_wait_commit); + DEBUG_SYNC(thd, "wait_for_prior_commit_waiting"); + old_msg= thd->enter_cond(&COND_wait_commit, &LOCK_wait_commit, + "Waiting for prior transaction to commit"); + while (waiting_for_commit && !thd->check_killed()) + mysql_cond_wait(&COND_wait_commit, &LOCK_wait_commit); + if (!waiting_for_commit) + { + if (wakeup_error) + my_error(ER_PRIOR_COMMIT_FAILED, MYF(0)); + goto end; + } + /* + Wait was interrupted by kill. We need to unregister our wait and give the + error. But if a wakeup is already in progress, then we must ignore the + kill and not give error, otherwise we get inconsistency between waitee and + waiter as to whether we succeed or fail (eg. we may roll back but waitee + might attempt to commit both us and any subsequent commits waiting for us). + */ + loc_waitee= this->waitee; + mysql_mutex_lock(&loc_waitee->LOCK_wait_commit); + if (loc_waitee->wakeup_subsequent_commits_running) + { + /* We are being woken up; ignore the kill and just wait. */ + mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); + do + { + mysql_cond_wait(&COND_wait_commit, &LOCK_wait_commit); + } while (waiting_for_commit); + goto end; + } + remove_from_list(&loc_waitee->subsequent_commits_list); + mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); + + DEBUG_SYNC(thd, "wait_for_prior_commit_killed"); + wakeup_error= thd->killed_errno(); + if (!wakeup_error) + wakeup_error= ER_QUERY_INTERRUPTED; + my_message(wakeup_error, ER(wakeup_error), MYF(0)); + +end: + thd->exit_cond(old_msg); + waitee= NULL; + return wakeup_error; +} + + +/* + Wakeup anyone waiting for us to have committed. + + Note about locking: + + We have a potential race or deadlock between wakeup_subsequent_commits() in + the waitee and unregister_wait_for_prior_commit() in the waiter. + + Both waiter and waitee needs to take their own lock before it is safe to take + a lock on the other party - else the other party might disappear and invalid + memory data could be accessed. But if we take the two locks in different + order, we may end up in a deadlock. + + The waiter needs to lock the waitee to delete itself from the list in + unregister_wait_for_prior_commit(). Thus wakeup_subsequent_commits() can not + hold its own lock while locking waiters, as this could lead to deadlock. + + So we need to prevent unregister_wait_for_prior_commit() running while wakeup + is in progress - otherwise the unregister could complete before the wakeup, + leading to incorrect spurious wakeup or accessing invalid memory. + + However, if we are in the middle of running wakeup_subsequent_commits(), then + there is no need for unregister_wait_for_prior_commit() in the first place - + the waiter can just do a normal wait_for_prior_commit(), as it will be + immediately woken up. + + So the solution to the potential race/deadlock is to set a flag in the waitee + that wakeup_subsequent_commits() is in progress. When this flag is set, + unregister_wait_for_prior_commit() becomes just wait_for_prior_commit(). + + Then also register_wait_for_prior_commit() needs to check if + wakeup_subsequent_commits() is running, and skip the registration if + so. This is needed in case a new waiter manages to register itself and + immediately try to unregister while wakeup_subsequent_commits() is + running. Else the new waiter would also wait rather than unregister, but it + would not be woken up until next wakeup, which could be potentially much + later than necessary. +*/ + +void +wait_for_commit::wakeup_subsequent_commits2(int wakeup_error) +{ + wait_for_commit *waiter; + + mysql_mutex_lock(&LOCK_wait_commit); + wakeup_subsequent_commits_running= true; + waiter= subsequent_commits_list; + subsequent_commits_list= NULL; + mysql_mutex_unlock(&LOCK_wait_commit); + + while (waiter) + { + /* + Important: we must grab the next pointer before waking up the waiter; + once the wakeup is done, the field could be invalidated at any time. + */ + wait_for_commit *next= waiter->next_subsequent_commit; + waiter->wakeup(wakeup_error); + waiter= next; + } + + /* + We need a full memory barrier between walking the list above, and clearing + the flag wakeup_subsequent_commits_running below. This barrier is needed + to ensure that no other thread will start to modify the list pointers + before we are done traversing the list. + + But wait_for_commit::wakeup() does a full memory barrier already (it locks + a mutex), so no extra explicit barrier is needed here. + */ + wakeup_subsequent_commits_running= false; +} + + +/* Cancel a previously registered wait for another THD to commit before us. */ +void +wait_for_commit::unregister_wait_for_prior_commit2() +{ + mysql_mutex_lock(&LOCK_wait_commit); + if (waiting_for_commit) + { + wait_for_commit *loc_waitee= this->waitee; + mysql_mutex_lock(&loc_waitee->LOCK_wait_commit); + if (loc_waitee->wakeup_subsequent_commits_running) + { + /* + When a wakeup is running, we cannot safely remove ourselves from the + list without corrupting it. Instead we can just wait, as wakeup is + already in progress and will thus be immediate. + + See comments on wakeup_subsequent_commits2() for more details. + */ + mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); + while (waiting_for_commit) + mysql_cond_wait(&COND_wait_commit, &LOCK_wait_commit); + } + else + { + /* Remove ourselves from the list in the waitee. */ + remove_from_list(&loc_waitee->subsequent_commits_list); + mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); + } + } + mysql_mutex_unlock(&LOCK_wait_commit); + this->waitee= NULL; +} + + bool Discrete_intervals_list::append(ulonglong start, ulonglong val, ulonglong incr) { diff --git a/sql/sql_class.h b/sql/sql_class.h index c25a5f82eac..726b920edb1 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -43,10 +43,13 @@ #include "violite.h" /* vio_is_connected */ #include "thr_lock.h" /* thr_lock_type, THR_LOCK_DATA, THR_LOCK_INFO */ - +#include "my_apc.h" +#include "rpl_gtid.h" class Reprepare_observer; class Relay_log_info; +struct rpl_group_info; +class Rpl_filter; class Query_log_event; class Load_log_event; @@ -57,7 +60,6 @@ class Lex_input_stream; class Parser_state; class Rows_log_event; class Sroutine_hash_entry; -class User_level_lock; class user_var_entry; enum enum_enable_or_disable { LEAVE_AS_IS, ENABLE, DISABLE }; @@ -75,38 +77,38 @@ enum enum_mark_columns enum enum_filetype { FILETYPE_CSV, FILETYPE_XML }; /* Bits for different SQL modes modes (including ANSI mode) */ -#define MODE_REAL_AS_FLOAT 1 -#define MODE_PIPES_AS_CONCAT 2 -#define MODE_ANSI_QUOTES 4 -#define MODE_IGNORE_SPACE 8 -#define MODE_IGNORE_BAD_TABLE_OPTIONS 16 -#define MODE_ONLY_FULL_GROUP_BY 32 -#define MODE_NO_UNSIGNED_SUBTRACTION 64 -#define MODE_NO_DIR_IN_CREATE 128 -#define MODE_POSTGRESQL 256 -#define MODE_ORACLE 512 -#define MODE_MSSQL 1024 -#define MODE_DB2 2048 -#define MODE_MAXDB 4096 -#define MODE_NO_KEY_OPTIONS 8192 -#define MODE_NO_TABLE_OPTIONS 16384 -#define MODE_NO_FIELD_OPTIONS 32768 -#define MODE_MYSQL323 65536L -#define MODE_MYSQL40 (MODE_MYSQL323*2) -#define MODE_ANSI (MODE_MYSQL40*2) -#define MODE_NO_AUTO_VALUE_ON_ZERO (MODE_ANSI*2) -#define MODE_NO_BACKSLASH_ESCAPES (MODE_NO_AUTO_VALUE_ON_ZERO*2) -#define MODE_STRICT_TRANS_TABLES (MODE_NO_BACKSLASH_ESCAPES*2) -#define MODE_STRICT_ALL_TABLES (MODE_STRICT_TRANS_TABLES*2) -#define MODE_NO_ZERO_IN_DATE (MODE_STRICT_ALL_TABLES*2) -#define MODE_NO_ZERO_DATE (MODE_NO_ZERO_IN_DATE*2) -#define MODE_INVALID_DATES (MODE_NO_ZERO_DATE*2) -#define MODE_ERROR_FOR_DIVISION_BY_ZERO (MODE_INVALID_DATES*2) -#define MODE_TRADITIONAL (MODE_ERROR_FOR_DIVISION_BY_ZERO*2) -#define MODE_NO_AUTO_CREATE_USER (MODE_TRADITIONAL*2) -#define MODE_HIGH_NOT_PRECEDENCE (MODE_NO_AUTO_CREATE_USER*2) -#define MODE_NO_ENGINE_SUBSTITUTION (MODE_HIGH_NOT_PRECEDENCE*2) -#define MODE_PAD_CHAR_TO_FULL_LENGTH (ULL(1) << 31) +#define MODE_REAL_AS_FLOAT (1ULL << 0) +#define MODE_PIPES_AS_CONCAT (1ULL << 1) +#define MODE_ANSI_QUOTES (1ULL << 2) +#define MODE_IGNORE_SPACE (1ULL << 3) +#define MODE_IGNORE_BAD_TABLE_OPTIONS (1ULL << 4) +#define MODE_ONLY_FULL_GROUP_BY (1ULL << 5) +#define MODE_NO_UNSIGNED_SUBTRACTION (1ULL << 6) +#define MODE_NO_DIR_IN_CREATE (1ULL << 7) +#define MODE_POSTGRESQL (1ULL << 8) +#define MODE_ORACLE (1ULL << 9) +#define MODE_MSSQL (1ULL << 10) +#define MODE_DB2 (1ULL << 11) +#define MODE_MAXDB (1ULL << 12) +#define MODE_NO_KEY_OPTIONS (1ULL << 13) +#define MODE_NO_TABLE_OPTIONS (1ULL << 14) +#define MODE_NO_FIELD_OPTIONS (1ULL << 15) +#define MODE_MYSQL323 (1ULL << 16) +#define MODE_MYSQL40 (1ULL << 17) +#define MODE_ANSI (1ULL << 18) +#define MODE_NO_AUTO_VALUE_ON_ZERO (1ULL << 19) +#define MODE_NO_BACKSLASH_ESCAPES (1ULL << 20) +#define MODE_STRICT_TRANS_TABLES (1ULL << 21) +#define MODE_STRICT_ALL_TABLES (1ULL << 22) +#define MODE_NO_ZERO_IN_DATE (1ULL << 23) +#define MODE_NO_ZERO_DATE (1ULL << 24) +#define MODE_INVALID_DATES (1ULL << 25) +#define MODE_ERROR_FOR_DIVISION_BY_ZERO (1ULL << 26) +#define MODE_TRADITIONAL (1ULL << 27) +#define MODE_NO_AUTO_CREATE_USER (1ULL << 28) +#define MODE_HIGH_NOT_PRECEDENCE (1ULL << 29) +#define MODE_NO_ENGINE_SUBSTITUTION (1ULL << 30) +#define MODE_PAD_CHAR_TO_FULL_LENGTH (1ULL << 31) /* Bits for different old style modes */ #define OLD_MODE_NO_DUP_KEY_WARNINGS_WITH_IGNORE 1 @@ -153,9 +155,6 @@ public: }; -#define TC_LOG_PAGE_SIZE 8192 -#define TC_LOG_MIN_SIZE (3*TC_LOG_PAGE_SIZE) - #define TC_HEURISTIC_RECOVER_COMMIT 1 #define TC_HEURISTIC_RECOVER_ROLLBACK 2 extern ulong tc_heuristic_recover; @@ -230,8 +229,9 @@ public: enum drop_type {KEY, COLUMN }; const char *name; enum drop_type type; - Alter_drop(enum drop_type par_type,const char *par_name) - :name(par_name), type(par_type) {} + bool drop_if_exists; + Alter_drop(enum drop_type par_type,const char *par_name, bool par_exists) + :name(par_name), type(par_type), drop_if_exists(par_exists) {} /** Used to make a clone of this object for ALTER/CREATE TABLE @sa comment for Key_part_spec::clone @@ -265,20 +265,23 @@ public: LEX_STRING name; engine_option_value *option_list; bool generated; + bool create_if_not_exists; Key(enum Keytype type_par, const LEX_STRING &name_arg, KEY_CREATE_INFO *key_info_arg, bool generated_arg, List<Key_part_spec> &cols, - engine_option_value *create_opt) + engine_option_value *create_opt, bool if_not_exists_opt) :type(type_par), key_create_info(*key_info_arg), columns(cols), - name(name_arg), option_list(create_opt), generated(generated_arg) + name(name_arg), option_list(create_opt), generated(generated_arg), + create_if_not_exists(if_not_exists_opt) {} Key(enum Keytype type_par, const char *name_arg, size_t name_len_arg, KEY_CREATE_INFO *key_info_arg, bool generated_arg, List<Key_part_spec> &cols, - engine_option_value *create_opt) + engine_option_value *create_opt, bool if_not_exists_opt) :type(type_par), key_create_info(*key_info_arg), columns(cols), - option_list(create_opt), generated(generated_arg) + option_list(create_opt), generated(generated_arg), + create_if_not_exists(if_not_exists_opt) { name.str= (char *)name_arg; name.length= name_len_arg; @@ -309,8 +312,10 @@ public: uint delete_opt, update_opt, match_opt; Foreign_key(const LEX_STRING &name_arg, List<Key_part_spec> &cols, Table_ident *table, List<Key_part_spec> &ref_cols, - uint delete_opt_arg, uint update_opt_arg, uint match_opt_arg) - :Key(FOREIGN_KEY, name_arg, &default_key_create_info, 0, cols, NULL), + uint delete_opt_arg, uint update_opt_arg, uint match_opt_arg, + bool if_not_exists_opt) + :Key(FOREIGN_KEY, name_arg, &default_key_create_info, 0, cols, NULL, + if_not_exists_opt), ref_table(table), ref_columns(ref_cols), delete_opt(delete_opt_arg), update_opt(update_opt_arg), match_opt(match_opt_arg) @@ -438,7 +443,8 @@ extern int killed_errno(killed_state killed); enum killed_type { KILL_TYPE_ID, - KILL_TYPE_USER + KILL_TYPE_USER, + KILL_TYPE_QUERY }; #include "sql_lex.h" /* Must be here */ @@ -508,6 +514,11 @@ typedef struct system_variables ulong net_write_timeout; ulong optimizer_prune_level; ulong optimizer_search_depth; + ulong optimizer_selectivity_sampling_limit; + ulong optimizer_use_condition_selectivity; + ulong use_stat_tables; + ulong histogram_size; + ulong histogram_type; ulong preload_buff_size; ulong profiling_history_size; ulong read_buff_size; @@ -537,11 +548,24 @@ typedef struct system_variables ulong tx_isolation; ulong updatable_views_with_limit; int max_user_connections; + ulong server_id; /** In slave thread we need to know in behalf of which thread the query is being run to replicate temp tables properly */ my_thread_id pseudo_thread_id; + /** + When replicating an event group with GTID, keep these values around so + slave binlog can receive the same GTID as the original. + */ + uint32 gtid_domain_id; + uint64 gtid_seq_no; + /** + Place holders to store Multi-source variables in sys_var.cc during + update and show of variables. + */ + ulong slave_skip_counter; + ulong max_relay_log_size; my_bool low_priority_updates; my_bool query_cache_wlock_invalidate; @@ -566,6 +590,9 @@ typedef struct system_variables CHARSET_INFO *collation_database; CHARSET_INFO *collation_connection; + /* Names. These will be allocated in buffers in thd */ + LEX_STRING default_master_connection; + /* Error messages */ MY_LOCALE *lc_messages; /* Locale Support */ @@ -685,6 +712,8 @@ typedef struct system_status_var ulonglong binlog_bytes_written; double last_query_cost; double cpu_time, busy_time; + /* Don't initialize */ + volatile int64 memory_used; /* This shouldn't be accumulated */ } STATUS_VAR; /* @@ -694,6 +723,7 @@ typedef struct system_status_var */ #define last_system_status_var questions +#define last_cleared_system_status_var memory_used void mark_transaction_to_rollback(THD *thd, bool all); @@ -1019,6 +1049,8 @@ public: char proxy_user[USERNAME_LENGTH + MAX_HOSTNAME + 5]; /* The host privilege we are using */ char priv_host[MAX_HOSTNAME]; + /* The role privilege we are using */ + char priv_role[USERNAME_LENGTH]; /* The external user (if available) */ char *external_user; /* points to host if host is available, otherwise points to ip */ @@ -1258,7 +1290,8 @@ enum enum_thread_type SYSTEM_THREAD_SLAVE_SQL= 4, SYSTEM_THREAD_NDBCLUSTER_BINLOG= 8, SYSTEM_THREAD_EVENT_SCHEDULER= 16, - SYSTEM_THREAD_EVENT_WORKER= 32 + SYSTEM_THREAD_EVENT_WORKER= 32, + SYSTEM_THREAD_BINLOG_BACKGROUND= 64 }; inline char const * @@ -1426,7 +1459,8 @@ public: m_reopen_array(NULL), m_locked_tables_count(0) { - init_sql_alloc(&m_locked_tables_root, MEM_ROOT_BLOCK_SIZE, 0); + init_sql_alloc(&m_locked_tables_root, MEM_ROOT_BLOCK_SIZE, 0, + MYF(MY_THREAD_SPECIFIC)); } void unlock_locked_tables(THD *thd); ~Locked_tables_list() @@ -1530,8 +1564,146 @@ private: }; +/* + Class to facilitate the commit of one transactions waiting for the commit of + another transaction to complete first. + + This is used during (parallel) replication, to allow different transactions + to be applied in parallel, but still commit in order. + + The transaction that wants to wait for a prior commit must first register + to wait with register_wait_for_prior_commit(waitee). Such registration + must be done holding the waitee->LOCK_wait_commit, to prevent the other + THD from disappearing during the registration. + + Then during commit, if a THD is registered to wait, it will call + wait_for_prior_commit() as part of ha_commit_trans(). If no wait is + registered, or if the waitee for has already completed commit, then + wait_for_prior_commit() returns immediately. + + And when a THD that may be waited for has completed commit (more precisely + commit_ordered()), then it must call wakeup_subsequent_commits() to wake + up any waiters. Note that this must be done at a point that is guaranteed + to be later than any waiters registering themselves. It is safe to call + wakeup_subsequent_commits() multiple times, as waiters are removed from + registration as part of the wakeup. + + The reason for separate register and wait calls is that this allows to + register the wait early, at a point where the waited-for THD is known to + exist. And then the actual wait can be done much later, where the + waited-for THD may have been long gone. By registering early, the waitee + can signal before disappearing. +*/ +struct wait_for_commit +{ + /* + The LOCK_wait_commit protects the fields subsequent_commits_list and + wakeup_subsequent_commits_running (for a waitee), and the flag + waiting_for_commit and associated COND_wait_commit (for a waiter). + */ + mysql_mutex_t LOCK_wait_commit; + mysql_cond_t COND_wait_commit; + /* List of threads that did register_wait_for_prior_commit() on us. */ + wait_for_commit *subsequent_commits_list; + /* Link field for entries in subsequent_commits_list. */ + wait_for_commit *next_subsequent_commit; + /* Our waitee, if we did register_wait_for_prior_commit(), else NULL. */ + wait_for_commit *waitee; + /* + Generic pointer for use by the transaction coordinator to optimise the + waiting for improved group commit. + + Currently used by binlog TC to signal that a waiter is ready to commit, so + that the waitee can grab it and group commit it directly. It is free to be + used by another transaction coordinator for similar purposes. + */ + void *opaque_pointer; + /* + The waiting_for_commit flag is cleared when a waiter has been woken + up. The COND_wait_commit condition is signalled when this has been + cleared. + */ + bool waiting_for_commit; + /* The wakeup error code from the waitee. 0 means no error. */ + int wakeup_error; + /* + Flag set when wakeup_subsequent_commits_running() is active, see comments + on that function for details. + */ + bool wakeup_subsequent_commits_running; + + void register_wait_for_prior_commit(wait_for_commit *waitee); + int wait_for_prior_commit(THD *thd) + { + /* + Quick inline check, to avoid function call and locking in the common case + where no wakeup is registered, or a registered wait was already signalled. + */ + if (waiting_for_commit) + return wait_for_prior_commit2(thd); + else + return wakeup_error; + } + void wakeup_subsequent_commits(int wakeup_error) + { + /* + Do the check inline, so only the wakeup case takes the cost of a function + call for every commmit. + + Note that the check is done without locking. It is the responsibility of + the user of the wakeup facility to ensure that no waiters can register + themselves after the last call to wakeup_subsequent_commits(). + + This avoids having to take another lock for every commit, which would be + pointless anyway - even if we check under lock, there is nothing to + prevent a waiter from arriving just after releasing the lock. + */ + if (subsequent_commits_list) + wakeup_subsequent_commits2(wakeup_error); + } + void unregister_wait_for_prior_commit() + { + if (waiting_for_commit) + unregister_wait_for_prior_commit2(); + } + /* + Remove a waiter from the list in the waitee. Used to unregister a wait. + The caller must be holding the locks of both waiter and waitee. + */ + void remove_from_list(wait_for_commit **next_ptr_ptr) + { + wait_for_commit *cur; + + while ((cur= *next_ptr_ptr) != NULL) + { + if (cur == this) + { + *next_ptr_ptr= this->next_subsequent_commit; + break; + } + next_ptr_ptr= &cur->next_subsequent_commit; + } + waiting_for_commit= false; + } + + void wakeup(int wakeup_error); + + int wait_for_prior_commit2(THD *thd); + void wakeup_subsequent_commits2(int wakeup_error); + void unregister_wait_for_prior_commit2(); + + wait_for_commit(); + ~wait_for_commit(); +}; + + extern "C" void my_message_sql(uint error, const char *str, myf MyFlags); +class THD; +#ifndef DBUG_OFF +void dbug_serve_apcs(THD *thd, int n_calls); +#endif + /** @class THD For each client connection we create a separate thread with THD serving as @@ -1559,8 +1731,12 @@ public: /* Used to execute base64 coded binlog events in MySQL server */ Relay_log_info* rli_fake; + rpl_group_info* rgi_fake; /* Slave applier execution context */ - Relay_log_info* rli_slave; + rpl_group_info* rgi_slave; + + /* Used to SLAVE SQL thread */ + Rpl_filter* rpl_filter; void reset_for_next_command(); /* @@ -1672,11 +1848,11 @@ public: HASH handler_tables_hash; /* - One thread can hold up to one named user-level lock. This variable - points to a lock object if the lock is present. See item_func.cc and + A thread can hold named user-level locks. This variable + contains granted tickets if a lock is present. See item_func.cc and chapter 'Miscellaneous functions', for functions GET_LOCK, RELEASE_LOCK. */ - User_level_lock *ull; + HASH ull_hash; #ifndef DBUG_OFF uint dbug_sentry; // watch out for memory corruption #endif @@ -1686,7 +1862,6 @@ public: first byte of the packet in do_command() */ enum enum_server_command command; - uint32 server_id; uint32 file_id; // for LOAD DATA INFILE /* remote (peer) port */ uint16 peer_port; @@ -1763,7 +1938,7 @@ public: MY_BITMAP const* cols, size_t colcnt, const uchar *old_data, const uchar *new_data); - void set_server_id(uint32 sid) { server_id = sid; } + void set_server_id(uint32 sid) { variables.server_id = sid; } /* Member functions to handle pending event for row-level logging. @@ -1878,7 +2053,8 @@ public: { bzero((char*)this, sizeof(*this)); xid_state.xid.null(); - init_sql_alloc(&mem_root, ALLOC_ROOT_MIN_BLOCK_SIZE, 0); + init_sql_alloc(&mem_root, ALLOC_ROOT_MIN_BLOCK_SIZE, 0, + MYF(MY_THREAD_SPECIFIC)); } } transaction; Global_read_lock global_read_lock; @@ -2202,9 +2378,25 @@ public: */ killed_state volatile killed; + /* See also thd_killed() */ + inline bool check_killed() + { + if (killed) + return TRUE; + if (apc_target.have_apc_requests()) + apc_target.process_apc_requests(); + return FALSE; + } + /* scramble - random string sent to client on handshake */ char scramble[SCRAMBLE_LENGTH+1]; + /* + If this is a slave, the name of the connection stored here. + This is used for taging error messages in the log files. + */ + LEX_STRING connection_name; + char default_master_connection_buff[MAX_CONNECTION_NAME+1]; bool slave_thread, one_shot_set; bool extra_port; /* If extra connection */ @@ -2401,10 +2593,21 @@ public: void close_active_vio(); #endif void awake(killed_state state_to_set); - + /** Disconnect the associated communication endpoint. */ void disconnect(); + + /* + Allows this thread to serve as a target for others to schedule Async + Procedure Calls on. + + It's possible to schedule any code to be executed this way, by + inheriting from the Apc_call object. Currently, only + Show_explain_request uses this. + */ + Apc_target apc_target; + #ifndef MYSQL_CLIENT enum enum_binlog_query_type { /* The query can be logged in row format or in statement format. */ @@ -2584,9 +2787,21 @@ public: return alloc_root(&transaction.mem_root,size); } - LEX_STRING *make_lex_string(LEX_STRING *lex_str, - const char* str, uint length, - bool allocate_lex_string); + LEX_STRING *make_lex_string(LEX_STRING *lex_str, const char* str, uint length) + { + if (!(lex_str->str= strmake_root(mem_root, str, length))) + return 0; + lex_str->length= length; + return lex_str; + } + + LEX_STRING *make_lex_string(const char* str, uint length) + { + LEX_STRING *lex_str; + if (!(lex_str= (LEX_STRING *)alloc_root(mem_root, sizeof(LEX_STRING)))) + return 0; + return make_lex_string(lex_str, str, length); + } bool convert_string(LEX_STRING *to, CHARSET_INFO *to_cs, const char *from, uint from_length, @@ -2598,7 +2813,7 @@ public: void add_changed_table(const char *key, long key_length); CHANGED_TABLE_LIST * changed_table_dup(const char *key, long key_length); int send_explain_fields(select_result *result); - + void make_explain_field_list(List<Item> &field_list); /** Clear the current error, if any. We do not clear is_fatal_error or is_fatal_sub_stmt_error since we @@ -2748,6 +2963,27 @@ public: void set_n_backup_active_arena(Query_arena *set, Query_arena *backup); void restore_active_arena(Query_arena *set, Query_arena *backup); + inline void get_binlog_format(enum_binlog_format *format, + enum_binlog_format *current_format) + { + *format= (enum_binlog_format) variables.binlog_format; + *current_format= current_stmt_binlog_format; + } + inline void set_binlog_format(enum_binlog_format format, + enum_binlog_format current_format) + { + DBUG_ENTER("set_binlog_format"); + variables.binlog_format= format; + current_stmt_binlog_format= current_format; + DBUG_VOID_RETURN; + } + inline void set_binlog_format_stmt() + { + DBUG_ENTER("set_binlog_format_stmt"); + variables.binlog_format= BINLOG_FORMAT_STMT; + current_stmt_binlog_format= BINLOG_FORMAT_STMT; + DBUG_VOID_RETURN; + } /* @todo Make these methods private or remove them completely. Only decide_logging_format should call them. /Sven @@ -2778,16 +3014,26 @@ public: DBUG_VOID_RETURN; } + inline void set_current_stmt_binlog_format_row() { DBUG_ENTER("set_current_stmt_binlog_format_row"); current_stmt_binlog_format= BINLOG_FORMAT_ROW; DBUG_VOID_RETURN; } - inline void clear_current_stmt_binlog_format_row() + /* Set binlog format temporarily to statement. Returns old format */ + inline enum_binlog_format set_current_stmt_binlog_format_stmt() { - DBUG_ENTER("clear_current_stmt_binlog_format_row"); + enum_binlog_format orig_format= current_stmt_binlog_format; + DBUG_ENTER("set_current_stmt_binlog_format_stmt"); current_stmt_binlog_format= BINLOG_FORMAT_STMT; + DBUG_RETURN(orig_format); + } + inline void restore_stmt_binlog_format(enum_binlog_format format) + { + DBUG_ENTER("restore_stmt_binlog_format"); + DBUG_ASSERT(!is_current_stmt_binlog_format_row()); + current_stmt_binlog_format= format; DBUG_VOID_RETURN; } inline void reset_current_stmt_binlog_format_row() @@ -2817,7 +3063,7 @@ public: if (variables.binlog_format == BINLOG_FORMAT_ROW) set_current_stmt_binlog_format_row(); else if (temporary_tables == NULL) - clear_current_stmt_binlog_format_row(); + set_current_stmt_binlog_format_stmt(); } DBUG_VOID_RETURN; } @@ -3022,7 +3268,10 @@ public: { set_query(CSET_STRING()); } void set_query_and_id(char *query_arg, uint32 query_length_arg, CHARSET_INFO *cs, query_id_t new_query_id); - void set_query_id(query_id_t new_query_id); + void set_query_id(query_id_t new_query_id) + { + query_id= new_query_id; + } void set_open_tables(TABLE *open_tables_arg) { mysql_mutex_lock(&LOCK_thd_data); @@ -3051,9 +3300,11 @@ public: } void leave_locked_tables_mode(); int decide_logging_format(TABLE_LIST *tables); - void binlog_invoker() { m_binlog_invoker= TRUE; } - bool need_binlog_invoker() { return m_binlog_invoker; } - void get_definer(LEX_USER *definer); + + enum need_invoker { INVOKER_NONE=0, INVOKER_USER, INVOKER_ROLE}; + void binlog_invoker(bool role) { m_binlog_invoker= role ? INVOKER_ROLE : INVOKER_USER; } + enum need_invoker need_binlog_invoker() { return m_binlog_invoker; } + void get_definer(LEX_USER *definer, bool role); void set_invoker(const LEX_STRING *user, const LEX_STRING *host) { invoker_user= *user; @@ -3103,6 +3354,20 @@ public: void wait_for_wakeup_ready(); /* Wake this thread up from wait_for_wakeup_ready(). */ void signal_wakeup_ready(); + + wait_for_commit *wait_for_commit_ptr; + int wait_for_prior_commit() + { + if (wait_for_commit_ptr) + return wait_for_commit_ptr->wait_for_prior_commit(this); + return 0; + } + void wakeup_subsequent_commits(int wakeup_error) + { + if (wait_for_commit_ptr) + wait_for_commit_ptr->wakeup_subsequent_commits(wakeup_error); + } + private: /** The current internal error handler for this thread, or NULL. */ @@ -3128,14 +3393,15 @@ private: Diagnostics_area main_da; /** - It will be set TURE if CURRENT_USER() is called in account management - statements or default definer is set in CREATE/ALTER SP, SF, Event, - TRIGGER or VIEW statements. + It will be set if CURRENT_USER() or CURRENT_ROLE() is called in account + management statements or default definer is set in CREATE/ALTER SP, SF, + Event, TRIGGER or VIEW statements. - Current user will be binlogged into Query_log_event if m_binlog_invoker - is TRUE; It will be stored into invoker_host and invoker_user by SQL thread. + Current user or role will be binlogged into Query_log_event if + m_binlog_invoker is not NONE; It will be stored into invoker_host and + invoker_user by SQL thread. */ - bool m_binlog_invoker; + enum need_invoker m_binlog_invoker; /** It points to the invoker in the Query_log_event. @@ -3145,6 +3411,12 @@ private: */ LEX_STRING invoker_user; LEX_STRING invoker_host; + + /* Protect against add/delete of temporary tables in parallel replication */ + void rgi_lock_temporary_tables(); + void rgi_unlock_temporary_tables(); + bool rgi_have_temporary_tables(); +public: /* Flag, mutex and condition for a thread to wait for a signal from another thread. @@ -3155,6 +3427,27 @@ private: bool wakeup_ready; mysql_mutex_t LOCK_wakeup_ready; mysql_cond_t COND_wakeup_ready; + /* + The GTID assigned to the last commit. If no GTID was assigned to any commit + so far, this is indicated by last_commit_gtid.seq_no == 0. + */ + rpl_gtid last_commit_gtid; + + inline void lock_temporary_tables() + { + if (rgi_slave) + rgi_lock_temporary_tables(); + } + inline void unlock_temporary_tables() + { + if (rgi_slave) + rgi_unlock_temporary_tables(); + } + inline bool have_temporary_tables() + { + return (temporary_tables || + (rgi_slave && rgi_have_temporary_tables())); + } }; @@ -3233,10 +3526,42 @@ public: class JOIN; -class select_result :public Sql_alloc { +/* Pure interface for sending tabular data */ +class select_result_sink: public Sql_alloc +{ +public: + /* + send_data returns 0 on ok, 1 on error and -1 if data was ignored, for + example for a duplicate row entry written to a temp table. + */ + virtual int send_data(List<Item> &items)=0; + virtual ~select_result_sink() {}; +}; + + +/* + Interface for sending tabular data, together with some other stuff: + + - Primary purpose seems to be seding typed tabular data: + = the DDL is sent with send_fields() + = the rows are sent with send_data() + Besides that, + - there seems to be an assumption that the sent data is a result of + SELECT_LEX_UNIT *unit, + - nest_level is used by SQL parser +*/ + +class select_result :public select_result_sink +{ protected: THD *thd; + /* + All descendant classes have their send_data() skip the first + unit->offset_limit_cnt rows sent. Select_materialize + also uses unit->get_unit_column_types(). + */ SELECT_LEX_UNIT *unit; + /* Something used only by the parser: */ public: select_result(); virtual ~select_result() {}; @@ -3254,11 +3579,6 @@ public: virtual uint field_count(List<Item> &fields) const { return fields.elements; } virtual bool send_result_set_metadata(List<Item> &list, uint flags)=0; - /* - send_data returns 0 on ok, 1 on error and -1 if data was ignored, for - example for a duplicate row entry written to a temp table. - */ - virtual int send_data(List<Item> &items)=0; virtual bool initialize_tables (JOIN *join=0) { return 0; } virtual void send_error(uint errcode,const char *err); virtual bool send_eof()=0; @@ -3283,6 +3603,57 @@ public: void begin_dataset() {} #endif virtual void update_used_tables() {} + + void reset_offset_limit() + { + unit->offset_limit_cnt= 0; + } +}; + + +/* + This is a select_result_sink which simply writes all data into a (temporary) + table. Creation/deletion of the table is outside of the scope of the class + + It is aimed at capturing SHOW EXPLAIN output, so: + - Unlike select_result class, we don't assume that the sent data is an + output of a SELECT_LEX_UNIT (and so we dont apply "LIMIT x,y" from the + unit) + - We don't try to convert the target table to MyISAM +*/ + +class select_result_explain_buffer : public select_result_sink +{ +public: + select_result_explain_buffer(THD *thd_arg, TABLE *table_arg) : + thd(thd_arg), dst_table(table_arg) {}; + + THD *thd; + TABLE *dst_table; /* table to write into */ + + /* The following is called in the child thread: */ + int send_data(List<Item> &items); +}; + + +/* + This is a select_result_sink which stores the data in text form. +*/ + +class select_result_text_buffer : public select_result_sink +{ +public: + select_result_text_buffer(THD *thd_arg) : thd(thd_arg) {} + int send_data(List<Item> &items); + bool send_result_set_metadata(List<Item> &fields, uint flag); + + void save_to(String *res); +private: + int append_row(List<Item> &items, bool send_names); + + THD *thd; + List<char*> rows; + int n_columns; }; @@ -3859,6 +4230,8 @@ class user_var_entry DTCollation collation; }; +user_var_entry *get_variable(HASH *hash, LEX_STRING &name, + bool create_if_not_exists); /* Unique -- class for unique (removing of duplicates). @@ -3881,6 +4254,7 @@ class Unique :public Sql_alloc uint size; uint full_size; uint min_dupl_count; /* always 0 for unions, > 0 for intersections */ + bool with_counters; bool merge(TABLE *table, uchar *buff, bool without_last_merge); @@ -4003,7 +4377,9 @@ class multi_update :public select_result_interceptor so that afterward send_error() needs to find out that. */ bool error_handled; - + + /* Need this to protect against multiple prepare() calls */ + bool prepared; public: multi_update(TABLE_LIST *ut, List<TABLE_LIST> *leaves_list, List<Item> *fields, List<Item> *values, @@ -4128,6 +4504,21 @@ public: */ #define CF_CAN_GENERATE_ROW_EVENTS (1U << 9) +/** + Statement that need the binlog format to be unchanged. +*/ +#define CF_FORCE_ORIGINAL_BINLOG_FORMAT (1U << 10) + +/** + Statement that inserts new rows (INSERT, REPLACE, LOAD, ALTER TABLE) +*/ +#define CF_INSERTS_DATA (1U << 11) + +/** + Statement that updates existing rows (UPDATE, multi-update) +*/ +#define CF_UPDATES_DATA (1U << 12) + /* Bits in server_command_flags */ /** diff --git a/sql/sql_connect.cc b/sql/sql_connect.cc index 041d2a545df..0e078f2aac4 100644 --- a/sql/sql_connect.cc +++ b/sql/sql_connect.cc @@ -226,7 +226,7 @@ void time_out_user_resource_limits(THD *thd, USER_CONN *uc) DBUG_ENTER("time_out_user_resource_limits"); /* If more than a hour since last check, reset resource checking */ - if (check_time - uc->reset_utime >= LL(3600000000)) + if (check_time - uc->reset_utime >= 3600000000ULL) { uc->questions=0; uc->updates=0; @@ -1187,7 +1187,7 @@ bool thd_prepare_connection(THD *thd) bool rc; lex_start(thd); rc= login_connection(thd); - MYSQL_AUDIT_NOTIFY_CONNECTION_CONNECT(thd); + mysql_audit_notify_connection_connect(thd); if (rc) return rc; diff --git a/sql/sql_const.h b/sql/sql_const.h index 9d227601a20..d0a7a83f3a1 100644 --- a/sql/sql_const.h +++ b/sql/sql_const.h @@ -38,8 +38,10 @@ #define MAX_REFLENGTH 4 /* Max length for record ref */ #endif #define MAX_HOSTNAME 61 /* len+1 in mysql.user */ +#define MAX_CONNECTION_NAME NAME_LEN #define MAX_MBWIDTH 3 /* Max multibyte sequence */ +#define MAX_FILENAME_MBWIDTH 5 #define MAX_FIELD_CHARLENGTH 255 #define MAX_FIELD_VARCHARLENGTH 65535 #define MAX_FIELD_BLOBLENGTH UINT_MAX32 /* cf field_blob::get_length() */ diff --git a/sql/sql_db.cc b/sql/sql_db.cc index 0464128bd97..0235ea408b3 100644 --- a/sql/sql_db.cc +++ b/sql/sql_db.cc @@ -37,6 +37,7 @@ #include "sp.h" #include "events.h" #include "sql_handler.h" +#include "sql_statistics.h" #include <my_dir.h> #include <m_ctype.h> #include "log.h" @@ -47,15 +48,12 @@ #define MAX_DROP_TABLE_Q_LEN 1024 -const char *del_exts[]= {".frm", ".BAK", ".TMD",".opt", NullS}; +const char *del_exts[]= {".BAK", ".TMD",".opt", NullS}; static TYPELIB deletable_extentions= {array_elements(del_exts)-1,"del_exts", del_exts, NULL}; -static bool find_db_tables_and_rm_known_files(THD *thd, MY_DIR *dirp, - const char *db, - const char *path, - TABLE_LIST **tables, - bool *found_other_files); +static bool find_db_tables_and_rm_known_files(THD *, MY_DIR *, char *, + const char *, TABLE_LIST **); long mysql_rm_arc_files(THD *thd, MY_DIR *dirp, const char *org_path); static my_bool rm_dir_w_symlink(const char *org_path, my_bool send_error); @@ -758,7 +756,6 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) char path[FN_REFLEN + 16]; MY_DIR *dirp; uint length; - bool found_other_files= false; TABLE_LIST *tables= NULL; TABLE_LIST *table; Drop_table_error_handler err_handler; @@ -790,8 +787,7 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) } } - if (find_db_tables_and_rm_known_files(thd, dirp, db, path, &tables, - &found_other_files)) + if (find_db_tables_and_rm_known_files(thd, dirp, db, path, &tables)) goto exit; /* @@ -817,16 +813,23 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) lock_db_routines(thd, db)) goto exit; + if (!in_bootstrap) + { + for (table= tables; table; table= table->next_local) + { + LEX_STRING db_name= { table->db, table->db_length }; + LEX_STRING table_name= { table->table_name, table->table_name_length }; + if (table->open_type == OT_BASE_ONLY || !find_temporary_table(thd, table)) + (void) delete_statistics_for_table(thd, &db_name, &table_name); + } + } + /* mysql_ha_rm_tables() requires a non-null TABLE_LIST. */ if (tables) mysql_ha_rm_tables(thd, tables); for (table= tables; table; table= table->next_local) - { - tdc_remove_table(thd, TDC_RT_REMOVE_ALL, table->db, table->table_name, - false); deleted_tables++; - } thd->push_internal_handler(&err_handler); if (!thd->killed && @@ -865,10 +868,7 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) If the directory is a symbolic link, remove the link first, then remove the directory the symbolic link pointed at */ - if (found_other_files) - my_error(ER_DB_DROP_RMDIR, MYF(0), path, EEXIST); - else - error= rm_dir_w_symlink(path, true); + error= rm_dir_w_symlink(path, true); } thd->pop_internal_handler(); @@ -924,16 +924,10 @@ update_binlog: for (tbl= tables; tbl; tbl= tbl->next_local) { uint tbl_name_len; - bool exists; char quoted_name[FN_REFLEN+3]; // Only write drop table to the binlog for tables that no longer exist. - if (check_if_table_exists(thd, tbl, 0, &exists)) - { - error= true; - goto exit; - } - if (exists) + if (ha_table_exists(thd, tbl->db, tbl->table_name)) continue; my_snprintf(quoted_name, sizeof(quoted_name), "%`s", tbl->table_name); @@ -985,31 +979,66 @@ exit: static bool find_db_tables_and_rm_known_files(THD *thd, MY_DIR *dirp, - const char *db, + char *dbname, const char *path, - TABLE_LIST **tables, - bool *found_other_files) + TABLE_LIST **tables) { char filePath[FN_REFLEN]; + LEX_STRING db= { dbname, strlen(dbname) }; TABLE_LIST *tot_list=0, **tot_list_next_local, **tot_list_next_global; DBUG_ENTER("find_db_tables_and_rm_known_files"); DBUG_PRINT("enter",("path: %s", path)); + /* first, get the list of tables */ + Dynamic_array<LEX_STRING*> files(dirp->number_of_files); + Discovered_table_list tl(thd, &files); + if (ha_discover_table_names(thd, &db, dirp, &tl, true)) + DBUG_RETURN(1); + + /* Now put the tables in the list */ tot_list_next_local= tot_list_next_global= &tot_list; + for (size_t idx=0; idx < files.elements(); idx++) + { + LEX_STRING *table= files.at(idx); + + /* Drop the table nicely */ + TABLE_LIST *table_list=(TABLE_LIST*)thd->calloc(sizeof(*table_list)); + + if (!table_list) + DBUG_RETURN(true); + table_list->db= db.str; + table_list->db_length= db.length; + table_list->table_name= table->str; + table_list->table_name_length= table->length; + table_list->open_type= OT_BASE_ONLY; + + /* To be able to correctly look up the table in the table cache. */ + if (lower_case_table_names) + table_list->table_name_length= my_casedn_str(files_charset_info, + table_list->table_name); + + table_list->alias= table_list->table_name; // If lower_case_table_names=2 + table_list->mdl_request.init(MDL_key::TABLE, table_list->db, + table_list->table_name, MDL_EXCLUSIVE, + MDL_TRANSACTION); + /* Link into list */ + (*tot_list_next_local)= table_list; + (*tot_list_next_global)= table_list; + tot_list_next_local= &table_list->next_local; + tot_list_next_global= &table_list->next_global; + } + *tables= tot_list; + + /* and at last delete all non-table files */ for (uint idx=0 ; - idx < (uint) dirp->number_off_files && !thd->killed ; + idx < (uint) dirp->number_of_files && !thd->killed ; idx++) { FILEINFO *file=dirp->dir_entry+idx; char *extension; DBUG_PRINT("info",("Examining: %s", file->name)); - /* skiping . and .. */ - if (file->name[0] == '.' && (!file->name[1] || - (file->name[1] == '.' && !file->name[2]))) - continue; - if (file->name[0] == 'a' && file->name[1] == 'r' && file->name[2] == 'c' && file->name[3] == '\0') { @@ -1026,59 +1055,12 @@ static bool find_db_tables_and_rm_known_files(THD *thd, MY_DIR *dirp, DBUG_PRINT("my",("Archive subdir found: %s", newpath)); if ((mysql_rm_arc_files(thd, new_dirp, newpath)) < 0) DBUG_RETURN(true); - continue; } - *found_other_files= true; continue; } if (!(extension= strrchr(file->name, '.'))) extension= strend(file->name); - if (find_type(extension, &deletable_extentions, FIND_TYPE_NO_PREFIX) <= 0) - { - if (find_type(extension, ha_known_exts(), FIND_TYPE_NO_PREFIX) <= 0) - *found_other_files= true; - continue; - } - /* just for safety we use files_charset_info */ - if (db && !my_strcasecmp(files_charset_info, - extension, reg_ext)) - { - /* Drop the table nicely */ - *extension= 0; // Remove extension - TABLE_LIST *table_list=(TABLE_LIST*) - thd->calloc(sizeof(*table_list) + - strlen(db) + 1 + - MYSQL50_TABLE_NAME_PREFIX_LENGTH + - strlen(file->name) + 1); - - if (!table_list) - DBUG_RETURN(true); - table_list->db= (char*) (table_list+1); - table_list->db_length= strmov(table_list->db, db) - table_list->db; - table_list->table_name= table_list->db + table_list->db_length + 1; - table_list->table_name_length= filename_to_tablename(file->name, - table_list->table_name, - MYSQL50_TABLE_NAME_PREFIX_LENGTH + - strlen(file->name) + 1); - table_list->open_type= OT_BASE_ONLY; - - /* To be able to correctly look up the table in the table cache. */ - if (lower_case_table_names) - table_list->table_name_length= my_casedn_str(files_charset_info, - table_list->table_name); - - table_list->alias= table_list->table_name; // If lower_case_table_names=2 - table_list->internal_tmp_table= is_prefix(file->name, tmp_file_prefix); - table_list->mdl_request.init(MDL_key::TABLE, table_list->db, - table_list->table_name, MDL_EXCLUSIVE, - MDL_TRANSACTION); - /* Link into list */ - (*tot_list_next_local)= table_list; - (*tot_list_next_global)= table_list; - tot_list_next_local= &table_list->next_local; - tot_list_next_global= &table_list->next_global; - } - else + if (find_type(extension, &deletable_extentions, FIND_TYPE_NO_PREFIX) > 0) { strxmov(filePath, path, "/", file->name, NullS); /* @@ -1093,7 +1075,7 @@ static bool find_db_tables_and_rm_known_files(THD *thd, MY_DIR *dirp, } } } - *tables= tot_list; + DBUG_RETURN(false); } @@ -1177,18 +1159,13 @@ long mysql_rm_arc_files(THD *thd, MY_DIR *dirp, const char *org_path) DBUG_PRINT("enter", ("path: %s", org_path)); for (uint idx=0 ; - idx < (uint) dirp->number_off_files && !thd->killed ; + idx < (uint) dirp->number_of_files && !thd->killed ; idx++) { FILEINFO *file=dirp->dir_entry+idx; char *extension, *revision; DBUG_PRINT("info",("Examining: %s", file->name)); - /* skiping . and .. */ - if (file->name[0] == '.' && (!file->name[1] || - (file->name[1] == '.' && !file->name[2]))) - continue; - extension= fn_ext(file->name); if (extension[0] != '.' || extension[1] != 'f' || extension[2] != 'r' || @@ -1492,14 +1469,18 @@ bool mysql_change_db(THD *thd, const LEX_STRING *new_db_name, bool force_switch) DBUG_PRINT("info",("Use database: %s", new_db_file_name.str)); #ifndef NO_EMBEDDED_ACCESS_CHECKS - db_access= - test_all_bits(sctx->master_access, DB_ACLS) ? - DB_ACLS : - acl_get(sctx->host, - sctx->ip, - sctx->priv_user, - new_db_file_name.str, - FALSE) | sctx->master_access; + if (test_all_bits(sctx->master_access, DB_ACLS)) + db_access= DB_ACLS; + else + { + db_access= acl_get(sctx->host, sctx->ip, sctx->priv_user, + new_db_file_name.str, FALSE) | sctx->master_access; + if (sctx->priv_role[0]) + { + /* include a possible currently set role for access */ + db_access|= acl_get("", "", sctx->priv_role, new_db_file_name.str, FALSE); + } + } if (!force_switch && !(db_access & DB_ACLS) && @@ -1674,7 +1655,7 @@ bool mysql_upgrade_db(THD *thd, LEX_STRING *old_db) /* Step2: Move tables to the new database */ if ((dirp = my_dir(path,MYF(MY_DONT_SORT)))) { - uint nfiles= (uint) dirp->number_off_files; + uint nfiles= (uint) dirp->number_of_files; for (uint idx=0 ; idx < nfiles && !thd->killed ; idx++) { FILEINFO *file= dirp->dir_entry + idx; @@ -1765,17 +1746,15 @@ bool mysql_upgrade_db(THD *thd, LEX_STRING *old_db) if ((dirp = my_dir(path,MYF(MY_DONT_SORT)))) { - uint nfiles= (uint) dirp->number_off_files; + uint nfiles= (uint) dirp->number_of_files; for (uint idx=0 ; idx < nfiles ; idx++) { FILEINFO *file= dirp->dir_entry + idx; char oldname[FN_REFLEN + 1], newname[FN_REFLEN + 1]; DBUG_PRINT("info",("Examining: %s", file->name)); - /* skiping . and .. and MY_DB_OPT_FILE */ - if ((file->name[0] == '.' && - (!file->name[1] || (file->name[1] == '.' && !file->name[2]))) || - !my_strcasecmp(files_charset_info, file->name, MY_DB_OPT_FILE)) + /* skiping MY_DB_OPT_FILE */ + if (!my_strcasecmp(files_charset_info, file->name, MY_DB_OPT_FILE)) continue; /* pass empty file name, and file->name as extension to avoid encoding */ diff --git a/sql/sql_db.h b/sql/sql_db.h index 1f447c11a52..62d379c515d 100644 --- a/sql/sql_db.h +++ b/sql/sql_db.h @@ -19,7 +19,6 @@ #include "hash.h" /* HASH */ class THD; -typedef struct st_ha_create_information HA_CREATE_INFO; int mysql_create_db(THD *thd, char *db, HA_CREATE_INFO *create, bool silent); bool mysql_alter_db(THD *thd, const char *db, HA_CREATE_INFO *create); diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index 64fd33bbc2c..e1cd8bed44c 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -35,10 +35,166 @@ #include "sql_select.h" #include "sp_head.h" #include "sql_trigger.h" +#include "sql_statistics.h" #include "transaction.h" #include "records.h" // init_read_record, #include "sql_derived.h" // mysql_handle_list_of_derived // end_read_record +#include "sql_partition.h" // make_used_partitions_str + +/* + @brief + Print query plan of a single-table DELETE command + + @detail + This function is used by EXPLAIN DELETE and by SHOW EXPLAIN when it is + invoked on a running DELETE statement. +*/ + +void Delete_plan::save_explain_data(Explain_query *query) +{ + Explain_delete* explain= new Explain_delete; + + if (deleting_all_rows) + { + explain->deleting_all_rows= true; + explain->select_type= "SIMPLE"; + explain->rows= scanned_rows; + } + else + { + explain->deleting_all_rows= false; + Update_plan::save_explain_data_intern(query, explain); + } + + query->add_upd_del_plan(explain); +} + + +void Update_plan::save_explain_data(Explain_query *query) +{ + Explain_update* explain= new Explain_update; + save_explain_data_intern(query, explain); + query->add_upd_del_plan(explain); +} + + +void Update_plan::save_explain_data_intern(Explain_query *query, + Explain_update *explain) +{ + explain->select_type= "SIMPLE"; + explain->table_name.append(table->pos_in_table_list->alias); + + explain->impossible_where= false; + explain->no_partitions= false; + + if (impossible_where) + { + explain->impossible_where= true; + return; + } + + if (no_partitions) + { + explain->no_partitions= true; + return; + } + + select_lex->set_explain_type(TRUE); + explain->select_type= select_lex->type; + /* Partitions */ + { +#ifdef WITH_PARTITION_STORAGE_ENGINE + partition_info *part_info; + if ((part_info= table->part_info)) + { + make_used_partitions_str(part_info, &explain->used_partitions); + explain->used_partitions_set= true; + } + else + explain->used_partitions_set= false; +#else + /* just produce empty column if partitioning is not compiled in */ + explain->used_partitions_set= false; +#endif + } + + + /* Set jtype */ + if (select && select->quick) + { + int quick_type= select->quick->get_type(); + if ((quick_type == QUICK_SELECT_I::QS_TYPE_INDEX_MERGE) || + (quick_type == QUICK_SELECT_I::QS_TYPE_INDEX_INTERSECT) || + (quick_type == QUICK_SELECT_I::QS_TYPE_ROR_INTERSECT) || + (quick_type == QUICK_SELECT_I::QS_TYPE_ROR_UNION)) + explain->jtype= JT_INDEX_MERGE; + else + explain->jtype= JT_RANGE; + } + else + { + if (index == MAX_KEY) + explain->jtype= JT_ALL; + else + explain->jtype= JT_NEXT; + } + + explain->using_where= test(select && select->cond); + explain->using_filesort= using_filesort; + explain->using_io_buffer= using_io_buffer; + + make_possible_keys_line(table, possible_keys, &explain->possible_keys_line); + + explain->quick_info= NULL; + + /* Calculate key_len */ + if (select && select->quick) + { + explain->quick_info= select->quick->get_explain(mem_root); + } + else + { + if (index != MAX_KEY) + { + explain->key_str.append(table->key_info[index].name); + char buf[64]; + size_t length; + length= longlong10_to_str(table->key_info[index].key_length, buf, 10) - buf; + explain->key_len_str.append(buf, length); + } + } + explain->rows= scanned_rows; + + if (select && select->quick && + select->quick->get_type() == QUICK_SELECT_I::QS_TYPE_RANGE) + { + explain_append_mrr_info((QUICK_RANGE_SELECT*)select->quick, + &explain->mrr_type); + } + + bool skip= updating_a_view; + + /* Save subquery children */ + for (SELECT_LEX_UNIT *unit= select_lex->first_inner_unit(); + unit; + unit= unit->next_unit()) + { + if (skip) + { + skip= false; + continue; + } + /* + Display subqueries only if they are not parts of eliminated WHERE/ON + clauses. + */ + if (!(unit->item && unit->item->eliminated)) + explain->add_child(unit->first_select()->select_number); + } +} + + /** Implement DELETE SQL word. @@ -48,7 +204,8 @@ */ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, - SQL_I_List<ORDER> *order_list, ha_rows limit, ulonglong options) + SQL_I_List<ORDER> *order_list, ha_rows limit, + ulonglong options, select_result *result) { bool will_batch; int error, loc_error; @@ -62,12 +219,16 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, bool reverse= FALSE; ORDER *order= (ORDER *) ((order_list && order_list->elements) ? order_list->first : NULL); - uint usable_index= MAX_KEY; SELECT_LEX *select_lex= &thd->lex->select_lex; killed_state killed_status= NOT_KILLED; THD::enum_binlog_query_type query_type= THD::ROW_QUERY_TYPE; + bool with_select= !select_lex->item_list.is_empty(); + Delete_plan query_plan(thd->mem_root); + query_plan.index= MAX_KEY; + query_plan.using_filesort= FALSE; DBUG_ENTER("mysql_delete"); + create_explain_query(thd->lex, thd->mem_root); if (open_and_lock_tables(thd, table_list, TRUE, 0)) DBUG_RETURN(TRUE); @@ -89,10 +250,16 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, } thd_proc_info(thd, "init"); table->map=1; + query_plan.select_lex= &thd->lex->select_lex; + query_plan.table= table; + query_plan.updating_a_view= test(table_list->view); - if (mysql_prepare_delete(thd, table_list, &conds)) + if (mysql_prepare_delete(thd, table_list, select_lex->with_wild, + select_lex->item_list, &conds)) DBUG_RETURN(TRUE); + (void) result->prepare(select_lex->item_list, NULL); + if (thd->lex->current_select->first_cond_optimization) { thd->lex->current_select->save_leaf_tables(thd); @@ -154,14 +321,19 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, - We should not be binlogging this statement in row-based, and - there should be no delete triggers associated with the table. */ - if (!using_limit && const_cond_result && - (!thd->is_current_stmt_binlog_format_row() && - !(table->triggers && table->triggers->has_delete_triggers()))) + if (!with_select && !using_limit && const_cond_result && + (!thd->is_current_stmt_binlog_format_row() && + !(table->triggers && table->triggers->has_delete_triggers()))) { /* Update the table->file->stats.records number */ table->file->info(HA_STATUS_VARIABLE | HA_STATUS_NO_LOCK); ha_rows const maybe_deleted= table->file->stats.records; DBUG_PRINT("debug", ("Trying to use delete_all_rows()")); + + query_plan.set_delete_all_rows(maybe_deleted); + if (thd->lex->describe) + goto exit_without_my_ok; + if (!(error=table->file->ha_delete_all_rows())) { /* @@ -186,20 +358,30 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, Item::cond_result result; conds= remove_eq_conds(thd, conds, &result); if (result == Item::COND_FALSE) // Impossible where + { limit= 0; + query_plan.set_impossible_where(); + if (thd->lex->describe) + goto exit_without_my_ok; + } } #ifdef WITH_PARTITION_STORAGE_ENGINE if (prune_partitions(thd, table, conds)) { free_underlaid_joins(thd, select_lex); - // No matching record + + query_plan.set_no_partitions(); + if (thd->lex->describe) + goto exit_without_my_ok; + my_ok(thd, 0); DBUG_RETURN(0); } #endif /* Update the table->file->stats.records number */ table->file->info(HA_STATUS_VARIABLE | HA_STATUS_NO_LOCK); + set_statistics_for_table(thd, table); table->covering_keys.clear_all(); table->quick_keys.clear_all(); // Can't use 'only index' @@ -209,6 +391,10 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, DBUG_RETURN(TRUE); if ((select && select->check_quick(thd, safe_update, limit)) || !limit) { + query_plan.set_impossible_where(); + if (thd->lex->describe) + goto exit_without_my_ok; + delete select; free_underlaid_joins(thd, select_lex); /* @@ -239,34 +425,63 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, if (options & OPTION_QUICK) (void) table->file->extra(HA_EXTRA_QUICK); + query_plan.scanned_rows= select? select->records: table->file->stats.records; if (order) { - uint length= 0; - SORT_FIELD *sortorder; - ha_rows examined_rows; - table->update_const_key_parts(conds); order= simple_remove_const(order, conds); - bool need_sort; if (select && select->quick && select->quick->unique_key_range()) { // Single row select (always "ordered") - need_sort= FALSE; - usable_index= MAX_KEY; + query_plan.using_filesort= FALSE; + query_plan.index= MAX_KEY; } else - usable_index= get_index_for_order(order, table, select, limit, - &need_sort, &reverse); - if (need_sort) { - DBUG_ASSERT(usable_index == MAX_KEY); + ha_rows scanned_limit= query_plan.scanned_rows; + query_plan.index= get_index_for_order(order, table, select, limit, + &scanned_limit, + &query_plan.using_filesort, + &reverse); + if (!query_plan.using_filesort) + query_plan.scanned_rows= scanned_limit; + } + } + + query_plan.select= select; + query_plan.possible_keys= select? select->possible_keys: key_map(0); + + /* + Ok, we have generated a query plan for the DELETE. + - if we're running EXPLAIN DELETE, goto produce explain output + - otherwise, execute the query plan + */ + if (thd->lex->describe) + goto exit_without_my_ok; + + query_plan.save_explain_data(thd->lex->explain); + + DBUG_EXECUTE_IF("show_explain_probe_delete_exec_start", + dbug_serve_apcs(thd, 1);); + + if (query_plan.using_filesort) + { + ha_rows examined_rows; + ha_rows found_rows; + uint length= 0; + SORT_FIELD *sortorder; + + { + DBUG_ASSERT(query_plan.index == MAX_KEY); table->sort.io_cache= (IO_CACHE *) my_malloc(sizeof(IO_CACHE), - MYF(MY_FAE | MY_ZEROFILL)); + MYF(MY_FAE | MY_ZEROFILL | + MY_THREAD_SPECIFIC)); if (!(sortorder= make_unireg_sortorder(order, &length, NULL)) || - (table->sort.found_records = filesort(thd, table, sortorder, length, - select, HA_POS_ERROR, 1, - &examined_rows)) + (table->sort.found_records= filesort(thd, table, sortorder, length, + select, HA_POS_ERROR, + true, + &examined_rows, &found_rows)) == HA_POS_ERROR) { delete select; @@ -291,7 +506,7 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, free_underlaid_joins(thd, select_lex); DBUG_RETURN(TRUE); } - if (usable_index == MAX_KEY || (select && select->quick)) + if (query_plan.index == MAX_KEY || (select && select->quick)) { if (init_read_record(&info, thd, table, select, 1, 1, FALSE)) { @@ -301,7 +516,7 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, } } else - init_read_record_idx(&info, thd, table, 1, usable_index, reverse); + init_read_record_idx(&info, thd, table, 1, query_plan.index, reverse); init_ftfuncs(thd, select_lex, 1); thd_proc_info(thd, "updating"); @@ -321,9 +536,16 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, else will_batch= !table->file->start_bulk_delete(); - table->mark_columns_needed_for_delete(); + if (with_select) + { + if (result->send_result_set_metadata(select_lex->item_list, + Protocol::SEND_NUM_ROWS | + Protocol::SEND_EOF)) + goto cleanup; + } + while (!(error=info.read_record(&info)) && !thd->killed && ! thd->is_error()) { @@ -343,6 +565,12 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, break; } + if (with_select && result->send_data(select_lex->item_list) < 0) + { + error=1; + break; + } + if (!(error= table->file->ha_delete_row(table->record[0]))) { deleted++; @@ -449,10 +677,23 @@ cleanup: if (error < 0 || (thd->lex->ignore && !thd->is_error() && !thd->is_fatal_error)) { - my_ok(thd, deleted); + if (!with_select) + my_ok(thd, deleted); + else + result->send_eof(); DBUG_PRINT("info",("%ld records deleted",(long) deleted)); } DBUG_RETURN(error >= 0 || thd->is_error()); + + /* Special exits */ +exit_without_my_ok: + query_plan.save_explain_data(thd->lex->explain); + int err2= thd->lex->explain->send_explain(thd); + + delete select; + free_underlaid_joins(thd, select_lex); + //table->set_keyread(false); + DBUG_RETURN((err2 || thd->is_error() || thd->killed) ? 1 : 0); } @@ -463,13 +704,16 @@ cleanup: mysql_prepare_delete() thd - thread handler table_list - global/local table list + wild_num - number of wildcards used in optional SELECT clause + field_list - list of items in optional SELECT clause conds - conditions RETURN VALUE FALSE OK TRUE error */ -int mysql_prepare_delete(THD *thd, TABLE_LIST *table_list, Item **conds) + int mysql_prepare_delete(THD *thd, TABLE_LIST *table_list, + uint wild_num, List<Item> &field_list, Item **conds) { Item *fake_conds= 0; SELECT_LEX *select_lex= &thd->lex->select_lex; @@ -481,7 +725,10 @@ int mysql_prepare_delete(THD *thd, TABLE_LIST *table_list, Item **conds) &thd->lex->select_lex.top_join_list, table_list, select_lex->leaf_tables, FALSE, - DELETE_ACL, SELECT_ACL, TRUE) || + DELETE_ACL, SELECT_ACL, TRUE)) + DBUG_RETURN(TRUE); + if ((wild_num && setup_wild(thd, table_list, field_list, NULL, wild_num)) || + setup_fields(thd, NULL, field_list, MARK_COLUMNS_READ, NULL, 0) || setup_conds(thd, table_list, select_lex->leaf_tables, conds) || setup_ftfuncs(select_lex)) DBUG_RETURN(TRUE); diff --git a/sql/sql_delete.h b/sql/sql_delete.h index 6147e0ea367..9cd09dc5722 100644 --- a/sql/sql_delete.h +++ b/sql/sql_delete.h @@ -21,12 +21,15 @@ class THD; struct TABLE_LIST; class Item; +class select_result; typedef class Item COND; template <typename T> class SQL_I_List; -int mysql_prepare_delete(THD *thd, TABLE_LIST *table_list, Item **conds); +int mysql_prepare_delete(THD *thd, TABLE_LIST *table_list, + uint wild_num, List<Item> &field_list, Item **conds); bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, - SQL_I_List<ORDER> *order, ha_rows rows, ulonglong options); + SQL_I_List<ORDER> *order, ha_rows rows, + ulonglong options, select_result *result); #endif /* SQL_DELETE_INCLUDED */ diff --git a/sql/sql_error.cc b/sql/sql_error.cc index e12723402a8..acb61fe68c5 100644 --- a/sql/sql_error.cc +++ b/sql/sql_error.cc @@ -461,23 +461,38 @@ Diagnostics_area::disable_status() m_status= DA_DISABLED; } -Warning_info::Warning_info(ulonglong warn_id_arg, bool allow_unlimited_warnings) +Warning_info::Warning_info(ulonglong warn_id_arg, + bool allow_unlimited_warnings, bool initialize) :m_statement_warn_count(0), m_current_row_for_warning(1), m_warn_id(warn_id_arg), m_allow_unlimited_warnings(allow_unlimited_warnings), + initialized(0), m_read_only(FALSE) { - /* Initialize sub structures */ - init_sql_alloc(&m_warn_root, WARN_ALLOC_BLOCK_SIZE, WARN_ALLOC_PREALLOC_SIZE); m_warn_list.empty(); bzero((char*) m_warn_count, sizeof(m_warn_count)); + if (initialize) + init(); } +void Warning_info::init() +{ + /* Initialize sub structures */ + init_sql_alloc(&m_warn_root, WARN_ALLOC_BLOCK_SIZE, + WARN_ALLOC_PREALLOC_SIZE, MYF(MY_THREAD_SPECIFIC)); + initialized= 1; +} + +void Warning_info::free_memory() +{ + if (initialized) + free_root(&m_warn_root,MYF(0)); +} Warning_info::~Warning_info() { - free_root(&m_warn_root,MYF(0)); + free_memory(); } @@ -488,7 +503,7 @@ Warning_info::~Warning_info() void Warning_info::clear_warning_info(ulonglong warn_id_arg) { m_warn_id= warn_id_arg; - free_root(&m_warn_root, MYF(0)); + free_memory(); bzero((char*) m_warn_count, sizeof(m_warn_count)); m_warn_list.empty(); m_statement_warn_count= 0; diff --git a/sql/sql_error.h b/sql/sql_error.h index 9efd77328ea..bd0cb308603 100644 --- a/sql/sql_error.h +++ b/sql/sql_error.h @@ -369,15 +369,21 @@ class Warning_info /** Indicates if push_warning() allows unlimited number of warnings. */ bool m_allow_unlimited_warnings; + bool initialized; /* Set to 1 if init() has been called */ private: Warning_info(const Warning_info &rhs); /* Not implemented */ Warning_info& operator=(const Warning_info &rhs); /* Not implemented */ public: - Warning_info(ulonglong warn_id_arg, bool allow_unlimited_warnings); + Warning_info(ulonglong warn_id_arg, bool allow_unlimited_warnings, + bool initialize=true); ~Warning_info(); + /* Allocate memory for structures */ + void init(); + void free_memory(); + /** Reset the warning information. Clear all warnings, the number of warnings, reset current row counter diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc new file mode 100644 index 00000000000..ad0fdf59ad4 --- /dev/null +++ b/sql/sql_explain.cc @@ -0,0 +1,955 @@ +/* + Copyright (c) 2013 Monty Program Ab + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#ifdef USE_PRAGMA_IMPLEMENTATION +#pragma implementation // gcc: Class implementation +#endif + +#include "sql_priv.h" +#include "sql_select.h" + + +Explain_query::Explain_query(THD *thd_arg) : + upd_del_plan(NULL), insert_plan(NULL), thd(thd_arg), apc_enabled(false) +{ + operations= 0; +} + + +Explain_query::~Explain_query() +{ + if (apc_enabled) + thd->apc_target.disable(); + + delete upd_del_plan; + delete insert_plan; + uint i; + for (i= 0 ; i < unions.elements(); i++) + delete unions.at(i); + for (i= 0 ; i < selects.elements(); i++) + delete selects.at(i); +} + + +Explain_node *Explain_query::get_node(uint select_id) +{ + Explain_union *u; + if ((u= get_union(select_id))) + return u; + else + return get_select(select_id); +} + +Explain_union *Explain_query::get_union(uint select_id) +{ + return (unions.elements() > select_id) ? unions.at(select_id) : NULL; +} + +Explain_select *Explain_query::get_select(uint select_id) +{ + return (selects.elements() > select_id) ? selects.at(select_id) : NULL; +} + + +void Explain_query::add_node(Explain_node *node) +{ + uint select_id; + operations++; + if (node->get_type() == Explain_node::EXPLAIN_UNION) + { + Explain_union *u= (Explain_union*)node; + select_id= u->get_select_id(); + if (unions.elements() <= select_id) + unions.resize(max(select_id+1, unions.elements()*2), NULL); + + Explain_union *old_node; + if ((old_node= get_union(select_id))) + delete old_node; + + unions.at(select_id)= u; + } + else + { + Explain_select *sel= (Explain_select*)node; + if (sel->select_id == FAKE_SELECT_LEX_ID) + { + DBUG_ASSERT(0); // this is a "fake select" from a UNION. + } + else + { + select_id= sel->select_id; + Explain_select *old_node; + + if (selects.elements() <= select_id) + selects.resize(max(select_id+1, selects.elements()*2), NULL); + + if ((old_node= get_select(select_id))) + delete old_node; + + selects.at(select_id)= sel; + } + } +} + + +void Explain_query::add_insert_plan(Explain_insert *insert_plan_arg) +{ + insert_plan= insert_plan_arg; + query_plan_ready(); +} + + +void Explain_query::add_upd_del_plan(Explain_update *upd_del_plan_arg) +{ + upd_del_plan= upd_del_plan_arg; + query_plan_ready(); +} + + +void Explain_query::query_plan_ready() +{ + if (!apc_enabled) + thd->apc_target.enable(); + apc_enabled= true; +} + +/* + Send EXPLAIN output to the client. +*/ + +int Explain_query::send_explain(THD *thd) +{ + select_result *result; + LEX *lex= thd->lex; + + if (!(result= new select_send()) || + thd->send_explain_fields(result)) + return 1; + + int res; + if ((res= print_explain(result, lex->describe))) + result->abort_result_set(); + else + result->send_eof(); + + return res; +} + + +/* + The main entry point to print EXPLAIN of the entire query +*/ + +int Explain_query::print_explain(select_result_sink *output, + uint8 explain_flags) +{ + if (upd_del_plan) + { + upd_del_plan->print_explain(this, output, explain_flags); + return 0; + } + else if (insert_plan) + { + insert_plan->print_explain(this, output, explain_flags); + return 0; + } + else + { + /* Start printing from node with id=1 */ + Explain_node *node= get_node(1); + if (!node) + return 1; /* No query plan */ + return node->print_explain(this, output, explain_flags); + } +} + + +bool print_explain_query(LEX *lex, THD *thd, String *str) +{ + return lex->explain->print_explain_str(thd, str); +} + + +/* + Return tabular EXPLAIN output as a text string +*/ + +bool Explain_query::print_explain_str(THD *thd, String *out_str) +{ + List<Item> fields; + thd->make_explain_field_list(fields); + + select_result_text_buffer output_buf(thd); + output_buf.send_result_set_metadata(fields, thd->lex->describe); + if (print_explain(&output_buf, thd->lex->describe)) + return true; + output_buf.save_to(out_str); + return false; +} + + +static void push_str(List<Item> *item_list, const char *str) +{ + item_list->push_back(new Item_string(str, + strlen(str), system_charset_info)); +} + + +static void push_string(List<Item> *item_list, String *str) +{ + item_list->push_back(new Item_string(str->ptr(), str->length(), + system_charset_info)); +} + + +int Explain_union::print_explain(Explain_query *query, + select_result_sink *output, + uint8 explain_flags) +{ + char table_name_buffer[SAFE_NAME_LEN]; + + /* print all UNION children, in order */ + for (int i= 0; i < (int) union_members.elements(); i++) + { + Explain_select *sel= query->get_select(union_members.at(i)); + sel->print_explain(query, output, explain_flags); + } + + /* Print a line with "UNION RESULT" */ + List<Item> item_list; + Item *item_null= new Item_null(); + + /* `id` column */ + item_list.push_back(item_null); + + /* `select_type` column */ + push_str(&item_list, fake_select_type); + + /* `table` column: something like "<union1,2>" */ + { + uint childno= 0; + uint len= 6, lastop= 0; + memcpy(table_name_buffer, STRING_WITH_LEN("<union")); + + for (; childno < union_members.elements() && len + lastop + 5 < NAME_LEN; + childno++) + { + len+= lastop; + lastop= my_snprintf(table_name_buffer + len, NAME_LEN - len, + "%u,", union_members.at(childno)); + } + + if (childno < union_members.elements() || len + lastop >= NAME_LEN) + { + memcpy(table_name_buffer + len, STRING_WITH_LEN("...>") + 1); + len+= 4; + } + else + { + len+= lastop; + table_name_buffer[len - 1]= '>'; // change ',' to '>' + } + const CHARSET_INFO *cs= system_charset_info; + item_list.push_back(new Item_string(table_name_buffer, len, cs)); + } + + /* `partitions` column */ + if (explain_flags & DESCRIBE_PARTITIONS) + item_list.push_back(item_null); + + /* `type` column */ + push_str(&item_list, join_type_str[JT_ALL]); + + /* `possible_keys` column */ + item_list.push_back(item_null); + + /* `key` */ + item_list.push_back(item_null); + + /* `key_len` */ + item_list.push_back(item_null); + + /* `ref` */ + item_list.push_back(item_null); + + /* `rows` */ + item_list.push_back(item_null); + + /* `filtered` */ + if (explain_flags & DESCRIBE_EXTENDED) + item_list.push_back(item_null); + + /* `Extra` */ + StringBuffer<256> extra_buf; + if (using_filesort) + { + extra_buf.append(STRING_WITH_LEN("Using filesort")); + } + const CHARSET_INFO *cs= system_charset_info; + item_list.push_back(new Item_string(extra_buf.ptr(), extra_buf.length(), cs)); + + //output->unit.offset_limit_cnt= 0; + if (output->send_data(item_list)) + return 1; + + /* + Print all subquery children (UNION children have already been printed at + the start of this function) + */ + return print_explain_for_children(query, output, explain_flags); +} + + +/* + Print EXPLAINs for all children nodes (i.e. for subqueries) +*/ + +int Explain_node::print_explain_for_children(Explain_query *query, + select_result_sink *output, + uint8 explain_flags) +{ + for (int i= 0; i < (int) children.elements(); i++) + { + Explain_node *node= query->get_node(children.at(i)); + if (node->print_explain(query, output, explain_flags)) + return 1; + } + return 0; +} + + +Explain_select::~Explain_select() +{ + if (join_tabs) + { + for (uint i= 0; i< n_join_tabs; i++) + delete join_tabs[i]; + my_free(join_tabs); + } +} + + +int Explain_select::print_explain(Explain_query *query, + select_result_sink *output, + uint8 explain_flags) +{ + if (message) + { + List<Item> item_list; + const CHARSET_INFO *cs= system_charset_info; + Item *item_null= new Item_null(); + + item_list.push_back(new Item_int((int32) select_id)); + item_list.push_back(new Item_string(select_type, + strlen(select_type), cs)); + for (uint i=0 ; i < 7; i++) + item_list.push_back(item_null); + if (explain_flags & DESCRIBE_PARTITIONS) + item_list.push_back(item_null); + if (explain_flags & DESCRIBE_EXTENDED) + item_list.push_back(item_null); + + item_list.push_back(new Item_string(message,strlen(message),cs)); + + if (output->send_data(item_list)) + return 1; + } + else + { + bool using_tmp= using_temporary; + bool using_fs= using_filesort; + for (uint i=0; i< n_join_tabs; i++) + { + join_tabs[i]->print_explain(output, explain_flags, select_id, + select_type, using_tmp, using_fs); + if (i == 0) + { + /* + "Using temporary; Using filesort" should only be shown near the 1st + table + */ + using_tmp= false; + using_fs= false; + } + } + } + + return print_explain_for_children(query, output, explain_flags); +} + + +void Explain_table_access::push_extra(enum explain_extra_tag extra_tag) +{ + extra_tags.append(extra_tag); +} + + +int Explain_table_access::print_explain(select_result_sink *output, uint8 explain_flags, + uint select_id, const char *select_type, + bool using_temporary, bool using_filesort) +{ + const CHARSET_INFO *cs= system_charset_info; + const char *hash_key_prefix= "#hash#"; + bool is_hj= (type == JT_HASH || type == JT_HASH_NEXT || + type == JT_HASH_RANGE || type == JT_HASH_INDEX_MERGE); + + List<Item> item_list; + Item *item_null= new Item_null(); + + if (sjm_nest_select_id) + select_id= sjm_nest_select_id; + + /* `id` column */ + item_list.push_back(new Item_int((int32) select_id)); + + /* `select_type` column */ + if (sjm_nest_select_id) + push_str(&item_list, "MATERIALIZED"); + else + push_str(&item_list, select_type); + + /* `table` column */ + push_string(&item_list, &table_name); + + /* `partitions` column */ + if (explain_flags & DESCRIBE_PARTITIONS) + { + if (used_partitions_set) + { + push_string(&item_list, &used_partitions); + } + else + item_list.push_back(item_null); + } + + /* `type` column */ + push_str(&item_list, join_type_str[type]); + + /* `possible_keys` column */ + if (possible_keys_str.length() > 0) + push_string(&item_list, &possible_keys_str); + else + item_list.push_back(item_null); + + /* `key` */ + StringBuffer<64> key_str; + if (key.get_key_name()) + { + if (is_hj) + key_str.append(hash_key_prefix, strlen(hash_key_prefix), cs); + + key_str.append(key.get_key_name()); + + if (is_hj && type != JT_HASH) + key_str.append(':'); + } + + if (quick_info) + { + StringBuffer<64> buf2; + quick_info->print_key(&buf2); + key_str.append(buf2); + } + if (type == JT_HASH_NEXT) + key_str.append(hash_next_key.get_key_name()); + + if (key_str.length() > 0) + push_string(&item_list, &key_str); + else + item_list.push_back(item_null); + + /* `key_len` */ + StringBuffer<64> key_len_str; + + if (key.get_key_len() != (uint)-1) + { + char buf[64]; + size_t length; + length= longlong10_to_str(key.get_key_len(), buf, 10) - buf; + key_len_str.append(buf, length); + if (is_hj && type != JT_HASH) + key_len_str.append(':'); + } + + if (quick_info) + { + StringBuffer<64> buf2; + quick_info->print_key_len(&buf2); + key_len_str.append(buf2); + } + + if (type == JT_HASH_NEXT) + { + char buf[64]; + size_t length; + length= longlong10_to_str(hash_next_key.get_key_len(), buf, 10) - buf; + key_len_str.append(buf, length); + } + + if (key_len_str.length() > 0) + push_string(&item_list, &key_len_str); + else + item_list.push_back(item_null); + + /* `ref` */ + if (ref_set) + push_string(&item_list, &ref); + else + item_list.push_back(item_null); + + /* `rows` */ + if (rows_set) + { + item_list.push_back(new Item_int((longlong) (ulonglong) rows, + MY_INT64_NUM_DECIMAL_DIGITS)); + } + else + item_list.push_back(item_null); + + /* `filtered` */ + if (explain_flags & DESCRIBE_EXTENDED) + { + if (filtered_set) + { + item_list.push_back(new Item_float(filtered, 2)); + } + else + item_list.push_back(item_null); + } + + /* `Extra` */ + StringBuffer<256> extra_buf; + bool first= true; + for (int i=0; i < (int)extra_tags.elements(); i++) + { + if (first) + first= false; + else + extra_buf.append(STRING_WITH_LEN("; ")); + append_tag_name(&extra_buf, extra_tags.at(i)); + } + + if (using_temporary) + { + if (first) + first= false; + else + extra_buf.append(STRING_WITH_LEN("; ")); + extra_buf.append(STRING_WITH_LEN("Using temporary")); + } + + if (using_filesort) + { + if (first) + first= false; + else + extra_buf.append(STRING_WITH_LEN("; ")); + extra_buf.append(STRING_WITH_LEN("Using filesort")); + } + + item_list.push_back(new Item_string(extra_buf.ptr(), extra_buf.length(), cs)); + + if (output->send_data(item_list)) + return 1; + + return 0; +} + + +/* + Elements in this array match members of enum Extra_tag, defined in + sql_explain.h +*/ + +const char * extra_tag_text[]= +{ + "ET_none", + "Using index condition", + "Using index condition(BKA)", + "Using ", // special handling + "Range checked for each record (index map: 0x", // special handling + "Using where with pushed condition", + "Using where", + "Not exists", + + "Using index", + "Full scan on NULL key", + "Skip_open_table", + "Open_frm_only", + "Open_full_table", + + "Scanned 0 databases", + "Scanned 1 database", + "Scanned all databases", + + "Using index for group-by", // special handling + + "USING MRR: DONT PRINT ME", // special handling + + "Distinct", + "LooseScan", + "Start temporary", + "End temporary", + "FirstMatch", // special handling + + "Using join buffer", // special handling + + "const row not found", + "unique row not found", + "Impossible ON condition" +}; + + +void Explain_table_access::append_tag_name(String *str, enum explain_extra_tag tag) +{ + switch (tag) { + case ET_USING: + { + // quick select + str->append(STRING_WITH_LEN("Using ")); + quick_info->print_extra(str); + break; + } + case ET_RANGE_CHECKED_FOR_EACH_RECORD: + { + /* 4 bits per 1 hex digit + terminating '\0' */ + char buf[MAX_KEY / 4 + 1]; + str->append(STRING_WITH_LEN("Range checked for each " + "record (index map: 0x")); + str->append(range_checked_map.print(buf)); + str->append(')'); + break; + } + case ET_USING_MRR: + { + str->append(mrr_type); + break; + } + case ET_USING_JOIN_BUFFER: + { + str->append(extra_tag_text[tag]); + + str->append(STRING_WITH_LEN(" (")); + const char *buffer_type= bka_type.incremental ? "incremental" : "flat"; + str->append(buffer_type); + str->append(STRING_WITH_LEN(", ")); + str->append(bka_type.join_alg); + str->append(STRING_WITH_LEN(" join")); + str->append(STRING_WITH_LEN(")")); + if (bka_type.mrr_type.length()) + str->append(bka_type.mrr_type); + + break; + } + case ET_FIRST_MATCH: + { + if (firstmatch_table_name.length()) + { + str->append("FirstMatch("); + str->append(firstmatch_table_name); + str->append(")"); + } + else + str->append(extra_tag_text[tag]); + break; + } + case ET_USING_INDEX_FOR_GROUP_BY: + { + str->append(extra_tag_text[tag]); + if (loose_scan_is_scanning) + str->append(" (scanning)"); + break; + } + default: + str->append(extra_tag_text[tag]); + } +} + + +/* + This is called for top-level Explain_quick_select only. The point of this + function is: + - index_merge should print $index_merge_type (child, ...) + - 'range' should not print anything. +*/ + +void Explain_quick_select::print_extra(String *str) +{ + if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || + quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || + quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + { + /* print nothing */ + } + else + print_extra_recursive(str); +} + + +void Explain_quick_select::print_extra_recursive(String *str) +{ + if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || + quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC) + { + str->append(range.get_key_name()); + } + else + { + str->append(get_name_by_type()); + str->append('('); + List_iterator_fast<Explain_quick_select> it (children); + Explain_quick_select* child; + bool first= true; + while ((child = it++)) + { + if (first) + first= false; + else + str->append(','); + + child->print_extra_recursive(str); + } + str->append(')'); + } +} + + +const char * Explain_quick_select::get_name_by_type() +{ + switch (quick_type) { + case QUICK_SELECT_I::QS_TYPE_INDEX_MERGE: + return "sort_union"; + case QUICK_SELECT_I::QS_TYPE_ROR_UNION: + return "union"; + case QUICK_SELECT_I::QS_TYPE_ROR_INTERSECT: + return "intersect"; + case QUICK_SELECT_I::QS_TYPE_INDEX_INTERSECT: + return "sort_intersect"; + default: + DBUG_ASSERT(0); + return "unknown quick select type"; + } +} + + +/* + This prints a comma-separated list of used indexes, ignoring nesting +*/ + +void Explain_quick_select::print_key(String *str) +{ + if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || + quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || + quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + { + if (str->length() > 0) + str->append(','); + str->append(range.get_key_name()); + } + else + { + List_iterator_fast<Explain_quick_select> it (children); + Explain_quick_select* child; + while ((child = it++)) + { + child->print_key(str); + } + } +} + + +/* + This prints a comma-separated list of used key_lengths, ignoring nesting +*/ + +void Explain_quick_select::print_key_len(String *str) +{ + if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || + quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || + quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + { + char buf[64]; + size_t length; + length= longlong10_to_str(range.get_key_len(), buf, 10) - buf; + if (str->length() > 0) + str->append(','); + str->append(buf, length); + } + else + { + List_iterator_fast<Explain_quick_select> it (children); + Explain_quick_select* child; + while ((child = it++)) + { + child->print_key_len(str); + } + } +} + + +int Explain_delete::print_explain(Explain_query *query, + select_result_sink *output, + uint8 explain_flags) +{ + if (deleting_all_rows) + { + const char *msg= "Deleting all rows"; + int res= print_explain_message_line(output, explain_flags, + 1 /*select number*/, + select_type, &rows, msg); + return res; + + } + else + { + return Explain_update::print_explain(query, output, explain_flags); + } +} + + +int Explain_update::print_explain(Explain_query *query, + select_result_sink *output, + uint8 explain_flags) +{ + StringBuffer<64> key_buf; + StringBuffer<64> key_len_buf; + StringBuffer<64> extra_str; + if (impossible_where || no_partitions) + { + const char *msg= impossible_where ? + "Impossible WHERE" : + "No matching rows after partition pruning"; + int res= print_explain_message_line(output, explain_flags, + 1 /*select number*/, + select_type, + NULL, /* rows */ + msg); + return res; + } + + + if (quick_info) + { + quick_info->print_key(&key_buf); + quick_info->print_key_len(&key_len_buf); + + StringBuffer<64> quick_buf; + quick_info->print_extra(&quick_buf); + if (quick_buf.length()) + { + extra_str.append(STRING_WITH_LEN("Using ")); + extra_str.append(quick_buf); + } + } + else + { + key_buf.copy(key_str); + key_len_buf.copy(key_len_str); + } + + if (using_where) + { + if (extra_str.length() !=0) + extra_str.append(STRING_WITH_LEN("; ")); + extra_str.append(STRING_WITH_LEN("Using where")); + } + + if (mrr_type.length() != 0) + { + if (extra_str.length() !=0) + extra_str.append(STRING_WITH_LEN("; ")); + extra_str.append(mrr_type); + } + + if (using_filesort) + { + if (extra_str.length() !=0) + extra_str.append(STRING_WITH_LEN("; ")); + extra_str.append(STRING_WITH_LEN("Using filesort")); + } + + if (using_io_buffer) + { + if (extra_str.length() !=0) + extra_str.append(STRING_WITH_LEN("; ")); + extra_str.append(STRING_WITH_LEN("Using buffer")); + } + + /* + Single-table DELETE commands do not do "Using temporary". + "Using index condition" is also not possible (which is an unjustified limitation) + */ + + print_explain_row(output, explain_flags, + 1, /* id */ + select_type, + table_name.c_ptr(), + used_partitions_set? used_partitions.c_ptr() : NULL, + jtype, + possible_keys_line.length()? possible_keys_line.c_ptr(): NULL, + key_buf.length()? key_buf.c_ptr() : NULL, + key_len_buf.length() ? key_len_buf.c_ptr() : NULL, + NULL, /* 'ref' is always NULL in single-table EXPLAIN DELETE */ + &rows, + extra_str.c_ptr_safe()); + + return print_explain_for_children(query, output, explain_flags); +} + + +int Explain_insert::print_explain(Explain_query *query, + select_result_sink *output, + uint8 explain_flags) +{ + const char *select_type="INSERT"; + print_explain_row(output, explain_flags, + 1, /* id */ + select_type, + table_name.c_ptr(), + NULL, // partitions + JT_ALL, + NULL, // possible_keys + NULL, // key + NULL, // key_len + NULL, // ref + NULL, // rows + NULL); + + return print_explain_for_children(query, output, explain_flags); +} + + +void delete_explain_query(LEX *lex) +{ + delete lex->explain; + lex->explain= NULL; +} + + +void create_explain_query(LEX *lex, MEM_ROOT *mem_root) +{ + DBUG_ASSERT(!lex->explain); + lex->explain= new Explain_query(lex->thd); + DBUG_ASSERT(mem_root == current_thd->mem_root); + lex->explain->mem_root= mem_root; +} + +void create_explain_query_if_not_exists(LEX *lex, MEM_ROOT *mem_root) +{ + if (!lex->explain) + create_explain_query(lex, mem_root); +} + diff --git a/sql/sql_explain.h b/sql/sql_explain.h new file mode 100644 index 00000000000..12621b88d23 --- /dev/null +++ b/sql/sql_explain.h @@ -0,0 +1,550 @@ +/* + Copyright (c) 2013 Monty Program Ab + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + + +/************************************************************************************** + + Data structures for producing EXPLAIN outputs. + + These structures + - Can be produced inexpensively from query plan. + - Store sufficient information to produce a tabular and/or a json EXPLAIN + - Have methods that produce a tabular output. + +*************************************************************************************/ + + +const int FAKE_SELECT_LEX_ID= (int)UINT_MAX; + +class Explain_query; + +/* + A node can be either a SELECT, or a UNION. +*/ +class Explain_node : public Sql_alloc +{ +public: + enum explain_node_type + { + EXPLAIN_UNION, + EXPLAIN_SELECT, + EXPLAIN_UPDATE, + EXPLAIN_DELETE, + EXPLAIN_INSERT + }; + + virtual enum explain_node_type get_type()= 0; + virtual int get_select_id()= 0; + + /* + A node may have children nodes. When a node's explain structure is + created, children nodes may not yet have QPFs. This is why we store ids. + */ + Dynamic_array<int> children; + void add_child(int select_no) + { + children.append(select_no); + } + + virtual int print_explain(Explain_query *query, select_result_sink *output, + uint8 explain_flags)=0; + + int print_explain_for_children(Explain_query *query, select_result_sink *output, + uint8 explain_flags); + virtual ~Explain_node(){} +}; + + +class Explain_table_access; + + +/* + EXPLAIN structure for a SELECT. + + A select can be: + 1. A degenerate case. In this case, message!=NULL, and it contains a + description of what kind of degenerate case it is (e.g. "Impossible + WHERE"). + 2. a non-degenrate join. In this case, join_tabs describes the join. + + In the non-degenerate case, a SELECT may have a GROUP BY/ORDER BY operation. + + In both cases, the select may have children nodes. class Explain_node provides + a way get node's children. +*/ + +class Explain_select : public Explain_node +{ +public: + enum explain_node_type get_type() { return EXPLAIN_SELECT; } + + Explain_select() : + message(NULL), join_tabs(NULL), + using_temporary(false), using_filesort(false) + {} + + ~Explain_select(); + + bool add_table(Explain_table_access *tab) + { + if (!join_tabs) + { + join_tabs= (Explain_table_access**) my_malloc(sizeof(Explain_table_access*) * + MAX_TABLES, MYF(0)); + n_join_tabs= 0; + } + join_tabs[n_join_tabs++]= tab; + return false; + } + +public: + int select_id; + const char *select_type; + + int get_select_id() { return select_id; } + + /* + If message != NULL, this is a degenerate join plan, and all subsequent + members have no info + */ + const char *message; + + /* + A flat array of Explain structs for tables. The order is "just like EXPLAIN + would print them". + */ + Explain_table_access** join_tabs; + uint n_join_tabs; + + /* Global join attributes. In tabular form, they are printed on the first row */ + bool using_temporary; + bool using_filesort; + + int print_explain(Explain_query *query, select_result_sink *output, + uint8 explain_flags); +}; + + +/* + Explain structure for a UNION. + + A UNION may or may not have "Using filesort". +*/ + +class Explain_union : public Explain_node +{ +public: + enum explain_node_type get_type() { return EXPLAIN_UNION; } + + int get_select_id() + { + DBUG_ASSERT(union_members.elements() > 0); + return union_members.at(0); + } + /* + Members of the UNION. Note: these are different from UNION's "children". + Example: + + (select * from t1) union + (select * from t2) order by (select col1 from t3 ...) + + here + - select-from-t1 and select-from-t2 are "union members", + - select-from-t3 is the only "child". + */ + Dynamic_array<int> union_members; + + void add_select(int select_no) + { + union_members.append(select_no); + } + int print_explain(Explain_query *query, select_result_sink *output, + uint8 explain_flags); + + const char *fake_select_type; + bool using_filesort; +}; + + +class Explain_update; +class Explain_delete; +class Explain_insert; + +/* + Explain structure for a query (i.e. a statement). + + This should be able to survive when the query plan was deleted. Currently, + we do not intend for it survive until after query's MEM_ROOT is freed. It + does surivive freeing of query's items. + + For reference, the process of post-query cleanup is as follows: + + >dispatch_command + | >mysql_parse + | | ... + | | lex_end() + | | ... + | | >THD::cleanup_after_query + | | | ... + | | | free_items() + | | | ... + | | <THD::cleanup_after_query + | | + | <mysql_parse + | + | log_slow_statement() + | + | free_root() + | + >dispatch_command + + That is, the order of actions is: + - free query's Items + - write to slow query log + - free query's MEM_ROOT + +*/ + +class Explain_query : public Sql_alloc +{ +public: + Explain_query(THD *thd); + ~Explain_query(); + + /* Add a new node */ + void add_node(Explain_node *node); + void add_insert_plan(Explain_insert *insert_plan_arg); + void add_upd_del_plan(Explain_update *upd_del_plan_arg); + + /* This will return a select, or a union */ + Explain_node *get_node(uint select_id); + + /* This will return a select (even if there is a union with this id) */ + Explain_select *get_select(uint select_id); + + Explain_union *get_union(uint select_id); + + /* Produce a tabular EXPLAIN output */ + int print_explain(select_result_sink *output, uint8 explain_flags); + + /* Send tabular EXPLAIN to the client */ + int send_explain(THD *thd); + + /* Return tabular EXPLAIN output as a text string */ + bool print_explain_str(THD *thd, String *out_str); + + /* If true, at least part of EXPLAIN can be printed */ + bool have_query_plan() { return insert_plan || upd_del_plan|| get_node(1) != NULL; } + + void query_plan_ready(); + + MEM_ROOT *mem_root; +private: + /* Explain_delete inherits from Explain_update */ + Explain_update *upd_del_plan; + + /* Query "plan" for INSERTs */ + Explain_insert *insert_plan; + + Dynamic_array<Explain_union*> unions; + Dynamic_array<Explain_select*> selects; + + THD *thd; // for APC start/stop + bool apc_enabled; + /* + Debugging aid: count how many times add_node() was called. Ideally, it + should be one, we currently allow O(1) query plan saves for each + select or union. The goal is not to have O(#rows_in_some_table), which + is unacceptable. + */ + longlong operations; +}; + + +/* + Some of the tags have matching text. See extra_tag_text for text names, and + Explain_table_access::append_tag_name() for code to convert from tag form to text + form. +*/ +enum explain_extra_tag +{ + ET_none= 0, /* not-a-tag */ + ET_USING_INDEX_CONDITION, + ET_USING_INDEX_CONDITION_BKA, + ET_USING, /* For quick selects of various kinds */ + ET_RANGE_CHECKED_FOR_EACH_RECORD, + ET_USING_WHERE_WITH_PUSHED_CONDITION, + ET_USING_WHERE, + ET_NOT_EXISTS, + + ET_USING_INDEX, + ET_FULL_SCAN_ON_NULL_KEY, + ET_SKIP_OPEN_TABLE, + ET_OPEN_FRM_ONLY, + ET_OPEN_FULL_TABLE, + + ET_SCANNED_0_DATABASES, + ET_SCANNED_1_DATABASE, + ET_SCANNED_ALL_DATABASES, + + ET_USING_INDEX_FOR_GROUP_BY, + + ET_USING_MRR, // does not print "Using mrr". + + ET_DISTINCT, + ET_LOOSESCAN, + ET_START_TEMPORARY, + ET_END_TEMPORARY, + ET_FIRST_MATCH, + + ET_USING_JOIN_BUFFER, + + ET_CONST_ROW_NOT_FOUND, + ET_UNIQUE_ROW_NOT_FOUND, + ET_IMPOSSIBLE_ON_CONDITION, + + ET_total +}; + + +typedef struct st_explain_bka_type +{ + bool incremental; + const char *join_alg; + StringBuffer<64> mrr_type; + +} EXPLAIN_BKA_TYPE; + + +/* + Data about how an index is used by some access method +*/ +class Explain_index_use : public Sql_alloc +{ + char *key_name; + uint key_len; + /* will add #keyparts here if we implement EXPLAIN FORMAT=JSON */ +public: + + void set(MEM_ROOT *root, const char *key_name_arg, uint key_len_arg) + { + if (key_name_arg) + { + size_t name_len= strlen(key_name_arg); + if ((key_name= (char*)alloc_root(root, name_len+1))) + memcpy(key_name, key_name_arg, name_len+1); + } + else + key_name= NULL; + key_len= key_len_arg; + } + + inline const char *get_key_name() { return key_name; } + inline uint get_key_len() { return key_len; } +}; + + +/* + QPF for quick range selects, as well as index_merge select +*/ +class Explain_quick_select : public Sql_alloc +{ +public: + Explain_quick_select(int quick_type_arg) : quick_type(quick_type_arg) + {} + + const int quick_type; + + /* This is used when quick_type == QUICK_SELECT_I::QS_TYPE_RANGE */ + Explain_index_use range; + + /* Used in all other cases */ + List<Explain_quick_select> children; + + void print_extra(String *str); + void print_key(String *str); + void print_key_len(String *str); +private: + void print_extra_recursive(String *str); + const char *get_name_by_type(); +}; + + +/* + Query Plan Footprint for a JOIN_TAB. +*/ +class Explain_table_access : public Sql_alloc +{ +public: + void push_extra(enum explain_extra_tag extra_tag); + + /* Internals */ +public: + /* + 0 means this tab is not inside SJM nest and should use Explain_select's id + other value means the tab is inside an SJM nest. + */ + int sjm_nest_select_id; + + /* id and 'select_type' are cared-of by the parent Explain_select */ + StringBuffer<32> table_name; + + enum join_type type; + + StringBuffer<32> used_partitions; + bool used_partitions_set; + + /* Empty string means "NULL" will be printed */ + StringBuffer<32> possible_keys_str; + + /* + Index use: key name and length. + Note: that when one is accessing I_S tables, those may show use of + non-existant indexes. + + key.key_name == NULL means 'NULL' will be shown in tabular output. + key.key_len == (uint)-1 means 'NULL' will be shown in tabular output. + */ + Explain_index_use key; + + /* + when type==JT_HASH_NEXT, 'key' stores the hash join pseudo-key. + hash_next_key stores the table's key. + */ + Explain_index_use hash_next_key; + + bool ref_set; /* not set means 'NULL' should be printed */ + StringBuffer<32> ref; + + bool rows_set; /* not set means 'NULL' should be printed */ + ha_rows rows; + + bool filtered_set; /* not set means 'NULL' should be printed */ + double filtered; + + /* + Contents of the 'Extra' column. Some are converted into strings, some have + parameters, values for which are stored below. + */ + Dynamic_array<enum explain_extra_tag> extra_tags; + + // Valid if ET_USING tag is present + Explain_quick_select *quick_info; + + // Valid if ET_USING_INDEX_FOR_GROUP_BY is present + bool loose_scan_is_scanning; + + // valid with ET_RANGE_CHECKED_FOR_EACH_RECORD + key_map range_checked_map; + + // valid with ET_USING_MRR + StringBuffer<32> mrr_type; + + // valid with ET_USING_JOIN_BUFFER + EXPLAIN_BKA_TYPE bka_type; + + StringBuffer<32> firstmatch_table_name; + + int print_explain(select_result_sink *output, uint8 explain_flags, + uint select_id, const char *select_type, + bool using_temporary, bool using_filesort); +private: + void append_tag_name(String *str, enum explain_extra_tag tag); +}; + + +/* + EXPLAIN structure for single-table UPDATE. + + This is similar to Explain_table_access, except that it is more restrictive. + Also, it can have UPDATE operation options, but currently there aren't any. +*/ + +class Explain_update : public Explain_node +{ +public: + virtual enum explain_node_type get_type() { return EXPLAIN_UPDATE; } + virtual int get_select_id() { return 1; /* always root */ } + + const char *select_type; + + StringBuffer<32> used_partitions; + bool used_partitions_set; + + bool impossible_where; + bool no_partitions; + StringBuffer<64> table_name; + + enum join_type jtype; + StringBuffer<128> possible_keys_line; + StringBuffer<128> key_str; + StringBuffer<128> key_len_str; + StringBuffer<64> mrr_type; + + Explain_quick_select *quick_info; + + bool using_where; + ha_rows rows; + + bool using_filesort; + bool using_io_buffer; + + virtual int print_explain(Explain_query *query, select_result_sink *output, + uint8 explain_flags); +}; + + +/* + EXPLAIN data structure for an INSERT. + + At the moment this doesn't do much as we don't really have any query plans + for INSERT statements. +*/ + +class Explain_insert : public Explain_node +{ +public: + StringBuffer<64> table_name; + + enum explain_node_type get_type() { return EXPLAIN_INSERT; } + int get_select_id() { return 1; /* always root */ } + + int print_explain(Explain_query *query, select_result_sink *output, + uint8 explain_flags); +}; + + +/* + EXPLAIN data of a single-table DELETE. +*/ + +class Explain_delete: public Explain_update +{ +public: + /* + TRUE means we're going to call handler->delete_all_rows() and not read any + rows. + */ + bool deleting_all_rows; + + virtual enum explain_node_type get_type() { return EXPLAIN_DELETE; } + virtual int get_select_id() { return 1; /* always root */ } + + virtual int print_explain(Explain_query *query, select_result_sink *output, + uint8 explain_flags); +}; + + diff --git a/sql/sql_expression_cache.cc b/sql/sql_expression_cache.cc index f3c96ee2d2f..1e64bc10a7c 100644 --- a/sql/sql_expression_cache.cc +++ b/sql/sql_expression_cache.cc @@ -257,7 +257,7 @@ my_bool Expression_cache_tmptable::put_value(Item *value) } *(items.head_ref())= value; - fill_record(table_thd, cache_table->field, items, TRUE, TRUE); + fill_record(table_thd, cache_table, cache_table->field, items, TRUE, TRUE); if (table_thd->is_error()) goto err;; diff --git a/sql/sql_handler.cc b/sql/sql_handler.cc index c099a2af496..87ca0f9056d 100644 --- a/sql/sql_handler.cc +++ b/sql/sql_handler.cc @@ -304,7 +304,8 @@ bool mysql_ha_open(THD *thd, TABLE_LIST *tables, SQL_HANDLER *reopen) /* There can be only one table in '*tables'. */ if (! (table->file->ha_table_flags() & HA_CAN_SQL_HANDLER)) { - my_error(ER_ILLEGAL_HA, MYF(0), tables->alias); + my_error(ER_ILLEGAL_HA, MYF(0), table->file->table_type(), + table->s->db.str, table->s->table_name.str); goto err; } @@ -323,7 +324,7 @@ bool mysql_ha_open(THD *thd, TABLE_LIST *tables, SQL_HANDLER *reopen) /* copy data to sql_handler */ if (!(sql_handler= new SQL_HANDLER(thd))) goto err; - init_alloc_root(&sql_handler->mem_root, 1024, 0); + init_alloc_root(&sql_handler->mem_root, 1024, 0, MYF(MY_THREAD_SPECIFIC)); sql_handler->db.length= strlen(tables->db); sql_handler->table_name.length= strlen(tables->table_name); @@ -904,7 +905,8 @@ retry: break; } default: - my_message(ER_ILLEGAL_HA, ER(ER_ILLEGAL_HA), MYF(0)); + my_error(ER_ILLEGAL_HA, MYF(0), table->file->table_type(), + table->s->db.str, table->s->table_name.str); goto err; } diff --git a/sql/sql_hset.h b/sql/sql_hset.h index f3a1467737f..dc3bd487ce5 100644 --- a/sql/sql_hset.h +++ b/sql/sql_hset.h @@ -23,19 +23,19 @@ A type-safe wrapper around mysys HASH. */ -template <typename T, my_hash_get_key K> +template <typename T> class Hash_set { public: - typedef T Value_type; enum { START_SIZE= 8 }; /** Constructs an empty hash. Does not allocate memory, it is done upon the first insert. Thus does not cause or return errors. */ - Hash_set() + Hash_set(uchar *(*K)(const T *, size_t *, my_bool)) { my_hash_clear(&m_hash); + m_hash.get_key= (my_hash_get_key)K; } /** Destroy the hash by freeing the buckets table. Does @@ -56,13 +56,19 @@ public: */ bool insert(T *value) { - my_hash_init_opt(&m_hash, &my_charset_bin, START_SIZE, 0, 0, K, 0, MYF(0)); + my_hash_init_opt(&m_hash, &my_charset_bin, START_SIZE, 0, 0, + m_hash.get_key, 0, MYF(0)); size_t key_len; - const uchar *key= K(reinterpret_cast<uchar*>(value), &key_len, FALSE); - if (my_hash_search(&m_hash, key, key_len) == NULL) - return my_hash_insert(&m_hash, reinterpret_cast<uchar *>(value)); + uchar *v= reinterpret_cast<uchar *>(value); + const uchar *key= m_hash.get_key(v, &key_len, FALSE); + if (find(key, key_len) == NULL) + return my_hash_insert(&m_hash, v); return FALSE; } + T *find(const void *key, size_t klen) const + { + return (T*)my_hash_search(&m_hash, reinterpret_cast<const uchar *>(key), klen); + } /** Is this hash set empty? */ bool is_empty() const { return m_hash.records == 0; } /** Returns the number of unique elements. */ diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 0bc1453ddef..a20596f017b 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2010, 2013, Monty Program Ab. + Copyright (c) 2009, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -184,10 +184,6 @@ error: @param fields_and_values_from_different_maps If 'values' are allowed to refer to other tables than those of 'fields' @param map See check_view_single_update - NOTE - Clears TIMESTAMP_AUTO_SET_ON_INSERT from table->timestamp_field_type - or leaves it as is, depending on if timestamp should be updated or - not. @returns 0 if success, -1 if error */ @@ -225,8 +221,6 @@ static int check_insert_fields(THD *thd, TABLE_LIST *table_list, if (check_grant_all_columns(thd, INSERT_ACL, &field_it)) return -1; #endif - clear_timestamp_auto_bits(table->timestamp_field_type, - TIMESTAMP_AUTO_SET_ON_INSERT); /* No fields are provided so all fields must be provided in the values. Thus we set all bits in the write set. @@ -286,18 +280,8 @@ static int check_insert_fields(THD *thd, TABLE_LIST *table_list, my_error(ER_FIELD_SPECIFIED_TWICE, MYF(0), thd->dup_field->field_name); return -1; } - if (table->timestamp_field) // Don't automaticly set timestamp if used - { - if (bitmap_is_set(table->write_set, - table->timestamp_field->field_index)) - clear_timestamp_auto_bits(table->timestamp_field_type, - TIMESTAMP_AUTO_SET_ON_INSERT); - else - { - bitmap_set_bit(table->write_set, - table->timestamp_field->field_index); - } - } + if (table->default_field) + table->mark_default_fields_for_write(); } /* Mark virtual columns used in the insert statement */ if (table->vfield) @@ -330,9 +314,9 @@ static int check_insert_fields(THD *thd, TABLE_LIST *table_list, refer to other tables than those of 'update_fields' @param map See check_view_single_update - NOTE - If the update fields include the timestamp field, - remove TIMESTAMP_AUTO_SET_ON_UPDATE from table->timestamp_field_type. + @note + If the update fields include an autoinc field, set the + table->next_number_field_updated flag. @returns 0 if success, -1 if error */ @@ -344,21 +328,9 @@ static int check_update_fields(THD *thd, TABLE_LIST *insert_table_list, table_map *map) { TABLE *table= insert_table_list->table; - my_bool timestamp_mark; my_bool autoinc_mark; - LINT_INIT(timestamp_mark); LINT_INIT(autoinc_mark); - if (table->timestamp_field) - { - /* - Unmark the timestamp field so that we can check if this is modified - by update_fields - */ - timestamp_mark= bitmap_test_and_clear(table->write_set, - table->timestamp_field->field_index); - } - table->next_number_field_updated= FALSE; if (table->found_next_number_field) @@ -384,17 +356,8 @@ static int check_update_fields(THD *thd, TABLE_LIST *insert_table_list, insert_table_list, map, false)) return -1; - if (table->timestamp_field) - { - /* Don't set timestamp column if this is modified. */ - if (bitmap_is_set(table->write_set, - table->timestamp_field->field_index)) - clear_timestamp_auto_bits(table->timestamp_field_type, - TIMESTAMP_AUTO_SET_ON_UPDATE); - if (timestamp_mark) - bitmap_set_bit(table->write_set, - table->timestamp_field->field_index); - } + if (table->default_field) + table->mark_default_fields_for_write(); if (table->found_next_number_field) { @@ -498,7 +461,8 @@ void upgrade_lock_type(THD *thd, thr_lock_type *lock_type, if (specialflag & (SPECIAL_NO_NEW_FUNC | SPECIAL_SAFE_MODE) || thd->variables.max_insert_delayed_threads == 0 || thd->locked_tables_mode > LTM_LOCK_TABLES || - thd->lex->uses_stored_routines()) + thd->lex->uses_stored_routines() /*|| + thd->lex->describe*/) { *lock_type= TL_WRITE; return; @@ -681,6 +645,36 @@ create_insert_stmt_from_insert_delayed(THD *thd, String *buf) } +static void save_insert_query_plan(THD* thd, TABLE_LIST *table_list) +{ + Explain_insert* explain= new Explain_insert; + explain->table_name.append(table_list->table->alias); + + thd->lex->explain->add_insert_plan(explain); + + /* See Update_plan::updating_a_view for details */ + bool skip= test(table_list->view); + + /* Save subquery children */ + for (SELECT_LEX_UNIT *unit= thd->lex->select_lex.first_inner_unit(); + unit; + unit= unit->next_unit()) + { + if (skip) + { + skip= false; + continue; + } + /* + Table elimination doesn't work for INSERTS, but let's still have this + here for consistency + */ + if (!(unit->item && unit->item->eliminated)) + explain->add_child(unit->first_select()->select_number); + } +} + + /** INSERT statement implementation @@ -697,10 +691,11 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, enum_duplicates duplic, bool ignore) { + bool retval= true; int error, res; bool transactional_table, joins_freed= FALSE; bool changed; - bool was_insert_delayed= (table_list->lock_type == TL_WRITE_DELAYED); + const bool was_insert_delayed= (table_list->lock_type == TL_WRITE_DELAYED); bool using_bulk_insert= 0; uint value_count; ulong counter = 1; @@ -724,6 +719,7 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, Item *unused_conds= 0; DBUG_ENTER("mysql_insert"); + create_explain_query(thd->lex, thd->mem_root); /* Upgrade lock type if the requested lock is incompatible with the current connection mode or table operation. @@ -817,6 +813,17 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, /* Restore the current context. */ ctx_state.restore_state(context, table_list); + + if (thd->lex->unit.first_select()->optimize_unflattened_subqueries(false)) + { + goto abort; + } + save_insert_query_plan(thd, table_list); + if (thd->lex->describe) + { + retval= thd->lex->explain->send_explain(thd); + goto abort; + } /* Fill in the given fields and dump it to the table file @@ -841,10 +848,10 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, table->next_number_field=table->found_next_number_field; #ifdef HAVE_REPLICATION - if (thd->slave_thread && + if (thd->rgi_slave && (info.handle_duplicates == DUP_UPDATE) && (table->next_number_field != NULL) && - rpl_master_has_bug(&active_mi->rli, 24432, TRUE, NULL, NULL)) + rpl_master_has_bug(thd->rgi_slave->rli, 24432, TRUE, NULL, NULL)) goto abort; #endif @@ -904,8 +911,7 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, if (fields.elements || !value_count) { restore_record(table,s->default_values); // Get empty record - if (fill_record_n_invoke_before_triggers(thd, fields, *values, 0, - table->triggers, + if (fill_record_n_invoke_before_triggers(thd, table, fields, *values, 0, TRG_EVENT_INSERT)) { if (values_list.elements != 1 && ! thd->is_error()) @@ -949,8 +955,7 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, share->default_values[share->null_bytes - 1]; } } - if (fill_record_n_invoke_before_triggers(thd, table->field, *values, 0, - table->triggers, + if (fill_record_n_invoke_before_triggers(thd, table, table->field, *values, 0, TRG_EVENT_INSERT)) { if (values_list.elements != 1 && ! thd->is_error()) @@ -962,6 +967,11 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, break; } } + if (table->default_field && table->update_default_fields()) + { + error= 1; + break; + } if ((res= table_list->view_check_option(thd, (values_list.elements == 1 ? @@ -978,7 +988,9 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, if (lock_type == TL_WRITE_DELAYED) { LEX_STRING const st_query = { query, thd->query_length() }; + DEBUG_SYNC(thd, "before_write_delayed"); error=write_delayed(thd, table, duplic, st_query, ignore, log_on); + DEBUG_SYNC(thd, "after_write_delayed"); query=0; } else @@ -1166,10 +1178,11 @@ abort: #endif if (table != NULL) table->file->ha_release_auto_increment(); + if (!joins_freed) free_underlaid_joins(thd, &thd->lex->select_lex); thd->abort_on_warning= 0; - DBUG_RETURN(TRUE); + DBUG_RETURN(retval); } @@ -1688,13 +1701,32 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) restore_record(table,record[1]); DBUG_ASSERT(info->update_fields->elements == info->update_values->elements); - if (fill_record_n_invoke_before_triggers(thd, *info->update_fields, + if (fill_record_n_invoke_before_triggers(thd, table, *info->update_fields, *info->update_values, info->ignore, - table->triggers, TRG_EVENT_UPDATE)) goto before_trg_err; + bool different_records= (!records_are_comparable(table) || + compare_record(table)); + /* + Default fields must be updated before checking view updateability. + This branch of INSERT is executed only when a UNIQUE key was violated + with the ON DUPLICATE KEY UPDATE option. In this case the INSERT + operation is transformed to an UPDATE, and the default fields must + be updated as if this is an UPDATE. + */ + if (different_records && table->default_field) + { + bool res; + enum_sql_command cmd= thd->lex->sql_command; + thd->lex->sql_command= SQLCOM_UPDATE; + res= table->update_default_fields(); + thd->lex->sql_command= cmd; + if (res) + goto err; + } + /* CHECK OPTION for VIEW ... ON DUPLICATE KEY UPDATE ... */ if (info->view && (res= info->view->view_check_option(current_thd, info->ignore)) == @@ -1705,7 +1737,7 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) table->file->restore_auto_increment(prev_insert_id); info->touched++; - if (!records_are_comparable(table) || compare_record(table)) + if (different_records) { if ((error=table->file->ha_update_row(table->record[1], table->record[0])) && @@ -1778,8 +1810,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) */ if (last_uniq_key(table,key_nr) && !table->file->referenced_by_foreign_key() && - (table->timestamp_field_type == TIMESTAMP_NO_AUTO_SET || - table->timestamp_field_type == TIMESTAMP_AUTO_SET_ON_BOTH) && (!table->triggers || !table->triggers->has_delete_triggers())) { if ((error=table->file->ha_update_row(table->record[1], @@ -1944,7 +1974,6 @@ public: ulonglong forced_insert_id; ulong auto_increment_increment; ulong auto_increment_offset; - timestamp_auto_set_type timestamp_field_type; LEX_STRING query; Time_zone *time_zone; @@ -2051,9 +2080,9 @@ public: thd.unlink(); // Must be unlinked under lock my_free(thd.query()); thd.security_ctx->user= thd.security_ctx->host=0; - thread_count--; delayed_insert_threads--; mysql_mutex_unlock(&LOCK_thread_count); + thread_safe_decrement32(&thread_count, &thread_count_lock); mysql_cond_broadcast(&COND_thread_count); /* Tell main we are ready */ } @@ -2188,9 +2217,9 @@ bool delayed_get_table(THD *thd, MDL_request *grl_protection_request, { if (!(di= new Delayed_insert())) goto end_create; - mysql_mutex_lock(&LOCK_thread_count); - thread_count++; - mysql_mutex_unlock(&LOCK_thread_count); + + thread_safe_increment32(&thread_count, &thread_count_lock); + /* Annotating delayed inserts is not supported. */ @@ -2314,7 +2343,7 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) { my_ptrdiff_t adjust_ptrs; Field **field,**org_field, *found_next_number_field; - Field **UNINIT_VAR(vfield); + Field **UNINIT_VAR(vfield), **UNINIT_VAR(dfield_ptr); TABLE *copy; TABLE_SHARE *share; uchar *bitmap; @@ -2390,6 +2419,15 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) bitmap= (uchar*) (field + share->fields + 1); copy->record[0]= (bitmap + share->column_bitmap_size*3); memcpy((char*) copy->record[0], (char*) table->record[0], share->reclength); + if (share->default_fields) + { + copy->default_field= (Field**) client_thd->alloc((share->default_fields+1)* + sizeof(Field**)); + if (!copy->default_field) + goto error; + dfield_ptr= copy->default_field; + } + /* Ensure we don't use the table list of the original table */ copy->pos_in_table_list= 0; @@ -2410,6 +2448,15 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) (*field)->move_field_offset(adjust_ptrs); // Point at copy->record[0] if (*org_field == found_next_number_field) (*field)->table->found_next_number_field= *field; + if (share->default_fields && + ((*org_field)->has_insert_default_function() || + (*org_field)->has_update_default_function())) + { + /* Put the newly copied field into the set of default fields. */ + *dfield_ptr= *field; + (*dfield_ptr)->unireg_check= (*org_field)->unireg_check; + dfield_ptr++; + } } *field=0; @@ -2434,15 +2481,8 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) *vfield= 0; } - /* Adjust timestamp */ - if (table->timestamp_field) - { - /* Restore offset as this may have been reset in handle_inserts */ - copy->timestamp_field= - (Field_timestamp*) copy->field[share->timestamp_field_offset]; - copy->timestamp_field->unireg_check= table->timestamp_field->unireg_check; - copy->timestamp_field_type= copy->timestamp_field->get_auto_set_type(); - } + if (share->default_fields) + *dfield_ptr= NULL; /* Adjust in_use for pointing to client thread */ copy->in_use= client_thd; @@ -2515,7 +2555,9 @@ int write_delayed(THD *thd, TABLE *table, enum_duplicates duplic, goto err; } - if (!(row->record= (char*) my_malloc(table->s->reclength, MYF(MY_WME)))) + /* This can't be THREAD_SPECIFIC as it's freed in delayed thread */ + if (!(row->record= (char*) my_malloc(table->s->reclength, + MYF(MY_WME)))) goto err; memcpy(row->record, table->record[0], table->s->reclength); row->start_time= thd->start_time; @@ -2532,7 +2574,6 @@ int write_delayed(THD *thd, TABLE *table, enum_duplicates duplic, thd->stmt_depends_on_first_successful_insert_id_in_prev_stmt; row->first_successful_insert_id_in_prev_stmt= thd->first_successful_insert_id_in_prev_stmt; - row->timestamp_field_type= table->timestamp_field_type; /* Add session variable timezone Time_zone object will not be freed even the thread is ended. @@ -2940,24 +2981,28 @@ pthread_handler_t handle_delayed_insert(void *arg) DBUG_LEAVE; } - di->table=0; - thd->killed= KILL_CONNECTION; // If error - mysql_mutex_unlock(&di->mutex); + { + DBUG_ENTER("handle_delayed_insert-cleanup"); + di->table=0; + thd->killed= KILL_CONNECTION; // If error + mysql_mutex_unlock(&di->mutex); - close_thread_tables(thd); // Free the table - thd->mdl_context.release_transactional_locks(); - mysql_cond_broadcast(&di->cond_client); // Safety + close_thread_tables(thd); // Free the table + thd->mdl_context.release_transactional_locks(); + mysql_cond_broadcast(&di->cond_client); // Safety - mysql_mutex_lock(&LOCK_delayed_create); // Because of delayed_get_table - mysql_mutex_lock(&LOCK_delayed_insert); - /* - di should be unlinked from the thread handler list and have no active - clients - */ - delete di; - mysql_mutex_unlock(&LOCK_delayed_insert); - mysql_mutex_unlock(&LOCK_delayed_create); + mysql_mutex_lock(&LOCK_delayed_create); // Because of delayed_get_table + mysql_mutex_lock(&LOCK_delayed_insert); + /* + di should be unlinked from the thread handler list and have no active + clients + */ + delete di; + mysql_mutex_unlock(&LOCK_delayed_insert); + mysql_mutex_unlock(&LOCK_delayed_create); + DBUG_LEAVE; + } my_thread_end(); pthread_exit(0); @@ -3082,7 +3127,6 @@ bool Delayed_insert::handle_inserts(void) row->first_successful_insert_id_in_prev_stmt; thd.stmt_depends_on_first_successful_insert_id_in_prev_stmt= row->stmt_depends_on_first_successful_insert_id_in_prev_stmt; - table->timestamp_field_type= row->timestamp_field_type; table->auto_increment_field_not_null= row->auto_increment_field_not_null; /* Copy the session variables. */ @@ -3474,10 +3518,10 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u) table->next_number_field=table->found_next_number_field; #ifdef HAVE_REPLICATION - if (thd->slave_thread && + if (thd->rgi_slave && (info.handle_duplicates == DUP_UPDATE) && (table->next_number_field != NULL) && - rpl_master_has_bug(&active_mi->rli, 24432, TRUE, NULL, NULL)) + rpl_master_has_bug(thd->rgi_slave->rli, 24432, TRUE, NULL, NULL)) DBUG_RETURN(1); #endif @@ -3565,6 +3609,8 @@ int select_insert::send_data(List<Item> &values) thd->count_cuted_fields= CHECK_FIELD_WARN; // Calculate cuted fields store_values(values); + if (table->default_field && table->update_default_fields()) + DBUG_RETURN(1); thd->count_cuted_fields= CHECK_FIELD_ERROR_FOR_NULL; if (thd->is_error()) { @@ -3624,11 +3670,11 @@ int select_insert::send_data(List<Item> &values) void select_insert::store_values(List<Item> &values) { if (fields->elements) - fill_record_n_invoke_before_triggers(thd, *fields, values, 1, - table->triggers, TRG_EVENT_INSERT); + fill_record_n_invoke_before_triggers(thd, table, *fields, values, 1, + TRG_EVENT_INSERT); else - fill_record_n_invoke_before_triggers(thd, table->field, values, 1, - table->triggers, TRG_EVENT_INSERT); + fill_record_n_invoke_before_triggers(thd, table, table->field, values, 1, + TRG_EVENT_INSERT); } void select_insert::send_error(uint errcode,const char *err) @@ -3846,7 +3892,6 @@ static TABLE *create_table_from_items(THD *thd, HA_CREATE_INFO *create_info, DBUG_ENTER("create_table_from_items"); tmp_table.alias= 0; - tmp_table.timestamp_field= 0; tmp_table.s= &share; init_tmp_table_share(thd, &share, "", 0, "", ""); @@ -3855,6 +3900,8 @@ static TABLE *create_table_from_items(THD *thd, HA_CREATE_INFO *create_info, tmp_table.null_row= 0; tmp_table.maybe_null= 0; + promote_first_timestamp_column(&alter_info->create_list); + while ((item=it++)) { Create_field *cr_field; @@ -3901,8 +3948,8 @@ static TABLE *create_table_from_items(THD *thd, HA_CREATE_INFO *create_info, { if (!mysql_create_table_no_lock(thd, create_table->db, create_table->table_name, - create_info, alter_info, 0, - select_field_count, NULL)) + create_info, alter_info, NULL, + select_field_count)) { DEBUG_SYNC(thd,"create_table_select_before_open"); @@ -4094,8 +4141,6 @@ select_create::prepare(List<Item> &values, SELECT_LEX_UNIT *u) for (Field **f= field ; *f ; f++) bitmap_set_bit(table->write_set, (*f)->field_index); - /* Don't set timestamp if used */ - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; table->next_number_field=table->found_next_number_field; restore_record(table,s->default_values); // Get empty record @@ -4171,8 +4216,8 @@ select_create::binlog_show_create_table(TABLE **tables, uint count) void select_create::store_values(List<Item> &values) { - fill_record_n_invoke_before_triggers(thd, field, values, 1, - table->triggers, TRG_EVENT_INSERT); + fill_record_n_invoke_before_triggers(thd, table, field, values, 1, + TRG_EVENT_INSERT); } @@ -4281,16 +4326,3 @@ void select_create::abort_result_set() DBUG_VOID_RETURN; } - -/***************************************************************************** - Instansiate templates -*****************************************************************************/ - -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class List_iterator_fast<List_item>; -#ifndef EMBEDDED_LIBRARY -template class I_List<Delayed_insert>; -template class I_List_iterator<Delayed_insert>; -template class I_List<delayed_row>; -#endif /* EMBEDDED_LIBRARY */ -#endif /* HAVE_EXPLICIT_TEMPLATE_INSTANTIATION */ diff --git a/sql/sql_join_cache.cc b/sql/sql_join_cache.cc index e97f0e185c6..a66ab42c855 100644 --- a/sql/sql_join_cache.cc +++ b/sql/sql_join_cache.cc @@ -934,7 +934,7 @@ int JOIN_CACHE::alloc_buffer() { ulong next_buff_size; - if ((buff= (uchar*) my_malloc(buff_size, MYF(0)))) + if ((buff= (uchar*) my_malloc(buff_size, MYF(MY_THREAD_SPECIFIC)))) break; next_buff_size= buff_size > buff_size_decr ? buff_size-buff_size_decr : 0; @@ -1012,7 +1012,7 @@ int JOIN_CACHE::realloc_buffer() { int rc; free(); - rc= test(!(buff= (uchar*) my_malloc(buff_size, MYF(0)))); + rc= test(!(buff= (uchar*) my_malloc(buff_size, MYF(MY_THREAD_SPECIFIC)))); reset(TRUE); return rc; } @@ -2257,7 +2257,7 @@ enum_nested_loop_state JOIN_CACHE::join_matching_records(bool skip_last) while (!(error= join_tab_scan->next())) { - if (join->thd->killed) + if (join->thd->check_killed()) { /* The user has aborted the execution of the query */ join->thd->send_kill_message(); @@ -2527,7 +2527,7 @@ enum_nested_loop_state JOIN_CACHE::join_null_complements(bool skip_last) for ( ; cnt; cnt--) { - if (join->thd->killed) + if (join->thd->check_killed()) { /* The user has aborted the execution of the query */ join->thd->send_kill_message(); @@ -2553,49 +2553,41 @@ finish: /* - Add a comment on the join algorithm employed by the join cache + Save data on the join algorithm employed by the join cache SYNOPSIS - print_explain_comment() + save_explain_data() str string to add the comment on the employed join algorithm to DESCRIPTION - This function adds info on the type of the used join buffer (flat or + This function puts info about the type of the used join buffer (flat or incremental) and on the type of the the employed join algorithm (BNL, - BNLH, BKA or BKAH) to the the end of the sring str. + BNLH, BKA or BKAH) to the data structure RETURN VALUE none */ -void JOIN_CACHE::print_explain_comment(String *str) +void JOIN_CACHE::save_explain_data(struct st_explain_bka_type *explain) { - str->append(STRING_WITH_LEN(" (")); - const char *buffer_type= prev_cache ? "incremental" : "flat"; - str->append(buffer_type); - str->append(STRING_WITH_LEN(", ")); - - const char *join_alg=""; + explain->incremental= test(prev_cache); + switch (get_join_alg()) { case BNL_JOIN_ALG: - join_alg= "BNL"; + explain->join_alg= "BNL"; break; case BNLH_JOIN_ALG: - join_alg= "BNLH"; + explain->join_alg= "BNLH"; break; case BKA_JOIN_ALG: - join_alg= "BKA"; + explain->join_alg= "BKA"; break; case BKAH_JOIN_ALG: - join_alg= "BKAH"; + explain->join_alg= "BKAH"; break; default: DBUG_ASSERT(0); } - - str->append(join_alg); - str->append(STRING_WITH_LEN(" join")); - str->append(STRING_WITH_LEN(")")); } /** @@ -2621,18 +2613,17 @@ static void add_mrr_explain_info(String *str, uint mrr_mode, handler *file) } } - -void JOIN_CACHE_BKA::print_explain_comment(String *str) +void JOIN_CACHE_BKA::save_explain_data(struct st_explain_bka_type *explain) { - JOIN_CACHE::print_explain_comment(str); - add_mrr_explain_info(str, mrr_mode, join_tab->table->file); + JOIN_CACHE::save_explain_data(explain); + add_mrr_explain_info(&explain->mrr_type, mrr_mode, join_tab->table->file); } -void JOIN_CACHE_BKAH::print_explain_comment(String *str) +void JOIN_CACHE_BKAH::save_explain_data(struct st_explain_bka_type *explain) { - JOIN_CACHE::print_explain_comment(str); - add_mrr_explain_info(str, mrr_mode, join_tab->table->file); + JOIN_CACHE::save_explain_data(explain); + add_mrr_explain_info(&explain->mrr_type, mrr_mode, join_tab->table->file); } @@ -2801,7 +2792,7 @@ int JOIN_CACHE_HASHED::realloc_buffer() { int rc; free(); - rc= test(!(buff= (uchar*) my_malloc(buff_size, MYF(0)))); + rc= test(!(buff= (uchar*) my_malloc(buff_size, MYF(MY_THREAD_SPECIFIC)))); init_hash_table(); reset(TRUE); return rc; @@ -3377,7 +3368,7 @@ int JOIN_TAB_SCAN::next() update_virtual_fields(thd, table); while (!err && select && (skip_rc= select->skip_record(thd)) <= 0) { - if (thd->killed || skip_rc < 0) + if (thd->check_killed() || skip_rc < 0) return 1; /* Move to the next record if the last retrieved record does not @@ -3812,7 +3803,8 @@ uint JOIN_TAB_SCAN_MRR::aux_buffer_incr(ulong recno) uint incr= 0; TABLE_REF *ref= &join_tab->ref; TABLE *tab= join_tab->table; - uint rec_per_key= tab->key_info[ref->key].rec_per_key[ref->key_parts-1]; + ha_rows rec_per_key= + (ha_rows) tab->key_info[ref->key].actual_rec_per_key(ref->key_parts-1); set_if_bigger(rec_per_key, 1); if (recno == 1) incr= ref->key_length + tab->file->ref_length; @@ -3893,13 +3885,15 @@ int JOIN_TAB_SCAN_MRR::next() int rc= join_tab->table->file->multi_range_read_next((range_id_t*)ptr) ? -1 : 0; if (!rc) { - /* + /* If a record in in an incremental cache contains no fields then the association for the last record in cache will be equal to cache->end_pos - */ - DBUG_ASSERT((!(mrr_mode & HA_MRR_NO_ASSOCIATION))? - (cache->buff <= (uchar *) (*ptr) && - (uchar *) (*ptr) <= cache->end_pos): TRUE); + */ + /* + psergey: this makes no sense where HA_MRR_NO_ASSOC is used. + DBUG_ASSERT(cache->buff <= (uchar *) (*ptr) && + (uchar *) (*ptr) <= cache->end_pos); + */ if (join_tab->table->vfield) update_virtual_fields(join->thd, join_tab->table); } @@ -3909,663 +3903,663 @@ int JOIN_TAB_SCAN_MRR::next() static void bka_range_seq_key_info(void *init_params, uint *length, - key_part_map *map) + key_part_map *map) { - TABLE_REF *ref= &(((JOIN_CACHE*)init_params)->join_tab->ref); - *length= ref->key_length; - *map= (key_part_map(1) << ref->key_parts) - 1; +TABLE_REF *ref= &(((JOIN_CACHE*)init_params)->join_tab->ref); +*length= ref->key_length; +*map= (key_part_map(1) << ref->key_parts) - 1; } /* - Initialize retrieval of range sequence for BKA join algorithm - - SYNOPSIS - bka_range_seq_init() - init_params pointer to the BKA join cache object - n_ranges the number of ranges obtained - flags combination of MRR flags - - DESCRIPTION - The function interprets init_param as a pointer to a JOIN_CACHE_BKA - object. The function prepares for an iteration over the join keys - built for all records from the cache join buffer. - - NOTE - This function are used only as a callback function. - - RETURN VALUE - init_param value that is to be used as a parameter of bka_range_seq_next() +Initialize retrieval of range sequence for BKA join algorithm + +SYNOPSIS + bka_range_seq_init() + init_params pointer to the BKA join cache object + n_ranges the number of ranges obtained + flags combination of MRR flags + +DESCRIPTION + The function interprets init_param as a pointer to a JOIN_CACHE_BKA + object. The function prepares for an iteration over the join keys + built for all records from the cache join buffer. + +NOTE + This function are used only as a callback function. + +RETURN VALUE + init_param value that is to be used as a parameter of bka_range_seq_next() */ static range_seq_t bka_range_seq_init(void *init_param, uint n_ranges, uint flags) { - DBUG_ENTER("bka_range_seq_init"); - JOIN_CACHE_BKA *cache= (JOIN_CACHE_BKA *) init_param; - cache->reset(0); - DBUG_RETURN((range_seq_t) init_param); +DBUG_ENTER("bka_range_seq_init"); +JOIN_CACHE_BKA *cache= (JOIN_CACHE_BKA *) init_param; +cache->reset(0); +DBUG_RETURN((range_seq_t) init_param); } /* - Get the next range/key over records from the join buffer used by a BKA cache - - SYNOPSIS - bka_range_seq_next() - seq the value returned by bka_range_seq_init - range OUT reference to the next range +Get the next range/key over records from the join buffer used by a BKA cache - DESCRIPTION - The function interprets seq as a pointer to a JOIN_CACHE_BKA - object. The function returns a pointer to the range descriptor - for the key built over the next record from the join buffer. - - NOTE - This function are used only as a callback function. - - RETURN VALUE - FALSE ok, the range structure filled with info about the next range/key - TRUE no more ranges +SYNOPSIS + bka_range_seq_next() + seq the value returned by bka_range_seq_init + range OUT reference to the next range + +DESCRIPTION + The function interprets seq as a pointer to a JOIN_CACHE_BKA + object. The function returns a pointer to the range descriptor + for the key built over the next record from the join buffer. + +NOTE + This function are used only as a callback function. + +RETURN VALUE + FALSE ok, the range structure filled with info about the next range/key + TRUE no more ranges */ static bool bka_range_seq_next(range_seq_t rseq, KEY_MULTI_RANGE *range) { - DBUG_ENTER("bka_range_seq_next"); - JOIN_CACHE_BKA *cache= (JOIN_CACHE_BKA *) rseq; - TABLE_REF *ref= &cache->join_tab->ref; - key_range *start_key= &range->start_key; - if ((start_key->length= cache->get_next_key((uchar **) &start_key->key))) - { - start_key->keypart_map= (1 << ref->key_parts) - 1; - start_key->flag= HA_READ_KEY_EXACT; - range->end_key= *start_key; - range->end_key.flag= HA_READ_AFTER_KEY; - range->ptr= (char *) cache->get_curr_rec(); - range->range_flag= EQ_RANGE; - DBUG_RETURN(0); - } - DBUG_RETURN(1); +DBUG_ENTER("bka_range_seq_next"); +JOIN_CACHE_BKA *cache= (JOIN_CACHE_BKA *) rseq; +TABLE_REF *ref= &cache->join_tab->ref; +key_range *start_key= &range->start_key; +if ((start_key->length= cache->get_next_key((uchar **) &start_key->key))) +{ + start_key->keypart_map= (1 << ref->key_parts) - 1; + start_key->flag= HA_READ_KEY_EXACT; + range->end_key= *start_key; + range->end_key.flag= HA_READ_AFTER_KEY; + range->ptr= (char *) cache->get_curr_rec(); + range->range_flag= EQ_RANGE; + DBUG_RETURN(0); +} +DBUG_RETURN(1); } /* - Check whether range_info orders to skip the next record from BKA buffer - - SYNOPSIS - bka_range_seq_skip_record() - seq value returned by bka_range_seq_init() - range_info information about the next range - rowid [NOT USED] rowid of the record to be checked +Check whether range_info orders to skip the next record from BKA buffer - - DESCRIPTION - The function interprets seq as a pointer to a JOIN_CACHE_BKA object. - The function returns TRUE if the record with this range_info - is to be filtered out from the stream of records returned by - multi_range_read_next(). +SYNOPSIS + bka_range_seq_skip_record() + seq value returned by bka_range_seq_init() + range_info information about the next range + rowid [NOT USED] rowid of the record to be checked - NOTE - This function are used only as a callback function. - - RETURN VALUE - 1 record with this range_info is to be filtered out from the stream - of records returned by multi_range_read_next() - 0 the record is to be left in the stream + +DESCRIPTION + The function interprets seq as a pointer to a JOIN_CACHE_BKA object. + The function returns TRUE if the record with this range_info + is to be filtered out from the stream of records returned by + multi_range_read_next(). + +NOTE + This function are used only as a callback function. + +RETURN VALUE + 1 record with this range_info is to be filtered out from the stream + of records returned by multi_range_read_next() + 0 the record is to be left in the stream */ static bool bka_range_seq_skip_record(range_seq_t rseq, range_id_t range_info, uchar *rowid) { - DBUG_ENTER("bka_range_seq_skip_record"); - JOIN_CACHE_BKA *cache= (JOIN_CACHE_BKA *) rseq; - bool res= cache->get_match_flag_by_pos((uchar *) range_info) == - JOIN_CACHE::MATCH_FOUND; - DBUG_RETURN(res); +DBUG_ENTER("bka_range_seq_skip_record"); +JOIN_CACHE_BKA *cache= (JOIN_CACHE_BKA *) rseq; +bool res= cache->get_match_flag_by_pos((uchar *) range_info) == + JOIN_CACHE::MATCH_FOUND; +DBUG_RETURN(res); } /* - Check if the record combination from BKA cache matches the index condition +Check if the record combination from BKA cache matches the index condition - SYNOPSIS - bka_skip_index_tuple() - rseq value returned by bka_range_seq_init() - range_info record chain for the next range/key returned by MRR - - DESCRIPTION - This is wrapper for JOIN_CACHE_BKA::skip_index_tuple method, - see comments there. +SYNOPSIS + bka_skip_index_tuple() + rseq value returned by bka_range_seq_init() + range_info record chain for the next range/key returned by MRR + +DESCRIPTION + This is wrapper for JOIN_CACHE_BKA::skip_index_tuple method, + see comments there. - NOTE - This function is used as a RANGE_SEQ_IF::skip_index_tuple callback. - - RETURN VALUE - 0 The record combination satisfies the index condition - 1 Otherwise +NOTE + This function is used as a RANGE_SEQ_IF::skip_index_tuple callback. + +RETURN VALUE + 0 The record combination satisfies the index condition + 1 Otherwise */ static bool bka_skip_index_tuple(range_seq_t rseq, range_id_t range_info) { - DBUG_ENTER("bka_skip_index_tuple"); - JOIN_CACHE_BKA *cache= (JOIN_CACHE_BKA *) rseq; - THD *thd= cache->thd(); - bool res; - status_var_increment(thd->status_var.ha_icp_attempts); - if (!(res= cache->skip_index_tuple(range_info))) - status_var_increment(thd->status_var.ha_icp_match); - DBUG_RETURN(res); +DBUG_ENTER("bka_skip_index_tuple"); +JOIN_CACHE_BKA *cache= (JOIN_CACHE_BKA *) rseq; +THD *thd= cache->thd(); +bool res; +status_var_increment(thd->status_var.ha_icp_attempts); +if (!(res= cache->skip_index_tuple(range_info))) + status_var_increment(thd->status_var.ha_icp_match); +DBUG_RETURN(res); } /* - Prepare to read the record from BKA cache matching the current joined record - - SYNOPSIS - prepare_look_for_matches() - skip_last <-> ignore the last record in the buffer (always unused here) - - DESCRIPTION - The function prepares to iterate over records in the join cache buffer - matching the record loaded into the record buffer for join_tab when - performing join operation by BKA join algorithm. With BKA algorithms the - record loaded into the record buffer for join_tab always has a direct - reference to the matching records from the join buffer. When the regular - BKA join algorithm is employed the record from join_tab can refer to - only one such record. - The function sets the counter of the remaining records from the cache - buffer that would match the current join_tab record to 1. - - RETURN VALUE - TRUE there are no records in the buffer to iterate over - FALSE otherwise +Prepare to read the record from BKA cache matching the current joined record + +SYNOPSIS + prepare_look_for_matches() + skip_last <-> ignore the last record in the buffer (always unused here) + +DESCRIPTION + The function prepares to iterate over records in the join cache buffer + matching the record loaded into the record buffer for join_tab when + performing join operation by BKA join algorithm. With BKA algorithms the + record loaded into the record buffer for join_tab always has a direct + reference to the matching records from the join buffer. When the regular + BKA join algorithm is employed the record from join_tab can refer to + only one such record. + The function sets the counter of the remaining records from the cache + buffer that would match the current join_tab record to 1. + +RETURN VALUE + TRUE there are no records in the buffer to iterate over + FALSE otherwise */ - + bool JOIN_CACHE_BKA::prepare_look_for_matches(bool skip_last) { - if (!records) - return TRUE; - rem_records= 1; - return FALSE; +if (!records) + return TRUE; +rem_records= 1; +return FALSE; } /* - Get the record from the BKA cache matching the current joined record - - SYNOPSIS - get_next_candidate_for_match - - DESCRIPTION - This method is used for iterations over the records from the join - cache buffer when looking for matches for records from join_tab. - The method performs the necessary preparations to read the next record - from the join buffer into the record buffer by the method - read_next_candidate_for_match, or, to skip the next record from the join - buffer by the method skip_if_not_needed_match. - This implementation of the virtual method get_next_candidate_for_match - just decrements the counter of the records that are to be iterated over - and returns the value of curr_association as a reference to the position - of the beginning of the record fields in the buffer. - - RETURN VALUE - pointer to the start of the record fields in the join buffer - if the there is another record to iterate over, 0 - otherwise. +Get the record from the BKA cache matching the current joined record + +SYNOPSIS + get_next_candidate_for_match + +DESCRIPTION + This method is used for iterations over the records from the join + cache buffer when looking for matches for records from join_tab. + The method performs the necessary preparations to read the next record + from the join buffer into the record buffer by the method + read_next_candidate_for_match, or, to skip the next record from the join + buffer by the method skip_if_not_needed_match. + This implementation of the virtual method get_next_candidate_for_match + just decrements the counter of the records that are to be iterated over + and returns the value of curr_association as a reference to the position + of the beginning of the record fields in the buffer. + +RETURN VALUE + pointer to the start of the record fields in the join buffer + if the there is another record to iterate over, 0 - otherwise. */ uchar *JOIN_CACHE_BKA::get_next_candidate_for_match() { - if (!rem_records) - return 0; - rem_records--; - return curr_association; +if (!rem_records) + return 0; +rem_records--; +return curr_association; } /* - Check whether the matching record from the BKA cache is to be skipped - - SYNOPSIS - skip_next_candidate_for_match - rec_ptr pointer to the position in the join buffer right after - the previous record - - DESCRIPTION - This implementation of the virtual function just calls the - method get_match_flag_by_pos to check whether the record referenced - by ref_ptr has its match flag set to MATCH_FOUND. - - RETURN VALUE - TRUE the record referenced by rec_ptr has its match flag set to - MATCH_FOUND - FALSE otherwise +Check whether the matching record from the BKA cache is to be skipped + +SYNOPSIS + skip_next_candidate_for_match + rec_ptr pointer to the position in the join buffer right after + the previous record + +DESCRIPTION + This implementation of the virtual function just calls the + method get_match_flag_by_pos to check whether the record referenced + by ref_ptr has its match flag set to MATCH_FOUND. + +RETURN VALUE + TRUE the record referenced by rec_ptr has its match flag set to + MATCH_FOUND + FALSE otherwise */ bool JOIN_CACHE_BKA::skip_next_candidate_for_match(uchar *rec_ptr) { - return join_tab->check_only_first_match() && - (get_match_flag_by_pos(rec_ptr) == MATCH_FOUND); +return join_tab->check_only_first_match() && + (get_match_flag_by_pos(rec_ptr) == MATCH_FOUND); } /* - Read the next record from the BKA join cache buffer when looking for matches - - SYNOPSIS - read_next_candidate_for_match - rec_ptr pointer to the position in the join buffer right after - the previous record - - DESCRIPTION - This implementation of the virtual method read_next_candidate_for_match - calls the method get_record_by_pos to read the record referenced by rec_ptr - from the join buffer into the record buffer. If this record refers to - fields in the other join buffers the call of get_record_by_po ensures that - these fields are read into the corresponding record buffers as well. - This function is supposed to be called after a successful call of - the method get_next_candidate_for_match. - - RETURN VALUE - none +Read the next record from the BKA join cache buffer when looking for matches + +SYNOPSIS + read_next_candidate_for_match + rec_ptr pointer to the position in the join buffer right after + the previous record + +DESCRIPTION + This implementation of the virtual method read_next_candidate_for_match + calls the method get_record_by_pos to read the record referenced by rec_ptr + from the join buffer into the record buffer. If this record refers to + fields in the other join buffers the call of get_record_by_po ensures that + these fields are read into the corresponding record buffers as well. + This function is supposed to be called after a successful call of + the method get_next_candidate_for_match. + +RETURN VALUE + none */ void JOIN_CACHE_BKA::read_next_candidate_for_match(uchar *rec_ptr) { - get_record_by_pos(rec_ptr); +get_record_by_pos(rec_ptr); } /* - Initialize the BKA join cache +Initialize the BKA join cache - SYNOPSIS - init +SYNOPSIS + init - DESCRIPTION - The function initializes the cache structure. It is supposed to be called - right after a constructor for the JOIN_CACHE_BKA. +DESCRIPTION + The function initializes the cache structure. It is supposed to be called + right after a constructor for the JOIN_CACHE_BKA. - NOTES - The function first constructs a companion object of the type - JOIN_TAB_SCAN_MRR, then it calls the init method of the parent class. - - RETURN VALUE - 0 initialization with buffer allocations has been succeeded - 1 otherwise +NOTES + The function first constructs a companion object of the type + JOIN_TAB_SCAN_MRR, then it calls the init method of the parent class. + +RETURN VALUE + 0 initialization with buffer allocations has been succeeded + 1 otherwise */ int JOIN_CACHE_BKA::init() { - int res; - bool check_only_first_match= join_tab->check_only_first_match(); +int res; +bool check_only_first_match= join_tab->check_only_first_match(); - RANGE_SEQ_IF rs_funcs= { bka_range_seq_key_info, - bka_range_seq_init, - bka_range_seq_next, - check_only_first_match ? - bka_range_seq_skip_record : 0, - bka_skip_index_tuple }; +RANGE_SEQ_IF rs_funcs= { bka_range_seq_key_info, + bka_range_seq_init, + bka_range_seq_next, + check_only_first_match ? + bka_range_seq_skip_record : 0, + bka_skip_index_tuple }; - DBUG_ENTER("JOIN_CACHE_BKA::init"); +DBUG_ENTER("JOIN_CACHE_BKA::init"); - JOIN_TAB_SCAN_MRR *jsm; - if (!(join_tab_scan= jsm= new JOIN_TAB_SCAN_MRR(join, join_tab, - mrr_mode, rs_funcs))) - DBUG_RETURN(1); +JOIN_TAB_SCAN_MRR *jsm; +if (!(join_tab_scan= jsm= new JOIN_TAB_SCAN_MRR(join, join_tab, + mrr_mode, rs_funcs))) + DBUG_RETURN(1); - if ((res= JOIN_CACHE::init())) - DBUG_RETURN(res); +if ((res= JOIN_CACHE::init())) + DBUG_RETURN(res); - if (use_emb_key) - jsm->mrr_mode |= HA_MRR_MATERIALIZED_KEYS; +if (use_emb_key) + jsm->mrr_mode |= HA_MRR_MATERIALIZED_KEYS; - DBUG_RETURN(0); +DBUG_RETURN(0); } /* - Get the key built over the next record from BKA join buffer - - SYNOPSIS - get_next_key() - key pointer to the buffer where the key value is to be placed - - DESCRIPTION - The function reads key fields from the current record in the join buffer. - and builds the key value out of these fields that will be used to access - the 'join_tab' table. Some of key fields may belong to previous caches. - They are accessed via record references to the record parts stored in the - previous join buffers. The other key fields always are placed right after - the flag fields of the record. - If the key is embedded, which means that its value can be read directly - from the join buffer, then *key is set to the beginning of the key in - this buffer. Otherwise the key is built in the join_tab->ref->key_buff. - The function returns the length of the key if it succeeds ro read it. - If is assumed that the functions starts reading at the position of - the record length which is provided for each records in a BKA cache. - After the key is built the 'pos' value points to the first position after - the current record. - The function just skips the records with MATCH_IMPOSSIBLE in the - match flag field if there is any. - The function returns 0 if the initial position is after the beginning - of the record fields for last record from the join buffer. - - RETURN VALUE - length of the key value - if the starting value of 'pos' points to - the position before the fields for the last record, - 0 - otherwise. +Get the key built over the next record from BKA join buffer + +SYNOPSIS + get_next_key() + key pointer to the buffer where the key value is to be placed + +DESCRIPTION + The function reads key fields from the current record in the join buffer. + and builds the key value out of these fields that will be used to access + the 'join_tab' table. Some of key fields may belong to previous caches. + They are accessed via record references to the record parts stored in the + previous join buffers. The other key fields always are placed right after + the flag fields of the record. + If the key is embedded, which means that its value can be read directly + from the join buffer, then *key is set to the beginning of the key in + this buffer. Otherwise the key is built in the join_tab->ref->key_buff. + The function returns the length of the key if it succeeds ro read it. + If is assumed that the functions starts reading at the position of + the record length which is provided for each records in a BKA cache. + After the key is built the 'pos' value points to the first position after + the current record. + The function just skips the records with MATCH_IMPOSSIBLE in the + match flag field if there is any. + The function returns 0 if the initial position is after the beginning + of the record fields for last record from the join buffer. + +RETURN VALUE + length of the key value - if the starting value of 'pos' points to + the position before the fields for the last record, + 0 - otherwise. */ uint JOIN_CACHE_BKA::get_next_key(uchar ** key) { - uint len; - uint32 rec_len; - uchar *init_pos; - JOIN_CACHE *cache; - +uint len; +uint32 rec_len; +uchar *init_pos; +JOIN_CACHE *cache; + start: - /* Any record in a BKA cache is prepended with its length */ - DBUG_ASSERT(with_length); - - if ((pos+size_of_rec_len) > last_rec_pos || !records) - return 0; +/* Any record in a BKA cache is prepended with its length */ +DBUG_ASSERT(with_length); + +if ((pos+size_of_rec_len) > last_rec_pos || !records) + return 0; - /* Read the length of the record */ - rec_len= get_rec_length(pos); - pos+= size_of_rec_len; - init_pos= pos; +/* Read the length of the record */ +rec_len= get_rec_length(pos); +pos+= size_of_rec_len; +init_pos= pos; - /* Read a reference to the previous cache if any */ - if (prev_cache) - pos+= prev_cache->get_size_of_rec_offset(); +/* Read a reference to the previous cache if any */ +if (prev_cache) + pos+= prev_cache->get_size_of_rec_offset(); - curr_rec_pos= pos; +curr_rec_pos= pos; - /* Read all flag fields of the record */ - read_flag_fields(); +/* Read all flag fields of the record */ +read_flag_fields(); - if (with_match_flag && - (Match_flag) curr_rec_pos[0] == MATCH_IMPOSSIBLE ) - { - pos= init_pos+rec_len; - goto start; - } - - if (use_emb_key) - { - /* An embedded key is taken directly from the join buffer */ - *key= pos; - len= emb_key_length; - } - else +if (with_match_flag && + (Match_flag) curr_rec_pos[0] == MATCH_IMPOSSIBLE ) +{ + pos= init_pos+rec_len; + goto start; +} + +if (use_emb_key) +{ + /* An embedded key is taken directly from the join buffer */ + *key= pos; + len= emb_key_length; +} +else +{ + /* Read key arguments from previous caches if there are any such fields */ + if (external_key_arg_fields) { - /* Read key arguments from previous caches if there are any such fields */ - if (external_key_arg_fields) - { - uchar *rec_ptr= curr_rec_pos; - uint key_arg_count= external_key_arg_fields; - CACHE_FIELD **copy_ptr= blob_ptr-key_arg_count; - for (cache= prev_cache; key_arg_count; cache= cache->prev_cache) - { - uint len= 0; + uchar *rec_ptr= curr_rec_pos; + uint key_arg_count= external_key_arg_fields; + CACHE_FIELD **copy_ptr= blob_ptr-key_arg_count; + for (cache= prev_cache; key_arg_count; cache= cache->prev_cache) + { + uint len= 0; + DBUG_ASSERT(cache); + rec_ptr= cache->get_rec_ref(rec_ptr); + while (!cache->referenced_fields) + { + cache= cache->prev_cache; DBUG_ASSERT(cache); rec_ptr= cache->get_rec_ref(rec_ptr); - while (!cache->referenced_fields) - { - cache= cache->prev_cache; - DBUG_ASSERT(cache); - rec_ptr= cache->get_rec_ref(rec_ptr); - } - while (key_arg_count && - cache->read_referenced_field(*copy_ptr, rec_ptr, &len)) - { - copy_ptr++; - --key_arg_count; - } + } + while (key_arg_count && + cache->read_referenced_field(*copy_ptr, rec_ptr, &len)) + { + copy_ptr++; + --key_arg_count; } } - - /* - Read the other key arguments from the current record. The fields for - these arguments are always first in the sequence of the record's fields. - */ - CACHE_FIELD *copy= field_descr+flag_fields; - CACHE_FIELD *copy_end= copy+local_key_arg_fields; - bool blob_in_rec_buff= blob_data_is_in_rec_buff(curr_rec_pos); - for ( ; copy < copy_end; copy++) - read_record_field(copy, blob_in_rec_buff); - - /* Build the key over the fields read into the record buffers */ - TABLE_REF *ref= &join_tab->ref; - cp_buffer_from_ref(join->thd, join_tab->table, ref); - *key= ref->key_buff; - len= ref->key_length; } + + /* + Read the other key arguments from the current record. The fields for + these arguments are always first in the sequence of the record's fields. + */ + CACHE_FIELD *copy= field_descr+flag_fields; + CACHE_FIELD *copy_end= copy+local_key_arg_fields; + bool blob_in_rec_buff= blob_data_is_in_rec_buff(curr_rec_pos); + for ( ; copy < copy_end; copy++) + read_record_field(copy, blob_in_rec_buff); + + /* Build the key over the fields read into the record buffers */ + TABLE_REF *ref= &join_tab->ref; + cp_buffer_from_ref(join->thd, join_tab->table, ref); + *key= ref->key_buff; + len= ref->key_length; +} - pos= init_pos+rec_len; +pos= init_pos+rec_len; - return len; +return len; } /* - Check the index condition of the joined table for a record from the BKA cache - - SYNOPSIS - skip_index_tuple() - range_info pointer to the record returned by MRR - - DESCRIPTION - This function is invoked from MRR implementation to check if an index - tuple matches the index condition. It is used in the case where the index - condition actually depends on both columns of the used index and columns - from previous tables. - - NOTES - Accessing columns of the previous tables requires special handling with - BKA. The idea of BKA is to collect record combinations in a buffer and - then do a batch of ref access lookups, i.e. by the time we're doing a - lookup its previous-records-combination is not in prev_table->record[0] - but somewhere in the join buffer. - We need to get it from there back into prev_table(s)->record[0] before we - can evaluate the index condition, and that's why we need this function - instead of regular IndexConditionPushdown. +Check the index condition of the joined table for a record from the BKA cache - NOTES - Possible optimization: - Before we unpack the record from a previous table - check if this table is used in the condition. - If so then unpack the record otherwise skip the unpacking. - This should be done by a special virtual method - get_partial_record_by_pos(). - - RETURN VALUE - 1 the record combination does not satisfies the index condition - 0 otherwise +SYNOPSIS + skip_index_tuple() + range_info pointer to the record returned by MRR + +DESCRIPTION + This function is invoked from MRR implementation to check if an index + tuple matches the index condition. It is used in the case where the index + condition actually depends on both columns of the used index and columns + from previous tables. + +NOTES + Accessing columns of the previous tables requires special handling with + BKA. The idea of BKA is to collect record combinations in a buffer and + then do a batch of ref access lookups, i.e. by the time we're doing a + lookup its previous-records-combination is not in prev_table->record[0] + but somewhere in the join buffer. + We need to get it from there back into prev_table(s)->record[0] before we + can evaluate the index condition, and that's why we need this function + instead of regular IndexConditionPushdown. + +NOTES + Possible optimization: + Before we unpack the record from a previous table + check if this table is used in the condition. + If so then unpack the record otherwise skip the unpacking. + This should be done by a special virtual method + get_partial_record_by_pos(). + +RETURN VALUE + 1 the record combination does not satisfies the index condition + 0 otherwise */ bool JOIN_CACHE_BKA::skip_index_tuple(range_id_t range_info) { - DBUG_ENTER("JOIN_CACHE_BKA::skip_index_tuple"); - get_record_by_pos((uchar*)range_info); - DBUG_RETURN(!join_tab->cache_idx_cond->val_int()); +DBUG_ENTER("JOIN_CACHE_BKA::skip_index_tuple"); +get_record_by_pos((uchar*)range_info); +DBUG_RETURN(!join_tab->cache_idx_cond->val_int()); } /* - Initialize retrieval of range sequence for the BKAH join algorithm - - SYNOPSIS - bkah_range_seq_init() - init_params pointer to the BKAH join cache object - n_ranges the number of ranges obtained - flags combination of MRR flags - - DESCRIPTION - The function interprets init_param as a pointer to a JOIN_CACHE_BKAH - object. The function prepares for an iteration over distinct join keys - built over the records from the cache join buffer. - - NOTE - This function are used only as a callback function. - - RETURN VALUE - init_param value that is to be used as a parameter of - bkah_range_seq_next() +Initialize retrieval of range sequence for the BKAH join algorithm + +SYNOPSIS + bkah_range_seq_init() + init_params pointer to the BKAH join cache object + n_ranges the number of ranges obtained + flags combination of MRR flags + +DESCRIPTION + The function interprets init_param as a pointer to a JOIN_CACHE_BKAH + object. The function prepares for an iteration over distinct join keys + built over the records from the cache join buffer. + +NOTE + This function are used only as a callback function. + +RETURN VALUE + init_param value that is to be used as a parameter of + bkah_range_seq_next() */ static range_seq_t bkah_range_seq_init(void *init_param, uint n_ranges, uint flags) { - DBUG_ENTER("bkah_range_seq_init"); - JOIN_CACHE_BKAH *cache= (JOIN_CACHE_BKAH *) init_param; - cache->reset(0); - DBUG_RETURN((range_seq_t) init_param); +DBUG_ENTER("bkah_range_seq_init"); +JOIN_CACHE_BKAH *cache= (JOIN_CACHE_BKAH *) init_param; +cache->reset(0); +DBUG_RETURN((range_seq_t) init_param); } /* - Get the next range/key over records from the join buffer of a BKAH cache - - SYNOPSIS - bkah_range_seq_next() - seq value returned by bkah_range_seq_init() - range OUT reference to the next range +Get the next range/key over records from the join buffer of a BKAH cache - DESCRIPTION - The function interprets seq as a pointer to a JOIN_CACHE_BKAH - object. The function returns a pointer to the range descriptor - for the next unique key built over records from the join buffer. - - NOTE - This function are used only as a callback function. - - RETURN VALUE - FALSE ok, the range structure filled with info about the next range/key - TRUE no more ranges +SYNOPSIS + bkah_range_seq_next() + seq value returned by bkah_range_seq_init() + range OUT reference to the next range + +DESCRIPTION + The function interprets seq as a pointer to a JOIN_CACHE_BKAH + object. The function returns a pointer to the range descriptor + for the next unique key built over records from the join buffer. + +NOTE + This function are used only as a callback function. + +RETURN VALUE + FALSE ok, the range structure filled with info about the next range/key + TRUE no more ranges */ static bool bkah_range_seq_next(range_seq_t rseq, KEY_MULTI_RANGE *range) { - DBUG_ENTER("bkah_range_seq_next"); - JOIN_CACHE_BKAH *cache= (JOIN_CACHE_BKAH *) rseq; - TABLE_REF *ref= &cache->join_tab->ref; - key_range *start_key= &range->start_key; - if ((start_key->length= cache->get_next_key((uchar **) &start_key->key))) - { - start_key->keypart_map= (1 << ref->key_parts) - 1; - start_key->flag= HA_READ_KEY_EXACT; - range->end_key= *start_key; - range->end_key.flag= HA_READ_AFTER_KEY; - range->ptr= (char *) cache->get_curr_key_chain(); - range->range_flag= EQ_RANGE; - DBUG_RETURN(0); - } - DBUG_RETURN(1); +DBUG_ENTER("bkah_range_seq_next"); +JOIN_CACHE_BKAH *cache= (JOIN_CACHE_BKAH *) rseq; +TABLE_REF *ref= &cache->join_tab->ref; +key_range *start_key= &range->start_key; +if ((start_key->length= cache->get_next_key((uchar **) &start_key->key))) +{ + start_key->keypart_map= (1 << ref->key_parts) - 1; + start_key->flag= HA_READ_KEY_EXACT; + range->end_key= *start_key; + range->end_key.flag= HA_READ_AFTER_KEY; + range->ptr= (char *) cache->get_curr_key_chain(); + range->range_flag= EQ_RANGE; + DBUG_RETURN(0); +} +DBUG_RETURN(1); } /* - Check whether range_info orders to skip the next record from BKAH join buffer +Check whether range_info orders to skip the next record from BKAH join buffer - SYNOPSIS - bkah_range_seq_skip_record() - seq value returned by bkah_range_seq_init() - range_info information about the next range/key returned by MRR - rowid [NOT USED] rowid of the record to be checked (not used) - - DESCRIPTION - The function interprets seq as a pointer to a JOIN_CACHE_BKAH - object. The function returns TRUE if the record with this range_info - is to be filtered out from the stream of records returned by - multi_range_read_next(). - - NOTE - This function are used only as a callback function. - - RETURN VALUE - 1 record with this range_info is to be filtered out from the stream - of records returned by multi_range_read_next() - 0 the record is to be left in the stream +SYNOPSIS + bkah_range_seq_skip_record() + seq value returned by bkah_range_seq_init() + range_info information about the next range/key returned by MRR + rowid [NOT USED] rowid of the record to be checked (not used) + +DESCRIPTION + The function interprets seq as a pointer to a JOIN_CACHE_BKAH + object. The function returns TRUE if the record with this range_info + is to be filtered out from the stream of records returned by + multi_range_read_next(). + +NOTE + This function are used only as a callback function. + +RETURN VALUE + 1 record with this range_info is to be filtered out from the stream + of records returned by multi_range_read_next() + 0 the record is to be left in the stream */ static bool bkah_range_seq_skip_record(range_seq_t rseq, range_id_t range_info, - uchar *rowid) + uchar *rowid) { - DBUG_ENTER("bkah_range_seq_skip_record"); - JOIN_CACHE_BKAH *cache= (JOIN_CACHE_BKAH *) rseq; - bool res= cache->check_all_match_flags_for_key((uchar *) range_info); - DBUG_RETURN(res); +DBUG_ENTER("bkah_range_seq_skip_record"); +JOIN_CACHE_BKAH *cache= (JOIN_CACHE_BKAH *) rseq; +bool res= cache->check_all_match_flags_for_key((uchar *) range_info); +DBUG_RETURN(res); } - + /* - Check if the record combination from BKAH cache matches the index condition +Check if the record combination from BKAH cache matches the index condition - SYNOPSIS - bkah_skip_index_tuple() - rseq value returned by bka_range_seq_init() - range_info record chain for the next range/key returned by MRR - - DESCRIPTION - This is wrapper for JOIN_CACHE_BKA_UNIQUE::skip_index_tuple method, - see comments there. +SYNOPSIS + bkah_skip_index_tuple() + rseq value returned by bka_range_seq_init() + range_info record chain for the next range/key returned by MRR + +DESCRIPTION + This is wrapper for JOIN_CACHE_BKA_UNIQUE::skip_index_tuple method, + see comments there. - NOTE - This function is used as a RANGE_SEQ_IF::skip_index_tuple callback. - - RETURN VALUE - 0 some records from the chain satisfy the index condition - 1 otherwise +NOTE + This function is used as a RANGE_SEQ_IF::skip_index_tuple callback. + +RETURN VALUE + 0 some records from the chain satisfy the index condition + 1 otherwise */ static bool bkah_skip_index_tuple(range_seq_t rseq, range_id_t range_info) { - DBUG_ENTER("bka_unique_skip_index_tuple"); - JOIN_CACHE_BKAH *cache= (JOIN_CACHE_BKAH *) rseq; - THD *thd= cache->thd(); - bool res; - status_var_increment(thd->status_var.ha_icp_attempts); - if (!(res= cache->skip_index_tuple(range_info))) - status_var_increment(thd->status_var.ha_icp_match); - DBUG_RETURN(res); +DBUG_ENTER("bka_unique_skip_index_tuple"); +JOIN_CACHE_BKAH *cache= (JOIN_CACHE_BKAH *) rseq; +THD *thd= cache->thd(); +bool res; +status_var_increment(thd->status_var.ha_icp_attempts); +if (!(res= cache->skip_index_tuple(range_info))) + status_var_increment(thd->status_var.ha_icp_match); +DBUG_RETURN(res); } /* - Prepare to read record from BKAH cache matching the current joined record - - SYNOPSIS - prepare_look_for_matches() - skip_last <-> ignore the last record in the buffer (always unused here) - - DESCRIPTION - The function prepares to iterate over records in the join cache buffer - matching the record loaded into the record buffer for join_tab when - performing join operation by BKAH join algorithm. With BKAH algorithm, if - association labels are used, then record loaded into the record buffer - for join_tab always has a direct reference to the chain of the mathing - records from the join buffer. If association labels are not used then - then the chain of the matching records is obtained by the call of the - get_key_chain_by_join_key function. - - RETURN VALUE - TRUE there are no records in the buffer to iterate over - FALSE otherwise +Prepare to read record from BKAH cache matching the current joined record + +SYNOPSIS + prepare_look_for_matches() + skip_last <-> ignore the last record in the buffer (always unused here) + +DESCRIPTION + The function prepares to iterate over records in the join cache buffer + matching the record loaded into the record buffer for join_tab when + performing join operation by BKAH join algorithm. With BKAH algorithm, if + association labels are used, then record loaded into the record buffer + for join_tab always has a direct reference to the chain of the mathing + records from the join buffer. If association labels are not used then + then the chain of the matching records is obtained by the call of the + get_key_chain_by_join_key function. + +RETURN VALUE + TRUE there are no records in the buffer to iterate over + FALSE otherwise */ - + bool JOIN_CACHE_BKAH::prepare_look_for_matches(bool skip_last) { - last_matching_rec_ref_ptr= next_matching_rec_ref_ptr= 0; - if (no_association && - !(curr_matching_chain= get_matching_chain_by_join_key())) +last_matching_rec_ref_ptr= next_matching_rec_ref_ptr= 0; +if (no_association && + !(curr_matching_chain= get_matching_chain_by_join_key())) //psergey: added '!' return 1; last_matching_rec_ref_ptr= get_next_rec_ref(curr_matching_chain); return 0; diff --git a/sql/sql_join_cache.h b/sql/sql_join_cache.h index 6953f6881ee..9dae7fb24e0 100644 --- a/sql/sql_join_cache.h +++ b/sql/sql_join_cache.h @@ -63,6 +63,7 @@ typedef struct st_cache_field { class JOIN_TAB_SCAN; +struct st_explain_bka_type; /* JOIN_CACHE is the base class to support the implementations of @@ -657,7 +658,7 @@ public: enum_nested_loop_state join_records(bool skip_last); /* Add a comment on the join algorithm employed by the join cache */ - virtual void print_explain_comment(String *str); + virtual void save_explain_data(struct st_explain_bka_type *explain); THD *thd(); @@ -1335,7 +1336,7 @@ public: /* Check index condition of the joined table for a record from BKA cache */ bool skip_index_tuple(range_id_t range_info); - void print_explain_comment(String *str); + void save_explain_data(struct st_explain_bka_type *explain); }; @@ -1426,5 +1427,5 @@ public: /* Check index condition of the joined table for a record from BKAH cache */ bool skip_index_tuple(range_id_t range_info); - void print_explain_comment(String *str); + void save_explain_data(struct st_explain_bka_type *explain); }; diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 8dab2191224..850dbc813bb 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -448,6 +448,8 @@ void lex_start(THD *thd) DBUG_ENTER("lex_start"); lex->thd= lex->unit.thd= thd; + + DBUG_ASSERT(!lex->explain); lex->context_stack.empty(); lex->unit.init_query(); @@ -460,6 +462,9 @@ void lex_start(THD *thd) lex->set_var_list.empty(); lex->param_list.empty(); lex->view_list.empty(); + lex->with_persistent_for_clause= FALSE; + lex->column_list= NULL; + lex->index_list= NULL; lex->prepared_stmt_params.empty(); lex->auxiliary_table_list.empty(); lex->unit.next= lex->unit.master= @@ -506,6 +511,8 @@ void lex_start(THD *thd) lex->expr_allows_subselect= TRUE; lex->use_only_table_context= FALSE; lex->parse_vcol_expr= FALSE; + lex->check_exists= FALSE; + lex->verbose= 0; lex->name.str= 0; lex->name.length= 0; @@ -1546,7 +1553,14 @@ int lex_one_token(void *arg, THD *thd) version= (ulong) my_strtoll10(lip->get_ptr(), &end_ptr, &error); - if (version <= MYSQL_VERSION_ID) + /* + MySQL-5.7 has new features and might have new SQL syntax that + MariaDB-10.0 does not understand. Ignore all versioned comments + with MySQL versions in the range 50700-999999, but + do not ignore MariaDB specific comments for the same versions. + */ + if (version <= MYSQL_VERSION_ID && + (version < 50700 || version > 99999 || maria_comment_syntax)) { /* Accept 'M' 'm' 'm' 'd' 'd' */ lip->yySkipn(length); @@ -1868,6 +1882,7 @@ void st_select_lex::init_query() ref_pointer_array= 0; ref_pointer_array_size= 0; select_n_where_fields= 0; + select_n_reserved= 0; select_n_having_items= 0; n_sum_items= 0; n_child_sum_items= 0; @@ -1881,6 +1896,7 @@ void st_select_lex::init_query() nest_level= 0; link_next= 0; is_prep_leaf_list_saved= FALSE; + have_merged_subqueries= FALSE; bzero((char*) expr_cache_may_be_used, sizeof(expr_cache_may_be_used)); m_non_agg_field_used= false; m_agg_func_used= false; @@ -2316,6 +2332,7 @@ bool st_select_lex::setup_ref_array(THD *thd, uint order_group_num) const uint n_elems= (n_sum_items + n_child_sum_items + item_list.elements + + select_n_reserved + select_n_having_items + select_n_where_fields + order_group_num) * 5; @@ -2561,14 +2578,15 @@ void Query_tables_list::destroy_query_tables_list() */ LEX::LEX() - :result(0), option_type(OPT_DEFAULT), is_lex_started(0), + : explain(NULL), + result(0), option_type(OPT_DEFAULT), is_lex_started(0), limit_rows_examined_cnt(ULONGLONG_MAX) { my_init_dynamic_array2(&plugins, sizeof(plugin_ref), plugins_static_buffer, INITIAL_LEX_PLUGIN_LIST_SIZE, - INITIAL_LEX_PLUGIN_LIST_SIZE); + INITIAL_LEX_PLUGIN_LIST_SIZE, 0); reset_query_tables_list(TRUE); mi.init(); } @@ -2891,7 +2909,7 @@ void st_select_lex_unit::set_limit(st_select_lex *sl) val= fix_fields_successful ? item->val_uint() : 0; } else - val= ULL(0); + val= 0; offset_limit_cnt= (ha_rows)val; #ifndef BIG_TABLES @@ -3498,7 +3516,7 @@ bool st_select_lex::optimize_unflattened_subqueries(bool const_only) if (options & SELECT_DESCRIBE) { /* Optimize the subquery in the context of EXPLAIN. */ - sl->set_explain_type(); + sl->set_explain_type(FALSE); sl->options|= SELECT_DESCRIBE; inner_join->select_options|= SELECT_DESCRIBE; } @@ -3507,6 +3525,18 @@ bool st_select_lex::optimize_unflattened_subqueries(bool const_only) is_correlated_unit|= sl->is_correlated; inner_join->select_options= save_options; un->thd->lex->current_select= save_select; + + Explain_query *eq; + if ((eq= inner_join->thd->lex->explain)) + { + Explain_select *expl_sel; + if ((expl_sel= eq->get_select(inner_join->select_lex->select_number))) + { + sl->set_explain_type(TRUE); + expl_sel->select_type= sl->type; + } + } + if (empty_union_result) { /* @@ -3946,9 +3976,12 @@ void st_select_lex::update_correlated_cache() /** Set the EXPLAIN type for this subquery. + + @param on_the_fly TRUE<=> We're running a SHOW EXPLAIN command, so we must + not change any variables */ -void st_select_lex::set_explain_type() +void st_select_lex::set_explain_type(bool on_the_fly) { bool is_primary= FALSE; if (next_select()) @@ -3970,6 +4003,9 @@ void st_select_lex::set_explain_type() } } + if (on_the_fly && !is_primary && have_merged_subqueries) + is_primary= TRUE; + SELECT_LEX *first= master_unit()->first_select(); /* drop UNCACHEABLE_EXPLAIN, because it is for internal usage only */ uint8 is_uncacheable= (uncacheable & ~UNCACHEABLE_EXPLAIN); @@ -4022,10 +4058,15 @@ void st_select_lex::set_explain_type() else { type= is_uncacheable ? "UNCACHEABLE UNION": "UNION"; + if (this == master_unit()->fake_select_lex) + type= "UNION RESULT"; + } } } - options|= SELECT_DESCRIBE; + + if (!on_the_fly) + options|= SELECT_DESCRIBE; } @@ -4066,7 +4107,8 @@ void SELECT_LEX::increase_derived_records(ha_rows records) void SELECT_LEX::mark_const_derived(bool empty) { TABLE_LIST *derived= master_unit()->derived; - if (!join->thd->lex->describe && derived) + /* join == NULL in DELETE ... RETURNING */ + if (!(join && join->thd->lex->describe) && derived) { if (!empty) increase_derived_records(1); @@ -4165,6 +4207,87 @@ bool st_select_lex::is_merged_child_of(st_select_lex *ancestor) return all_merged; } +/* + This is used by SHOW EXPLAIN. It assuses query plan has been already + collected into QPF structures and we only need to print it out. +*/ + +int LEX::print_explain(select_result_sink *output, uint8 explain_flags, + bool *printed_anything) +{ + int res; + if (explain && explain->have_query_plan()) + { + res= explain->print_explain(output, explain_flags); + *printed_anything= true; + } + else + { + res= 0; + *printed_anything= false; + } + return res; +} + + +/* + Save explain structures of a UNION. The only variable member is whether the + union has "Using filesort". + + There is also save_union_explain_part2() function, which is called before we read + UNION's output. + + The reason for it is examples like this: + + SELECT col1 FROM t1 UNION SELECT col2 FROM t2 ORDER BY (select ... from t3 ...) + + Here, the (select ... from t3 ...) subquery must be a child of UNION's + st_select_lex. However, it is not connected as child until a very late + stage in execution. +*/ + +int st_select_lex_unit::save_union_explain(Explain_query *output) +{ + SELECT_LEX *first= first_select(); + Explain_union *eu= new (output->mem_root) Explain_union; + + for (SELECT_LEX *sl= first; sl; sl= sl->next_select()) + eu->add_select(sl->select_number); + + eu->fake_select_type= "UNION RESULT"; + eu->using_filesort= test(global_parameters->order_list.first); + + // Save the UNION node + output->add_node(eu); + + if (eu->get_select_id() == 1) + output->query_plan_ready(); + + return 0; +} + + +/* + @see st_select_lex_unit::save_union_explain +*/ + +int st_select_lex_unit::save_union_explain_part2(Explain_query *output) +{ + Explain_union *eu= output->get_union(first_select()->select_number); + if (fake_select_lex) + { + for (SELECT_LEX_UNIT *unit= fake_select_lex->first_inner_unit(); + unit; unit= unit->next_unit()) + { + if (!(unit->item && unit->item->eliminated)) + { + eu->add_child(unit->first_select()->select_number); + } + } + } + return 0; +} + /** A routine used by the parser to decide whether we are specifying a full @@ -4331,6 +4454,3 @@ void binlog_unsafe_map_init() } #endif -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class Mem_root_array<ORDER*, true>; -#endif diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 458e990f043..944435d0acf 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -157,7 +157,8 @@ enum enum_sql_command { SQLCOM_FLUSH, SQLCOM_KILL, SQLCOM_ANALYZE, SQLCOM_ROLLBACK, SQLCOM_ROLLBACK_TO_SAVEPOINT, SQLCOM_COMMIT, SQLCOM_SAVEPOINT, SQLCOM_RELEASE_SAVEPOINT, - SQLCOM_SLAVE_START, SQLCOM_SLAVE_STOP, + SQLCOM_SLAVE_START, SQLCOM_SLAVE_ALL_START, + SQLCOM_SLAVE_STOP, SQLCOM_SLAVE_ALL_STOP, SQLCOM_BEGIN, SQLCOM_CHANGE_MASTER, SQLCOM_RENAME_TABLE, SQLCOM_RESET, SQLCOM_PURGE, SQLCOM_PURGE_BEFORE, SQLCOM_SHOW_BINLOGS, @@ -194,6 +195,8 @@ enum enum_sql_command { SQLCOM_SHOW_RELAYLOG_EVENTS, SQLCOM_SHOW_USER_STATS, SQLCOM_SHOW_TABLE_STATS, SQLCOM_SHOW_INDEX_STATS, SQLCOM_SHOW_CLIENT_STATS, + SQLCOM_SHOW_EXPLAIN, SQLCOM_SHUTDOWN, + SQLCOM_CREATE_ROLE, SQLCOM_DROP_ROLE, SQLCOM_GRANT_ROLE, SQLCOM_REVOKE_ROLE, /* When a command is added here, be sure it's also added in mysqld.cc @@ -287,6 +290,9 @@ struct LEX_MASTER_INFO char *host, *user, *password, *log_file_name; char *ssl_key, *ssl_cert, *ssl_ca, *ssl_capath, *ssl_cipher; char *relay_log_name; + LEX_STRING connection_name; + /* Value in START SLAVE UNTIL master_gtid_pos=xxx */ + LEX_STRING gtid_pos_str; ulonglong pos; ulong relay_log_pos; ulong server_id; @@ -298,12 +304,15 @@ struct LEX_MASTER_INFO */ enum {LEX_MI_UNCHANGED, LEX_MI_DISABLE, LEX_MI_ENABLE} ssl, ssl_verify_server_cert, heartbeat_opt, repl_ignore_server_ids_opt; + enum { + LEX_GTID_UNCHANGED, LEX_GTID_NO, LEX_GTID_CURRENT_POS, LEX_GTID_SLAVE_POS + } use_gtid_opt; void init() { bzero(this, sizeof(*this)); my_init_dynamic_array(&repl_ignore_server_ids, - sizeof(::server_id), 0, 16); + sizeof(::server_id), 0, 16, MYF(0)); } void reset() { @@ -314,6 +323,9 @@ struct LEX_MASTER_INFO heartbeat_period= 0; ssl= ssl_verify_server_cert= heartbeat_opt= repl_ignore_server_ids_opt= LEX_MI_UNCHANGED; + gtid_pos_str.length= 0; + gtid_pos_str.str= NULL; + use_gtid_opt= LEX_GTID_UNCHANGED; } }; @@ -356,6 +368,8 @@ typedef uchar index_clause_map; #define INDEX_HINT_MASK_ALL (INDEX_HINT_MASK_JOIN | INDEX_HINT_MASK_GROUP | \ INDEX_HINT_MASK_ORDER) +class select_result_sink; + /* Single element of an USE/FORCE/IGNORE INDEX list specified as a SQL hint */ class Index_hint : public Sql_alloc { @@ -605,7 +619,12 @@ class select_result; class JOIN; class select_union; class Procedure; +class Explain_query; +void delete_explain_query(LEX *lex); +void create_explain_query(LEX *lex, MEM_ROOT *mem_root); +void create_explain_query_if_not_exists(LEX *lex, MEM_ROOT *mem_root); +bool print_explain_query(LEX *lex, THD *thd, String *str); class st_select_lex_unit: public st_select_lex_node { protected: @@ -716,6 +735,9 @@ public: friend int subselect_union_engine::exec(); List<Item> *get_unit_column_types(); + + int save_union_explain(Explain_query *output); + int save_union_explain_part2(Explain_query *output); }; typedef class st_select_lex_unit SELECT_LEX_UNIT; @@ -776,6 +798,12 @@ public: those converted to jtbm nests. The list is emptied when conversion is done. */ List<Item_in_subselect> sj_subselects; + + /* + Needed to correctly generate 'PRIMARY' or 'SIMPLE' for select_type column + of EXPLAIN + */ + bool have_merged_subqueries; List<TABLE_LIST> leaf_tables; List<TABLE_LIST> leaf_tables_exec; @@ -808,6 +836,8 @@ public: and all inner subselects. */ uint select_n_where_fields; + /* reserved for exists 2 in */ + uint select_n_reserved; enum_parsing_place parsing_place; /* where we are parsing expression */ bool with_sum_func; /* sum function indicator */ @@ -1003,9 +1033,13 @@ public: void clear_index_hints(void) { index_hints= NULL; } bool is_part_of_union() { return master_unit()->is_union(); } + bool is_top_level_node() + { + return (select_number == 1) && !is_part_of_union(); + } bool optimize_unflattened_subqueries(bool const_only); /* Set the EXPLAIN type for this subquery. */ - void set_explain_type(); + void set_explain_type(bool on_the_fly); bool handle_derived(LEX *lex, uint phases); void append_table_to_list(TABLE_LIST *TABLE_LIST::*link, TABLE_LIST *table); bool get_free_table_map(table_map *map, uint *tablenr); @@ -1029,6 +1063,7 @@ public: bool save_leaf_tables(THD *thd); bool save_prep_leaf_tables(THD *thd); + bool is_merged_child_of(st_select_lex *ancestor); /* @@ -2335,6 +2370,89 @@ protected: LEX *m_lex; }; + +class Delete_plan; +class SQL_SELECT; + +class Explain_query; +class Explain_update; + +/* + Query plan of a single-table UPDATE. + (This is actually a plan for single-table DELETE also) +*/ + +class Update_plan +{ +protected: + bool impossible_where; + bool no_partitions; +public: + /* + When single-table UPDATE updates a VIEW, that VIEW's select is still + listed as the first child. When we print EXPLAIN, it looks like a + subquery. + In order to get rid of it, updating_a_view=TRUE means that first child + select should not be shown when printing EXPLAIN. + */ + bool updating_a_view; + + /* Allocate things there */ + MEM_ROOT *mem_root; + + TABLE *table; + SQL_SELECT *select; + uint index; + ha_rows scanned_rows; + /* + Top-level select_lex. Most of its fields are not used, we need it only to + get to the subqueries. + */ + SELECT_LEX *select_lex; + + key_map possible_keys; + bool using_filesort; + bool using_io_buffer; + + /* Set this plan to be a plan to do nothing because of impossible WHERE */ + void set_impossible_where() { impossible_where= true; } + void set_no_partitions() { no_partitions= true; } + + void save_explain_data(Explain_query *query); + void save_explain_data_intern(Explain_query *query, Explain_update *eu); + virtual ~Update_plan() {} + + Update_plan(MEM_ROOT *mem_root_arg) : + impossible_where(false), no_partitions(false), + mem_root(mem_root_arg), + using_filesort(false), using_io_buffer(false) + {} +}; + + +/* Query plan of a single-table DELETE */ +class Delete_plan : public Update_plan +{ + bool deleting_all_rows; +public: + + /* Construction functions */ + Delete_plan(MEM_ROOT *mem_root_arg) : + Update_plan(mem_root_arg), + deleting_all_rows(false) + {} + + /* Set this query plan to be a plan to make a call to h->delete_all_rows() */ + void set_delete_all_rows(ha_rows rows_arg) + { + deleting_all_rows= true; + scanned_rows= rows_arg; + } + + void save_explain_data(Explain_query *query); +}; + + /* The state of the lex parsing. This is saved in the THD struct */ struct LEX: public Query_tables_list @@ -2345,6 +2463,9 @@ struct LEX: public Query_tables_list SELECT_LEX *current_select; /* list of all SELECT_LEX */ SELECT_LEX *all_selects_list; + + /* Query Plan Footprint of a currently running select */ + Explain_query *explain; char *length,*dec,*change; LEX_STRING name; @@ -2352,7 +2473,7 @@ struct LEX: public Query_tables_list char *backup_dir; /* For RESTORE/BACKUP */ char* to_log; /* For PURGE MASTER LOGS TO */ char* x509_subject,*x509_issuer,*ssl_cipher; - String *wild; + String *wild; /* Wildcard in SHOW {something} LIKE 'wild'*/ sql_exchange *exchange; select_result *result; Item *default_value, *on_update_value; @@ -2397,6 +2518,8 @@ struct LEX: public Query_tables_list List<Item_func_set_user_var> set_var_list; // in-query assignment list List<Item_param> param_list; List<LEX_STRING> view_list; // view list (list of field names in view) + List<LEX_STRING> *column_list; // list of column names (in ANALYZE) + List<LEX_STRING> *index_list; // list of index names (in ANALYZE) /* A stack of name resolution contexts for the query. This stack is used at parse time to set local name resolution contexts for various parts @@ -2422,6 +2545,7 @@ struct LEX: public Query_tables_list KEY_CREATE_INFO key_create_info; LEX_MASTER_INFO mi; // used by CHANGE MASTER LEX_SERVER_OPTIONS server_options; + LEX_STRING relay_log_connection_name; USER_RESOURCES mqh; LEX_RESET_SLAVE reset_slave_info; ulong type; @@ -2464,6 +2588,8 @@ struct LEX: public Query_tables_list union { enum ha_rkey_function ha_rkey_mode; enum xa_option_words xa_opt; + bool with_admin_option; // GRANT role + bool with_persistent_for_clause; // uses PERSISTENT FOR clause (in ANALYZE) }; enum enum_var_type option_type; enum enum_view_create_mode create_view_mode; @@ -2494,7 +2620,8 @@ struct LEX: public Query_tables_list uint16 create_view_algorithm; uint8 create_view_check; uint8 context_analysis_only; - bool drop_if_exists, drop_temporary, local_file, one_shot_set; + bool drop_temporary, local_file, one_shot_set; + bool check_exists; bool autocommit; bool verbose, no_write_to_binlog; @@ -2757,6 +2884,9 @@ struct LEX: public Query_tables_list } return FALSE; } + + int print_explain(select_result_sink *output, uint8 explain_flags, + bool *printed_anything); }; diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 493a38ce0f5..0bd01d4ea2e 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -274,7 +274,6 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, for (field=table->field; *field ; field++) fields_vars.push_back(new Item_field(*field)); bitmap_set_all(table->write_set); - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; /* Let us also prepare SET clause, altough it is probably empty in this case. @@ -290,21 +289,9 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, setup_fields(thd, 0, set_fields, MARK_COLUMNS_WRITE, 0, 0) || check_that_all_fields_are_given_values(thd, table, table_list)) DBUG_RETURN(TRUE); - /* - Check whenever TIMESTAMP field with auto-set feature specified - explicitly. - */ - if (table->timestamp_field) - { - if (bitmap_is_set(table->write_set, - table->timestamp_field->field_index)) - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - else - { - bitmap_set_bit(table->write_set, - table->timestamp_field->field_index); - } - } + /* Add all fields with default functions to table->write_set. */ + if (table->default_field) + table->mark_default_fields_for_write(); /* Fix the expressions in SET clause */ if (setup_fields(thd, 0, set_values, MARK_COLUMNS_READ, 0, 0)) DBUG_RETURN(TRUE); @@ -376,11 +363,11 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, MY_RETURN_REAL_PATH); } - if (thd->slave_thread) + if (thd->rgi_slave) { #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) - if (strncmp(active_mi->rli.slave_patternload_file, name, - active_mi->rli.slave_patternload_file_size)) + if (strncmp(thd->rgi_slave->rli->slave_patternload_file, name, + thd->rgi_slave->rli->slave_patternload_file_size)) { /* LOAD DATA INFILE in the slave SQL Thread can only read from @@ -850,8 +837,12 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, ER_WARN_TOO_FEW_RECORDS, ER(ER_WARN_TOO_FEW_RECORDS), thd->warning_info->current_row_for_warning()); + /* + Timestamp fields that are NOT NULL are autoupdated if there is no + corresponding value in the data file. + */ if (!field->maybe_null() && field->type() == FIELD_TYPE_TIMESTAMP) - ((Field_timestamp*) field)->set_time(); + field->set_time(); } else { @@ -866,6 +857,8 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, if ((pos+=length) > read_info.row_end) pos= read_info.row_end; /* Fills rest with space */ } + /* Do not auto-update this field. */ + field->set_has_explicit_value(); } if (pos != read_info.row_end) { @@ -877,10 +870,10 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, } if (thd->killed || - fill_record_n_invoke_before_triggers(thd, set_fields, set_values, + fill_record_n_invoke_before_triggers(thd, table, set_fields, set_values, ignore_check_option_errors, - table->triggers, - TRG_EVENT_INSERT)) + TRG_EVENT_INSERT) || + (table->default_field && table->update_default_fields())) DBUG_RETURN(1); switch (table_list->view_check_option(thd, @@ -997,12 +990,18 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, field->set_null(); if (!field->maybe_null()) { + /* + Timestamp fields that are NOT NULL are autoupdated if there is no + corresponding value in the data file. + */ if (field->type() == MYSQL_TYPE_TIMESTAMP) - ((Field_timestamp*) field)->set_time(); + field->set_time(); else if (field != table->next_number_field) field->set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_NULL_TO_NOTNULL, 1); } + /* Do not auto-update this field. */ + field->set_has_explicit_value(); } else if (item->type() == Item::STRING_ITEM) { @@ -1026,6 +1025,7 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, if (field == table->next_number_field) table->auto_increment_field_not_null= TRUE; field->store((char*) pos, length, read_info.read_charset); + field->set_has_explicit_value(); } else if (item->type() == Item::STRING_ITEM) { @@ -1067,7 +1067,8 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, DBUG_RETURN(1); } if (!field->maybe_null() && field->type() == FIELD_TYPE_TIMESTAMP) - ((Field_timestamp*) field)->set_time(); + field->set_time(); + field->set_has_explicit_value(); /* TODO: We probably should not throw warning for each field. But how about intention to always have the same number @@ -1094,10 +1095,10 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, } if (thd->killed || - fill_record_n_invoke_before_triggers(thd, set_fields, set_values, + fill_record_n_invoke_before_triggers(thd, table, set_fields, set_values, ignore_check_option_errors, - table->triggers, - TRG_EVENT_INSERT)) + TRG_EVENT_INSERT) || + (table->default_field && table->update_default_fields())) DBUG_RETURN(1); switch (table_list->view_check_option(thd, @@ -1207,11 +1208,13 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, if (!field->maybe_null()) { if (field->type() == FIELD_TYPE_TIMESTAMP) - ((Field_timestamp *) field)->set_time(); + field->set_time(); else if (field != table->next_number_field) field->set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_NULL_TO_NOTNULL, 1); } + /* Do not auto-update this field. */ + field->set_has_explicit_value(); } else ((Item_user_var_as_out_param *) item)->set_null_value(cs); @@ -1226,6 +1229,7 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, if (field == table->next_number_field) table->auto_increment_field_not_null= TRUE; field->store((char *) tag->value.ptr(), tag->value.length(), cs); + field->set_has_explicit_value(); } else ((Item_user_var_as_out_param *) item)->set_value( @@ -1270,10 +1274,10 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, } if (thd->killed || - fill_record_n_invoke_before_triggers(thd, set_fields, set_values, + fill_record_n_invoke_before_triggers(thd, table, set_fields, set_values, ignore_check_option_errors, - table->triggers, - TRG_EVENT_INSERT)) + TRG_EVENT_INSERT) || + (table->default_field && table->update_default_fields())) DBUG_RETURN(1); switch (table_list->view_check_option(thd, @@ -1368,7 +1372,7 @@ READ_INFO::READ_INFO(File file_par, uint tot_length, CHARSET_INFO *cs, set_if_bigger(length,line_start.length()); stack=stack_pos=(int*) sql_alloc(sizeof(int)*length); - if (!(buffer=(uchar*) my_malloc(buff_length+1,MYF(0)))) + if (!(buffer=(uchar*) my_malloc(buff_length+1,MYF(MY_THREAD_SPECIFIC)))) error=1; /* purecov: inspected */ else { @@ -1376,7 +1380,7 @@ READ_INFO::READ_INFO(File file_par, uint tot_length, CHARSET_INFO *cs, if (init_io_cache(&cache,(get_it_from_net) ? -1 : file, 0, (get_it_from_net) ? READ_NET : (is_fifo ? READ_FIFO : READ_CACHE),0L,1, - MYF(MY_WME))) + MYF(MY_WME | MY_THREAD_SPECIFIC))) { my_free(buffer); /* purecov: inspected */ buffer= NULL; @@ -1602,7 +1606,7 @@ int READ_INFO::read_field() ** We come here if buffer is too small. Enlarge it and continue */ if (!(new_buffer=(uchar*) my_realloc((char*) buffer,buff_length+1+IO_SIZE, - MYF(MY_WME)))) + MYF(MY_WME | MY_THREAD_SPECIFIC)))) return (error=1); to=new_buffer + (to-buffer); buffer=new_buffer; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 61c822f0d69..02723647717 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -82,6 +82,7 @@ #include <myisam.h> #include <my_dir.h> #include "rpl_handler.h" +#include "rpl_mi.h" #include "sp_head.h" #include "sp.h" @@ -118,7 +119,7 @@ "FUNCTION" : "PROCEDURE") static bool execute_sqlcom_select(THD *thd, TABLE_LIST *all_tables); -static void sql_kill(THD *thd, ulong id, killed_state state); +static void sql_kill(THD *thd, longlong id, killed_state state, killed_type type); static void sql_kill_user(THD *thd, LEX_USER *user, killed_state state); static bool execute_show_status(THD *, TABLE_LIST *); static bool execute_rename_table(THD *, TABLE_LIST *, TABLE_LIST *); @@ -169,8 +170,8 @@ const char *xa_state_names[]={ */ inline bool all_tables_not_ok(THD *thd, TABLE_LIST *tables) { - return rpl_filter->is_on() && tables && !thd->spcont && - !rpl_filter->tables_ok(thd->db, tables); + return thd->rpl_filter->is_on() && tables && !thd->spcont && + !thd->rpl_filter->tables_ok(thd->db, tables); } #endif @@ -268,12 +269,14 @@ void init_update_queries(void) CF_CAN_GENERATE_ROW_EVENTS; sql_command_flags[SQLCOM_CREATE_INDEX]= CF_CHANGES_DATA | CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_ALTER_TABLE]= CF_CHANGES_DATA | CF_WRITE_LOGS_COMMAND | - CF_AUTO_COMMIT_TRANS | CF_REPORT_PROGRESS ; + CF_AUTO_COMMIT_TRANS | CF_REPORT_PROGRESS | + CF_INSERTS_DATA; sql_command_flags[SQLCOM_TRUNCATE]= CF_CHANGES_DATA | CF_WRITE_LOGS_COMMAND | CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_DROP_TABLE]= CF_CHANGES_DATA | CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_LOAD]= CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | - CF_CAN_GENERATE_ROW_EVENTS | CF_REPORT_PROGRESS; + CF_CAN_GENERATE_ROW_EVENTS | CF_REPORT_PROGRESS | + CF_INSERTS_DATA; sql_command_flags[SQLCOM_CREATE_DB]= CF_CHANGES_DATA | CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_DROP_DB]= CF_CHANGES_DATA | CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_ALTER_DB_UPGRADE]= CF_AUTO_COMMIT_TRANS; @@ -290,21 +293,21 @@ void init_update_queries(void) sql_command_flags[SQLCOM_DROP_EVENT]= CF_CHANGES_DATA | CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_UPDATE]= CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | - CF_CAN_GENERATE_ROW_EVENTS; + CF_CAN_GENERATE_ROW_EVENTS | CF_UPDATES_DATA; sql_command_flags[SQLCOM_UPDATE_MULTI]= CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | - CF_CAN_GENERATE_ROW_EVENTS; + CF_CAN_GENERATE_ROW_EVENTS | CF_UPDATES_DATA; sql_command_flags[SQLCOM_INSERT]= CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | - CF_CAN_GENERATE_ROW_EVENTS; + CF_CAN_GENERATE_ROW_EVENTS | CF_INSERTS_DATA; sql_command_flags[SQLCOM_INSERT_SELECT]= CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | - CF_CAN_GENERATE_ROW_EVENTS; + CF_CAN_GENERATE_ROW_EVENTS | CF_INSERTS_DATA; sql_command_flags[SQLCOM_DELETE]= CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | CF_CAN_GENERATE_ROW_EVENTS; sql_command_flags[SQLCOM_DELETE_MULTI]= CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | CF_CAN_GENERATE_ROW_EVENTS; sql_command_flags[SQLCOM_REPLACE]= CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | - CF_CAN_GENERATE_ROW_EVENTS; + CF_CAN_GENERATE_ROW_EVENTS | CF_INSERTS_DATA; sql_command_flags[SQLCOM_REPLACE_SELECT]= CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | - CF_CAN_GENERATE_ROW_EVENTS; + CF_CAN_GENERATE_ROW_EVENTS | CF_INSERTS_DATA; sql_command_flags[SQLCOM_SELECT]= CF_REEXECUTION_FRAGILE | CF_CAN_GENERATE_ROW_EVENTS; sql_command_flags[SQLCOM_SET_OPTION]= CF_REEXECUTION_FRAGILE | CF_AUTO_COMMIT_TRANS; @@ -335,6 +338,7 @@ void init_update_queries(void) sql_command_flags[SQLCOM_SHOW_ENGINE_STATUS]= CF_STATUS_COMMAND; sql_command_flags[SQLCOM_SHOW_ENGINE_MUTEX]= CF_STATUS_COMMAND; sql_command_flags[SQLCOM_SHOW_ENGINE_LOGS]= CF_STATUS_COMMAND; + sql_command_flags[SQLCOM_SHOW_EXPLAIN]= CF_STATUS_COMMAND; sql_command_flags[SQLCOM_SHOW_PROCESSLIST]= CF_STATUS_COMMAND; sql_command_flags[SQLCOM_SHOW_GRANTS]= CF_STATUS_COMMAND; sql_command_flags[SQLCOM_SHOW_CREATE_DB]= CF_STATUS_COMMAND; @@ -350,7 +354,7 @@ void init_update_queries(void) sql_command_flags[SQLCOM_SHOW_CREATE_EVENT]= CF_STATUS_COMMAND; sql_command_flags[SQLCOM_SHOW_PROFILES]= CF_STATUS_COMMAND; sql_command_flags[SQLCOM_SHOW_PROFILE]= CF_STATUS_COMMAND; - sql_command_flags[SQLCOM_BINLOG_BASE64_EVENT]= CF_STATUS_COMMAND; + sql_command_flags[SQLCOM_BINLOG_BASE64_EVENT]= CF_STATUS_COMMAND | CF_CAN_GENERATE_ROW_EVENTS; sql_command_flags[SQLCOM_SHOW_CLIENT_STATS]= CF_STATUS_COMMAND; sql_command_flags[SQLCOM_SHOW_USER_STATS]= CF_STATUS_COMMAND; sql_command_flags[SQLCOM_SHOW_TABLE_STATS]= CF_STATUS_COMMAND; @@ -362,8 +366,11 @@ void init_update_queries(void) sql_command_flags[SQLCOM_CREATE_USER]= CF_CHANGES_DATA; sql_command_flags[SQLCOM_RENAME_USER]= CF_CHANGES_DATA; sql_command_flags[SQLCOM_DROP_USER]= CF_CHANGES_DATA; + sql_command_flags[SQLCOM_CREATE_ROLE]= CF_CHANGES_DATA; sql_command_flags[SQLCOM_GRANT]= CF_CHANGES_DATA; + sql_command_flags[SQLCOM_GRANT_ROLE]= CF_CHANGES_DATA; sql_command_flags[SQLCOM_REVOKE]= CF_CHANGES_DATA; + sql_command_flags[SQLCOM_REVOKE_ROLE]= CF_CHANGES_DATA; sql_command_flags[SQLCOM_OPTIMIZE]= CF_CHANGES_DATA; sql_command_flags[SQLCOM_CREATE_FUNCTION]= CF_CHANGES_DATA; sql_command_flags[SQLCOM_CREATE_PROCEDURE]= CF_CHANGES_DATA | CF_AUTO_COMMIT_TRANS; @@ -386,6 +393,19 @@ void init_update_queries(void) sql_command_flags[SQLCOM_EXECUTE]= CF_CAN_GENERATE_ROW_EVENTS; /* + We don't want to change to statement based replication for these commands + */ + sql_command_flags[SQLCOM_ROLLBACK]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT; + /* We don't want to replicate ALTER TABLE for temp tables in row format */ + sql_command_flags[SQLCOM_ALTER_TABLE]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT; + /* We don't want to replicate TRUNCATE for temp tables in row format */ + sql_command_flags[SQLCOM_TRUNCATE]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT; + /* We don't want to replicate DROP for temp tables in row format */ + sql_command_flags[SQLCOM_DROP_TABLE]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT; + /* One can change replication mode with SET */ + sql_command_flags[SQLCOM_SET_OPTION]|= CF_FORCE_ORIGINAL_BINLOG_FORMAT; + + /* The following admin table operations are allowed on log tables. */ @@ -398,9 +418,13 @@ void init_update_queries(void) sql_command_flags[SQLCOM_CREATE_USER]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_DROP_USER]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_RENAME_USER]|= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_CREATE_ROLE]|= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_DROP_ROLE]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_REVOKE_ALL]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_REVOKE]|= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_REVOKE_ROLE]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_GRANT]|= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_GRANT_ROLE]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_ASSIGN_TO_KEYCACHE]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_PRELOAD_KEYS]= CF_AUTO_COMMIT_TRANS; @@ -495,7 +519,8 @@ static void handle_bootstrap_impl(THD *thd) thd_proc_info(thd, 0); thd->security_ctx->user= (char*) my_strdup("boot", MYF(MY_WME)); - thd->security_ctx->priv_user[0]= thd->security_ctx->priv_host[0]=0; + thd->security_ctx->priv_user[0]= thd->security_ctx->priv_host[0]= + thd->security_ctx->priv_role[0]= 0; /* Make the "client" handle multiple results. This is necessary to enable stored procedures with SELECTs and Dynamic SQL @@ -581,6 +606,7 @@ static void handle_bootstrap_impl(THD *thd) #if defined(ENABLED_PROFILING) thd->profiling.finish_current_query(); #endif + delete_explain_query(thd->lex); if (bootstrap_error) break; @@ -625,14 +651,13 @@ void do_handle_bootstrap(THD *thd) handle_bootstrap_impl(thd); end: - net_end(&thd->net); - thd->cleanup(); delete thd; #ifndef EMBEDDED_LIBRARY - mysql_mutex_lock(&LOCK_thread_count); - thread_count--; + thread_safe_decrement32(&thread_count, &thread_count_lock); in_bootstrap= FALSE; + + mysql_mutex_lock(&LOCK_thread_count); mysql_cond_broadcast(&COND_thread_count); mysql_mutex_unlock(&LOCK_thread_count); my_thread_end(); @@ -790,7 +815,9 @@ bool do_command(THD *thd) my_net_set_read_timeout(net, thd->variables.net_read_timeout); DBUG_ASSERT(packet_length); + DBUG_ASSERT(!thd->apc_target.is_enabled()); return_value= dispatch_command(command, thd, packet+1, (uint) (packet_length-1)); + DBUG_ASSERT(!thd->apc_target.is_enabled()); out: DBUG_RETURN(return_value); @@ -917,10 +944,16 @@ bool dispatch_command(enum enum_server_command command, THD *thd, DEBUG_SYNC(thd,"dispatch_command_before_set_time"); thd->set_time(); - if (server_command_flags[command] & CF_SKIP_QUERY_ID) - thd->set_query_id(get_query_id()); - else + if (!(server_command_flags[command] & CF_SKIP_QUERY_ID)) thd->set_query_id(next_query_id()); + else + { + /* + ping, get statistics or similar stateless command. + No reason to increase query id here. + */ + thd->set_query_id(get_query_id()); + } inc_thread_running(); if (!(server_command_flags[command] & CF_SKIP_QUESTIONS)) @@ -999,7 +1032,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd, else rc= acl_authenticate(thd, 0, packet_length); - MYSQL_AUDIT_NOTIFY_CONNECTION_CHANGE_USER(thd); + mysql_audit_notify_connection_change_user(thd); if (rc) { /* Free user if allocated by acl_authenticate */ @@ -1102,6 +1135,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd, ulong length= (ulong)(packet_end - beginning_of_next_stmt); log_slow_statement(thd); + DBUG_ASSERT(!thd->apc_target.is_enabled()); /* Remove garbage at start of query */ while (length > 0 && my_isspace(thd->charset(), *beginning_of_next_stmt)) @@ -1270,10 +1304,10 @@ bool dispatch_command(enum enum_server_command command, THD *thd, /* TODO: The following has to be changed to an 8 byte integer */ pos = uint4korr(packet); flags = uint2korr(packet + 4); - thd->server_id=0; /* avoid suicide */ + thd->variables.server_id=0; /* avoid suicide */ if ((slave_server_id= uint4korr(packet+6))) // mysqlbinlog.server_id==0 kill_zombie_dump_threads(slave_server_id); - thd->server_id = slave_server_id; + thd->variables.server_id = slave_server_id; general_log_print(thd, command, "Log: '%s' Pos: %ld", packet+10, (long) pos); @@ -1315,17 +1349,21 @@ bool dispatch_command(enum enum_server_command command, THD *thd, and flushes tables. */ bool res; - my_pthread_setspecific_ptr(THR_THD, NULL); + set_current_thd(0); res= reload_acl_and_cache(NULL, options | REFRESH_FAST, NULL, ¬_used); - my_pthread_setspecific_ptr(THR_THD, thd); + set_current_thd(thd); if (res) break; } else #endif - if (reload_acl_and_cache(thd, options, (TABLE_LIST*) 0, ¬_used)) - break; + { + thd->lex->relay_log_connection_name.str= (char*) ""; + thd->lex->relay_log_connection_name.length= 0; + if (reload_acl_and_cache(thd, options, (TABLE_LIST*) 0, ¬_used)) + break; + } if (trans_commit_implicit(thd)) break; close_thread_tables(thd); @@ -1380,7 +1418,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd, if (!(uptime= (ulong) (thd->start_time - server_start_time))) queries_per_second1000= 0; else - queries_per_second1000= thd->query_id * LL(1000) / uptime; + queries_per_second1000= thd->query_id * 1000 / uptime; length= my_snprintf(buff, buff_len - 1, "Uptime: %lu Threads: %d Questions: %lu " @@ -1422,7 +1460,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd, { status_var_increment(thd->status_var.com_stat[SQLCOM_KILL]); ulong id=(ulong) uint4korr(packet); - sql_kill(thd,id, KILL_CONNECTION_HARD); + sql_kill(thd, id, KILL_CONNECTION_HARD, KILL_TYPE_ID); break; } case COM_SET_OPTION: @@ -1485,6 +1523,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd, thd_proc_info(thd, "cleaning up"); thd->reset_query(); + thd->examined_row_count= 0; // For processlist thd->command=COM_SLEEP; thd->set_time(); dec_thread_running(); @@ -1512,23 +1551,31 @@ bool dispatch_command(enum enum_server_command command, THD *thd, } +/* + @note + This function must call delete_explain_query(). +*/ void log_slow_statement(THD *thd) { DBUG_ENTER("log_slow_statement"); + /* The following should never be true with our current code base, but better to keep this here so we don't accidently try to log a statement in a trigger or stored function */ if (unlikely(thd->in_sub_stmt)) - DBUG_VOID_RETURN; // Don't set time for sub stmt + goto end; // Don't set time for sub stmt + /* Follow the slow log filter configuration. */ if (!thd->enable_slow_log || (thd->variables.log_slow_filter && !(thd->variables.log_slow_filter & thd->query_plan_flags))) - DBUG_VOID_RETURN; + { + goto end; + } if (((thd->server_status & SERVER_QUERY_WAS_SLOW) || ((thd->server_status & @@ -1544,13 +1591,16 @@ void log_slow_statement(THD *thd) */ if (thd->variables.log_slow_rate_limit > 1 && (global_query_id % thd->variables.log_slow_rate_limit) != 0) - DBUG_VOID_RETURN; + goto end; thd_proc_info(thd, "logging slow query"); slow_log_print(thd, thd->query(), thd->query_length(), thd->utime_after_query); thd_proc_info(thd, 0); } + +end: + delete_explain_query(thd->lex); DBUG_VOID_RETURN; } @@ -1647,8 +1697,8 @@ int prepare_schema_table(THD *thd, LEX *lex, Table_ident *table_ident, DBUG_RETURN(1); lex->query_tables_last= query_tables_last; break; - } #endif + } case SCH_PROFILES: /* Mark this current profiling record to be discarded. We don't @@ -1775,7 +1825,6 @@ static void reset_one_shot_variables(THD *thd) } -static bool sp_process_definer(THD *thd) { DBUG_ENTER("sp_process_definer"); @@ -1812,7 +1861,7 @@ bool sp_process_definer(THD *thd) Query_arena original_arena; Query_arena *ps_arena= thd->activate_stmt_arena_if_needed(&original_arena); - lex->definer= create_default_definer(thd); + lex->definer= create_default_definer(thd, false); if (ps_arena) thd->restore_active_arena(ps_arena, &original_arena); @@ -1826,20 +1875,24 @@ bool sp_process_definer(THD *thd) } else { + LEX_USER *d= lex->definer= get_current_user(thd, lex->definer); + if (!d) + DBUG_RETURN(TRUE); + /* - If the specified definer differs from the current user, we + If the specified definer differs from the current user or role, we should check that the current user has SUPER privilege (in order to create a stored routine under another user one must have SUPER privilege). */ - if ((strcmp(lex->definer->user.str, thd->security_ctx->priv_user) || - my_strcasecmp(system_charset_info, lex->definer->host.str, - thd->security_ctx->priv_host)) && - check_global_access(thd, SUPER_ACL, true)) - { - my_error(ER_SPECIFIC_ACCESS_DENIED_ERROR, MYF(0), "SUPER"); + bool curuser= !strcmp(d->user.str, thd->security_ctx->priv_user); + bool currole= !curuser && !strcmp(d->user.str, thd->security_ctx->priv_role); + bool curuserhost= curuser && d->host.str && + !my_strcasecmp(system_charset_info, d->host.str, + thd->security_ctx->priv_host); + if (!curuserhost && !currole && + check_global_access(thd, SUPER_ACL, false)) DBUG_RETURN(TRUE); - } } /* Check that the specified definer exists. Emit a warning if not. */ @@ -1956,6 +2009,8 @@ mysql_execute_command(THD *thd) #ifdef HAVE_REPLICATION /* have table map for update for multi-update statement (BUG#37051) */ bool have_table_map_for_update= FALSE; + /* */ + Rpl_filter *rpl_filter= thd->rpl_filter; #endif DBUG_ENTER("mysql_execute_command"); @@ -2094,7 +2149,7 @@ mysql_execute_command(THD *thd) if (!(lex->sql_command == SQLCOM_UPDATE_MULTI) && !(lex->sql_command == SQLCOM_SET_OPTION) && !(lex->sql_command == SQLCOM_DROP_TABLE && - lex->drop_temporary && lex->drop_if_exists) && + lex->drop_temporary && lex->check_exists) && all_tables_not_ok(thd, all_tables)) { /* we warn the slave SQL thread */ @@ -2146,6 +2201,22 @@ mysql_execute_command(THD *thd) DBUG_ASSERT(thd->transaction.stmt.modified_non_trans_table == FALSE); + /* store old value of binlog format */ + enum_binlog_format orig_binlog_format,orig_current_stmt_binlog_format; + + thd->get_binlog_format(&orig_binlog_format, + &orig_current_stmt_binlog_format); + + /* + Force statement logging for DDL commands to allow us to update + privilege, system or statistic tables directly without the updates + getting logged. + */ + if (!(sql_command_flags[lex->sql_command] & + (CF_CAN_GENERATE_ROW_EVENTS | CF_FORCE_ORIGINAL_BINLOG_FORMAT | + CF_STATUS_COMMAND))) + thd->set_binlog_format_stmt(); + /* End a active transaction so that this command will have it's own transaction and will also sync the binary log. If a DDL is @@ -2167,7 +2238,7 @@ mysql_execute_command(THD *thd) /* Release metadata locks acquired in this transaction. */ thd->mdl_context.release_transactional_locks(); } - + #ifndef DBUG_OFF if (lex->sql_command != SQLCOM_SET_OPTION) DEBUG_SYNC(thd,"before_execute_sql_command"); @@ -2192,6 +2263,33 @@ mysql_execute_command(THD *thd) execute_show_status(thd, all_tables); break; } + case SQLCOM_SHOW_EXPLAIN: + { + if (!thd->security_ctx->priv_user[0] && + check_global_access(thd,PROCESS_ACL)) + break; + + /* + The select should use only one table, it's the SHOW EXPLAIN pseudo-table + */ + if (lex->sroutines.records || lex->query_tables->next_global) + { + my_message(ER_SET_CONSTANTS_ONLY, ER(ER_SET_CONSTANTS_ONLY), + MYF(0)); + goto error; + } + + Item **it= lex->value_list.head_ref(); + if (!(*it)->basic_const_item() || + (!(*it)->fixed && (*it)->fix_fields(lex->thd, it)) || + (*it)->check_cols(1)) + { + my_message(ER_SET_CONSTANTS_ONLY, ER(ER_SET_CONSTANTS_ONLY), + MYF(0)); + goto error; + } + /* no break; fall through */ + } case SQLCOM_SHOW_DATABASES: case SQLCOM_SHOW_TABLES: case SQLCOM_SHOW_TRIGGERS: @@ -2365,10 +2463,52 @@ case SQLCOM_PREPARE: #ifdef HAVE_REPLICATION case SQLCOM_CHANGE_MASTER: { + LEX_MASTER_INFO *lex_mi= &thd->lex->mi; + Master_info *mi; + bool new_master= 0; + bool master_info_added; + if (check_global_access(thd, SUPER_ACL)) goto error; mysql_mutex_lock(&LOCK_active_mi); - res = change_master(thd,active_mi); + + mi= master_info_index->get_master_info(&lex_mi->connection_name, + MYSQL_ERROR::WARN_LEVEL_NOTE); + + if (mi == NULL) + { + /* New replication created */ + mi= new Master_info(&lex_mi->connection_name, relay_log_recovery); + if (!mi || mi->error()) + { + delete mi; + res= 1; + mysql_mutex_unlock(&LOCK_active_mi); + break; + } + new_master= 1; + } + + res= change_master(thd, mi, &master_info_added); + if (res && new_master) + { + /* + If the new master was added by change_master(), remove it as it didn't + work (this will free mi as well). + + If new master was not added, we still need to free mi. + */ + if (master_info_added) + master_info_index->remove_master_info(&lex_mi->connection_name); + else + delete mi; + } + else + { + mi->rpl_filter= get_or_create_rpl_filter(lex_mi->connection_name.str, + lex_mi->connection_name.length); + } + mysql_mutex_unlock(&LOCK_active_mi); break; } @@ -2378,15 +2518,19 @@ case SQLCOM_PREPARE: if (check_global_access(thd, SUPER_ACL | REPL_CLIENT_ACL)) goto error; mysql_mutex_lock(&LOCK_active_mi); - if (active_mi != NULL) - { - res = show_master_info(thd, active_mi); - } + + if (lex->verbose) + res= show_all_master_info(thd); else { - push_warning(thd, MYSQL_ERROR::WARN_LEVEL_WARN, - WARN_NO_MASTER_INFO, ER(WARN_NO_MASTER_INFO)); - my_ok(thd); + LEX_MASTER_INFO *lex_mi= &thd->lex->mi; + Master_info *mi; + mi= master_info_index->get_master_info(&lex_mi->connection_name, + MYSQL_ERROR::WARN_LEVEL_ERROR); + if (mi != NULL) + { + res= show_master_info(thd, mi, 0); + } } mysql_mutex_unlock(&LOCK_active_mi); break; @@ -2693,40 +2837,100 @@ end_with_restore_list: #ifdef HAVE_REPLICATION case SQLCOM_SLAVE_START: { + LEX_MASTER_INFO* lex_mi= &thd->lex->mi; + Master_info *mi; + int load_error; + + load_error= rpl_load_gtid_slave_state(thd); + mysql_mutex_lock(&LOCK_active_mi); - start_slave(thd,active_mi,1 /* net report*/); + + if ((mi= (master_info_index-> + get_master_info(&lex_mi->connection_name, + MYSQL_ERROR::WARN_LEVEL_ERROR)))) + { + if (load_error) + { + /* + We cannot start a slave using GTID if we cannot load the GTID position + from the mysql.gtid_slave_pos table. But we can allow non-GTID + replication (useful eg. during upgrade). + */ + if (mi->using_gtid != Master_info::USE_GTID_NO) + { + mysql_mutex_unlock(&LOCK_active_mi); + break; + } + else + thd->clear_error(); + } + if (!start_slave(thd, mi, 1 /* net report*/)) + my_ok(thd); + } mysql_mutex_unlock(&LOCK_active_mi); break; } case SQLCOM_SLAVE_STOP: - /* - If the client thread has locked tables, a deadlock is possible. - Assume that - - the client thread does LOCK TABLE t READ. - - then the master updates t. - - then the SQL slave thread wants to update t, - so it waits for the client thread because t is locked by it. + { + LEX_MASTER_INFO *lex_mi; + Master_info *mi; + /* + If the client thread has locked tables, a deadlock is possible. + Assume that + - the client thread does LOCK TABLE t READ. + - then the master updates t. + - then the SQL slave thread wants to update t, + so it waits for the client thread because t is locked by it. - then the client thread does SLAVE STOP. SLAVE STOP waits for the SQL slave thread to terminate its update t, which waits for the client thread because t is locked by it. - To prevent that, refuse SLAVE STOP if the - client thread has locked tables - */ - if (thd->locked_tables_mode || - thd->in_active_multi_stmt_transaction() || thd->global_read_lock.is_acquired()) + To prevent that, refuse SLAVE STOP if the + client thread has locked tables + */ + if (thd->locked_tables_mode || + thd->in_active_multi_stmt_transaction() || + thd->global_read_lock.is_acquired()) + { + my_message(ER_LOCK_OR_ACTIVE_TRANSACTION, + ER(ER_LOCK_OR_ACTIVE_TRANSACTION), MYF(0)); + goto error; + } + + lex_mi= &thd->lex->mi; + mysql_mutex_lock(&LOCK_active_mi); + if ((mi= (master_info_index-> + get_master_info(&lex_mi->connection_name, + MYSQL_ERROR::WARN_LEVEL_ERROR)))) + if (!stop_slave(thd, mi, 1/* net report*/)) + my_ok(thd); + mysql_mutex_unlock(&LOCK_active_mi); + break; + } + case SQLCOM_SLAVE_ALL_START: { - my_message(ER_LOCK_OR_ACTIVE_TRANSACTION, - ER(ER_LOCK_OR_ACTIVE_TRANSACTION), MYF(0)); - goto error; + mysql_mutex_lock(&LOCK_active_mi); + if (!master_info_index->start_all_slaves(thd)) + my_ok(thd); + mysql_mutex_unlock(&LOCK_active_mi); + break; } + case SQLCOM_SLAVE_ALL_STOP: { + if (thd->locked_tables_mode || + thd->in_active_multi_stmt_transaction() || + thd->global_read_lock.is_acquired()) + { + my_message(ER_LOCK_OR_ACTIVE_TRANSACTION, + ER(ER_LOCK_OR_ACTIVE_TRANSACTION), MYF(0)); + goto error; + } mysql_mutex_lock(&LOCK_active_mi); - stop_slave(thd,active_mi,1/* net report*/); + if (!master_info_index->stop_all_slaves(thd)) + my_ok(thd); mysql_mutex_unlock(&LOCK_active_mi); break; } #endif /* HAVE_REPLICATION */ - case SQLCOM_RENAME_TABLE: { if (execute_rename_table(thd, first_table, all_tables)) @@ -3005,6 +3209,7 @@ end_with_restore_list: case SQLCOM_INSERT_SELECT: { select_result *sel_result; + bool explain= test(lex->describe); DBUG_ASSERT(first_table == all_tables && first_table != 0); if ((res= insert_precheck(thd, all_tables))) break; @@ -3074,6 +3279,10 @@ end_with_restore_list: } delete sel_result; } + + if (!res && explain) + res= thd->lex->explain->send_explain(thd); + /* revert changes for SP */ MYSQL_INSERT_SELECT_DONE(res, (ulong) thd->get_row_count_func()); select_lex->table_list.first= first_table; @@ -3092,6 +3301,7 @@ end_with_restore_list: } case SQLCOM_DELETE: { + select_result *sel_result=lex->result; DBUG_ASSERT(first_table == all_tables && first_table != 0); if ((res= delete_precheck(thd, all_tables))) break; @@ -3099,9 +3309,13 @@ end_with_restore_list: unit->set_limit(select_lex); MYSQL_DELETE_START(thd->query()); - res = mysql_delete(thd, all_tables, select_lex->where, - &select_lex->order_list, - unit->select_limit_cnt, select_lex->options); + if (!(sel_result= lex->result) && !(sel_result= new select_send())) + return 1; + res = mysql_delete(thd, all_tables, + select_lex->where, &select_lex->order_list, + unit->select_limit_cnt, select_lex->options, + sel_result); + delete sel_result; MYSQL_DELETE_DONE(res, (ulong) thd->get_row_count_func()); break; } @@ -3109,7 +3323,8 @@ end_with_restore_list: { DBUG_ASSERT(first_table == all_tables && first_table != 0); TABLE_LIST *aux_tables= thd->lex->auxiliary_table_list.first; - multi_delete *del_result; + bool explain= test(lex->describe); + multi_delete *result; if ((res= multi_delete_precheck(thd, all_tables))) break; @@ -3131,25 +3346,34 @@ end_with_restore_list: goto error; } - if (!thd->is_fatal_error && - (del_result= new multi_delete(aux_tables, lex->table_count))) - { - res= mysql_select(thd, &select_lex->ref_pointer_array, - select_lex->get_table_list(), - select_lex->with_wild, - select_lex->item_list, - select_lex->where, - 0, (ORDER *)NULL, (ORDER *)NULL, (Item *)NULL, - (ORDER *)NULL, - (select_lex->options | thd->variables.option_bits | - SELECT_NO_JOIN_CACHE | SELECT_NO_UNLOCK | - OPTION_SETUP_TABLES_DONE) & ~OPTION_BUFFER_RESULT, - del_result, unit, select_lex); - res|= thd->is_error(); - MYSQL_MULTI_DELETE_DONE(res, del_result->num_deleted()); - if (res) - del_result->abort_result_set(); - delete del_result; + if (!thd->is_fatal_error) + { + result= new multi_delete(aux_tables, lex->table_count); + if (result) + { + res= mysql_select(thd, &select_lex->ref_pointer_array, + select_lex->get_table_list(), + select_lex->with_wild, + select_lex->item_list, + select_lex->where, + 0, (ORDER *)NULL, (ORDER *)NULL, (Item *)NULL, + (ORDER *)NULL, + (select_lex->options | thd->variables.option_bits | + SELECT_NO_JOIN_CACHE | SELECT_NO_UNLOCK | + OPTION_SETUP_TABLES_DONE) & ~OPTION_BUFFER_RESULT, + result, unit, select_lex); + res|= thd->is_error(); + + MYSQL_MULTI_DELETE_DONE(res, result->num_deleted()); + if (res) + result->abort_result_set(); /* for both DELETE and EXPLAIN DELETE */ + else + { + if (explain) + res= thd->lex->explain->send_explain(thd); + } + delete result; + } } else { @@ -3172,7 +3396,7 @@ end_with_restore_list: thd->variables.option_bits|= OPTION_KEEP_LOG; } /* DDL and binlog write order are protected by metadata locks. */ - res= mysql_rm_table(thd, first_table, lex->drop_if_exists, + res= mysql_rm_table(thd, first_table, lex->check_exists, lex->drop_temporary); } break; @@ -3386,7 +3610,7 @@ end_with_restore_list: #endif if (check_access(thd, DROP_ACL, lex->name.str, NULL, NULL, 1, 0)) break; - res= mysql_rm_db(thd, lex->name.str, lex->drop_if_exists, 0); + res= mysql_rm_db(thd, lex->name.str, lex->check_exists, 0); break; } case SQLCOM_ALTER_DB_UPGRADE: @@ -3514,7 +3738,7 @@ end_with_restore_list: case SQLCOM_DROP_EVENT: if (!(res= Events::drop_event(thd, lex->spname->m_db, lex->spname->m_name, - lex->drop_if_exists))) + lex->check_exists))) my_ok(thd); break; #else @@ -3536,22 +3760,26 @@ end_with_restore_list: } #ifndef NO_EMBEDDED_ACCESS_CHECKS case SQLCOM_CREATE_USER: + case SQLCOM_CREATE_ROLE: { if (check_access(thd, INSERT_ACL, "mysql", NULL, NULL, 1, 1) && check_global_access(thd,CREATE_USER_ACL)) break; /* Conditionally writes to binlog */ - if (!(res= mysql_create_user(thd, lex->users_list))) + if (!(res= mysql_create_user(thd, lex->users_list, + lex->sql_command == SQLCOM_CREATE_ROLE))) my_ok(thd); break; } case SQLCOM_DROP_USER: + case SQLCOM_DROP_ROLE: { if (check_access(thd, DELETE_ACL, "mysql", NULL, NULL, 1, 1) && check_global_access(thd,CREATE_USER_ACL)) break; /* Conditionally writes to binlog */ - if (!(res= mysql_drop_user(thd, lex->users_list))) + if (!(res= mysql_drop_user(thd, lex->users_list, + lex->sql_command == SQLCOM_DROP_ROLE))) my_ok(thd); break; } @@ -3571,9 +3799,6 @@ end_with_restore_list: check_global_access(thd,CREATE_USER_ACL)) break; - /* Replicate current user as grantor */ - thd->binlog_invoker(); - /* Conditionally writes to binlog */ if (!(res = mysql_revoke_all(thd, lex->users_list))) my_ok(thd); @@ -3591,42 +3816,58 @@ end_with_restore_list: goto error; /* Replicate current user as grantor */ - thd->binlog_invoker(); + thd->binlog_invoker(false); if (thd->security_ctx->user) // If not replication { - LEX_USER *user, *tmp_user; + LEX_USER *user; bool first_user= TRUE; List_iterator <LEX_USER> user_list(lex->users_list); - while ((tmp_user= user_list++)) + while ((user= user_list++)) { - if (!(user= get_current_user(thd, tmp_user))) - goto error; if (specialflag & SPECIAL_NO_RESOLVE && hostname_requires_resolving(user->host.str)) push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_HOSTNAME_WONT_WORK, ER(ER_WARN_HOSTNAME_WONT_WORK)); - // Are we trying to change a password of another user - DBUG_ASSERT(user->host.str != 0); /* GRANT/REVOKE PROXY has the target user as a first entry in the list. */ if (lex->type == TYPE_ENUM_PROXY && first_user) { + if (!(user= get_current_user(thd, user)) || !user->host.str) + goto error; + first_user= FALSE; if (acl_check_proxy_grant_access (thd, user->host.str, user->user.str, lex->grant & GRANT_ACL)) goto error; } - else if (is_acl_user(user->host.str, user->user.str) && - user->password.str && - check_change_password (thd, user->host.str, user->user.str, - user->password.str, - user->password.length)) - goto error; + else if (user->password.str) + { + // Are we trying to change a password of another user? + const char *hostname= user->host.str, *username=user->user.str; + bool userok; + if (username == current_user.str) + { + username= thd->security_ctx->priv_user; + hostname= thd->security_ctx->priv_host; + userok= true; + } + else + { + if (!hostname) + hostname= host_not_specified.str; + userok= is_acl_user(hostname, username); + } + + if (userok && check_change_password (thd, hostname, username, + user->password.str, + user->password.length)) + goto error; + } } } if (first_table) @@ -3670,9 +3911,9 @@ end_with_restore_list: else { /* Conditionally writes to binlog */ - res = mysql_grant(thd, select_lex->db, lex->users_list, lex->grant, - lex->sql_command == SQLCOM_REVOKE, - lex->type == TYPE_ENUM_PROXY); + res= mysql_grant(thd, select_lex->db, lex->users_list, lex->grant, + lex->sql_command == SQLCOM_REVOKE, + lex->type == TYPE_ENUM_PROXY); } if (!res) { @@ -3691,6 +3932,14 @@ end_with_restore_list: } break; } + case SQLCOM_REVOKE_ROLE: + case SQLCOM_GRANT_ROLE: + { + if (!(res= mysql_grant_role(thd, lex->users_list, + lex->sql_command != SQLCOM_GRANT_ROLE))) + my_ok(thd); + break; + } #endif /*!NO_EMBEDDED_ACCESS_CHECKS*/ case SQLCOM_RESET: /* @@ -3758,7 +4007,7 @@ end_with_restore_list: break; } - if (lex->kill_type == KILL_TYPE_ID) + if (lex->kill_type == KILL_TYPE_ID || lex->kill_type == KILL_TYPE_QUERY) { Item *it= (Item *)lex->value_list.head(); if ((!it->fixed && it->fix_fields(lex->thd, &it)) || it->check_cols(1)) @@ -3767,21 +4016,38 @@ end_with_restore_list: MYF(0)); goto error; } - sql_kill(thd, (ulong) it->val_int(), lex->kill_signal); + sql_kill(thd, it->val_int(), lex->kill_signal, lex->kill_type); } else sql_kill_user(thd, get_current_user(thd, lex->users_list.head()), lex->kill_signal); break; } + case SQLCOM_SHUTDOWN: +#ifndef EMBEDDED_LIBRARY + if (check_global_access(thd,SHUTDOWN_ACL)) + goto error; + kill_mysql(); + my_ok(thd); +#else + my_error(ER_NOT_SUPPORTED_YET, MYF(0), "embedded server"); +#endif + break; + #ifndef NO_EMBEDDED_ACCESS_CHECKS case SQLCOM_SHOW_GRANTS: { - LEX_USER *grant_user= get_current_user(thd, lex->grant_user); + LEX_USER *grant_user= lex->grant_user; if (!grant_user) goto error; - if ((thd->security_ctx->priv_user && - !strcmp(thd->security_ctx->priv_user, grant_user->user.str)) || + + if (grant_user->user.str && + !strcmp(thd->security_ctx->priv_user, grant_user->user.str)) + grant_user->user= current_user; + + if (grant_user->user.str == current_user.str || + grant_user->user.str == current_role.str || + grant_user->user.str == current_user_and_current_role.str || !check_access(thd, SELECT_ACL, "mysql", NULL, NULL, 1, 0)) { res = mysql_show_grants(thd, grant_user); @@ -3813,6 +4079,7 @@ end_with_restore_list: break; case SQLCOM_BEGIN: + DBUG_PRINT("info", ("Executing SQLCOM_BEGIN thd: %p", thd)); if (trans_begin(thd, lex->start_transaction_opt)) goto error; my_ok(thd); @@ -4201,7 +4468,7 @@ create_sp_error: if (lex->spname->m_db.str == NULL) { - if (lex->drop_if_exists) + if (lex->check_exists) { push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, ER_SP_DOES_NOT_EXIST, ER(ER_SP_DOES_NOT_EXIST), @@ -4270,7 +4537,7 @@ create_sp_error: my_ok(thd); break; case SP_KEY_NOT_FOUND: - if (lex->drop_if_exists) + if (lex->check_exists) { res= write_bin_log(thd, TRUE, thd->query(), thd->query_length()); push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, @@ -4483,7 +4750,7 @@ create_sp_error: if ((err_code= drop_server(thd, &lex->server_options))) { - if (! lex->drop_if_exists && err_code == ER_FOREIGN_SERVER_DOESNT_EXIST) + if (! lex->check_exists && err_code == ER_FOREIGN_SERVER_DOESNT_EXIST) { DBUG_PRINT("info", ("problem dropping server %s", lex->server_options.server_name)); @@ -4587,6 +4854,11 @@ finish: if (lex->sql_command != SQLCOM_SET_OPTION && ! thd->in_sub_stmt) DEBUG_SYNC(thd, "execute_command_after_close_tables"); #endif + if (!(sql_command_flags[lex->sql_command] & + (CF_CAN_GENERATE_ROW_EVENTS | CF_FORCE_ORIGINAL_BINLOG_FORMAT | + CF_STATUS_COMMAND))) + thd->set_binlog_format(orig_binlog_format, + orig_current_stmt_binlog_format); if (! thd->in_sub_stmt && thd->transaction_rollback_request) { @@ -4657,24 +4929,37 @@ static bool execute_sqlcom_select(THD *thd, TABLE_LIST *all_tables) if (!(result= new select_send())) return 1; /* purecov: inspected */ thd->send_explain_fields(result); - res= mysql_explain_union(thd, &thd->lex->unit, result); + /* - The code which prints the extended description is not robust - against malformed queries, so skip it if we have an error. + This will call optimize() for all parts of query. The query plan is + printed out below. */ - if (!res && (lex->describe & DESCRIBE_EXTENDED)) + res= mysql_explain_union(thd, &thd->lex->unit, result); + + /* Print EXPLAIN only if we don't have an error */ + if (!res) { - char buff[1024]; - String str(buff,(uint32) sizeof(buff), system_charset_info); - str.length(0); - /* - The warnings system requires input in utf8, @see - mysqld_show_warnings(). - */ - thd->lex->unit.print(&str, QT_TO_SYSTEM_CHARSET); - push_warning(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, - ER_YES, str.c_ptr_safe()); + /* + Do like the original select_describe did: remove OFFSET from the + top-level LIMIT + */ + result->reset_offset_limit(); + thd->lex->explain->print_explain(result, thd->lex->describe); + if (lex->describe & DESCRIBE_EXTENDED) + { + char buff[1024]; + String str(buff,(uint32) sizeof(buff), system_charset_info); + str.length(0); + /* + The warnings system requires input in utf8, @see + mysqld_show_warnings(). + */ + thd->lex->unit.print(&str, QT_TO_SYSTEM_CHARSET); + push_warning(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_YES, str.c_ptr_safe()); + } } + if (res) result->abort_result_set(); else @@ -4718,7 +5003,8 @@ static bool execute_show_status(THD *thd, TABLE_LIST *all_tables) mysql_mutex_lock(&LOCK_status); add_diff_to_status(&global_status_var, &thd->status_var, &old_status_var); - thd->status_var= old_status_var; + memcpy(&thd->status_var, &old_status_var, + offsetof(STATUS_VAR, last_cleared_system_status_var)); mysql_mutex_unlock(&LOCK_status); return res; } @@ -4923,6 +5209,10 @@ check_access(THD *thd, ulong want_access, const char *db, ulong *save_priv, if ((db != NULL) && (db != any_db)) { + /* + Check if this is reserved database, like information schema or + performance schema + */ const ACL_internal_schema_access *access; access= get_cached_schema_access(grant_internal_info, db); if (access) @@ -4965,8 +5255,12 @@ check_access(THD *thd, ulong want_access, const char *db, ulong *save_priv, if (!(sctx->master_access & SELECT_ACL)) { if (db && (!thd->db || db_is_pattern || strcmp(db, thd->db))) + { db_access= acl_get(sctx->host, sctx->ip, sctx->priv_user, db, db_is_pattern); + if (sctx->priv_role[0]) + db_access|= acl_get("", "", sctx->priv_role, db, db_is_pattern); + } else { /* get access for current db */ @@ -5010,8 +5304,14 @@ check_access(THD *thd, ulong want_access, const char *db, ulong *save_priv, } if (db && (!thd->db || db_is_pattern || strcmp(db,thd->db))) + { db_access= acl_get(sctx->host, sctx->ip, sctx->priv_user, db, db_is_pattern); + if (sctx->priv_role[0]) + { + db_access|= acl_get("", "", sctx->priv_role, db, db_is_pattern); + } + } else db_access= sctx->db_access; DBUG_PRINT("info",("db_access: %lu want_access: %lu", @@ -5883,6 +6183,7 @@ bool add_field_to_list(THD *thd, LEX_STRING *field_name, enum_field_types type, { register Create_field *new_field; LEX *lex= thd->lex; + uint8 datetime_precision= length ? atoi(length) : 0; DBUG_ENTER("add_field_to_list"); if (check_string_char_length(field_name, "", NAME_CHAR_LEN, @@ -5897,7 +6198,7 @@ bool add_field_to_list(THD *thd, LEX_STRING *field_name, enum_field_types type, lex->col_list.push_back(new Key_part_spec(*field_name, 0)); key= new Key(Key::PRIMARY, null_lex_str, &default_key_create_info, - 0, lex->col_list, NULL); + 0, lex->col_list, NULL, lex->check_exists); lex->alter_info.key_list.push_back(key); lex->col_list.empty(); } @@ -5907,7 +6208,7 @@ bool add_field_to_list(THD *thd, LEX_STRING *field_name, enum_field_types type, lex->col_list.push_back(new Key_part_spec(*field_name, 0)); key= new Key(Key::UNIQUE, null_lex_str, &default_key_create_info, 0, - lex->col_list, NULL); + lex->col_list, NULL, lex->check_exists); lex->alter_info.key_list.push_back(key); lex->col_list.empty(); } @@ -5919,11 +6220,13 @@ bool add_field_to_list(THD *thd, LEX_STRING *field_name, enum_field_types type, no need fix_fields() We allow only one function as part of default value - - NOW() as default for TIMESTAMP type. + NOW() as default for TIMESTAMP and DATETIME type. */ if (default_value->type() == Item::FUNC_ITEM && - !(((Item_func*)default_value)->functype() == Item_func::NOW_FUNC && - type == MYSQL_TYPE_TIMESTAMP)) + (static_cast<Item_func*>(default_value)->functype() != + Item_func::NOW_FUNC || + (mysql_type_to_time_type(type) != MYSQL_TIMESTAMP_DATETIME) || + default_value->decimals < datetime_precision)) { my_error(ER_INVALID_DEFAULT, MYF(0), field_name->str); DBUG_RETURN(1); @@ -5945,7 +6248,9 @@ bool add_field_to_list(THD *thd, LEX_STRING *field_name, enum_field_types type, } } - if (on_update_value && type != MYSQL_TYPE_TIMESTAMP) + if (on_update_value && + (mysql_type_to_time_type(type) != MYSQL_TIMESTAMP_DATETIME || + on_update_value->decimals < datetime_precision)) { my_error(ER_INVALID_ON_UPDATE, MYF(0), field_name->str); DBUG_RETURN(1); @@ -5955,7 +6260,7 @@ bool add_field_to_list(THD *thd, LEX_STRING *field_name, enum_field_types type, new_field->init(thd, field_name->str, type, length, decimals, type_modifier, default_value, on_update_value, comment, change, interval_list, cs, uint_geom_type, vcol_info, - create_options)) + create_options, lex->check_exists)) DBUG_RETURN(1); lex->alter_info.create_list.push_back(new_field); @@ -6636,37 +6941,56 @@ void add_join_natural(TABLE_LIST *a, TABLE_LIST *b, List<String> *using_fields, /** - kill on thread. + Find a thread by id and return it, locking it LOCK_thd_data - @param thd Thread class - @param id Thread id - @param only_kill_query Should it kill the query or the connection + @param id Identifier of the thread we're looking for + @param query_id If true, search by query_id instead of thread_id - @note - This is written such that we have a short lock on LOCK_thread_count + @return NULL - not found + pointer - thread found, and its LOCK_thd_data is locked. */ -uint kill_one_thread(THD *thd, ulong id, killed_state kill_signal) +THD *find_thread_by_id(longlong id, bool query_id) { THD *tmp; - uint error=ER_NO_SUCH_THREAD; - DBUG_ENTER("kill_one_thread"); - DBUG_PRINT("enter", ("id: %lu signal: %u", id, (uint) kill_signal)); - mysql_mutex_lock(&LOCK_thread_count); // For unlink from list I_List_iterator<THD> it(threads); while ((tmp=it++)) { if (tmp->command == COM_DAEMON) continue; - if (tmp->thread_id == id) + if (id == (query_id ? tmp->query_id : (longlong) tmp->thread_id)) { mysql_mutex_lock(&tmp->LOCK_thd_data); // Lock from delete break; } } mysql_mutex_unlock(&LOCK_thread_count); - if (tmp) + return tmp; +} + + +/** + kill one thread. + + @param thd Thread class + @param id Thread id or query id + @param kill_signal Should it kill the query or the connection + @param type Type of id: thread id or query id + + @note + This is written such that we have a short lock on LOCK_thread_count +*/ + +uint +kill_one_thread(THD *thd, longlong id, killed_state kill_signal, killed_type type) +{ + THD *tmp; + uint error= (type == KILL_TYPE_QUERY ? ER_NO_SUCH_QUERY : ER_NO_SUCH_THREAD); + DBUG_ENTER("kill_one_thread"); + DBUG_PRINT("enter", ("id: %lld signal: %u", id, (uint) kill_signal)); + + if (id && (tmp= find_thread_by_id(id, type == KILL_TYPE_QUERY))) { /* If we're SUPER, we can KILL anything, including system-threads. @@ -6754,7 +7078,7 @@ static uint kill_threads_for_user(THD *thd, LEX_USER *user, mysql_mutex_unlock(&LOCK_thread_count); DBUG_RETURN(ER_KILL_DENIED_ERROR); } - if (!threads_to_kill.push_back(tmp, tmp->mem_root)) + if (!threads_to_kill.push_back(tmp, thd->mem_root)) mysql_mutex_lock(&tmp->LOCK_thd_data); // Lock from delete } } @@ -6784,21 +7108,20 @@ static uint kill_threads_for_user(THD *thd, LEX_USER *user, } -/* - kills a thread and sends response +/** + kills a thread and sends response. - SYNOPSIS - sql_kill() - thd Thread class - id Thread id - only_kill_query Should it kill the query or the connection + @param thd Thread class + @param id Thread id or query id + @param state Should it kill the query or the connection + @param type Type of id: thread id or query id */ static -void sql_kill(THD *thd, ulong id, killed_state state) +void sql_kill(THD *thd, longlong id, killed_state state, killed_type type) { uint error; - if (!(error= kill_one_thread(thd, id, state))) + if (!(error= kill_one_thread(thd, id, state, type))) { if ((!thd->killed)) my_ok(thd); @@ -7395,16 +7718,23 @@ Item *negate_expression(THD *thd, Item *expr) @param[out] definer definer */ -void get_default_definer(THD *thd, LEX_USER *definer) +void get_default_definer(THD *thd, LEX_USER *definer, bool role) { const Security_context *sctx= thd->security_ctx; - definer->user.str= (char *) sctx->priv_user; + if (role) + { + definer->user.str= const_cast<char*>(sctx->priv_role); + definer->host= empty_lex_str; + } + else + { + definer->user.str= const_cast<char*>(sctx->priv_user); + definer->host.str= const_cast<char*>(sctx->priv_host); + definer->host.length= strlen(definer->host.str); + } definer->user.length= strlen(definer->user.str); - definer->host.str= (char *) sctx->priv_host; - definer->host.length= strlen(definer->host.str); - definer->password= null_lex_str; definer->plugin= empty_lex_str; definer->auth= empty_lex_str; @@ -7422,16 +7752,22 @@ void get_default_definer(THD *thd, LEX_USER *definer) - On error, return 0. */ -LEX_USER *create_default_definer(THD *thd) +LEX_USER *create_default_definer(THD *thd, bool role) { LEX_USER *definer; if (! (definer= (LEX_USER*) thd->alloc(sizeof(LEX_USER)))) return 0; - thd->get_definer(definer); + thd->get_definer(definer, role); - return definer; + if (role && definer->user.length == 0) + { + my_error(ER_MALFORMED_DEFINER, MYF(0)); + return 0; + } + else + return definer; } @@ -7467,27 +7803,6 @@ LEX_USER *create_definer(THD *thd, LEX_STRING *user_name, LEX_STRING *host_name) /** - Retuns information about user or current user. - - @param[in] thd thread handler - @param[in] user user - - @return - - On success, return a valid pointer to initialized - LEX_USER, which contains user information. - - On error, return 0. -*/ - -LEX_USER *get_current_user(THD *thd, LEX_USER *user) -{ - if (!user->user.str) // current_user - return create_default_definer(thd); - - return user; -} - - -/** Check that byte length of a string does not exceed some limit. @param str string to be checked @@ -7550,6 +7865,7 @@ bool check_string_char_length(LEX_STRING *str, const char *err_msg, return TRUE; } +C_MODE_START /* Check if path does not contain mysql data home directory @@ -7562,7 +7878,6 @@ bool check_string_char_length(LEX_STRING *str, const char *err_msg, 0 ok 1 error ; Given path contains data directory */ -C_MODE_START int test_if_data_home_dir(const char *dir) { @@ -7573,6 +7888,22 @@ int test_if_data_home_dir(const char *dir) if (!dir) DBUG_RETURN(0); + /* + data_file_name and index_file_name include the table name without + extension. Mostly this does not refer to an existing file. When + comparing data_file_name or index_file_name against the data + directory, we try to resolve all symbolic links. On some systems, + we use realpath(3) for the resolution. This returns ENOENT if the + resolved path does not refer to an existing file. my_realpath() + does then copy the requested path verbatim, without symlink + resolution. Thereafter the comparison can fail even if the + requested path is within the data directory. E.g. if symlinks to + another file system are used. To make realpath(3) return the + resolved path, we strip the table name and compare the directory + path only. If the directory doesn't exist either, table creation + will fail anyway. + */ + (void) fn_format(path, dir, "", "", (MY_RETURN_REAL_PATH|MY_RESOLVE_SYMLINKS)); dir_len= strlen(path); @@ -7606,6 +7937,22 @@ int test_if_data_home_dir(const char *dir) C_MODE_END +int error_if_data_home_dir(const char *path, const char *what) +{ + size_t dirlen; + char dirpath[FN_REFLEN]; + if (path) + { + dirname_part(dirpath, path, &dirlen); + if (test_if_data_home_dir(dirpath)) + { + my_error(ER_WRONG_ARGUMENTS, MYF(0), what); + return 1; + } + } + return 0; +} + /** Check that host name string is valid. diff --git a/sql/sql_parse.h b/sql/sql_parse.h index d1d6458d22c..b56edf5b4f7 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -34,6 +34,7 @@ enum enum_mysql_completiontype { }; extern "C" int test_if_data_home_dir(const char *dir); +int error_if_data_home_dir(const char *path, const char *what); bool multi_update_precheck(THD *thd, TABLE_LIST *tables); bool multi_delete_precheck(THD *thd, TABLE_LIST *tables); @@ -62,10 +63,11 @@ Comp_creator *comp_ne_creator(bool invert); int prepare_schema_table(THD *thd, LEX *lex, Table_ident *table_ident, enum enum_schema_tables schema_table_idx); -void get_default_definer(THD *thd, LEX_USER *definer); -LEX_USER *create_default_definer(THD *thd); +void get_default_definer(THD *thd, LEX_USER *definer, bool role); +LEX_USER *create_default_definer(THD *thd, bool role); LEX_USER *create_definer(THD *thd, LEX_STRING *user_name, LEX_STRING *host_name); -LEX_USER *get_current_user(THD *thd, LEX_USER *user); +LEX_USER *get_current_user(THD *thd, LEX_USER *user, bool lock=true); +bool sp_process_definer(THD *thd); bool check_string_byte_length(LEX_STRING *str, const char *err_msg, uint max_byte_length); bool check_string_char_length(LEX_STRING *str, const char *err_msg, diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index 357bd8efc27..752ad99908c 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2005, 2013, Oracle and/or its affiliates. - Copyright (c) 2009, 2011, Monty Program Ab. + Copyright (c) 2009, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -4672,7 +4672,7 @@ uint prep_alter_part_table(THD *thd, TABLE *table, Alter_info *alter_info, */ DBUG_ASSERT(thd->mdl_context.is_lock_owner(MDL_key::TABLE, db, table_name, MDL_INTENTION_EXCLUSIVE)); - new_table= open_table_uncached(thd, path, db, table_name, 0); + new_table= open_table_uncached(thd, old_db_type, path, db, table_name, 0); if (!new_table) DBUG_RETURN(TRUE); @@ -4853,8 +4853,7 @@ uint prep_alter_part_table(THD *thd, TABLE *table, Alter_info *alter_info, } alt_part_info->part_type= tab_part_info->part_type; alt_part_info->subpart_type= tab_part_info->subpart_type; - if (alt_part_info->set_up_defaults_for_partitioning(new_table->file, - ULL(0), + if (alt_part_info->set_up_defaults_for_partitioning(new_table->file, 0, tab_part_info->num_parts)) { goto err; @@ -5272,8 +5271,7 @@ state of p1. alt_part_info->num_subparts= tab_part_info->num_subparts; DBUG_ASSERT(!alt_part_info->use_default_partitions); if (alt_part_info->set_up_defaults_for_partitioning(new_table->file, - ULL(0), - 0)) + 0, 0)) { goto err; } @@ -5406,7 +5404,7 @@ the generated partition syntax in a correct manner. tab_part_info->use_default_num_subpartitions= FALSE; } if (tab_part_info->check_partition_info(thd, (handlerton**)NULL, - new_table->file, ULL(0), TRUE)) + new_table->file, 0, TRUE)) { goto err; } @@ -6675,9 +6673,6 @@ uint fast_alter_partition_table(THD *thd, TABLE *table, lpt->pack_frm_data= NULL; lpt->pack_frm_len= 0; - /* Never update timestamp columns when alter */ - lpt->table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - if (table->file->alter_table_flags(alter_info->flags) & HA_PARTITION_ONE_PHASE) { diff --git a/sql/sql_partition.h b/sql/sql_partition.h index 951d58a655b..cf532c45c66 100644 --- a/sql/sql_partition.h +++ b/sql/sql_partition.h @@ -31,7 +31,6 @@ class partition_info; struct TABLE; struct TABLE_LIST; typedef struct st_bitmap MY_BITMAP; -typedef struct st_ha_create_information HA_CREATE_INFO; typedef struct st_key KEY; typedef struct st_key_range key_range; diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index c127b82b3b6..efc49c6286b 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -53,7 +53,6 @@ static TYPELIB global_plugin_typelib= { array_elements(global_plugin_typelib_names)-1, "", global_plugin_typelib_names, NULL }; - char *opt_plugin_load= NULL; char *opt_plugin_dir_ptr; char opt_plugin_dir[FN_REFLEN]; @@ -197,6 +196,8 @@ static bool reap_needed= false; static int plugin_array_version=0; static bool initialized= 0; +ulong dlopen_count; + /* write-lock on LOCK_system_variables_hash is required before modifying @@ -308,10 +309,6 @@ static void unlock_variables(THD *thd, struct system_variables *vars); static void cleanup_variables(THD *thd, struct system_variables *vars); static void plugin_vars_free_values(sys_var *vars); static void restore_pluginvar_names(sys_var *first); -static void plugin_opt_set_limits(struct my_option *, - const struct st_mysql_sys_var *); -#define my_intern_plugin_lock(A,B) intern_plugin_lock(A,B) -#define my_intern_plugin_lock_ci(A,B) intern_plugin_lock(A,B) static plugin_ref intern_plugin_lock(LEX *lex, plugin_ref plugin); static void intern_plugin_unlock(LEX *lex, plugin_ref plugin); static void reap_plugins(void); @@ -509,7 +506,6 @@ static my_bool read_mysql_plugin_info(struct st_plugin_dl *plugin_dl, /* Determine interface version */ if (!sym) { - free_plugin_mem(plugin_dl); report_error(report, ER_CANT_FIND_DL_ENTRY, plugin_interface_version_sym); DBUG_RETURN(TRUE); } @@ -519,7 +515,6 @@ static my_bool read_mysql_plugin_info(struct st_plugin_dl *plugin_dl, if (plugin_dl->mysqlversion < min_plugin_interface_version || (plugin_dl->mysqlversion >> 8) > (MYSQL_PLUGIN_INTERFACE_VERSION >> 8)) { - free_plugin_mem(plugin_dl); report_error(report, ER_CANT_OPEN_LIBRARY, dlpath, 0, "plugin interface version mismatch"); DBUG_RETURN(TRUE); @@ -527,7 +522,6 @@ static my_bool read_mysql_plugin_info(struct st_plugin_dl *plugin_dl, /* Find plugin declarations */ if (!(sym= dlsym(plugin_dl->handle, plugin_declarations_sym))) { - free_plugin_mem(plugin_dl); report_error(report, ER_CANT_FIND_DL_ENTRY, plugin_declarations_sym); DBUG_RETURN(TRUE); } @@ -558,7 +552,6 @@ static my_bool read_mysql_plugin_info(struct st_plugin_dl *plugin_dl, MYF(MY_ZEROFILL|MY_WME)); if (!cur) { - free_plugin_mem(plugin_dl); report_error(report, ER_OUTOFMEMORY, static_cast<int>(plugin_dl->dl.length)); DBUG_RETURN(TRUE); @@ -633,7 +626,6 @@ static my_bool read_maria_plugin_info(struct st_plugin_dl *plugin_dl, Actually this branch impossible because in case of absence of maria version we try mysql version. */ - free_plugin_mem(plugin_dl); report_error(report, ER_CANT_FIND_DL_ENTRY, maria_plugin_interface_version_sym); DBUG_RETURN(TRUE); @@ -644,7 +636,6 @@ static my_bool read_maria_plugin_info(struct st_plugin_dl *plugin_dl, if (plugin_dl->mariaversion < min_maria_plugin_interface_version || (plugin_dl->mariaversion >> 8) > (MARIA_PLUGIN_INTERFACE_VERSION >> 8)) { - free_plugin_mem(plugin_dl); report_error(report, ER_CANT_OPEN_LIBRARY, dlpath, ENOEXEC, "plugin interface version mismatch"); DBUG_RETURN(TRUE); @@ -652,7 +643,6 @@ static my_bool read_maria_plugin_info(struct st_plugin_dl *plugin_dl, /* Find plugin declarations */ if (!(sym= dlsym(plugin_dl->handle, maria_plugin_declarations_sym))) { - free_plugin_mem(plugin_dl); report_error(report, ER_CANT_FIND_DL_ENTRY, maria_plugin_declarations_sym); DBUG_RETURN(TRUE); } @@ -666,7 +656,6 @@ static my_bool read_maria_plugin_info(struct st_plugin_dl *plugin_dl, sizeof_st_plugin= *(int *)sym; else { - free_plugin_mem(plugin_dl); report_error(report, ER_CANT_FIND_DL_ENTRY, maria_sizeof_st_plugin_sym); DBUG_RETURN(TRUE); } @@ -684,7 +673,6 @@ static my_bool read_maria_plugin_info(struct st_plugin_dl *plugin_dl, MYF(MY_ZEROFILL|MY_WME)); if (!cur) { - free_plugin_mem(plugin_dl); report_error(report, ER_OUTOFMEMORY, static_cast<int>(plugin_dl->dl.length)); DBUG_RETURN(TRUE); @@ -716,11 +704,12 @@ static st_plugin_dl *plugin_dl_add(const LEX_STRING *dl, int report) #ifdef HAVE_DLOPEN char dlpath[FN_REFLEN]; uint plugin_dir_len, dummy_errors, dlpathlen, i; - struct st_plugin_dl *tmp, plugin_dl; + struct st_plugin_dl *tmp= 0, plugin_dl; void *sym; DBUG_ENTER("plugin_dl_add"); DBUG_PRINT("enter", ("dl->str: '%s', dl->length: %d", dl->str, (int) dl->length)); + mysql_mutex_assert_owner(&LOCK_plugin); plugin_dir_len= strlen(opt_plugin_dir); /* Ensure that the dll doesn't have a path. @@ -758,8 +747,9 @@ static st_plugin_dl *plugin_dl_add(const LEX_STRING *dl, int report) if (*errmsg == ' ') errmsg++; } report_error(report, ER_CANT_OPEN_LIBRARY, dlpath, errno, errmsg); - DBUG_RETURN(0); + goto ret; } + dlopen_count++; /* Checks which plugin interface present and reads info */ if (!(sym= dlsym(plugin_dl.handle, maria_plugin_interface_version_sym))) @@ -769,12 +759,12 @@ static st_plugin_dl *plugin_dl_add(const LEX_STRING *dl, int report) plugin_interface_version_sym), dlpath, report)) - DBUG_RETURN(0); + goto ret; } else { if (read_maria_plugin_info(&plugin_dl, sym, dlpath, report)) - DBUG_RETURN(0); + goto ret; } /* link the services in */ @@ -791,7 +781,7 @@ static st_plugin_dl *plugin_dl_add(const LEX_STRING *dl, int report) "service '%s' interface version mismatch", list_of_services[i].name); report_error(report, ER_CANT_OPEN_LIBRARY, dlpath, ENOEXEC, buf); - DBUG_RETURN(0); + goto ret; } *(void**)sym= list_of_services[i].service; } @@ -801,10 +791,9 @@ static st_plugin_dl *plugin_dl_add(const LEX_STRING *dl, int report) plugin_dl.dl.length= dl->length * files_charset_info->mbmaxlen + 1; if (! (plugin_dl.dl.str= (char*) my_malloc(plugin_dl.dl.length, MYF(0)))) { - free_plugin_mem(&plugin_dl); report_error(report, ER_OUTOFMEMORY, static_cast<int>(plugin_dl.dl.length)); - DBUG_RETURN(0); + goto ret; } plugin_dl.dl.length= copy_and_convert(plugin_dl.dl.str, plugin_dl.dl.length, files_charset_info, dl->str, dl->length, system_charset_info, @@ -813,12 +802,17 @@ static st_plugin_dl *plugin_dl_add(const LEX_STRING *dl, int report) /* Add this dll to array */ if (! (tmp= plugin_dl_insert_or_reuse(&plugin_dl))) { - free_plugin_mem(&plugin_dl); report_error(report, ER_OUTOFMEMORY, static_cast<int>(sizeof(struct st_plugin_dl))); - DBUG_RETURN(0); + goto ret; } + +ret: + if (!tmp) + free_plugin_mem(&plugin_dl); + DBUG_RETURN(tmp); + #else DBUG_ENTER("plugin_dl_add"); report_error(report, ER_FEATURE_DISABLED, "plugin", "HAVE_DLOPEN"); @@ -827,34 +821,23 @@ static st_plugin_dl *plugin_dl_add(const LEX_STRING *dl, int report) } -static void plugin_dl_del(const LEX_STRING *dl) +static void plugin_dl_del(struct st_plugin_dl *plugin_dl) { -#ifdef HAVE_DLOPEN - uint i; DBUG_ENTER("plugin_dl_del"); + if (!plugin_dl) + DBUG_VOID_RETURN; + mysql_mutex_assert_owner(&LOCK_plugin); - for (i= 0; i < plugin_dl_array.elements; i++) + /* Do not remove this element, unless no other plugin uses this dll. */ + if (! --plugin_dl->ref_count) { - struct st_plugin_dl *tmp= *dynamic_element(&plugin_dl_array, i, - struct st_plugin_dl **); - if (tmp->ref_count && - ! my_strnncoll(files_charset_info, - (const uchar *)dl->str, dl->length, - (const uchar *)tmp->dl.str, tmp->dl.length)) - { - /* Do not remove this element, unless no other plugin uses this dll. */ - if (! --tmp->ref_count) - { - free_plugin_mem(tmp); - bzero(tmp, sizeof(struct st_plugin_dl)); - } - break; - } + free_plugin_mem(plugin_dl); + bzero(plugin_dl, sizeof(struct st_plugin_dl)); } + DBUG_VOID_RETURN; -#endif } @@ -925,7 +908,8 @@ static plugin_ref intern_plugin_lock(LEX *lex, plugin_ref rc) mysql_mutex_assert_owner(&LOCK_plugin); - if (pi->state & (PLUGIN_IS_READY | PLUGIN_IS_UNINITIALIZED)) + if (pi->state & (PLUGIN_IS_READY | PLUGIN_IS_UNINITIALIZED | + PLUGIN_IS_DELETED)) { plugin_ref plugin; #ifdef DBUG_OFF @@ -990,7 +974,7 @@ plugin_ref plugin_lock(THD *thd, plugin_ref ptr) #endif mysql_mutex_lock(&LOCK_plugin); plugin_ref_to_int(ptr)->locks_total++; - rc= my_intern_plugin_lock_ci(lex, ptr); + rc= intern_plugin_lock(lex, ptr); mysql_mutex_unlock(&LOCK_plugin); DBUG_RETURN(rc); } @@ -1004,7 +988,7 @@ plugin_ref plugin_lock_by_name(THD *thd, const LEX_STRING *name, int type) DBUG_ENTER("plugin_lock_by_name"); mysql_mutex_lock(&LOCK_plugin); if ((plugin= plugin_find_internal(name, type))) - rc= my_intern_plugin_lock_ci(lex, plugin_int_to_ref(plugin)); + rc= intern_plugin_lock(lex, plugin_int_to_ref(plugin)); mysql_mutex_unlock(&LOCK_plugin); DBUG_RETURN(rc); } @@ -1045,6 +1029,8 @@ static bool plugin_add(MEM_ROOT *tmp_root, struct st_maria_plugin *plugin; uint oks= 0, errs= 0; DBUG_ENTER("plugin_add"); + DBUG_PRINT("enter", ("name: %s dl: %s", name->str, dl->str)); + if (name->str && plugin_find_internal(name, MYSQL_ANY_PLUGIN)) { report_error(report, ER_UDF_EXISTS, name->str); @@ -1108,7 +1094,7 @@ static bool plugin_add(MEM_ROOT *tmp_root, plugin_array_version++; if (my_hash_insert(&plugin_hash[plugin->type], (uchar*)tmp_plugin_ptr)) tmp_plugin_ptr->state= PLUGIN_IS_FREED; - init_alloc_root(&tmp_plugin_ptr->mem_root, 4096, 4096); + init_alloc_root(&tmp_plugin_ptr->mem_root, 4096, 4096, MYF(0)); if (name->str) DBUG_RETURN(FALSE); // all done @@ -1126,7 +1112,7 @@ err: if (errs == 0 && oks == 0) // no plugin was found report_error(report, ER_CANT_FIND_DL_ENTRY, name->str); - plugin_dl_del(dl); + plugin_dl_del(tmp.plugin_dl); DBUG_RETURN(errs > 0 || oks == 0); } @@ -1142,22 +1128,21 @@ static void plugin_deinitialize(struct st_plugin_int *plugin, bool ref_check) if (plugin->plugin->status_vars) { -#ifdef FIX_LATER - /** - @todo - unfortunately, status variables were introduced without a - pluginname_ namespace, that is pluginname_ was not added automatically - to status variable names. It should be fixed together with the next - incompatible API change. + /* + historical ndb behavior caused MySQL plugins to specify + status var names in full, with the plugin name prefix. + this was never fixed in MySQL. + MariaDB fixes that but support MySQL style too. */ - SHOW_VAR array[2]= { + SHOW_VAR *show_vars= plugin->plugin->status_vars; + SHOW_VAR tmp_array[2]= { {plugin->plugin->name, (char*)plugin->plugin->status_vars, SHOW_ARRAY}, {0, 0, SHOW_UNDEF} }; - remove_status_vars(array); -#else - remove_status_vars(plugin->plugin->status_vars); -#endif /* FIX_LATER */ + if (strncasecmp(show_vars->name, plugin->name.str, plugin->name.length)) + show_vars= tmp_array; + + remove_status_vars(show_vars); } if (plugin_type_deinitialize[plugin->plugin->type]) @@ -1201,8 +1186,7 @@ static void plugin_del(struct st_plugin_int *plugin) /* Free allocated strings before deleting the plugin. */ plugin_vars_free_values(plugin->system_vars); my_hash_delete(&plugin_hash[plugin->plugin->type], (uchar*)plugin); - if (plugin->plugin_dl) - plugin_dl_del(&plugin->plugin_dl->dl); + plugin_dl_del(plugin->plugin_dl); plugin->state= PLUGIN_IS_FREED; plugin_array_version++; free_root(&plugin->mem_root, MYF(0)); @@ -1389,24 +1373,22 @@ static int plugin_initialize(MEM_ROOT *tmp_root, struct st_plugin_int *plugin, if (plugin->plugin->status_vars) { -#ifdef FIX_LATER /* - We have a problem right now where we can not prepend without - breaking backwards compatibility. We will fix this shortly so - that engines have "use names" and we wil use those for - CREATE TABLE, and use the plugin name then for adding automatic - variable names. + historical ndb behavior caused MySQL plugins to specify + status var names in full, with the plugin name prefix. + this was never fixed in MySQL. + MariaDB fixes that, but supports MySQL style too. */ - SHOW_VAR array[2]= { + SHOW_VAR *show_vars= plugin->plugin->status_vars; + SHOW_VAR tmp_array[2]= { {plugin->plugin->name, (char*)plugin->plugin->status_vars, SHOW_ARRAY}, {0, 0, SHOW_UNDEF} }; - if (add_status_vars(array)) // add_status_vars makes a copy - goto err; -#else - if (add_status_vars(plugin->plugin->status_vars)) + if (strncasecmp(show_vars->name, plugin->name.str, plugin->name.length)) + show_vars= tmp_array; + + if (add_status_vars(show_vars)) goto err; -#endif /* FIX_LATER */ } /* @@ -1521,13 +1503,15 @@ int plugin_init(int *argc, char **argv, int flags) if (initialized) DBUG_RETURN(0); + dlopen_count =0; + #ifdef HAVE_PSI_INTERFACE init_plugin_psi_keys(); #endif - init_alloc_root(&plugin_mem_root, 4096, 4096); - init_alloc_root(&plugin_vars_mem_root, 4096, 4096); - init_alloc_root(&tmp_root, 4096, 4096); + init_alloc_root(&plugin_mem_root, 4096, 4096, MYF(0)); + init_alloc_root(&plugin_vars_mem_root, 4096, 4096, MYF(0)); + init_alloc_root(&tmp_root, 4096, 4096, MYF(0)); if (my_hash_init(&bookmark_hash, &my_charset_bin, 16, 0, 0, get_bookmark_hash_key, NULL, HASH_UNIQUE)) @@ -1537,9 +1521,9 @@ int plugin_init(int *argc, char **argv, int flags) mysql_mutex_init(key_LOCK_plugin, &LOCK_plugin, MY_MUTEX_INIT_FAST); if (my_init_dynamic_array(&plugin_dl_array, - sizeof(struct st_plugin_dl *),16,16) || + sizeof(struct st_plugin_dl *), 16, 16, MYF(0)) || my_init_dynamic_array(&plugin_array, - sizeof(struct st_plugin_int *),16,16)) + sizeof(struct st_plugin_int *), 16, 16, MYF(0))) goto err; for (i= 0; i < MYSQL_MAX_PLUGIN_TYPE_NUM; i++) @@ -1550,8 +1534,8 @@ int plugin_init(int *argc, char **argv, int flags) } /* prepare debug_sync service */ - DBUG_ASSERT(strcmp(list_of_services[5].name, "debug_sync_service") == 0); - list_of_services[5].service= *(void**)&debug_sync_C_callback_ptr; + DBUG_ASSERT(strcmp(list_of_services[4].name, "debug_sync_service") == 0); + list_of_services[4].service= *(void**)&debug_sync_C_callback_ptr; mysql_mutex_lock(&LOCK_plugin); @@ -1625,7 +1609,7 @@ int plugin_init(int *argc, char **argv, int flags) { DBUG_ASSERT(!global_system_variables.table_plugin); global_system_variables.table_plugin= - my_intern_plugin_lock(NULL, plugin_int_to_ref(plugin_ptr)); + intern_plugin_lock(NULL, plugin_int_to_ref(plugin_ptr)); DBUG_ASSERT(plugin_ptr->ref_count == 1); } } @@ -1726,40 +1710,27 @@ static bool register_builtin(struct st_maria_plugin *plugin, */ static void plugin_load(MEM_ROOT *tmp_root) { - THD thd; TABLE_LIST tables; TABLE *table; READ_RECORD read_record_info; int error; - THD *new_thd= &thd; + THD *new_thd= new THD; bool result; -#ifdef EMBEDDED_LIBRARY - No_such_table_error_handler error_handler; -#endif /* EMBEDDED_LIBRARY */ DBUG_ENTER("plugin_load"); new_thd->thread_stack= (char*) &tables; new_thd->store_globals(); new_thd->db= my_strdup("mysql", MYF(0)); new_thd->db_length= 5; - bzero((char*) &thd.net, sizeof(thd.net)); + bzero((char*) &new_thd->net, sizeof(new_thd->net)); tables.init_one_table("mysql", 5, "plugin", 6, "plugin", TL_READ); - -#ifdef EMBEDDED_LIBRARY - /* - When building an embedded library, if the mysql.plugin table - does not exist, we silently ignore the missing table - */ - new_thd->push_internal_handler(&error_handler); -#endif /* EMBEDDED_LIBRARY */ + tables.open_strategy= TABLE_LIST:: IF_EMBEDDED(OPEN_IF_EXISTS, OPEN_NORMAL); result= open_and_lock_tables(new_thd, &tables, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT); -#ifdef EMBEDDED_LIBRARY - new_thd->pop_internal_handler(); - if (error_handler.safely_trapped_errors()) + table= tables.table; + if (IF_EMBEDDED(!table, false)) goto end; -#endif /* EMBEDDED_LIBRARY */ if (result) { @@ -1771,7 +1742,7 @@ static void plugin_load(MEM_ROOT *tmp_root) sql_print_warning("Could not open mysql.plugin table. Some options may be missing from the help text"); goto end; } - table= tables.table; + if (init_read_record(&read_record_info, new_thd, table, NULL, 1, 0, FALSE)) { sql_print_error("Could not initialize init_read_record; Plugins not " @@ -1803,13 +1774,14 @@ static void plugin_load(MEM_ROOT *tmp_root) mysql_mutex_unlock(&LOCK_plugin); } if (error > 0) - sql_print_error(ER(ER_GET_ERRNO), my_errno); + sql_print_error(ER(ER_GET_ERRNO), my_errno, table->file->table_type()); end_read_record(&read_record_info); table->m_needs_reopen= TRUE; // Force close to free memory close_mysql_tables(new_thd); end: /* Remember that we don't have a THD */ - my_pthread_setspecific_ptr(THR_THD, 0); + delete new_thd; + set_current_thd(0); DBUG_VOID_RETURN; } @@ -2369,6 +2341,74 @@ err: } +static bool plugin_dl_foreach_internal(THD *thd, st_plugin_dl *plugin_dl, + st_maria_plugin *plug, + plugin_foreach_func *func, void *arg) +{ + for (; plug->name; plug++) + { + st_plugin_int tmp, *plugin; + + tmp.name.str= const_cast<char*>(plug->name); + tmp.name.length= strlen(plug->name); + tmp.plugin= plug; + tmp.plugin_dl= plugin_dl; + + mysql_mutex_lock(&LOCK_plugin); + if ((plugin= plugin_find_internal(&tmp.name, MYSQL_ANY_PLUGIN)) && + plugin->plugin == plug) + + { + tmp.state= plugin->state; + tmp.load_option= plugin->load_option; + } + else + { + tmp.state= PLUGIN_IS_FREED; + tmp.load_option= PLUGIN_OFF; + } + mysql_mutex_unlock(&LOCK_plugin); + + plugin= &tmp; + if (func(thd, plugin_int_to_ref(plugin), arg)) + return 1; + } + return 0; +} + +bool plugin_dl_foreach(THD *thd, const LEX_STRING *dl, + plugin_foreach_func *func, void *arg) +{ + bool err= 0; + + if (dl) + { + mysql_mutex_lock(&LOCK_plugin); + st_plugin_dl *plugin_dl= plugin_dl_add(dl, REPORT_TO_USER); + mysql_mutex_unlock(&LOCK_plugin); + + if (!plugin_dl) + return 1; + + err= plugin_dl_foreach_internal(thd, plugin_dl, plugin_dl->plugins, + func, arg); + + mysql_mutex_lock(&LOCK_plugin); + plugin_dl_del(plugin_dl); + mysql_mutex_unlock(&LOCK_plugin); + } + else + { + struct st_maria_plugin **builtins; + for (builtins= mysql_mandatory_plugins; !err && *builtins; builtins++) + err= plugin_dl_foreach_internal(thd, 0, *builtins, func, arg); + for (builtins= mysql_optional_plugins; !err && *builtins; builtins++) + err= plugin_dl_foreach_internal(thd, 0, *builtins, func, arg); + } + return err; +} + + /**************************************************************************** Internal type declarations for variables support ****************************************************************************/ @@ -2711,7 +2751,7 @@ sys_var *find_sys_var(THD *thd, const char *str, uint length) { mysql_rwlock_unlock(&LOCK_system_variables_hash); LEX *lex= thd ? thd->lex : 0; - if (!(plugin= my_intern_plugin_lock(lex, plugin_int_to_ref(pi->plugin)))) + if (!(plugin= intern_plugin_lock(lex, plugin_int_to_ref(pi->plugin)))) var= NULL; /* failed to lock it, it must be uninstalling */ else if (!(plugin_state(plugin) & PLUGIN_IS_READY)) @@ -3044,7 +3084,7 @@ void plugin_thdvar_init(THD *thd) mysql_mutex_lock(&LOCK_plugin); thd->variables.table_plugin= - my_intern_plugin_lock(NULL, global_system_variables.table_plugin); + intern_plugin_lock(NULL, global_system_variables.table_plugin); intern_plugin_unlock(NULL, old_table_plugin); mysql_mutex_unlock(&LOCK_plugin); DBUG_VOID_RETURN; @@ -3372,8 +3412,8 @@ bool sys_var_pluginvar::global_update(THD *thd, set_var *var) options->block_size= (long) (opt)->blk_sz; -static void plugin_opt_set_limits(struct my_option *options, - const struct st_mysql_sys_var *opt) +void plugin_opt_set_limits(struct my_option *options, + const struct st_mysql_sys_var *opt) { options->sub_size= 0; @@ -3479,17 +3519,6 @@ static void plugin_opt_set_limits(struct my_option *options, options->arg_type= OPT_ARG; } -extern "C" my_bool get_one_plugin_option(int optid, const struct my_option *, - char *); - -my_bool get_one_plugin_option(int optid __attribute__((unused)), - const struct my_option *opt, - char *argument) -{ - return 0; -} - - /** Creates a set of my_option objects associated with a specified plugin- handle. @@ -3978,3 +4007,18 @@ void add_plugin_options(DYNAMIC_ARRAY *options, MEM_ROOT *mem_root) } } + +/** + Returns a sys_var corresponding to a particular MYSQL_SYSVAR(...) +*/ +sys_var *find_plugin_sysvar(st_plugin_int *plugin, st_mysql_sys_var *plugin_var) +{ + for (sys_var *var= plugin->system_vars; var; var= var->next) + { + sys_var_pluginvar *pvar=var->cast_pluginvar(); + if (pvar->plugin_var == plugin_var) + return var; + } + return 0; +} + diff --git a/sql/sql_plugin.h b/sql/sql_plugin.h index be1cfcdcc4f..11c91fe93eb 100644 --- a/sql/sql_plugin.h +++ b/sql/sql_plugin.h @@ -39,6 +39,8 @@ enum enum_plugin_load_option { PLUGIN_OFF, PLUGIN_ON, PLUGIN_FORCE, PLUGIN_FORCE_PLUS_PERMANENT }; extern const char *global_plugin_typelib_names[]; +extern ulong dlopen_count; + #include <my_sys.h> #ifdef DBUG_OFF @@ -150,9 +152,7 @@ extern void plugin_shutdown(void); void add_plugin_options(DYNAMIC_ARRAY *options, MEM_ROOT *mem_root); extern bool plugin_is_ready(const LEX_STRING *name, int type); #define my_plugin_lock_by_name(A,B,C) plugin_lock_by_name(A,B,C) -#define my_plugin_lock_by_name_ci(A,B,C) plugin_lock_by_name(A,B,C) #define my_plugin_lock(A,B) plugin_lock(A,B) -#define my_plugin_lock_ci(A,B) plugin_lock(A,B) extern plugin_ref plugin_lock(THD *thd, plugin_ref ptr); extern plugin_ref plugin_lock_by_name(THD *thd, const LEX_STRING *name, int type); @@ -165,6 +165,8 @@ extern bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, extern bool plugin_register_builtin(struct st_mysql_plugin *plugin); extern void plugin_thdvar_init(THD *thd); extern void plugin_thdvar_cleanup(THD *thd); +sys_var *find_plugin_sysvar(st_plugin_int *plugin, st_mysql_sys_var *var); +void plugin_opt_set_limits(struct my_option *, const struct st_mysql_sys_var *); extern SHOW_COMP_OPTION plugin_status(const char *name, size_t len, int type); extern bool check_valid_path(const char *path, size_t length); @@ -174,4 +176,6 @@ typedef my_bool (plugin_foreach_func)(THD *thd, #define plugin_foreach(A,B,C,D) plugin_foreach_with_mask(A,B,C,PLUGIN_IS_READY,D) extern bool plugin_foreach_with_mask(THD *thd, plugin_foreach_func *func, int type, uint state_mask, void *arg); +extern bool plugin_dl_foreach(THD *thd, const LEX_STRING *dl, + plugin_foreach_func *func, void *arg); #endif diff --git a/sql/sql_plugin_services.h b/sql/sql_plugin_services.h index ede8d9a675e..d983b01a6bd 100644 --- a/sql/sql_plugin_services.h +++ b/sql/sql_plugin_services.h @@ -1,4 +1,5 @@ -/* Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2009, 2010, Oracle and/or its affiliates. + Copyright (c) 2012, 2013, Monty Program Ab This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -41,11 +42,6 @@ static struct thd_wait_service_st thd_wait_handler= { thd_wait_end }; -static struct my_thread_scheduler_service my_thread_scheduler_handler= { - my_thread_scheduler_set, - my_thread_scheduler_reset, -}; - static struct progress_report_service_st progress_report_handler= { thd_progress_init, thd_progress_report, @@ -73,7 +69,6 @@ static struct st_service_ref list_of_services[]= { "my_snprintf_service", VERSION_my_snprintf, &my_snprintf_handler }, { "thd_alloc_service", VERSION_thd_alloc, &thd_alloc_handler }, { "thd_wait_service", VERSION_thd_wait, &thd_wait_handler }, - { "my_thread_scheduler_service", VERSION_my_thread_scheduler, &my_thread_scheduler_handler }, { "progress_report_service", VERSION_progress_report, &progress_report_handler }, { "debug_sync_service", VERSION_debug_sync, 0 }, // updated in plugin_init() { "thd_kill_statement_service", VERSION_kill_statement, &thd_kill_statement_handler }, diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index e948813584d..12b74b40815 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -1460,7 +1460,10 @@ static bool mysql_test_delete(Prepared_statement *stmt, goto error; } - DBUG_RETURN(mysql_prepare_delete(thd, table_list, &lex->select_lex.where)); + DBUG_RETURN(mysql_prepare_delete(thd, table_list, + lex->select_lex.with_wild, + lex->select_lex.item_list, + &lex->select_lex.where)); error: DBUG_RETURN(TRUE); } @@ -2144,6 +2147,7 @@ static bool check_prepared_statement(Prepared_statement *stmt) Note that we don't need to have cases in this list if they are marked with CF_STATUS_COMMAND in sql_command_flags */ + case SQLCOM_SHOW_EXPLAIN: case SQLCOM_DROP_TABLE: case SQLCOM_RENAME_TABLE: case SQLCOM_ALTER_TABLE: @@ -2161,6 +2165,8 @@ static bool check_prepared_statement(Prepared_statement *stmt) case SQLCOM_FLUSH: case SQLCOM_SLAVE_START: case SQLCOM_SLAVE_STOP: + case SQLCOM_SLAVE_ALL_START: + case SQLCOM_SLAVE_ALL_STOP: case SQLCOM_INSTALL_PLUGIN: case SQLCOM_UNINSTALL_PLUGIN: case SQLCOM_CREATE_DB: @@ -2175,6 +2181,7 @@ static bool check_prepared_statement(Prepared_statement *stmt) case SQLCOM_GRANT: case SQLCOM_REVOKE: case SQLCOM_KILL: + case SQLCOM_SHUTDOWN: break; case SQLCOM_PREPARE: @@ -2485,6 +2492,7 @@ void reinit_stmt_before_use(THD *thd, LEX *lex) object and because of this can be used in different threads. */ lex->thd= thd; + DBUG_ASSERT(!lex->explain); if (lex->empty_field_list_on_rset) { @@ -3146,7 +3154,7 @@ Prepared_statement::Prepared_statement(THD *thd_arg) flags((uint) IS_IN_USE) { init_sql_alloc(&main_mem_root, thd_arg->variables.query_alloc_block_size, - thd_arg->variables.query_prealloc_size); + thd_arg->variables.query_prealloc_size, MYF(MY_THREAD_SPECIFIC)); *last_error= '\0'; } @@ -3952,6 +3960,21 @@ bool Prepared_statement::execute(String *expanded_query, bool open_cursor) if (! cursor) cleanup_stmt(); + + /* + EXECUTE command has its own dummy "explain data". We don't need it, + instead, we want to keep the query plan of the statement that was + executed. + */ + if (!stmt_backup.lex->explain || + !stmt_backup.lex->explain->have_query_plan()) + { + delete_explain_query(stmt_backup.lex); + stmt_backup.lex->explain = thd->lex->explain; + thd->lex->explain= NULL; + } + else + delete_explain_query(thd->lex); thd->set_statement(&stmt_backup); thd->stmt_arena= old_stmt_arena; @@ -4058,7 +4081,7 @@ Ed_result_set::Ed_result_set(List<Ed_row> *rows_arg, */ Ed_connection::Ed_connection(THD *thd) - :m_warning_info(thd->query_id, false), + :m_warning_info(thd->query_id, false, true), m_thd(thd), m_rsets(0), m_current_rset(0) @@ -4482,7 +4505,7 @@ bool Protocol_local::send_result_set_metadata(List<Item> *columns, uint) { DBUG_ASSERT(m_rset == 0 && !alloc_root_inited(&m_rset_root)); - init_sql_alloc(&m_rset_root, MEM_ROOT_BLOCK_SIZE, 0); + init_sql_alloc(&m_rset_root, MEM_ROOT_BLOCK_SIZE, 0, MYF(MY_THREAD_SPECIFIC)); if (! (m_rset= new (&m_rset_root) List<Ed_row>)) return TRUE; diff --git a/sql/sql_priv.h b/sql/sql_priv.h index 6e778c09bd8..c86b585fc45 100644 --- a/sql/sql_priv.h +++ b/sql/sql_priv.h @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates. - Copyright (c) 2010-2011 Monty Program Ab + Copyright (c) 2010, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -143,15 +143,15 @@ however, needs to rollback the effects of the succeeded statement to keep replication consistent. */ -#define OPTION_MASTER_SQL_ERROR (1ULL << 35) +#define OPTION_MASTER_SQL_ERROR (1ULL << 35) /* Dont report errors for individual rows, But just report error on commit (or read ofcourse) Note! Reserved for use in MySQL Cluster */ -#define OPTION_ALLOW_BATCH (ULL(1) << 36) // THD, intern (slave) -#define OPTION_SKIP_REPLICATION (ULL(1) << 37) // THD, user +#define OPTION_ALLOW_BATCH (1ULL << 36) // THD, intern (slave) +#define OPTION_SKIP_REPLICATION (1ULL << 37) // THD, user /* Check how many bytes are available on buffer. @@ -227,7 +227,8 @@ template <class T> bool valid_buffer_range(T jump, #define OPTIMIZER_SWITCH_OPTIMIZE_JOIN_BUFFER_SIZE (1ULL << 25) #define OPTIMIZER_SWITCH_TABLE_ELIMINATION (1ULL << 26) #define OPTIMIZER_SWITCH_EXTENDED_KEYS (1ULL << 27) -#define OPTIMIZER_SWITCH_LAST (1ULL << 27) +#define OPTIMIZER_SWITCH_EXISTS_TO_IN (1ULL << 28) +#define OPTIMIZER_SWITCH_USE_CONDITION_SELECTIVITY (1ULL << 29) #define OPTIMIZER_SWITCH_DEFAULT (OPTIMIZER_SWITCH_INDEX_MERGE | \ OPTIMIZER_SWITCH_INDEX_MERGE_UNION | \ @@ -237,6 +238,7 @@ template <class T> bool valid_buffer_range(T jump, OPTIMIZER_SWITCH_DERIVED_MERGE | \ OPTIMIZER_SWITCH_DERIVED_WITH_KEYS | \ OPTIMIZER_SWITCH_TABLE_ELIMINATION | \ + OPTIMIZER_SWITCH_EXTENDED_KEYS | \ OPTIMIZER_SWITCH_IN_TO_EXISTS | \ OPTIMIZER_SWITCH_MATERIALIZATION | \ OPTIMIZER_SWITCH_PARTIAL_MATCH_ROWID_MERGE|\ @@ -362,10 +364,12 @@ enum enum_yes_no_unknown }; #ifdef MYSQL_SERVER + /* External variables */ + /* sql_yacc.cc */ #ifndef DBUG_OFF extern void turn_parser_debug_on(); diff --git a/sql/sql_reload.cc b/sql/sql_reload.cc index 914b9026014..8a05732173d 100644 --- a/sql/sql_reload.cc +++ b/sql/sql_reload.cc @@ -26,6 +26,7 @@ #include "sql_repl.h" // reset_master, reset_slave #include "rpl_mi.h" // Master_info::data_lock #include "debug_sync.h" +#include "rpl_mi.h" static void disable_checkpoints(THD *thd); @@ -96,7 +97,7 @@ bool reload_acl_and_cache(THD *thd, unsigned long options, { delete tmp_thd; /* Remember that we don't have a THD */ - my_pthread_setspecific_ptr(THR_THD, 0); + set_current_thd(0); thd= 0; } reset_mqh((LEX_USER *)NULL, TRUE); @@ -157,10 +158,36 @@ bool reload_acl_and_cache(THD *thd, unsigned long options, if (options & REFRESH_RELAY_LOG) { #ifdef HAVE_REPLICATION - mysql_mutex_lock(&active_mi->data_lock); - if (rotate_relay_log(active_mi)) - *write_to_binlog= -1; - mysql_mutex_unlock(&active_mi->data_lock); + LEX_STRING connection_name; + Master_info *mi; + if (thd) + connection_name= thd->lex->relay_log_connection_name; + else + { + connection_name.str= (char*) ""; + connection_name.length= 0; + } + + /* + Writing this command to the binlog may cause problems as the + slave is not likely to have the same connection names. + */ + tmp_write_to_binlog= 0; + mysql_mutex_lock(&LOCK_active_mi); + if (!(mi= (master_info_index-> + get_master_info(&connection_name, + MYSQL_ERROR::WARN_LEVEL_ERROR)))) + { + result= 1; + } + else + { + mysql_mutex_lock(&mi->data_lock); + if (rotate_relay_log(mi)) + *write_to_binlog= -1; + mysql_mutex_unlock(&mi->data_lock); + } + mysql_mutex_unlock(&LOCK_active_mi); #endif } #ifdef HAVE_QUERY_CACHE @@ -178,6 +205,7 @@ bool reload_acl_and_cache(THD *thd, unsigned long options, DBUG_ASSERT(!thd || thd->locked_tables_mode || !thd->mdl_context.has_locks() || thd->handler_tables_hash.records || + thd->ull_hash.records || thd->global_read_lock.is_acquired()); /* @@ -294,7 +322,7 @@ bool reload_acl_and_cache(THD *thd, unsigned long options, { DBUG_ASSERT(thd); tmp_write_to_binlog= 0; - if (reset_master(thd)) + if (reset_master(thd, NULL, 0)) { /* NOTE: my_error() has been already called by reset_master(). */ result= 1; @@ -314,13 +342,27 @@ bool reload_acl_and_cache(THD *thd, unsigned long options, #ifdef HAVE_REPLICATION if (options & REFRESH_SLAVE) { + LEX_MASTER_INFO* lex_mi= &thd->lex->mi; + Master_info *mi; tmp_write_to_binlog= 0; mysql_mutex_lock(&LOCK_active_mi); - if (reset_slave(thd, active_mi)) + + if (!(mi= (master_info_index-> + get_master_info(&lex_mi->connection_name, + MYSQL_ERROR::WARN_LEVEL_ERROR)))) + { + result= 1; + } + else if (reset_slave(thd, mi)) { /* NOTE: my_error() has been already called by reset_slave(). */ result= 1; } + else if (mi->connection_name.length && thd->lex->reset_slave_info.all) + { + /* If not default connection and 'all' is used */ + master_info_index->remove_master_info(&mi->connection_name); + } mysql_mutex_unlock(&LOCK_active_mi); } #endif diff --git a/sql/sql_rename.cc b/sql/sql_rename.cc index ee7c0fd2f73..295a55e1eb1 100644 --- a/sql/sql_rename.cc +++ b/sql/sql_rename.cc @@ -29,10 +29,13 @@ #include "lock.h" // MYSQL_OPEN_SKIP_TEMPORARY #include "sql_base.h" // tdc_remove_table, lock_table_names, #include "sql_handler.h" // mysql_ha_rm_tables -#include "datadict.h" +#include "sql_statistics.h" static TABLE_LIST *rename_tables(THD *thd, TABLE_LIST *table_list, bool skip_error); +static bool do_rename(THD *thd, TABLE_LIST *ren_table, char *new_db, + char *new_table_name, char *new_table_alias, + bool skip_error); static TABLE_LIST *reverse_table_list(TABLE_LIST *table_list); @@ -145,10 +148,6 @@ bool mysql_rename_tables(THD *thd, TABLE_LIST *table_list, bool silent) MYSQL_OPEN_SKIP_TEMPORARY)) goto err; - for (ren_table= table_list; ren_table; ren_table= ren_table->next_local) - tdc_remove_table(thd, TDC_RT_REMOVE_ALL, ren_table->db, - ren_table->table_name, FALSE); - error=0; /* An exclusive lock on table names is satisfactory to ensure @@ -236,16 +235,14 @@ static TABLE_LIST *reverse_table_list(TABLE_LIST *table_list) true rename failed */ -bool +static bool do_rename(THD *thd, TABLE_LIST *ren_table, char *new_db, char *new_table_name, char *new_table_alias, bool skip_error) { int rc= 1; - char new_name[FN_REFLEN + 1], old_name[FN_REFLEN + 1]; + handlerton *hton; + bool new_exists, old_exists; const char *new_alias, *old_alias; - frm_type_enum frm_type; - enum legacy_db_type table_type; - DBUG_ENTER("do_rename"); if (lower_case_table_names == 2) @@ -260,47 +257,52 @@ do_rename(THD *thd, TABLE_LIST *ren_table, char *new_db, char *new_table_name, } DBUG_ASSERT(new_alias); - build_table_filename(new_name, sizeof(new_name) - 1, - new_db, new_alias, reg_ext, 0); - build_table_filename(old_name, sizeof(old_name) - 1, - ren_table->db, old_alias, reg_ext, 0); - if (check_table_file_presence(old_name, - new_name, new_db, new_alias, new_alias, TRUE)) + new_exists= ha_table_exists(thd, new_db, new_alias); + + if (new_exists) { - DBUG_RETURN(1); // This can't be skipped + my_error(ER_TABLE_EXISTS_ERROR, MYF(0), new_alias); + DBUG_RETURN(1); // This can't be skipped } - frm_type= dd_frm_type(thd, old_name, &table_type); - switch (frm_type) + old_exists= ha_table_exists(thd, ren_table->db, old_alias, &hton); + + if (old_exists) { - case FRMTYPE_TABLE: + DBUG_ASSERT(!thd->locked_tables_mode); + tdc_remove_table(thd, TDC_RT_REMOVE_ALL, + ren_table->db, ren_table->table_name, false); + + if (hton != view_pseudo_hton) + { + if (!(rc= mysql_rename_table(hton, ren_table->db, old_alias, + new_db, new_alias, 0))) { - if (!(rc= mysql_rename_table(ha_resolve_by_legacy_type(thd, - table_type), - ren_table->db, old_alias, - new_db, new_alias, 0))) + LEX_STRING db_name= { ren_table->db, ren_table->db_length }; + LEX_STRING table_name= { ren_table->table_name, + ren_table->table_name_length }; + LEX_STRING new_table= { (char *) new_alias, strlen(new_alias) }; + (void) rename_table_in_stat_tables(thd, &db_name, &table_name, + &db_name, &new_table); + if ((rc= Table_triggers_list::change_table_name(thd, ren_table->db, + old_alias, + ren_table->table_name, + new_db, + new_alias))) { - if ((rc= Table_triggers_list::change_table_name(thd, ren_table->db, - old_alias, - ren_table->table_name, - new_db, - new_alias))) - { - /* - We've succeeded in renaming table's .frm and in updating - corresponding handler data, but have failed to update table's - triggers appropriately. So let us revert operations on .frm - and handler's data and report about failure to rename table. - */ - (void) mysql_rename_table(ha_resolve_by_legacy_type(thd, - table_type), - new_db, new_alias, - ren_table->db, old_alias, NO_FK_CHECKS); - } + /* + We've succeeded in renaming table's .frm and in updating + corresponding handler data, but have failed to update table's + triggers appropriately. So let us revert operations on .frm + and handler's data and report about failure to rename table. + */ + (void) mysql_rename_table(hton, new_db, new_alias, + ren_table->db, old_alias, NO_FK_CHECKS); } } - break; - case FRMTYPE_VIEW: + } + else + { /* change of schema is not allowed except of ALTER ...UPGRADE DATA DIRECTORY NAME command @@ -308,22 +310,19 @@ do_rename(THD *thd, TABLE_LIST *ren_table, char *new_db, char *new_table_name, */ if (thd->lex->sql_command != SQLCOM_ALTER_DB_UPGRADE && strcmp(ren_table->db, new_db)) - my_error(ER_FORBID_SCHEMA_CHANGE, MYF(0), ren_table->db, - new_db); + my_error(ER_FORBID_SCHEMA_CHANGE, MYF(0), ren_table->db, new_db); else rc= mysql_rename_view(thd, new_db, new_alias, ren_table); - break; - default: - DBUG_ASSERT(0); // should never happen - case FRMTYPE_ERROR: - my_error(ER_FILE_NOT_FOUND, MYF(0), old_name, my_errno); - break; + } + } + else + { + my_error(ER_NO_SUCH_TABLE, MYF(0), ren_table->db, old_alias); } if (rc && !skip_error) DBUG_RETURN(1); DBUG_RETURN(0); - } /* Rename all tables in list; Return pointer to wrong entry if something goes diff --git a/sql/sql_rename.h b/sql/sql_rename.h index 039a3b8b4a1..aaf09a8d030 100644 --- a/sql/sql_rename.h +++ b/sql/sql_rename.h @@ -20,8 +20,5 @@ class THD; struct TABLE_LIST; bool mysql_rename_tables(THD *thd, TABLE_LIST *table_list, bool silent); -bool do_rename(THD *thd, TABLE_LIST *ren_table, char *new_db, - char *new_table_name, char *new_table_alias, - bool skip_error); #endif /* SQL_RENAME_INCLUDED */ diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index dac4077430c..33a0ab5b4f1 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2008, 2012, Monty Program Ab. + Copyright (c) 2008, 2013, Monty Program Ab This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -16,10 +16,12 @@ #include "sql_priv.h" #include "unireg.h" +#include "sql_base.h" #include "sql_parse.h" // check_access #ifdef HAVE_REPLICATION #include "rpl_mi.h" +#include "rpl_rli.h" #include "sql_repl.h" #include "sql_acl.h" // SUPER_ACL #include "log_event.h" @@ -28,22 +30,90 @@ #include "rpl_handler.h" #include "debug_sync.h" + +enum enum_gtid_until_state { + GTID_UNTIL_NOT_DONE, + GTID_UNTIL_STOP_AFTER_STANDALONE, + GTID_UNTIL_STOP_AFTER_TRANSACTION +}; + + int max_binlog_dump_events = 0; // unlimited my_bool opt_sporadic_binlog_dump_fail = 0; #ifndef DBUG_OFF static int binlog_dump_count = 0; #endif -/** - a copy of active_mi->rli->slave_skip_counter, for showing in SHOW VARIABLES, - INFORMATION_SCHEMA.GLOBAL_VARIABLES and @@sql_slave_skip_counter without - taking all the mutexes needed to access active_mi->rli->slave_skip_counter - properly. -*/ -uint sql_slave_skip_counter; - extern TYPELIB binlog_checksum_typelib; + +static int +fake_event_header(String* packet, Log_event_type event_type, ulong extra_len, + my_bool *do_checksum, ha_checksum *crc, const char** errmsg, + uint8 checksum_alg_arg, uint32 end_pos) +{ + char header[LOG_EVENT_HEADER_LEN]; + ulong event_len; + + *do_checksum= checksum_alg_arg != BINLOG_CHECKSUM_ALG_OFF && + checksum_alg_arg != BINLOG_CHECKSUM_ALG_UNDEF; + + /* + 'when' (the timestamp) is set to 0 so that slave could distinguish between + real and fake Rotate events (if necessary) + */ + memset(header, 0, 4); + header[EVENT_TYPE_OFFSET] = (uchar)event_type; + event_len= LOG_EVENT_HEADER_LEN + extra_len + + (*do_checksum ? BINLOG_CHECKSUM_LEN : 0); + int4store(header + SERVER_ID_OFFSET, global_system_variables.server_id); + int4store(header + EVENT_LEN_OFFSET, event_len); + int2store(header + FLAGS_OFFSET, LOG_EVENT_ARTIFICIAL_F); + // TODO: check what problems this may cause and fix them + int4store(header + LOG_POS_OFFSET, end_pos); + if (packet->append(header, sizeof(header))) + { + *errmsg= "Failed due to out-of-memory writing event"; + return -1; + } + if (*do_checksum) + { + *crc= my_checksum(0L, NULL, 0); + *crc= my_checksum(*crc, (uchar*)header, sizeof(header)); + } + return 0; +} + + +static int +fake_event_footer(String *packet, my_bool do_checksum, ha_checksum crc, const char **errmsg) +{ + if (do_checksum) + { + char b[BINLOG_CHECKSUM_LEN]; + int4store(b, crc); + if (packet->append(b, sizeof(b))) + { + *errmsg= "Failed due to out-of-memory writing event checksum"; + return -1; + } + } + return 0; +} + + +static int +fake_event_write(NET *net, String *packet, const char **errmsg) +{ + if (my_net_write(net, (uchar*) packet->ptr(), packet->length())) + { + *errmsg = "failed on my_net_write()"; + return -1; + } + return 0; +} + + /* fake_rotate_event() builds a fake (=which does not exist physically in any binlog) Rotate event, which contains the name of the binlog we are going to @@ -67,59 +137,71 @@ static int fake_rotate_event(NET* net, String* packet, char* log_file_name, uint8 checksum_alg_arg) { DBUG_ENTER("fake_rotate_event"); - char header[LOG_EVENT_HEADER_LEN], buf[ROTATE_HEADER_LEN+100]; - - /* - this Rotate is to be sent with checksum if and only if - slave's get_master_version_and_clock time handshake value - of master's @@global.binlog_checksum was TRUE - */ - - my_bool do_checksum= checksum_alg_arg != BINLOG_CHECKSUM_ALG_OFF && - checksum_alg_arg != BINLOG_CHECKSUM_ALG_UNDEF; - - /* - 'when' (the timestamp) is set to 0 so that slave could distinguish between - real and fake Rotate events (if necessary) - */ - memset(header, 0, 4); - header[EVENT_TYPE_OFFSET] = ROTATE_EVENT; - + char buf[ROTATE_HEADER_LEN+100]; + my_bool do_checksum; + int err; char* p = log_file_name+dirname_length(log_file_name); uint ident_len = (uint) strlen(p); - ulong event_len = ident_len + LOG_EVENT_HEADER_LEN + ROTATE_HEADER_LEN + - (do_checksum ? BINLOG_CHECKSUM_LEN : 0); - int4store(header + SERVER_ID_OFFSET, server_id); - int4store(header + EVENT_LEN_OFFSET, event_len); - int2store(header + FLAGS_OFFSET, LOG_EVENT_ARTIFICIAL_F); + ha_checksum crc; - // TODO: check what problems this may cause and fix them - int4store(header + LOG_POS_OFFSET, 0); + if ((err= fake_event_header(packet, ROTATE_EVENT, + ident_len + ROTATE_HEADER_LEN, &do_checksum, &crc, + errmsg, checksum_alg_arg, 0))) + DBUG_RETURN(err); - packet->append(header, sizeof(header)); int8store(buf+R_POS_OFFSET,position); packet->append(buf, ROTATE_HEADER_LEN); packet->append(p, ident_len); if (do_checksum) { - char b[BINLOG_CHECKSUM_LEN]; - ha_checksum crc= my_checksum(0L, NULL, 0); - crc= my_checksum(crc, (uchar*)header, sizeof(header)); crc= my_checksum(crc, (uchar*)buf, ROTATE_HEADER_LEN); crc= my_checksum(crc, (uchar*)p, ident_len); - int4store(b, crc); - packet->append(b, sizeof(b)); } - if (my_net_write(net, (uchar*) packet->ptr(), packet->length())) + if ((err= fake_event_footer(packet, do_checksum, crc, errmsg)) || + (err= fake_event_write(net, packet, errmsg))) + DBUG_RETURN(err); + + DBUG_RETURN(0); +} + + +static int fake_gtid_list_event(NET* net, String* packet, + Gtid_list_log_event *glev, const char** errmsg, + uint8 checksum_alg_arg, uint32 current_pos) +{ + my_bool do_checksum; + int err; + ha_checksum crc; + char buf[128]; + String str(buf, sizeof(buf), system_charset_info); + + str.length(0); + if (glev->to_packet(&str)) { - *errmsg = "failed on my_net_write()"; - DBUG_RETURN(-1); + *errmsg= "Failed due to out-of-memory writing Gtid_list event"; + return -1; } - DBUG_RETURN(0); + if ((err= fake_event_header(packet, GTID_LIST_EVENT, + str.length(), &do_checksum, &crc, + errmsg, checksum_alg_arg, current_pos))) + return err; + + packet->append(str); + if (do_checksum) + { + crc= my_checksum(crc, (uchar*)str.ptr(), str.length()); + } + + if ((err= fake_event_footer(packet, do_checksum, crc, errmsg)) || + (err= fake_event_write(net, packet, errmsg))) + return err; + + return 0; } + /* Reset thread transmit packet buffer for event sending @@ -492,6 +574,81 @@ static ulonglong get_heartbeat_period(THD * thd) } /* + Lookup the capabilities of the slave, which it announces by setting a value + MARIA_SLAVE_CAPABILITY_XXX in @mariadb_slave_capability. + + Older MariaDB slaves, and other MySQL slaves, do not set + @mariadb_slave_capability, corresponding to a capability of + MARIA_SLAVE_CAPABILITY_UNKNOWN (0). +*/ +static int +get_mariadb_slave_capability(THD *thd) +{ + bool null_value; + const LEX_STRING name= { C_STRING_WITH_LEN("mariadb_slave_capability") }; + const user_var_entry *entry= + (user_var_entry*) my_hash_search(&thd->user_vars, (uchar*) name.str, + name.length); + return entry ? + (int)(entry->val_int(&null_value)) : MARIA_SLAVE_CAPABILITY_UNKNOWN; +} + + +/* + Get the value of the @slave_connect_state user variable into the supplied + String (this is the GTID connect state requested by the connecting slave). + + Returns false if error (ie. slave did not set the variable and does not + want to use GTID to set start position), true if success. +*/ +static bool +get_slave_connect_state(THD *thd, String *out_str) +{ + bool null_value; + + const LEX_STRING name= { C_STRING_WITH_LEN("slave_connect_state") }; + user_var_entry *entry= + (user_var_entry*) my_hash_search(&thd->user_vars, (uchar*) name.str, + name.length); + return entry && entry->val_str(&null_value, out_str, 0) && !null_value; +} + + +static bool +get_slave_gtid_strict_mode(THD *thd) +{ + bool null_value; + + const LEX_STRING name= { C_STRING_WITH_LEN("slave_gtid_strict_mode") }; + user_var_entry *entry= + (user_var_entry*) my_hash_search(&thd->user_vars, (uchar*) name.str, + name.length); + return entry && entry->val_int(&null_value) && !null_value; +} + + +/* + Get the value of the @slave_until_gtid user variable into the supplied + String (this is the GTID position specified for START SLAVE UNTIL + master_gtid_pos='xxx'). + + Returns false if error (ie. slave did not set the variable and is not doing + START SLAVE UNTIL mater_gtid_pos='xxx'), true if success. +*/ +static bool +get_slave_until_gtid(THD *thd, String *out_str) +{ + bool null_value; + + const LEX_STRING name= { C_STRING_WITH_LEN("slave_until_gtid") }; + user_var_entry *entry= + (user_var_entry*) my_hash_search(&thd->user_vars, (uchar*) name.str, + name.length); + return entry && entry->val_str(&null_value, out_str, 0) && !null_value; +} + + +/* Function prepares and sends repliation heartbeat event. @param net net object of THD @@ -526,7 +683,7 @@ static int send_heartbeat_event(NET* net, String* packet, uint ident_len = strlen(p); ulong event_len = ident_len + LOG_EVENT_HEADER_LEN + (do_checksum ? BINLOG_CHECKSUM_LEN : 0); - int4store(header + SERVER_ID_OFFSET, server_id); + int4store(header + SERVER_ID_OFFSET, global_system_variables.server_id); int4store(header + EVENT_LEN_OFFSET, event_len); int2store(header + FLAGS_OFFSET, 0); @@ -554,6 +711,800 @@ static int send_heartbeat_event(NET* net, String* packet, } +struct binlog_file_entry +{ + binlog_file_entry *next; + char *name; +}; + +static binlog_file_entry * +get_binlog_list(MEM_ROOT *memroot) +{ + IO_CACHE *index_file; + char fname[FN_REFLEN]; + size_t length; + binlog_file_entry *current_list= NULL, *e; + DBUG_ENTER("get_binlog_list"); + + if (!mysql_bin_log.is_open()) + { + my_error(ER_NO_BINARY_LOGGING, MYF(0)); + DBUG_RETURN(NULL); + } + + mysql_bin_log.lock_index(); + index_file=mysql_bin_log.get_index_file(); + reinit_io_cache(index_file, READ_CACHE, (my_off_t) 0, 0, 0); + + /* The file ends with EOF or empty line */ + while ((length=my_b_gets(index_file, fname, sizeof(fname))) > 1) + { + --length; /* Remove the newline */ + if (!(e= (binlog_file_entry *)alloc_root(memroot, sizeof(*e))) || + !(e->name= strmake_root(memroot, fname, length))) + { + mysql_bin_log.unlock_index(); + my_error(ER_OUTOFMEMORY, MYF(0), length + 1 + sizeof(*e)); + DBUG_RETURN(NULL); + } + e->next= current_list; + current_list= e; + } + mysql_bin_log.unlock_index(); + + DBUG_RETURN(current_list); +} + +/* + Find the Gtid_list_log_event at the start of a binlog. + + NULL for ok, non-NULL error message for error. + + If ok, then the event is returned in *out_gtid_list. This can be NULL if we + get back to binlogs written by old server version without GTID support. If + so, it means we have reached the point to start from, as no GTID events can + exist in earlier binlogs. +*/ +static const char * +get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list) +{ + Format_description_log_event init_fdle(BINLOG_VERSION); + Format_description_log_event *fdle; + Log_event *ev; + const char *errormsg = NULL; + + *out_gtid_list= NULL; + + if (!(ev= Log_event::read_log_event(cache, 0, &init_fdle, + opt_master_verify_checksum)) || + ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) + { + if (ev) + delete ev; + return "Could not read format description log event while looking for " + "GTID position in binlog"; + } + + fdle= static_cast<Format_description_log_event *>(ev); + + for (;;) + { + Log_event_type typ; + + ev= Log_event::read_log_event(cache, 0, fdle, opt_master_verify_checksum); + if (!ev) + { + errormsg= "Could not read GTID list event while looking for GTID " + "position in binlog"; + break; + } + typ= ev->get_type_code(); + if (typ == GTID_LIST_EVENT) + break; /* Done, found it */ + delete ev; + if (typ == ROTATE_EVENT || typ == STOP_EVENT || + typ == FORMAT_DESCRIPTION_EVENT) + continue; /* Continue looking */ + + /* We did not find any Gtid_list_log_event, must be old binlog. */ + ev= NULL; + break; + } + + delete fdle; + *out_gtid_list= static_cast<Gtid_list_log_event *>(ev); + return errormsg; +} + + +/* + Check if every GTID requested by the slave is contained in this (or a later) + binlog file. Return true if so, false if not. + + We do the check with a single scan of the list of GTIDs, avoiding the need + to build an in-memory hash or stuff like that. + + We need to check that slave did not request GTID D-S-N1, when the + Gtid_list_log_event for this binlog file has D-S-N2 with N2 >= N1. + (Because this means that requested GTID is in an earlier binlog). + However, if the Gtid_list_log_event indicates that D-S-N1 is the very last + GTID for domain D in prior binlog files, then it is ok to start from the + very start of this binlog file. This special case is important, as it + allows to purge old logs even if some domain is unused for long. + + In addition, we need to check that we do not have a GTID D-S-N3 in the + Gtid_list_log_event where D is not present in the requested slave state at + all. Since if D is not in requested slave state, it means that slave needs + to start at the very first GTID in domain D. +*/ +static bool +contains_all_slave_gtid(slave_connection_state *st, Gtid_list_log_event *glev) +{ + uint32 i; + + for (i= 0; i < glev->count; ++i) + { + uint32 gl_domain_id= glev->list[i].domain_id; + const rpl_gtid *gtid= st->find(gl_domain_id); + if (!gtid) + { + /* + The slave needs to start from the very beginning of this domain, which + is in an earlier binlog file. So we need to search back further. + */ + return false; + } + if (gtid->server_id == glev->list[i].server_id && + gtid->seq_no <= glev->list[i].seq_no) + { + /* + The slave needs to start after gtid, but it is contained in an earlier + binlog file. So we need to search back further, unless it was the very + last gtid logged for the domain in earlier binlog files. + */ + if (gtid->seq_no < glev->list[i].seq_no) + return false; + + /* + The slave requested D-S-N1, which happens to be the last GTID logged + in prior binlog files with same domain id D and server id S. + + The Gtid_list is kept sorted on domain_id, with the last GTID in each + domain_id group being the last one logged. So if this is the last GTID + within the domain_id group, then it is ok to start from the very + beginning of this group, per the special case explained in comment at + the start of this function. If not, then we need to search back further. + */ + if (i+1 < glev->count && gl_domain_id == glev->list[i+1].domain_id) + return false; + } + } + + return true; +} + + +static void +give_error_start_pos_missing_in_binlog(int *err, const char **errormsg, + rpl_gtid *error_gtid) +{ + rpl_gtid binlog_gtid; + + if (mysql_bin_log.lookup_domain_in_binlog_state(error_gtid->domain_id, + &binlog_gtid) && + binlog_gtid.seq_no >= error_gtid->seq_no) + { + *errormsg= "Requested slave GTID state not found in binlog. The slave has " + "probably diverged due to executing errorneous transactions"; + *err= ER_GTID_POSITION_NOT_FOUND_IN_BINLOG2; + } + else + { + *errormsg= "Requested slave GTID state not found in binlog"; + *err= ER_GTID_POSITION_NOT_FOUND_IN_BINLOG; + } +} + + +/* + Check the start GTID state requested by the slave against our binlog state. + + Give an error if the slave requests something that we do not have in our + binlog. +*/ + +static int +check_slave_start_position(THD *thd, slave_connection_state *st, + const char **errormsg, rpl_gtid *error_gtid, + slave_connection_state *until_gtid_state) +{ + uint32 i; + int err; + slave_connection_state::entry **delete_list= NULL; + uint32 delete_idx= 0; + + if (rpl_load_gtid_slave_state(thd)) + { + *errormsg= "Failed to load replication slave GTID state"; + err= ER_CANNOT_LOAD_SLAVE_GTID_STATE; + goto end; + } + + for (i= 0; i < st->hash.records; ++i) + { + slave_connection_state::entry *slave_gtid_entry= + (slave_connection_state::entry *)my_hash_element(&st->hash, i); + rpl_gtid *slave_gtid= &slave_gtid_entry->gtid; + rpl_gtid master_gtid; + rpl_gtid master_replication_gtid; + rpl_gtid start_gtid; + bool start_at_own_slave_pos= + rpl_global_gtid_slave_state.domain_to_gtid(slave_gtid->domain_id, + &master_replication_gtid) && + slave_gtid->server_id == master_replication_gtid.server_id && + slave_gtid->seq_no == master_replication_gtid.seq_no; + + if (mysql_bin_log.find_in_binlog_state(slave_gtid->domain_id, + slave_gtid->server_id, + &master_gtid) && + master_gtid.seq_no >= slave_gtid->seq_no) + { + /* + If connecting slave requests to start at the GTID we last applied when + we were ourselves a slave, then this GTID may not exist in our binlog + (in case of --log-slave-updates=0). So set the flag to disable the + error about missing GTID in the binlog in this case. + */ + if (start_at_own_slave_pos) + slave_gtid_entry->flags|= slave_connection_state::START_OWN_SLAVE_POS; + continue; + } + + if (!start_at_own_slave_pos) + { + rpl_gtid domain_gtid; + rpl_gtid *until_gtid; + + if (!mysql_bin_log.lookup_domain_in_binlog_state(slave_gtid->domain_id, + &domain_gtid)) + { + /* + We do not have anything in this domain, neither in the binlog nor + in the slave state. So we are probably one master in a multi-master + setup, and this domain is served by a different master. + + But set a flag so that if we then ever _do_ happen to encounter + anything in this domain, then we will re-check that the requested + slave position exists, and give the error at that time if not. + */ + slave_gtid_entry->flags|= slave_connection_state::START_ON_EMPTY_DOMAIN; + continue; + } + + if (until_gtid_state && + ( !(until_gtid= until_gtid_state->find(slave_gtid->domain_id)) || + (mysql_bin_log.find_in_binlog_state(until_gtid->domain_id, + until_gtid->server_id, + &master_gtid) && + master_gtid.seq_no >= until_gtid->seq_no))) + { + /* + The slave requested to start from a position that is not (yet) in + our binlog, but it also specified an UNTIL condition that _is_ in + our binlog (or a missing UNTIL, which means stop at the very + beginning). So the stop position is before the start position, and + we just delete the entry from the UNTIL hash to mark that this + domain has already reached the UNTIL condition. + */ + if(until_gtid) + until_gtid_state->remove(until_gtid); + continue; + } + + *error_gtid= *slave_gtid; + give_error_start_pos_missing_in_binlog(&err, errormsg, error_gtid); + goto end; + } + + /* + Ok, so connecting slave asked to start at a GTID that we do not have in + our binlog, but it was in fact the last GTID we applied earlier, when we + were acting as a replication slave. + + So this means that we were running as a replication slave without + --log-slave-updates, but now we switched to be a master. It is worth it + to handle this special case, as it allows users to run a simple + master -> slave without --log-slave-updates, and then exchange slave and + master, as long as they make sure the slave is caught up before switching. + */ + + /* + First check if we logged something ourselves as a master after being a + slave. This will be seen as a GTID with our own server_id and bigger + seq_no than what is in the slave state. + + If we did not log anything ourselves, then start the connecting slave + replicating from the current binlog end position, which in this case + corresponds to our replication slave state and hence what the connecting + slave is requesting. + */ + if (mysql_bin_log.find_in_binlog_state(slave_gtid->domain_id, + global_system_variables.server_id, + &start_gtid) && + start_gtid.seq_no > slave_gtid->seq_no) + { + /* + Start replication within this domain at the first GTID that we logged + ourselves after becoming a master. + + Remember that this starting point is in fact a "fake" GTID which may + not exists in the binlog, so that we do not complain about it in + --gtid-strict-mode. + */ + slave_gtid->server_id= global_system_variables.server_id; + slave_gtid_entry->flags|= slave_connection_state::START_OWN_SLAVE_POS; + } + else if (mysql_bin_log.lookup_domain_in_binlog_state(slave_gtid->domain_id, + &start_gtid)) + { + slave_gtid->server_id= start_gtid.server_id; + slave_gtid->seq_no= start_gtid.seq_no; + } + else + { + /* + We do not have _anything_ in our own binlog for this domain. Just + delete the entry in the slave connection state, then it will pick up + anything new that arrives. + + We just queue up the deletion and do it later, after the loop, so that + we do not mess up the iteration over the hash. + */ + if (!delete_list) + { + if (!(delete_list= (slave_connection_state::entry **) + my_malloc(sizeof(*delete_list) * st->hash.records, MYF(MY_WME)))) + { + *errormsg= "Out of memory while checking slave start position"; + err= ER_OUT_OF_RESOURCES; + goto end; + } + } + delete_list[delete_idx++]= slave_gtid_entry; + } + } + + /* Do any delayed deletes from the hash. */ + if (delete_list) + { + for (i= 0; i < delete_idx; ++i) + st->remove(&(delete_list[i]->gtid)); + } + err= 0; + +end: + if (delete_list) + my_free(delete_list); + return err; +} + +/* + Find the name of the binlog file to start reading for a slave that connects + using GTID state. + + Returns the file name in out_name, which must be of size at least FN_REFLEN. + + Returns NULL on ok, error message on error. + + In case of non-error return, the returned binlog file is guaranteed to + contain the first event to be transmitted to the slave for every domain + present in our binlogs. It is still necessary to skip all GTIDs up to + and including the GTID requested by slave within each domain. + + However, as a special case, if the event to be sent to the slave is the very + first event (within that domain) in the returned binlog, then nothing should + be skipped, so that domain is deleted from the passed in slave connection + state. + + This is necessary in case the slave requests a GTID within a replication + domain that has long been inactive. The binlog file containing that GTID may + have been long since purged. However, as long as no GTIDs after that have + been purged, we have the GTID requested by slave in the Gtid_list_log_event + of the latest binlog. So we can start from there, as long as we delete the + corresponding entry in the slave state so we do not wrongly skip any events + that might turn up if that domain becomes active again, vainly looking for + the requested GTID that was already purged. +*/ +static const char * +gtid_find_binlog_file(slave_connection_state *state, char *out_name, + slave_connection_state *until_gtid_state) +{ + MEM_ROOT memroot; + binlog_file_entry *list; + Gtid_list_log_event *glev= NULL; + const char *errormsg= NULL; + char buf[FN_REFLEN]; + + init_alloc_root(&memroot, 10*(FN_REFLEN+sizeof(binlog_file_entry)), 0, + MYF(MY_THREAD_SPECIFIC)); + if (!(list= get_binlog_list(&memroot))) + { + errormsg= "Out of memory while looking for GTID position in binlog"; + goto end; + } + + while (list) + { + File file; + IO_CACHE cache; + + if (!list->next) + { + /* + It should be safe to read the currently used binlog, as we will only + read the header part that is already written. + + But if that does not work on windows, then we will need to cache the + event somewhere in memory I suppose - that could work too. + */ + } + /* + Read the Gtid_list_log_event at the start of the binlog file to + get the binlog state. + */ + if (normalize_binlog_name(buf, list->name, false)) + { + errormsg= "Failed to determine binlog file name while looking for " + "GTID position in binlog"; + goto end; + } + bzero((char*) &cache, sizeof(cache)); + if ((file= open_binlog(&cache, buf, &errormsg)) == (File)-1) + goto end; + errormsg= get_gtid_list_event(&cache, &glev); + end_io_cache(&cache); + mysql_file_close(file, MYF(MY_WME)); + if (errormsg) + goto end; + + if (!glev || contains_all_slave_gtid(state, glev)) + { + strmake(out_name, buf, FN_REFLEN); + + if (glev) + { + uint32 i; + + /* + As a special case, we allow to start from binlog file N if the + requested GTID is the last event (in the corresponding domain) in + binlog file (N-1), but then we need to remove that GTID from the slave + state, rather than skipping events waiting for it to turn up. + + If slave is doing START SLAVE UNTIL, check for any UNTIL conditions + that are already included in a previous binlog file. Delete any such + from the UNTIL hash, to mark that such domains have already reached + their UNTIL condition. + */ + for (i= 0; i < glev->count; ++i) + { + const rpl_gtid *gtid= state->find(glev->list[i].domain_id); + if (!gtid) + { + /* + Contains_all_slave_gtid() returns false if there is any domain in + Gtid_list_event which is not in the requested slave position. + + We may delete a domain from the slave state inside this loop, but + we only do this when it is the very last GTID logged for that + domain in earlier binlogs, and then we can not encounter it in any + further GTIDs in the Gtid_list. + */ + DBUG_ASSERT(0); + } else if (gtid->server_id == glev->list[i].server_id && + gtid->seq_no == glev->list[i].seq_no) + { + /* + The slave requested to start from the very beginning of this + domain in this binlog file. So delete the entry from the state, + we do not need to skip anything. + */ + state->remove(gtid); + } + + if (until_gtid_state && + (gtid= until_gtid_state->find(glev->list[i].domain_id)) && + gtid->server_id == glev->list[i].server_id && + gtid->seq_no <= glev->list[i].seq_no) + { + /* + We've already reached the stop position in UNTIL for this domain, + since it is before the start position. + */ + until_gtid_state->remove(gtid); + } + } + } + + goto end; + } + delete glev; + glev= NULL; + list= list->next; + } + + /* We reached the end without finding anything. */ + errormsg= "Could not find GTID state requested by slave in any binlog " + "files. Probably the slave state is too old and required binlog files " + "have been purged."; + +end: + if (glev) + delete glev; + + free_root(&memroot, MYF(0)); + return errormsg; +} + + +/* + Given an old-style binlog position with file name and file offset, find the + corresponding gtid position. If the offset is not at an event boundary, give + an error. + + Return NULL on ok, error message string on error. + + ToDo: Improve the performance of this by using binlog index files. +*/ +static const char * +gtid_state_from_pos(const char *name, uint32 offset, + slave_connection_state *gtid_state) +{ + IO_CACHE cache; + File file; + const char *errormsg= NULL; + bool found_gtid_list_event= false; + bool found_format_description_event= false; + bool valid_pos= false; + uint8 current_checksum_alg= BINLOG_CHECKSUM_ALG_UNDEF; + int err; + String packet; + Format_description_log_event *fdev= NULL; + + if (gtid_state->load((const rpl_gtid *)NULL, 0)) + { + errormsg= "Internal error (out of memory?) initializing slave state " + "while scanning binlog to find start position"; + return errormsg; + } + + if ((file= open_binlog(&cache, name, &errormsg)) == (File)-1) + return errormsg; + + if (!(fdev= new Format_description_log_event(3))) + { + errormsg= "Out of memory initializing format_description event " + "while scanning binlog to find start position"; + goto end; + } + + /* + First we need to find the initial GTID_LIST_EVENT. We need this even + if the offset is at the very start of the binlog file. + + But if we do not find any GTID_LIST_EVENT, then this is an old binlog + with no GTID information, so we return empty GTID state. + */ + for (;;) + { + Log_event_type typ; + uint32 cur_pos; + + cur_pos= (uint32)my_b_tell(&cache); + if (cur_pos == offset) + valid_pos= true; + if (found_format_description_event && found_gtid_list_event && + cur_pos >= offset) + break; + + packet.length(0); + err= Log_event::read_log_event(&cache, &packet, NULL, + current_checksum_alg); + if (err) + { + errormsg= "Could not read binlog while searching for slave start " + "position on master"; + goto end; + } + /* + The cast to uchar is needed to avoid a signed char being converted to a + negative number. + */ + typ= (Log_event_type)(uchar)packet[EVENT_TYPE_OFFSET]; + if (typ == FORMAT_DESCRIPTION_EVENT) + { + Format_description_log_event *tmp; + + if (found_format_description_event) + { + errormsg= "Duplicate format description log event found while " + "searching for old-style position in binlog"; + goto end; + } + + current_checksum_alg= get_checksum_alg(packet.ptr(), packet.length()); + found_format_description_event= true; + if (!(tmp= new Format_description_log_event(packet.ptr(), packet.length(), + fdev))) + { + errormsg= "Corrupt Format_description event found or out-of-memory " + "while searching for old-style position in binlog"; + goto end; + } + delete fdev; + fdev= tmp; + } + else if (typ != FORMAT_DESCRIPTION_EVENT && !found_format_description_event) + { + errormsg= "Did not find format description log event while searching " + "for old-style position in binlog"; + goto end; + } + else if (typ == ROTATE_EVENT || typ == STOP_EVENT || + typ == BINLOG_CHECKPOINT_EVENT) + continue; /* Continue looking */ + else if (typ == GTID_LIST_EVENT) + { + rpl_gtid *gtid_list; + bool status; + uint32 list_len; + + if (found_gtid_list_event) + { + errormsg= "Found duplicate Gtid_list_log_event while scanning binlog " + "to find slave start position"; + goto end; + } + status= Gtid_list_log_event::peek(packet.ptr(), packet.length(), + current_checksum_alg, + >id_list, &list_len, fdev); + if (status) + { + errormsg= "Error reading Gtid_list_log_event while searching " + "for old-style position in binlog"; + goto end; + } + err= gtid_state->load(gtid_list, list_len); + my_free(gtid_list); + if (err) + { + errormsg= "Internal error (out of memory?) initialising slave state " + "while scanning binlog to find start position"; + goto end; + } + found_gtid_list_event= true; + } + else if (!found_gtid_list_event) + { + /* We did not find any Gtid_list_log_event, must be old binlog. */ + goto end; + } + else if (typ == GTID_EVENT) + { + rpl_gtid gtid; + uchar flags2; + if (Gtid_log_event::peek(packet.ptr(), packet.length(), + current_checksum_alg, >id.domain_id, + >id.server_id, >id.seq_no, &flags2, fdev)) + { + errormsg= "Corrupt gtid_log_event found while scanning binlog to find " + "initial slave position"; + goto end; + } + if (gtid_state->update(>id)) + { + errormsg= "Internal error (out of memory?) updating slave state while " + "scanning binlog to find start position"; + goto end; + } + } + } + + if (!valid_pos) + { + errormsg= "Slave requested incorrect position in master binlog. " + "Requested position %u in file '%s', but this position does not " + "correspond to the location of any binlog event."; + } + +end: + delete fdev; + end_io_cache(&cache); + mysql_file_close(file, MYF(MY_WME)); + + return errormsg; +} + + +int +gtid_state_from_binlog_pos(const char *in_name, uint32 pos, String *out_str) +{ + slave_connection_state gtid_state; + const char *lookup_name; + char name_buf[FN_REFLEN]; + LOG_INFO linfo; + + if (!mysql_bin_log.is_open()) + { + my_error(ER_NO_BINARY_LOGGING, MYF(0)); + return 1; + } + + if (in_name && in_name[0]) + { + mysql_bin_log.make_log_name(name_buf, in_name); + lookup_name= name_buf; + } + else + lookup_name= NULL; + linfo.index_file_offset= 0; + if (mysql_bin_log.find_log_pos(&linfo, lookup_name, 1)) + return 1; + + if (pos < 4) + pos= 4; + + if (gtid_state_from_pos(linfo.log_file_name, pos, >id_state) || + gtid_state.to_string(out_str)) + return 1; + return 0; +} + + +static bool +is_until_reached(THD *thd, NET *net, String *packet, ulong *ev_offset, + enum_gtid_until_state gtid_until_group, + Log_event_type event_type, uint8 current_checksum_alg, + ushort flags, const char **errmsg, + rpl_binlog_state *until_binlog_state, uint32 current_pos) +{ + switch (gtid_until_group) + { + case GTID_UNTIL_NOT_DONE: + return false; + case GTID_UNTIL_STOP_AFTER_STANDALONE: + if (Log_event::is_part_of_group(event_type)) + return false; + break; + case GTID_UNTIL_STOP_AFTER_TRANSACTION: + if (event_type != XID_EVENT && + (event_type != QUERY_EVENT || + !Query_log_event::peek_is_commit_rollback(packet->ptr()+*ev_offset, + packet->length()-*ev_offset, + current_checksum_alg))) + return false; + break; + } + + /* + The last event group has been sent, now the START SLAVE UNTIL condition + has been reached. + + Send a last fake Gtid_list_log_event with a flag set to mark that we + stop due to UNTIL condition. + */ + if (reset_transmit_packet(thd, flags, ev_offset, errmsg)) + return true; + Gtid_list_log_event glev(until_binlog_state, + Gtid_list_log_event::FLAG_UNTIL_REACHED); + if (fake_gtid_list_event(net, packet, &glev, errmsg, current_checksum_alg, + current_pos)) + return true; + *errmsg= NULL; + return true; +} + + /* Helper function for mysql_binlog_send() to write an event down the slave connection. @@ -563,14 +1514,307 @@ static int send_heartbeat_event(NET* net, String* packet, static const char * send_event_to_slave(THD *thd, NET *net, String* const packet, ushort flags, Log_event_type event_type, char *log_file_name, - IO_CACHE *log) + IO_CACHE *log, int mariadb_slave_capability, + ulong ev_offset, uint8 current_checksum_alg, + bool using_gtid_state, slave_connection_state *gtid_state, + enum_gtid_skip_type *gtid_skip_group, + slave_connection_state *until_gtid_state, + enum_gtid_until_state *gtid_until_group, + rpl_binlog_state *until_binlog_state, + bool slave_gtid_strict_mode, rpl_gtid *error_gtid, + bool *send_fake_gtid_list, + Format_description_log_event *fdev) { my_off_t pos; + size_t len= packet->length(); - /* Do not send annotate_rows events unless slave requested it. */ - if (event_type == ANNOTATE_ROWS_EVENT && - !(flags & BINLOG_SEND_ANNOTATE_ROWS_EVENT)) + if (event_type == GTID_LIST_EVENT && using_gtid_state && until_gtid_state) + { + rpl_gtid *gtid_list; + uint32 list_len; + bool err; + + if (ev_offset > len || + Gtid_list_log_event::peek(packet->ptr()+ev_offset, len - ev_offset, + current_checksum_alg, + >id_list, &list_len, fdev)) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + return "Failed to read Gtid_list_log_event: corrupt binlog"; + } + err= until_binlog_state->load(gtid_list, list_len); + my_free(gtid_list); + if (err) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + return "Failed in internal GTID book-keeping: Out of memory"; + } + } + + /* Skip GTID event groups until we reach slave position within a domain_id. */ + if (event_type == GTID_EVENT && using_gtid_state) + { + uchar flags2; + slave_connection_state::entry *gtid_entry; + rpl_gtid *gtid; + + if (gtid_state->count() > 0 || until_gtid_state) + { + rpl_gtid event_gtid; + + if (ev_offset > len || + Gtid_log_event::peek(packet->ptr()+ev_offset, len - ev_offset, + current_checksum_alg, + &event_gtid.domain_id, &event_gtid.server_id, + &event_gtid.seq_no, &flags2, fdev)) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + return "Failed to read Gtid_log_event: corrupt binlog"; + } + + DBUG_EXECUTE_IF("gtid_force_reconnect_at_10_1_100", + { + rpl_gtid *dbug_gtid; + if ((dbug_gtid= until_binlog_state->find_nolock(10,1)) && + dbug_gtid->seq_no == 100) + { + DBUG_SET("-d,gtid_force_reconnect_at_10_1_100"); + DBUG_SET_INITIAL("-d,gtid_force_reconnect_at_10_1_100"); + my_errno= ER_UNKNOWN_ERROR; + return "DBUG-injected forced reconnect"; + } + }); + + if (until_binlog_state->update_nolock(&event_gtid, false)) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + return "Failed in internal GTID book-keeping: Out of memory"; + } + + if (gtid_state->count() > 0) + { + gtid_entry= gtid_state->find_entry(event_gtid.domain_id); + if (gtid_entry != NULL) + { + gtid= >id_entry->gtid; + if (gtid_entry->flags & slave_connection_state::START_ON_EMPTY_DOMAIN) + { + rpl_gtid master_gtid; + if (!mysql_bin_log.find_in_binlog_state(gtid->domain_id, + gtid->server_id, + &master_gtid) || + master_gtid.seq_no < gtid->seq_no) + { + int err; + const char *errormsg; + *error_gtid= *gtid; + give_error_start_pos_missing_in_binlog(&err, &errormsg, error_gtid); + my_errno= err; + return errormsg; + } + gtid_entry->flags&= ~(uint32)slave_connection_state::START_ON_EMPTY_DOMAIN; + } + + /* Skip this event group if we have not yet reached slave start pos. */ + if (event_gtid.server_id != gtid->server_id || + event_gtid.seq_no <= gtid->seq_no) + *gtid_skip_group = (flags2 & Gtid_log_event::FL_STANDALONE ? + GTID_SKIP_STANDALONE : GTID_SKIP_TRANSACTION); + if (event_gtid.server_id == gtid->server_id && + event_gtid.seq_no >= gtid->seq_no) + { + if (slave_gtid_strict_mode && event_gtid.seq_no > gtid->seq_no && + !(gtid_entry->flags & slave_connection_state::START_OWN_SLAVE_POS)) + { + /* + In strict mode, it is an error if the slave requests to start + in a "hole" in the master's binlog: a GTID that does not + exist, even though both the prior and subsequent seq_no exists + for same domain_id and server_id. + */ + my_errno= ER_GTID_START_FROM_BINLOG_HOLE; + *error_gtid= *gtid; + return "The binlog on the master is missing the GTID requested " + "by the slave (even though both a prior and a subsequent " + "sequence number does exist), and GTID strict mode is enabled."; + } + + /* + Send a fake Gtid_list event to the slave. + This allows the slave to update its current binlog position + so MASTER_POS_WAIT() and MASTER_GTID_WAIT() can work. + The fake event will be sent at the end of this event group. + */ + *send_fake_gtid_list= true; + + /* + Delete this entry if we have reached slave start position (so we + will not skip subsequent events and won't have to look them up + and check). + */ + gtid_state->remove(gtid); + } + } + } + + if (until_gtid_state) + { + gtid= until_gtid_state->find(event_gtid.domain_id); + if (gtid == NULL) + { + /* + This domain already reached the START SLAVE UNTIL stop condition, + so skip this event group. + */ + *gtid_skip_group = (flags2 & Gtid_log_event::FL_STANDALONE ? + GTID_SKIP_STANDALONE : GTID_SKIP_TRANSACTION); + } + else if (event_gtid.server_id == gtid->server_id && + event_gtid.seq_no >= gtid->seq_no) + { + /* + We have reached the stop condition. + Delete this domain_id from the hash, so we will skip all further + events in this domain and eventually stop when all domains are + done. + */ + uint64 until_seq_no= gtid->seq_no; + until_gtid_state->remove(gtid); + if (until_gtid_state->count() == 0) + *gtid_until_group= (flags2 & Gtid_log_event::FL_STANDALONE ? + GTID_UNTIL_STOP_AFTER_STANDALONE : + GTID_UNTIL_STOP_AFTER_TRANSACTION); + if (event_gtid.seq_no > until_seq_no) + { + /* + The GTID in START SLAVE UNTIL condition is missing in our binlog. + This should normally not happen (user error), but since we can be + sure that we are now beyond the position that the UNTIL condition + should be in, we can just stop now. And we also need to skip this + event group (as it is beyond the UNTIL condition). + */ + *gtid_skip_group = (flags2 & Gtid_log_event::FL_STANDALONE ? + GTID_SKIP_STANDALONE : GTID_SKIP_TRANSACTION); + } + } + } + } + } + + /* + Skip event group if we have not yet reached the correct slave GTID position. + + Note that slave that understands GTID can also tolerate holes, so there is + no need to supply dummy event. + */ + switch (*gtid_skip_group) + { + case GTID_SKIP_STANDALONE: + if (!Log_event::is_part_of_group(event_type)) + *gtid_skip_group= GTID_SKIP_NOT; + return NULL; + case GTID_SKIP_TRANSACTION: + if (event_type == XID_EVENT || + (event_type == QUERY_EVENT && + Query_log_event::peek_is_commit_rollback(packet->ptr() + ev_offset, + len - ev_offset, + current_checksum_alg))) + *gtid_skip_group= GTID_SKIP_NOT; return NULL; + case GTID_SKIP_NOT: + break; + } + + /* Do not send annotate_rows events unless slave requested it. */ + if (event_type == ANNOTATE_ROWS_EVENT && !(flags & BINLOG_SEND_ANNOTATE_ROWS_EVENT)) + { + if (mariadb_slave_capability >= MARIA_SLAVE_CAPABILITY_TOLERATE_HOLES) + { + /* This slave can tolerate events omitted from the binlog stream. */ + return NULL; + } + else if (mariadb_slave_capability >= MARIA_SLAVE_CAPABILITY_ANNOTATE) + { + /* + The slave did not request ANNOTATE_ROWS_EVENT (it does not need them as + it will not log them in its own binary log). However, it understands the + event and will just ignore it, and it would break if we omitted it, + leaving a hole in the binlog stream. So just send the event as-is. + */ + } + else + { + /* + The slave does not understand ANNOTATE_ROWS_EVENT. + + Older MariaDB slaves (and MySQL slaves) will break replication if there + are holes in the binlog stream (they will miscompute the binlog offset + and request the wrong position when reconnecting). + + So replace the event with a dummy event of the same size that will be + a no-operation on the slave. + */ + if (Query_log_event::dummy_event(packet, ev_offset, current_checksum_alg)) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + return "Failed to replace row annotate event with dummy: too small event."; + } + } + } + + /* + Replace GTID events with old-style BEGIN events for slaves that do not + understand global transaction IDs. For stand-alone events, where there is + no terminating COMMIT query event, omit the GTID event or replace it with + a dummy event, as appropriate. + */ + if (event_type == GTID_EVENT && + mariadb_slave_capability < MARIA_SLAVE_CAPABILITY_GTID) + { + bool need_dummy= + mariadb_slave_capability < MARIA_SLAVE_CAPABILITY_TOLERATE_HOLES; + bool err= Gtid_log_event::make_compatible_event(packet, &need_dummy, + ev_offset, + current_checksum_alg); + if (err) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + return "Failed to replace GTID event with backwards-compatible event: " + "currupt event."; + } + if (!need_dummy) + return NULL; + } + + /* + Do not send binlog checkpoint or gtid list events to a slave that does not + understand it. + */ + if ((unlikely(event_type == BINLOG_CHECKPOINT_EVENT) && + mariadb_slave_capability < MARIA_SLAVE_CAPABILITY_BINLOG_CHECKPOINT) || + (unlikely(event_type == GTID_LIST_EVENT) && + mariadb_slave_capability < MARIA_SLAVE_CAPABILITY_GTID)) + { + if (mariadb_slave_capability >= MARIA_SLAVE_CAPABILITY_TOLERATE_HOLES) + { + /* This slave can tolerate events omitted from the binlog stream. */ + return NULL; + } + else + { + /* + The slave does not understand BINLOG_CHECKPOINT_EVENT. Send a dummy + event instead, with same length so slave does not get confused about + binlog positions. + */ + if (Query_log_event::dummy_event(packet, ev_offset, current_checksum_alg)) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + return "Failed to replace binlog checkpoint or gtid list event with " + "dummy: too small event."; + } + } + } /* Skip events with the @@skip_replication flag set, if slave requested @@ -592,24 +1836,37 @@ send_event_to_slave(THD *thd, NET *net, String* const packet, ushort flags, pos= my_b_tell(log); if (RUN_HOOK(binlog_transmit, before_send_event, (thd, flags, packet, log_file_name, pos))) + { + my_errno= ER_UNKNOWN_ERROR; return "run 'before_send_event' hook failed"; + } - if (my_net_write(net, (uchar*) packet->ptr(), packet->length())) + if (my_net_write(net, (uchar*) packet->ptr(), len)) + { + my_errno= ER_UNKNOWN_ERROR; return "Failed on my_net_write()"; + } DBUG_PRINT("info", ("log event code %d", (*packet)[LOG_EVENT_OFFSET+1] )); if (event_type == LOAD_EVENT) { if (send_file(thd)) + { + my_errno= ER_UNKNOWN_ERROR; return "failed in send_file()"; + } } if (RUN_HOOK(binlog_transmit, after_send_event, (thd, flags, packet))) + { + my_errno= ER_UNKNOWN_ERROR; return "Failed to run hook 'after_send_event'"; + } return NULL; /* Success */ } + void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, ushort flags) { @@ -628,17 +1885,34 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, NET* net = &thd->net; mysql_mutex_t *log_lock; mysql_cond_t *log_cond; + int mariadb_slave_capability; + char str_buf[128]; + String connect_gtid_state(str_buf, sizeof(str_buf), system_charset_info); + bool using_gtid_state; + char str_buf2[128]; + String slave_until_gtid_str(str_buf2, sizeof(str_buf2), system_charset_info); + slave_connection_state gtid_state, until_gtid_state_obj; + slave_connection_state *until_gtid_state= NULL; + rpl_gtid error_gtid; + enum_gtid_skip_type gtid_skip_group= GTID_SKIP_NOT; + enum_gtid_until_state gtid_until_group= GTID_UNTIL_NOT_DONE; + rpl_binlog_state until_binlog_state; + bool slave_gtid_strict_mode= false; + bool send_fake_gtid_list= false; uint8 current_checksum_alg= BINLOG_CHECKSUM_ALG_UNDEF; int old_max_allowed_packet= thd->variables.max_allowed_packet; + Format_description_log_event *fdev= NULL; #ifndef DBUG_OFF int left_events = max_binlog_dump_events; + uint dbug_reconnect_counter= 0; #endif DBUG_ENTER("mysql_binlog_send"); DBUG_PRINT("enter",("log_ident: '%s' pos: %ld", log_ident, (long) pos)); bzero((char*) &log,sizeof(log)); + bzero(&error_gtid, sizeof(error_gtid)); /* heartbeat_period from @master_heartbeat_period user variable */ @@ -649,14 +1923,45 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, *p_start_coord= &start_coord; LOG_POS_COORD coord_buf= { log_file_name, BIN_LOG_HEADER_SIZE }, *p_coord= &coord_buf; - if (heartbeat_period != LL(0)) + if (heartbeat_period != 0) { heartbeat_ts= &heartbeat_buf; set_timespec_nsec(*heartbeat_ts, 0); } + mariadb_slave_capability= get_mariadb_slave_capability(thd); + + connect_gtid_state.length(0); + using_gtid_state= get_slave_connect_state(thd, &connect_gtid_state); + DBUG_EXECUTE_IF("simulate_non_gtid_aware_master", using_gtid_state= false;); + if (using_gtid_state) + { + slave_gtid_strict_mode= get_slave_gtid_strict_mode(thd); + if(get_slave_until_gtid(thd, &slave_until_gtid_str)) + until_gtid_state= &until_gtid_state_obj; + } + + DBUG_EXECUTE_IF("binlog_force_reconnect_after_22_events", + { + DBUG_SET("-d,binlog_force_reconnect_after_22_events"); + DBUG_SET_INITIAL("-d,binlog_force_reconnect_after_22_events"); + dbug_reconnect_counter= 22; + }); + + /* + We want to corrupt the first event, in Log_event::read_log_event(). + But we do not want the corruption to happen early, eg. when client does + BINLOG_GTID_POS(). So test case sets a DBUG trigger which causes us to + set the real DBUG injection here. + */ + DBUG_EXECUTE_IF("corrupt_read_log_event2_set", + { + DBUG_SET("-d,corrupt_read_log_event2_set"); + DBUG_SET("+d,corrupt_read_log_event2"); + }); + if (global_system_variables.log_warnings > 1) - sql_print_information("Start binlog_dump to slave_server(%u), pos(%s, %lu)", - thd->server_id, log_ident, (ulong)pos); + sql_print_information("Start binlog_dump to slave_server(%lu), pos(%s, %lu)", + thd->variables.server_id, log_ident, (ulong)pos); if (RUN_HOOK(binlog_transmit, transmit_start, (thd, flags, log_ident, pos))) { errmsg= "Failed to run hook 'transmit_start'"; @@ -673,6 +1978,13 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, } #endif + if (!(fdev= new Format_description_log_event(3))) + { + errmsg= "Out of memory initializing format_description event"; + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + goto err; + } + if (!mysql_bin_log.is_open()) { errmsg = "Binary log is not open"; @@ -687,10 +1999,46 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, } name=search_file_name; - if (log_ident[0]) - mysql_bin_log.make_log_name(search_file_name, log_ident); + if (using_gtid_state) + { + if (gtid_state.load(connect_gtid_state.c_ptr_quick(), + connect_gtid_state.length())) + { + errmsg= "Out of memory or malformed slave request when obtaining start " + "position from GTID state"; + my_errno= ER_UNKNOWN_ERROR; + goto err; + } + if (until_gtid_state && + until_gtid_state->load(slave_until_gtid_str.c_ptr_quick(), + slave_until_gtid_str.length())) + { + errmsg= "Out of memory or malformed slave request when obtaining UNTIL " + "position sent from slave"; + my_errno= ER_UNKNOWN_ERROR; + goto err; + } + if ((error= check_slave_start_position(thd, >id_state, &errmsg, + &error_gtid, until_gtid_state))) + { + my_errno= error; + goto err; + } + if ((errmsg= gtid_find_binlog_file(>id_state, search_file_name, + until_gtid_state))) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + goto err; + } + pos= 4; + } else - name=0; // Find first log + { + if (log_ident[0]) + mysql_bin_log.make_log_name(search_file_name, log_ident); + else + name=0; // Find first log + } linfo.index_file_offset = 0; @@ -800,6 +2148,8 @@ impossible position"; (*packet)[EVENT_TYPE_OFFSET+ev_offset])); if ((*packet)[EVENT_TYPE_OFFSET+ev_offset] == FORMAT_DESCRIPTION_EVENT) { + Format_description_log_event *tmp; + current_checksum_alg= get_checksum_alg(packet->ptr() + ev_offset, packet->length() - ev_offset); DBUG_ASSERT(current_checksum_alg == BINLOG_CHECKSUM_ALG_OFF || @@ -817,6 +2167,18 @@ impossible position"; "slaves that cannot process them"); goto err; } + + if (!(tmp= new Format_description_log_event(packet->ptr()+ev_offset, + packet->length()-ev_offset, + fdev))) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + errmsg= "Corrupt Format_description event found or out-of-memory"; + goto err; + } + delete fdev; + fdev= tmp; + (*packet)[FLAGS_OFFSET+ev_offset] &= ~LOG_EVENT_BINLOG_IN_USE_F; /* mark that this event with "log_pos=0", so the slave @@ -867,12 +2229,22 @@ impossible position"; /* The Format_description_log_event event will be found naturally. */ } + /* + Handle the case of START SLAVE UNTIL with an UNTIL condition already + fulfilled at the start position. + + We will send one event, the format_description, and then stop. + */ + if (until_gtid_state && until_gtid_state->count() == 0) + gtid_until_group= GTID_UNTIL_STOP_AFTER_STANDALONE; + /* seek to the requested position, to start the requested dump */ my_b_seek(&log, pos); // Seek will done on next read while (!net->error && net->vio != 0 && !thd->killed) { Log_event_type event_type= UNKNOWN_EVENT; + killed_state killed; /* reset the transmit packet for the event read from binary log file */ @@ -880,7 +2252,8 @@ impossible position"; goto err; bool is_active_binlog= false; - while (!(error= Log_event::read_log_event(&log, packet, log_lock, + while (!(killed= thd->killed) && + !(error = Log_event::read_log_event(&log, packet, log_lock, current_checksum_alg, log_file_name, &is_active_binlog))) @@ -923,6 +2296,8 @@ impossible position"; #endif if (event_type == FORMAT_DESCRIPTION_EVENT) { + Format_description_log_event *tmp; + current_checksum_alg= get_checksum_alg(packet->ptr() + ev_offset, packet->length() - ev_offset); DBUG_ASSERT(current_checksum_alg == BINLOG_CHECKSUM_ALG_OFF || @@ -941,16 +2316,71 @@ impossible position"; goto err; } + if (!(tmp= new Format_description_log_event(packet->ptr()+ev_offset, + packet->length()-ev_offset, + fdev))) + { + my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; + errmsg= "Corrupt Format_description event found or out-of-memory"; + goto err; + } + delete fdev; + fdev= tmp; + (*packet)[FLAGS_OFFSET+ev_offset] &= ~LOG_EVENT_BINLOG_IN_USE_F; } +#ifndef DBUG_OFF + if (dbug_reconnect_counter > 0) + { + --dbug_reconnect_counter; + if (dbug_reconnect_counter == 0) + { + errmsg= "DBUG-injected forced reconnect"; + my_errno= ER_UNKNOWN_ERROR; + goto err; + } + } +#endif + if ((tmp_msg= send_event_to_slave(thd, net, packet, flags, event_type, - log_file_name, &log))) + log_file_name, &log, + mariadb_slave_capability, ev_offset, + current_checksum_alg, using_gtid_state, + >id_state, >id_skip_group, + until_gtid_state, >id_until_group, + &until_binlog_state, + slave_gtid_strict_mode, &error_gtid, + &send_fake_gtid_list, fdev))) { errmsg= tmp_msg; - my_errno= ER_UNKNOWN_ERROR; goto err; } + if (unlikely(send_fake_gtid_list) && gtid_skip_group == GTID_SKIP_NOT) + { + Gtid_list_log_event glev(&until_binlog_state, 0); + + if (reset_transmit_packet(thd, flags, &ev_offset, &errmsg) || + fake_gtid_list_event(net, packet, &glev, &errmsg, + current_checksum_alg, my_b_tell(&log))) + { + my_errno= ER_UNKNOWN_ERROR; + goto err; + } + send_fake_gtid_list= false; + } + if (until_gtid_state && + is_until_reached(thd, net, packet, &ev_offset, gtid_until_group, + event_type, current_checksum_alg, flags, &errmsg, + &until_binlog_state, my_b_tell(&log))) + { + if (errmsg) + { + my_errno= ER_UNKNOWN_ERROR; + goto err; + } + goto end; + } DBUG_EXECUTE_IF("dump_thread_wait_before_send_xid", { @@ -964,6 +2394,8 @@ impossible position"; if (reset_transmit_packet(thd, flags, &ev_offset, &errmsg)) goto err; } + if (killed) + goto end; DBUG_EXECUTE_IF("wait_after_binlog_EOF", { @@ -1049,7 +2481,8 @@ impossible position"; int ret; ulong signal_cnt; DBUG_PRINT("wait",("waiting for data in binary log")); - if (thd->server_id==0) // for mysqlbinlog (mysqlbinlog.server_id==0) + /* For mysqlbinlog (mysqlbinlog.server_id==0). */ + if (thd->variables.server_id==0) { mysql_mutex_unlock(log_lock); goto end; @@ -1070,6 +2503,8 @@ impossible position"; thd->enter_cond(log_cond, log_lock, "Master has sent all binlog to slave; " "waiting for binlog to be updated"); + if (thd->killed) + break; ret= mysql_bin_log.wait_for_update_bin_log(thd, heartbeat_ts); DBUG_ASSERT(ret == 0 || (heartbeat_period != 0)); if (ret == ETIMEDOUT || ret == ETIME) @@ -1101,7 +2536,7 @@ impossible position"; { DBUG_PRINT("wait",("binary log received update or a broadcast signal caught")); } - } while (signal_cnt == mysql_bin_log.signal_cnt && !thd->killed); + } while (signal_cnt == mysql_bin_log.signal_cnt); thd->exit_cond(old_msg); } break; @@ -1112,14 +2547,47 @@ impossible position"; goto err; } - if (read_packet && - (tmp_msg= send_event_to_slave(thd, net, packet, flags, event_type, - log_file_name, &log))) + if (read_packet) { - errmsg= tmp_msg; - my_errno= ER_UNKNOWN_ERROR; - goto err; - } + if ((tmp_msg= send_event_to_slave(thd, net, packet, flags, event_type, + log_file_name, &log, + mariadb_slave_capability, ev_offset, + current_checksum_alg, + using_gtid_state, >id_state, + >id_skip_group, until_gtid_state, + >id_until_group, &until_binlog_state, + slave_gtid_strict_mode, &error_gtid, + &send_fake_gtid_list, fdev))) + { + errmsg= tmp_msg; + goto err; + } + if (unlikely(send_fake_gtid_list) && gtid_skip_group == GTID_SKIP_NOT) + { + Gtid_list_log_event glev(&until_binlog_state, 0); + + if (reset_transmit_packet(thd, flags, &ev_offset, &errmsg) || + fake_gtid_list_event(net, packet, &glev, &errmsg, + current_checksum_alg, my_b_tell(&log))) + { + my_errno= ER_UNKNOWN_ERROR; + goto err; + } + send_fake_gtid_list= false; + } + if (until_gtid_state && + is_until_reached(thd, net, packet, &ev_offset, gtid_until_group, + event_type, current_checksum_alg, flags, &errmsg, + &until_binlog_state, my_b_tell(&log))) + { + if (errmsg) + { + my_errno= ER_UNKNOWN_ERROR; + goto err; + } + goto end; + } + } log.error=0; } @@ -1187,6 +2655,7 @@ end: thd->current_linfo = 0; mysql_mutex_unlock(&LOCK_thread_count); thd->variables.max_allowed_packet= old_max_allowed_packet; + delete fdev; DBUG_VOID_RETURN; err: @@ -1206,6 +2675,45 @@ err: my_basename(p_coord->file_name), p_coord->pos, my_basename(log_file_name), my_b_tell(&log)); } + else if (my_errno == ER_GTID_POSITION_NOT_FOUND_IN_BINLOG) + { + my_snprintf(error_text, sizeof(error_text), + "Error: connecting slave requested to start from GTID " + "%u-%u-%llu, which is not in the master's binlog", + error_gtid.domain_id, error_gtid.server_id, error_gtid.seq_no); + /* Use this error code so slave will know not to try reconnect. */ + my_errno = ER_MASTER_FATAL_ERROR_READING_BINLOG; + } + else if (my_errno == ER_GTID_POSITION_NOT_FOUND_IN_BINLOG2) + { + my_snprintf(error_text, sizeof(error_text), + "Error: connecting slave requested to start from GTID " + "%u-%u-%llu, which is not in the master's binlog. Since the " + "master's binlog contains GTIDs with higher sequence numbers, " + "it probably means that the slave has diverged due to " + "executing extra errorneous transactions", + error_gtid.domain_id, error_gtid.server_id, error_gtid.seq_no); + /* Use this error code so slave will know not to try reconnect. */ + my_errno = ER_MASTER_FATAL_ERROR_READING_BINLOG; + } + else if (my_errno == ER_GTID_START_FROM_BINLOG_HOLE) + { + my_snprintf(error_text, sizeof(error_text), + "The binlog on the master is missing the GTID %u-%u-%llu " + "requested by the slave (even though both a prior and a " + "subsequent sequence number does exist), and GTID strict mode " + "is enabled", + error_gtid.domain_id, error_gtid.server_id, error_gtid.seq_no); + /* Use this error code so slave will know not to try reconnect. */ + my_errno = ER_MASTER_FATAL_ERROR_READING_BINLOG; + } + else if (my_errno == ER_CANNOT_LOAD_SLAVE_GTID_STATE) + { + my_snprintf(error_text, sizeof(error_text), + "Failed to load replication slave GTID state from table %s.%s", + "mysql", rpl_gtid_slave_state_table_name.str); + my_errno = ER_MASTER_FATAL_ERROR_READING_BINLOG; + } else strcpy(error_text, errmsg); end_io_cache(&log); @@ -1223,6 +2731,7 @@ err: if (file >= 0) mysql_file_close(file, MYF(MY_WME)); thd->variables.max_allowed_packet= old_max_allowed_packet; + delete fdev; my_message(my_errno, error_text, MYF(0)); DBUG_VOID_RETURN; @@ -1241,18 +2750,52 @@ err: @retval 0 success @retval 1 error + @retval -1 fatal error */ + int start_slave(THD* thd , Master_info* mi, bool net_report) { int slave_errno= 0; int thread_mask; + char master_info_file_tmp[FN_REFLEN]; + char relay_log_info_file_tmp[FN_REFLEN]; DBUG_ENTER("start_slave"); if (check_access(thd, SUPER_ACL, any_db, NULL, NULL, 0, 0)) - DBUG_RETURN(1); + DBUG_RETURN(-1); + + create_logfile_name_with_suffix(master_info_file_tmp, + sizeof(master_info_file_tmp), + master_info_file, 0, + &mi->cmp_connection_name); + create_logfile_name_with_suffix(relay_log_info_file_tmp, + sizeof(relay_log_info_file_tmp), + relay_log_info_file, 0, + &mi->cmp_connection_name); + lock_slave_threads(mi); // this allows us to cleanly read slave_running // Get a mask of _stopped_ threads init_thread_mask(&thread_mask,mi,1 /* inverse */); + + if (thd->lex->mi.gtid_pos_str.str) + { + if (thread_mask != (SLAVE_IO|SLAVE_SQL)) + { + slave_errno= ER_SLAVE_WAS_RUNNING; + goto err; + } + if (thd->lex->slave_thd_opt) + { + slave_errno= ER_BAD_SLAVE_UNTIL_COND; + goto err; + } + if (mi->using_gtid == Master_info::USE_GTID_NO) + { + slave_errno= ER_UNTIL_REQUIRES_USING_GTID; + goto err; + } + } + /* Below we will start all stopped threads. But if the user wants to start only one thread, do as if the other thread was running (as we @@ -1263,7 +2806,7 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) thread_mask&= thd->lex->slave_thd_opt; if (thread_mask) //some threads are stopped, start them { - if (init_master_info(mi,master_info_file,relay_log_info_file, 0, + if (init_master_info(mi,master_info_file_tmp,relay_log_info_file_tmp, 0, thread_mask)) slave_errno=ER_MASTER_INFO; else if (server_id_supplied && *mi->host) @@ -1297,10 +2840,22 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) mi->rli.until_log_pos= thd->lex->mi.relay_log_pos; strmake_buf(mi->rli.until_log_name, thd->lex->mi.relay_log_name); } + else if (thd->lex->mi.gtid_pos_str.str) + { + if (mi->rli.until_gtid_pos.load(thd->lex->mi.gtid_pos_str.str, + thd->lex->mi.gtid_pos_str.length)) + { + slave_errno= ER_INCORRECT_GTID_STATE; + mysql_mutex_unlock(&mi->rli.data_lock); + goto err; + } + mi->rli.until_condition= Relay_log_info::UNTIL_GTID; + } else mi->rli.clear_until_condition(); - if (mi->rli.until_condition != Relay_log_info::UNTIL_NONE) + if (mi->rli.until_condition == Relay_log_info::UNTIL_MASTER_POS || + mi->rli.until_condition == Relay_log_info::UNTIL_RELAY_POS) { /* Preparing members for effective until condition checking */ const char *p= fn_ext(mi->rli.until_log_name); @@ -1323,7 +2878,10 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) /* mark the cached result of the UNTIL comparison as "undefined" */ mi->rli.until_log_names_cmp_result= Relay_log_info::UNTIL_LOG_NAMES_CMP_UNKNOWN; + } + if (mi->rli.until_condition != Relay_log_info::UNTIL_NONE) + { /* Issuing warning then started without --skip-slave-start */ if (!opt_skip_slave_start) push_warning(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, @@ -1339,10 +2897,11 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) if (!slave_errno) slave_errno = start_slave_threads(0 /*no mutex */, - 1 /* wait for start */, - mi, - master_info_file,relay_log_info_file, - thread_mask); + 1 /* wait for start */, + mi, + master_info_file_tmp, + relay_log_info_file_tmp, + thread_mask); } else slave_errno = ER_BAD_SLAVE; @@ -1354,16 +2913,17 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) ER(ER_SLAVE_WAS_RUNNING)); } +err: unlock_slave_threads(mi); if (slave_errno) { if (net_report) - my_message(slave_errno, ER(slave_errno), MYF(0)); - DBUG_RETURN(1); + my_error(slave_errno, MYF(0), + (int) mi->connection_name.length, + mi->connection_name.str); + DBUG_RETURN(slave_errno == ER_BAD_SLAVE ? -1 : 1); } - else if (net_report) - my_ok(thd); DBUG_RETURN(0); } @@ -1381,17 +2941,17 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) @retval 0 success @retval 1 error + @retval -1 error */ + int stop_slave(THD* thd, Master_info* mi, bool net_report ) { - DBUG_ENTER("stop_slave"); - int slave_errno; - if (!thd) - thd = current_thd; + DBUG_ENTER("stop_slave"); + DBUG_PRINT("enter",("Connection: %s", mi->connection_name.str)); if (check_access(thd, SUPER_ACL, any_db, NULL, NULL, 0, 0)) - DBUG_RETURN(1); + DBUG_RETURN(-1); thd_proc_info(thd, "Killing slave"); int thread_mask; lock_slave_threads(mi); @@ -1427,8 +2987,6 @@ int stop_slave(THD* thd, Master_info* mi, bool net_report ) my_message(slave_errno, ER(slave_errno), MYF(0)); DBUG_RETURN(1); } - else if (net_report) - my_ok(thd); DBUG_RETURN(0); } @@ -1452,15 +3010,18 @@ int reset_slave(THD *thd, Master_info* mi) int thread_mask= 0, error= 0; uint sql_errno=ER_UNKNOWN_ERROR; const char* errmsg= "Unknown error occured while reseting slave"; + char master_info_file_tmp[FN_REFLEN]; + char relay_log_info_file_tmp[FN_REFLEN]; DBUG_ENTER("reset_slave"); lock_slave_threads(mi); init_thread_mask(&thread_mask,mi,0 /* not inverse */); if (thread_mask) // We refuse if any slave thread is running { - sql_errno= ER_SLAVE_MUST_STOP; - error=1; - goto err; + unlock_slave_threads(mi); + my_error(ER_SLAVE_MUST_STOP, MYF(0), (int) mi->connection_name.length, + mi->connection_name.str); + DBUG_RETURN(ER_SLAVE_MUST_STOP); } ha_reset_slave(thd); @@ -1487,22 +3048,37 @@ int reset_slave(THD *thd, Master_info* mi) // close master_info_file, relay_log_info_file, set mi->inited=rli->inited=0 end_master_info(mi); + // and delete these two files - fn_format(fname, master_info_file, mysql_data_home, "", 4+32); + create_logfile_name_with_suffix(master_info_file_tmp, + sizeof(master_info_file_tmp), + master_info_file, 0, + &mi->cmp_connection_name); + create_logfile_name_with_suffix(relay_log_info_file_tmp, + sizeof(relay_log_info_file_tmp), + relay_log_info_file, 0, + &mi->cmp_connection_name); + + fn_format(fname, master_info_file_tmp, mysql_data_home, "", 4+32); if (mysql_file_stat(key_file_master_info, fname, &stat_area, MYF(0)) && mysql_file_delete(key_file_master_info, fname, MYF(MY_WME))) { error=1; goto err; } + else if (global_system_variables.log_warnings > 1) + sql_print_information("Deleted Master_info file '%s'.", fname); + // delete relay_log_info_file - fn_format(fname, relay_log_info_file, mysql_data_home, "", 4+32); + fn_format(fname, relay_log_info_file_tmp, mysql_data_home, "", 4+32); if (mysql_file_stat(key_file_relay_log_info, fname, &stat_area, MYF(0)) && mysql_file_delete(key_file_relay_log_info, fname, MYF(MY_WME))) { error=1; goto err; } + else if (global_system_variables.log_warnings > 1) + sql_print_information("Deleted Master_info file '%s'.", fname); RUN_HOOK(binlog_relay_io, after_reset_slave, (thd, mi)); err: @@ -1541,7 +3117,7 @@ void kill_zombie_dump_threads(uint32 slave_server_id) while ((tmp=it++)) { if (tmp->command == COM_BINLOG_DUMP && - tmp->server_id == slave_server_id) + tmp->variables.server_id == slave_server_id) { mysql_mutex_lock(&tmp->LOCK_thd_data); // Lock from delete break; @@ -1596,10 +3172,14 @@ static bool get_string_parameter(char *to, const char *from, size_t length, @param mi Pointer to Master_info object belonging to the slave's IO thread. + @param master_info_added Out parameter saying if the Master_info *mi was + added to the global list of masters. This is useful in error conditions + to know if caller should free Master_info *mi. + @retval FALSE success @retval TRUE error */ -bool change_master(THD* thd, Master_info* mi) +bool change_master(THD* thd, Master_info* mi, bool *master_info_added) { int thread_mask; const char* errmsg= 0; @@ -1608,20 +3188,14 @@ bool change_master(THD* thd, Master_info* mi) char saved_host[HOSTNAME_LENGTH + 1]; uint saved_port; char saved_log_name[FN_REFLEN]; + Master_info::enum_using_gtid saved_using_gtid; + char master_info_file_tmp[FN_REFLEN]; + char relay_log_info_file_tmp[FN_REFLEN]; my_off_t saved_log_pos; - DBUG_ENTER("change_master"); - - lock_slave_threads(mi); - init_thread_mask(&thread_mask,mi,0 /*not inverse*/); LEX_MASTER_INFO* lex_mi= &thd->lex->mi; - if (thread_mask) // We refuse if any slave thread is running - { - my_message(ER_SLAVE_MUST_STOP, ER(ER_SLAVE_MUST_STOP), MYF(0)); - ret= TRUE; - goto err; - } + DBUG_ENTER("change_master"); - thd_proc_info(thd, "Changing master"); + *master_info_added= false; /* We need to check if there is an empty master_host. Otherwise change master succeeds, a master.info file is created containing @@ -1629,17 +3203,64 @@ bool change_master(THD* thd, Master_info* mi) is thrown stating that the server is not configured as slave. (See BUG#28796). */ - if(lex_mi->host && !*lex_mi->host) + if (lex_mi->host && !*lex_mi->host) { my_error(ER_WRONG_ARGUMENTS, MYF(0), "MASTER_HOST"); - unlock_slave_threads(mi); DBUG_RETURN(TRUE); } - // TODO: see if needs re-write - if (init_master_info(mi, master_info_file, relay_log_info_file, 0, + if (master_info_index->check_duplicate_master_info(&lex_mi->connection_name, + lex_mi->host, + lex_mi->port)) + DBUG_RETURN(TRUE); + + lock_slave_threads(mi); + init_thread_mask(&thread_mask,mi,0 /*not inverse*/); + if (thread_mask) // We refuse if any slave thread is running + { + my_error(ER_SLAVE_MUST_STOP, MYF(0), (int) mi->connection_name.length, + mi->connection_name.str); + ret= TRUE; + goto err; + } + + thd_proc_info(thd, "Changing master"); + + create_logfile_name_with_suffix(master_info_file_tmp, + sizeof(master_info_file_tmp), + master_info_file, 0, + &mi->cmp_connection_name); + create_logfile_name_with_suffix(relay_log_info_file_tmp, + sizeof(relay_log_info_file_tmp), + relay_log_info_file, 0, + &mi->cmp_connection_name); + + /* if new Master_info doesn't exists, add it */ + if (!master_info_index->get_master_info(&mi->connection_name, + MYSQL_ERROR::WARN_LEVEL_NOTE)) + { + if (master_info_index->add_master_info(mi, TRUE)) + { + my_error(ER_MASTER_INFO, MYF(0), + (int) lex_mi->connection_name.length, + lex_mi->connection_name.str); + ret= TRUE; + goto err; + } + *master_info_added= true; + } + if (global_system_variables.log_warnings > 1) + sql_print_information("Master: '%.*s' Master_info_file: '%s' " + "Relay_info_file: '%s'", + (int) mi->connection_name.length, + mi->connection_name.str, + master_info_file_tmp, relay_log_info_file_tmp); + + if (init_master_info(mi, master_info_file_tmp, relay_log_info_file_tmp, 0, thread_mask)) { - my_message(ER_MASTER_INFO, ER(ER_MASTER_INFO), MYF(0)); + my_error(ER_MASTER_INFO, MYF(0), + (int) lex_mi->connection_name.length, + lex_mi->connection_name.str); ret= TRUE; goto err; } @@ -1657,6 +3278,7 @@ bool change_master(THD* thd, Master_info* mi) saved_port= mi->port; strmake_buf(saved_log_name, mi->master_log_name); saved_log_pos= mi->master_log_pos; + saved_using_gtid= mi->using_gtid; /* If the user specified host or port without binlog or position, @@ -1698,7 +3320,7 @@ bool change_master(THD* thd, Master_info* mi) else mi->heartbeat_period= (float) min(SLAVE_MAX_HEARTBEAT_PERIOD, (slave_net_timeout/2.0)); - mi->received_heartbeats= LL(0); // counter lives until master is CHANGEd + mi->received_heartbeats= 0; // counter lives until master is CHANGEd /* reset the last time server_id list if the current CHANGE MASTER is mentioning IGNORE_SERVER_IDS= (...) @@ -1709,7 +3331,7 @@ bool change_master(THD* thd, Master_info* mi) { ulong s_id; get_dynamic(&lex_mi->repl_ignore_server_ids, (uchar*) &s_id, i); - if (s_id == ::server_id && replicate_same_server_id) + if (s_id == global_system_variables.server_id && replicate_same_server_id) { my_error(ER_SLAVE_IGNORE_SERVER_IDS, MYF(0), static_cast<int>(s_id)); ret= TRUE; @@ -1767,6 +3389,15 @@ bool change_master(THD* thd, Master_info* mi) mi->rli.group_relay_log_pos= mi->rli.event_relay_log_pos= lex_mi->relay_log_pos; } + if (lex_mi->use_gtid_opt == LEX_MASTER_INFO::LEX_GTID_SLAVE_POS) + mi->using_gtid= Master_info::USE_GTID_SLAVE_POS; + else if (lex_mi->use_gtid_opt == LEX_MASTER_INFO::LEX_GTID_CURRENT_POS) + mi->using_gtid= Master_info::USE_GTID_CURRENT_POS; + else if (lex_mi->use_gtid_opt == LEX_MASTER_INFO::LEX_GTID_NO || + lex_mi->log_file_name || lex_mi->pos || + lex_mi->relay_log_name || lex_mi->relay_log_pos) + mi->using_gtid= Master_info::USE_GTID_NO; + /* If user did specify neither host nor port nor any log name nor any log pos, i.e. he specified only user/password/master_connect_retry, he probably @@ -1796,6 +3427,7 @@ bool change_master(THD* thd, Master_info* mi) mi->rli.group_master_log_pos); strmake_buf(mi->master_log_name, mi->rli.group_master_log_name); } + /* Relay log's IO_CACHE may not be inited, if rli->inited==0 (server was never a slave before). @@ -1865,6 +3497,11 @@ bool change_master(THD* thd, Master_info* mi) "master_log_pos='%ld'.", saved_host, saved_port, saved_log_name, (ulong) saved_log_pos, mi->host, mi->port, mi->master_log_name, (ulong) mi->master_log_pos); + if (saved_using_gtid != Master_info::USE_GTID_NO || + mi->using_gtid != Master_info::USE_GTID_NO) + sql_print_information("Previous Using_Gtid=%s. New Using_Gtid=%s", + mi->using_gtid_astext(saved_using_gtid), + mi->using_gtid_astext(mi->using_gtid)); /* If we don't write new coordinates to disk now, then old will remain in @@ -1895,7 +3532,7 @@ err: @retval 0 success @retval 1 error */ -int reset_master(THD* thd) +int reset_master(THD* thd, rpl_gtid *init_state, uint32 init_state_len) { if (!mysql_bin_log.is_open()) { @@ -1904,7 +3541,7 @@ int reset_master(THD* thd) return 1; } - if (mysql_bin_log.reset_logs(thd)) + if (mysql_bin_log.reset_logs(thd, 1, init_state, init_state_len)) return 1; RUN_HOOK(binlog_transmit, after_reset_master, (thd, 0 /* flags */)); return 0; @@ -1930,6 +3567,7 @@ bool mysql_show_binlog_events(THD* thd) File file = -1; MYSQL_BIN_LOG *binary_log= NULL; int old_max_allowed_packet= thd->variables.max_allowed_packet; + Master_info *mi= 0; LOG_INFO linfo; DBUG_ENTER("mysql_show_binlog_events"); @@ -1959,10 +3597,15 @@ bool mysql_show_binlog_events(THD* thd) } else /* showing relay log contents */ { - if (!active_mi) + mysql_mutex_lock(&LOCK_active_mi); + if (!(mi= master_info_index-> + get_master_info(&thd->variables.default_master_connection, + MYSQL_ERROR::WARN_LEVEL_ERROR))) + { + mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(TRUE); - - binary_log= &(active_mi->rli.relay_log); + } + binary_log= &(mi->rli.relay_log); } if (binary_log->is_open()) @@ -1976,6 +3619,13 @@ bool mysql_show_binlog_events(THD* thd) mysql_mutex_t *log_lock = binary_log->get_log_lock(); Log_event* ev; + if (mi) + { + /* We can unlock the mutex as we have a lock on the file */ + mysql_mutex_unlock(&LOCK_active_mi); + mi= 0; + } + unit->set_limit(thd->lex->current_select); limit_start= unit->offset_limit_cnt; limit_end= unit->select_limit_cnt; @@ -2070,6 +3720,9 @@ bool mysql_show_binlog_events(THD* thd) mysql_mutex_unlock(log_lock); } + else if (mi) + mysql_mutex_unlock(&LOCK_active_mi); + // Check that linfo is still on the function scope. DEBUG_SYNC(thd, "after_show_binlog_events"); @@ -2276,4 +3929,204 @@ int log_loaded_block(IO_CACHE* file) DBUG_RETURN(0); } + +/** + Initialise the slave replication state from the mysql.gtid_slave_pos table. + + This is called each time an SQL thread starts, but the data is only actually + loaded on the first call. + + The slave state is the last GTID applied on the slave within each + replication domain. + + To avoid row lock contention, there are multiple rows for each domain_id. + The one containing the current slave state is the one with the maximal + sub_id value, within each domain_id. + + CREATE TABLE mysql.gtid_slave_pos ( + domain_id INT UNSIGNED NOT NULL, + sub_id BIGINT UNSIGNED NOT NULL, + server_id INT UNSIGNED NOT NULL, + seq_no BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (domain_id, sub_id)) +*/ + +void +rpl_init_gtid_slave_state() +{ + rpl_global_gtid_slave_state.init(); +} + + +void +rpl_deinit_gtid_slave_state() +{ + rpl_global_gtid_slave_state.deinit(); +} + + +void +rpl_init_gtid_waiting() +{ + rpl_global_gtid_waiting.init(); +} + + +void +rpl_deinit_gtid_waiting() +{ + rpl_global_gtid_waiting.destroy(); +} + + +/* + Format the current GTID state as a string, for returning the value of + @@global.gtid_slave_pos. + + If the flag use_binlog is true, then the contents of the binary log (if + enabled) is merged into the current GTID state (@@global.gtid_current_pos). +*/ +int +rpl_append_gtid_state(String *dest, bool use_binlog) +{ + int err; + rpl_gtid *gtid_list= NULL; + uint32 num_gtids= 0; + + if (use_binlog && opt_bin_log && + (err= mysql_bin_log.get_most_recent_gtid_list(>id_list, &num_gtids))) + return err; + + err= rpl_global_gtid_slave_state.tostring(dest, gtid_list, num_gtids); + my_free(gtid_list); + + return err; +} + + +/* + Load the current GTID position into a slave_connection_state, for use when + connecting to a master server with GTID. + + If the flag use_binlog is true, then the contents of the binary log (if + enabled) is merged into the current GTID state (master_use_gtid=current_pos). +*/ +int +rpl_load_gtid_state(slave_connection_state *state, bool use_binlog) +{ + int err; + rpl_gtid *gtid_list= NULL; + uint32 num_gtids= 0; + + if (use_binlog && opt_bin_log && + (err= mysql_bin_log.get_most_recent_gtid_list(>id_list, &num_gtids))) + return err; + + err= state->load(&rpl_global_gtid_slave_state, gtid_list, num_gtids); + my_free(gtid_list); + + return err; +} + + +bool +rpl_gtid_pos_check(THD *thd, char *str, size_t len) +{ + slave_connection_state tmp_slave_state; + bool gave_conflict_warning= false, gave_missing_warning= false; + + /* Check that we can parse the supplied string. */ + if (tmp_slave_state.load(str, len)) + return true; + + /* + Check our own binlog for any of our own transactions that are newer + than the GTID state the user is requesting. Any such transactions would + result in an out-of-order binlog, which could break anyone replicating + with us as master. + + So give an error if this is found, requesting the user to do a + RESET MASTER (to clean up the binlog) if they really want this. + */ + if (mysql_bin_log.is_open()) + { + rpl_gtid *binlog_gtid_list= NULL; + uint32 num_binlog_gtids= 0; + uint32 i; + + if (mysql_bin_log.get_most_recent_gtid_list(&binlog_gtid_list, + &num_binlog_gtids)) + { + my_error(ER_OUT_OF_RESOURCES, MYF(MY_WME)); + return true; + } + for (i= 0; i < num_binlog_gtids; ++i) + { + rpl_gtid *binlog_gtid= &binlog_gtid_list[i]; + rpl_gtid *slave_gtid; + if (binlog_gtid->server_id != global_system_variables.server_id) + continue; + if (!(slave_gtid= tmp_slave_state.find(binlog_gtid->domain_id))) + { + if (opt_gtid_strict_mode) + { + my_error(ER_MASTER_GTID_POS_MISSING_DOMAIN, MYF(0), + binlog_gtid->domain_id, binlog_gtid->domain_id, + binlog_gtid->server_id, binlog_gtid->seq_no); + break; + } + else if (!gave_missing_warning) + { + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, + ER_MASTER_GTID_POS_MISSING_DOMAIN, + ER(ER_MASTER_GTID_POS_MISSING_DOMAIN), + binlog_gtid->domain_id, binlog_gtid->domain_id, + binlog_gtid->server_id, binlog_gtid->seq_no); + gave_missing_warning= true; + } + } + else if (slave_gtid->seq_no < binlog_gtid->seq_no) + { + if (opt_gtid_strict_mode) + { + my_error(ER_MASTER_GTID_POS_CONFLICTS_WITH_BINLOG, MYF(0), + slave_gtid->domain_id, slave_gtid->server_id, + slave_gtid->seq_no, binlog_gtid->domain_id, + binlog_gtid->server_id, binlog_gtid->seq_no); + break; + } + else if (!gave_conflict_warning) + { + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, + ER_MASTER_GTID_POS_CONFLICTS_WITH_BINLOG, + ER(ER_MASTER_GTID_POS_CONFLICTS_WITH_BINLOG), + slave_gtid->domain_id, slave_gtid->server_id, + slave_gtid->seq_no, binlog_gtid->domain_id, + binlog_gtid->server_id, binlog_gtid->seq_no); + gave_conflict_warning= true; + } + } + } + my_free(binlog_gtid_list); + if (i != num_binlog_gtids) + return true; + } + + return false; +} + + +bool +rpl_gtid_pos_update(THD *thd, char *str, size_t len) +{ + if (rpl_global_gtid_slave_state.load(thd, str, len, true, true)) + { + my_error(ER_FAILED_GTID_STATE_INIT, MYF(0)); + return true; + } + else + return false; +} + + #endif /* HAVE_REPLICATION */ diff --git a/sql/sql_repl.h b/sql/sql_repl.h index c5a0b31388e..defb1b23f5b 100644 --- a/sql/sql_repl.h +++ b/sql/sql_repl.h @@ -32,6 +32,8 @@ typedef struct st_slave_info THD* thd; } SLAVE_INFO; +struct slave_connection_state; + extern my_bool opt_show_slave_auth_info; extern char *master_host, *master_info_file; extern bool server_id_supplied; @@ -41,10 +43,10 @@ extern my_bool opt_sporadic_binlog_dump_fail; int start_slave(THD* thd, Master_info* mi, bool net_report); int stop_slave(THD* thd, Master_info* mi, bool net_report); -bool change_master(THD* thd, Master_info* mi); +bool change_master(THD* thd, Master_info* mi, bool *master_info_added); bool mysql_show_binlog_events(THD* thd); int reset_slave(THD *thd, Master_info* mi); -int reset_master(THD* thd); +int reset_master(THD* thd, rpl_gtid *init_state, uint32 init_state_len); bool purge_master_logs(THD* thd, const char* to_log); bool purge_master_logs_before_date(THD* thd, time_t purge_time); bool log_in_use(const char* log_name); @@ -65,6 +67,17 @@ int log_loaded_block(IO_CACHE* file); int init_replication_sys_vars(); void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, ushort flags); +extern PSI_mutex_key key_LOCK_slave_state, key_LOCK_binlog_state; +void rpl_init_gtid_slave_state(); +void rpl_deinit_gtid_slave_state(); +void rpl_init_gtid_waiting(); +void rpl_deinit_gtid_waiting(); +int gtid_state_from_binlog_pos(const char *name, uint32 pos, String *out_str); +int rpl_append_gtid_state(String *dest, bool use_binlog); +int rpl_load_gtid_state(slave_connection_state *state, bool use_binlog); +bool rpl_gtid_pos_check(THD *thd, char *str, size_t len); +bool rpl_gtid_pos_update(THD *thd, char *str, size_t len); + #endif /* HAVE_REPLICATION */ #endif /* SQL_REPL_INCLUDED */ diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 35267e6617c..93e67c484e3 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -51,6 +51,7 @@ #include "opt_subselect.h" #include "log_slow.h" #include "sql_derived.h" +#include "sql_statistics.h" #include "debug_sync.h" // DEBUG_SYNC #include <m_ctype.h> @@ -88,12 +89,14 @@ void best_access_path(JOIN *join, JOIN_TAB *s, POSITION *pos, POSITION *loose_scan_pos); static void optimize_straight_join(JOIN *join, table_map join_tables); static bool greedy_search(JOIN *join, table_map remaining_tables, - uint depth, uint prune_level); + uint depth, uint prune_level, + uint use_cond_selectivity); static bool best_extension_by_limited_search(JOIN *join, table_map remaining_tables, uint idx, double record_count, double read_time, uint depth, - uint prune_level); + uint prune_level, + uint use_cond_selectivity); static uint determine_search_depth(JOIN* join); C_MODE_START static int join_tab_cmp(const void *dummy, const void* ptr1, const void* ptr2); @@ -105,7 +108,7 @@ C_MODE_END tested and approved. */ static bool find_best(JOIN *join,table_map rest_tables,uint index, - double record_count,double read_time); + double record_count,double read_time, uint use_cond_selectivity); static uint cache_record_length(JOIN *join,uint index); bool get_best_combination(JOIN *join); static store_key *get_store_key(THD *thd, @@ -133,7 +136,8 @@ static COND *build_equal_items(JOIN *join, COND *cond, COND_EQUAL *inherited, List<TABLE_LIST> *join_list, bool ignore_on_conds, - COND_EQUAL **cond_equal_ref); + COND_EQUAL **cond_equal_ref, + bool link_equal_fields= FALSE); static COND* substitute_for_best_equal_field(JOIN_TAB *context_tab, COND *cond, COND_EQUAL *cond_equal, @@ -149,8 +153,9 @@ static uint build_bitmap_for_nested_joins(List<TABLE_LIST> *join_list, static COND *optimize_cond(JOIN *join, COND *conds, List<TABLE_LIST> *join_list, bool ignore_on_conds, - Item::cond_result *cond_value, - COND_EQUAL **cond_equal); + Item::cond_result *cond_value, + COND_EQUAL **cond_equal, + int flags= 0); bool const_expression_in_where(COND *conds,Item *item, Item **comp_item); static bool create_internal_tmp_table_from_heap2(THD *, TABLE *, ENGINE_COLUMNDEF *, ENGINE_COLUMNDEF **, @@ -279,6 +284,59 @@ enum enum_exec_or_opt {WALK_OPTIMIZATION_TABS , WALK_EXECUTION_TABS}; JOIN_TAB *first_breadth_first_tab(JOIN *join, enum enum_exec_or_opt tabs_kind); JOIN_TAB *next_breadth_first_tab(JOIN *join, enum enum_exec_or_opt tabs_kind, JOIN_TAB *tab); +static double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, + table_map rem_tables); + +#ifndef DBUG_OFF + +/* + SHOW EXPLAIN testing: wait for, and serve n_calls APC requests. +*/ +void dbug_serve_apcs(THD *thd, int n_calls) +{ + const char *save_proc_info= thd->proc_info; + /* This is so that mysqltest knows we're ready to serve requests: */ + thd_proc_info(thd, "show_explain_trap"); + + /* Busy-wait for n_calls APC requests to arrive and be processed */ + int n_apcs= thd->apc_target.n_calls_processed + n_calls; + while (thd->apc_target.n_calls_processed < n_apcs) + { + my_sleep(300); + if (thd->check_killed()) + break; + } + thd_proc_info(thd, save_proc_info); +} + + +/* + Debugging: check if @name=value, comparing as integer + + Intended usage: + + DBUG_EXECUTE_IF("show_explain_probe_2", + if (dbug_user_var_equals_int(thd, "select_id", select_id)) + dbug_serve_apcs(thd, 1); + ); + +*/ + +bool dbug_user_var_equals_int(THD *thd, const char *name, int value) +{ + user_var_entry *var; + LEX_STRING varname= {(char*)name, strlen(name)}; + if ((var= get_variable(&thd->user_vars, varname, FALSE))) + { + bool null_value; + longlong var_value= var->val_int(&null_value); + if (!null_value && var_value == value) + return TRUE; + } + return FALSE; +} +#endif + /** This handles SELECT with and without UNION. @@ -553,7 +611,9 @@ inline int setup_without_group(THD *thd, Item **ref_pointer_array, List<Item> &all_fields, COND **conds, ORDER *order, - ORDER *group, bool *hidden_group_fields) + ORDER *group, + bool *hidden_group_fields, + uint *reserved) { int res; st_select_lex *const select= thd->lex->current_select; @@ -567,6 +627,13 @@ inline int setup_without_group(THD *thd, Item **ref_pointer_array, thd->lex->allow_sum_func&= ~((nesting_map)1 << select->nest_level); res= setup_conds(thd, tables, leaves, conds); + if (thd->lex->current_select->first_cond_optimization) + { + if (!res && *conds) + (*reserved)= (*conds)->exists2in_reserved_items(); + else + (*reserved)= 0; + } /* it's not wrong to have non-aggregated columns in a WHERE */ select->set_non_agg_field_used(saved_non_agg_field_used); @@ -714,7 +781,7 @@ JOIN::prepare(Item ***rref_pointer_array, setup_without_group(thd, (*rref_pointer_array), tables_list, select_lex->leaf_tables, fields_list, all_fields, &conds, order, group_list, - &hidden_group_fields)) + &hidden_group_fields, &select_lex->select_n_reserved)) DBUG_RETURN(-1); /* purecov: inspected */ ref_pointer_array= *rref_pointer_array; @@ -945,6 +1012,34 @@ err: DBUG_RETURN(res); /* purecov: inspected */ } +int JOIN::optimize() +{ + bool was_optimized= optimized; + int res= optimize_inner(); + /* + If we're inside a non-correlated subquery, this function may be + called for the second time after the subquery has been executed + and deleted. The second call will not produce a valid query plan, it will + short-circuit because optimized==TRUE. + + "was_optimized != optimized" is here to handle this case: + - first optimization starts, gets an error (from a const. cheap + subquery), returns 1 + - another JOIN::optimize() call made, and now join->optimize() will + return 0, even though we never had a query plan. + */ + if (was_optimized != optimized && !res && have_query_plan != QEP_DELETED) + { + create_explain_query_if_not_exists(thd->lex, thd->mem_root); + have_query_plan= QEP_AVAILABLE; + save_explain_data(thd->lex->explain, false /* can overwrite */, + need_tmp, + !skip_sort_order && !no_order && (order || group_list), + select_distinct); + } + return res; +} + /** global select optimisation. @@ -959,7 +1054,7 @@ err: */ int -JOIN::optimize() +JOIN::optimize_inner() { ulonglong select_opts_for_readinfo; uint no_jbuf_after; @@ -992,6 +1087,24 @@ JOIN::optimize() // Update used tables after all handling derived table procedures select_lex->update_used_tables(); + /* + In fact we transform underlying subqueries after their 'prepare' phase and + before 'optimize' from upper query 'optimize' to allow semijoin + conversion happened (which done in the same way. + */ + if(select_lex->first_cond_optimization && + conds && conds->walk(&Item::exists2in_processor, 0, (uchar *)thd)) + DBUG_RETURN(1); + /* +TODO: make view to decide if it is possible to write to WHERE directly or make Semi-Joins able to process ON condition if it is possible + for (TABLE_LIST *tbl= tables_list; tbl; tbl= tbl->next_local) + { + if (tbl->on_expr && + tbl->on_expr->walk(&Item::exists2in_processor, 0, (uchar *)thd)) + DBUG_RETURN(1); + } + */ + if (transform_max_min_subquery()) DBUG_RETURN(1); /* purecov: inspected */ @@ -1075,7 +1188,7 @@ JOIN::optimize() DBUG_RETURN(1); conds= optimize_cond(this, conds, join_list, FALSE, - &cond_value, &cond_equal); + &cond_value, &cond_equal, OPT_LINK_EQUAL_FIELDS); if (thd->is_error()) { @@ -1466,9 +1579,10 @@ JOIN::optimize() We have found that grouping can be removed since groups correspond to only one row anyway, but we still have to guarantee correct result order. The line below effectively rewrites the query from GROUP BY - <fields> to ORDER BY <fields>. There are two exceptions: + <fields> to ORDER BY <fields>. There are three exceptions: - if skip_sort_order is set (see above), then we can simply skip GROUP BY; + - if we are in a subquery, we don't have to maintain order - we can only rewrite ORDER BY if the ORDER BY fields are 'compatible' with the GROUP BY ones, i.e. either one is a prefix of another. We only check if the ORDER BY is a prefix of GROUP BY. In this case @@ -1478,7 +1592,13 @@ JOIN::optimize() 'order' as is. */ if (!order || test_if_subpart(group_list, order)) - order= skip_sort_order ? 0 : group_list; + { + if (skip_sort_order || + select_lex->master_unit()->item) // This is a subquery + order= NULL; + else + order= group_list; + } /* If we have an IGNORE INDEX FOR GROUP BY(fields) clause, this must be rewritten to IGNORE INDEX FOR ORDER BY(fields). @@ -1522,8 +1642,10 @@ JOIN::optimize() JOIN_TAB *tab= &join_tab[const_tables]; bool all_order_fields_used; if (order) + { skip_sort_order= test_if_skip_sort_order(tab, order, select_limit, 1, &tab->table->keys_in_use_for_order_by); + } if ((group_list=create_distinct_group(thd, select_lex->ref_pointer_array, order, fields_list, all_fields, &all_order_fields_used))) @@ -1891,6 +2013,7 @@ int JOIN::init_execution() if (!group_list && ! exec_tmp_table1->distinct && order && simple_order) { + DBUG_PRINT("info",("Sorting for order")); thd_proc_info(thd, "Sorting for order"); if (create_sort_index(thd, this, order, HA_POS_ERROR, HA_POS_ERROR, TRUE)) @@ -2102,8 +2225,7 @@ JOIN::reinit() DBUG_ENTER("JOIN::reinit"); unit->offset_limit_cnt= (ha_rows)(select_lex->offset_limit ? - select_lex->offset_limit->val_uint() : - ULL(0)); + select_lex->offset_limit->val_uint() : 0); first_record= 0; cleaned= false; @@ -2201,6 +2323,59 @@ JOIN::save_join_tab() } +void JOIN::save_explain_data(Explain_query *output, bool can_overwrite, + bool need_tmp_table, bool need_order, + bool distinct) +{ + if (select_lex->select_number != UINT_MAX && + select_lex->select_number != INT_MAX /* this is not a UNION's "fake select */ && + have_query_plan != JOIN::QEP_NOT_PRESENT_YET && + have_query_plan != JOIN::QEP_DELETED && // this happens when there was + // no QEP ever, but then + //cleanup() is called multiple times + output && // for "SET" command in SPs. + (can_overwrite? true: !output->get_select(select_lex->select_number))) + { + const char *message= NULL; + if (!table_count || !tables_list || zero_result_cause) + { + /* It's a degenerate join */ + message= zero_result_cause ? zero_result_cause : "No tables used"; + } + save_explain_data_intern(thd->lex->explain, need_tmp_table, need_order, + distinct, message); + } +} + + +void JOIN::exec() +{ + DBUG_EXECUTE_IF("show_explain_probe_join_exec_start", + if (dbug_user_var_equals_int(thd, + "show_explain_probe_select_id", + select_lex->select_number)) + dbug_serve_apcs(thd, 1); + ); + exec_inner(); + + if (!exec_saved_explain) + { + save_explain_data(thd->lex->explain, true /* can overwrite */, + need_tmp, + order != 0 && !skip_sort_order, + select_distinct); + exec_saved_explain= true; + } + + DBUG_EXECUTE_IF("show_explain_probe_join_exec_end", + if (dbug_user_var_equals_int(thd, + "show_explain_probe_select_id", + select_lex->select_number)) + dbug_serve_apcs(thd, 1); + ); +} + + /** Exec select. @@ -2212,14 +2387,16 @@ JOIN::save_join_tab() @todo When can we have here thd->net.report_error not zero? */ -void -JOIN::exec() + +void JOIN::exec_inner() { List<Item> *columns_list= &fields_list; int tmp_error; DBUG_ENTER("JOIN::exec"); + const bool has_group_by= this->group; + thd_proc_info(thd, "executing"); error= 0; if (procedure) @@ -2523,6 +2700,10 @@ JOIN::exec() DBUG_PRINT("info",("Creating group table")); /* Free first data from old join */ + + /* + psergey-todo: this is the place of pre-mature JOIN::free call. + */ curr_join->join_free(); if (curr_join->make_simple_join(this, curr_tmp_table)) DBUG_VOID_RETURN; @@ -2563,11 +2744,12 @@ JOIN::exec() } if (curr_join->group_list) { - thd_proc_info(thd, "Creating sort index"); if (curr_join->join_tab == join_tab && save_join_tab()) { DBUG_VOID_RETURN; } + DBUG_PRINT("info",("Sorting for index")); + thd_proc_info(thd, "Creating sort index"); if (create_sort_index(thd, curr_join, curr_join->group_list, HA_POS_ERROR, HA_POS_ERROR, FALSE) || make_group_fields(this, curr_join)) @@ -2731,6 +2913,7 @@ JOIN::exec() JOIN_TAB *curr_table= &curr_join->join_tab[curr_join->const_tables]; table_map used_tables= (curr_join->const_table_map | curr_table->table->map); + curr_join->tmp_having->update_used_tables(); Item* sort_table_cond= make_cond_for_table(thd, curr_join->tmp_having, used_tables, @@ -2820,13 +3003,39 @@ JOIN::exec() the query. XXX: it's never shown in EXPLAIN! OPTION_FOUND_ROWS supersedes LIMIT and is taken into account. */ - if (create_sort_index(thd, curr_join, - curr_join->group_list ? - curr_join->group_list : curr_join->order, - curr_join->select_limit, - (select_options & OPTION_FOUND_ROWS ? - HA_POS_ERROR : unit->select_limit_cnt), - curr_join->group_list ? TRUE : FALSE)) + DBUG_PRINT("info",("Sorting for order by/group by")); + ORDER *order_arg= + curr_join->group_list ? curr_join->group_list : curr_join->order; + /* + filesort_limit: Return only this many rows from filesort(). + We can use select_limit_cnt only if we have no group_by and 1 table. + This allows us to use Bounded_queue for queries like: + "select SQL_CALC_FOUND_ROWS * from t1 order by b desc limit 1;" + select_limit == HA_POS_ERROR (we need a full table scan) + unit->select_limit_cnt == 1 (we only need one row in the result set) + */ + const ha_rows filesort_limit_arg= + (has_group_by || curr_join->table_count > 1) + ? curr_join->select_limit : unit->select_limit_cnt; + const ha_rows select_limit_arg= + select_options & OPTION_FOUND_ROWS + ? HA_POS_ERROR : unit->select_limit_cnt; + + DBUG_PRINT("info", ("has_group_by %d " + "curr_join->table_count %d " + "curr_join->m_select_limit %d " + "unit->select_limit_cnt %d", + has_group_by, + curr_join->table_count, + (int) curr_join->select_limit, + (int) unit->select_limit_cnt)); + + if (create_sort_index(thd, + curr_join, + order_arg, + filesort_limit_arg, + select_limit_arg, + curr_join->group_list ? FALSE : TRUE)) DBUG_VOID_RETURN; sortorder= curr_join->sortorder; if (curr_join->const_tables != curr_join->table_count && @@ -2858,6 +3067,14 @@ JOIN::exec() Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF); error= do_select(curr_join, curr_fields_list, NULL, procedure); thd->limit_found_rows= curr_join->send_records; + if (curr_join->order && + curr_join->sortorder) + { + /* Use info provided by filesort. */ + DBUG_ASSERT(curr_join->table_count > curr_join->const_tables); + JOIN_TAB *tab= curr_join->join_tab + curr_join->const_tables; + thd->limit_found_rows= tab->records; + } /* Accumulate the counts from all join iterations of all join parts. */ thd->examined_row_count+= curr_join->examined_rows; @@ -3206,6 +3423,8 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, table_vector[i]=s->table=table=tables->table; table->pos_in_table_list= tables; error= tables->fetch_number_of_rows(); + set_statistics_for_table(join->thd, table); + bitmap_clear_all(&table->cond_set); #ifdef WITH_PARTITION_STORAGE_ENGINE const bool no_partitions_used= table->no_partitions_used; @@ -3231,8 +3450,8 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, s->dependent= tables->dep_tables; if (tables->schema_table) - table->file->stats.records= 2; - table->quick_condition_rows= table->file->stats.records; + table->file->stats.records= table->used_stat_records= 2; + table->quick_condition_rows= table->stat_records(); s->on_expr_ref= &tables->on_expr; if (*s->on_expr_ref) @@ -3670,6 +3889,8 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, all select distinct fields participate in one index. */ add_group_and_distinct_keys(join, s); + + s->table->cond_selectivity= 1.0; /* Perform range analysis if there are keys it could use (1). @@ -3678,7 +3899,8 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, Don't do range analysis for materialized subqueries (4). Don't do range analysis for materialized derived tables (5) */ - if (!s->const_keys.is_clear_all() && // (1) + if ((!s->const_keys.is_clear_all() || + !bitmap_is_clear_all(&s->table->cond_set)) && // (1) (!s->table->pos_in_table_list->embedding || // (2) (s->table->pos_in_table_list->embedding && // (3) s->table->pos_in_table_list->embedding->sj_on_expr)) && // (3) @@ -3686,20 +3908,37 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, !(s->table->pos_in_table_list->derived && // (5) s->table->pos_in_table_list->is_materialized_derived())) // (5) { - ha_rows records; - SQL_SELECT *select; - select= make_select(s->table, found_const_table_map, - found_const_table_map, - *s->on_expr_ref ? *s->on_expr_ref : conds, - 1, &error); - if (!select) - goto error; - records= get_quick_record_count(join->thd, select, s->table, - &s->const_keys, join->row_limit); - s->quick=select->quick; - s->needed_reg=select->needed_reg; - select->quick=0; - if (records == 0 && s->table->reginfo.impossible_range) + bool impossible_range= FALSE; + ha_rows records= HA_POS_ERROR; + SQL_SELECT *select= 0; + if (!s->const_keys.is_clear_all()) + { + select= make_select(s->table, found_const_table_map, + found_const_table_map, + *s->on_expr_ref ? *s->on_expr_ref : conds, + 1, &error); + if (!select) + goto error; + records= get_quick_record_count(join->thd, select, s->table, + &s->const_keys, join->row_limit); + s->quick=select->quick; + s->needed_reg=select->needed_reg; + select->quick=0; + impossible_range= records == 0 && s->table->reginfo.impossible_range; + } + if (!impossible_range) + { + if (join->thd->variables.optimizer_use_condition_selectivity > 1) + calculate_cond_selectivity_for_table(join->thd, s->table, + *s->on_expr_ref ? + *s->on_expr_ref : conds); + if (s->table->reginfo.impossible_range) + { + impossible_range= TRUE; + records= 0; + } + } + if (impossible_range) { /* Impossible WHERE or ON expression @@ -3713,7 +3952,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, if (*s->on_expr_ref) { /* Generate empty row */ - s->info= "Impossible ON condition"; + s->info= ET_IMPOSSIBLE_ON_CONDITION; found_const_table_map|= s->table->map; s->type= JT_CONST; mark_as_null_row(s->table); // All fields are NULL @@ -3724,8 +3963,10 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, s->found_records=records; s->read_time= s->quick ? s->quick->read_time : 0.0; } - delete select; + if (select) + delete select; } + } if (pull_out_semijoin_tables(join)) @@ -3796,7 +4037,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, DEBUG_SYNC(join->thd, "inside_make_join_statistics"); /* Generate an execution plan from the found optimal join order. */ - DBUG_RETURN(join->thd->killed || get_best_combination(join)); + DBUG_RETURN(join->thd->check_killed() || get_best_combination(join)); error: /* @@ -4095,11 +4336,12 @@ add_key_field(JOIN *join, else if (!(field->flags & PART_KEY_FLAG)) { // Don't remove column IS NULL on a LEFT JOIN table - if (!eq_func || (*value)->type() != Item::NULL_ITEM || - !field->table->maybe_null || field->null_ptr) - return; // Not a key. Skip it - optimize= KEY_OPTIMIZE_EXISTS; - DBUG_ASSERT(num_values == 1); + if (eq_func && (*value)->type() == Item::NULL_ITEM && + field->table->maybe_null && !field->null_ptr) + { + optimize= KEY_OPTIMIZE_EXISTS; + DBUG_ASSERT(num_values == 1); + } } if (optimize != KEY_OPTIMIZE_EXISTS) { @@ -4148,7 +4390,11 @@ add_key_field(JOIN *join, break; } if (is_const) + { stat[0].const_keys.merge(possible_keys); + if (possible_keys.is_clear_all()) + bitmap_set_bit(&field->table->cond_set, field->field_index); + } else if (!eq_func) { /* @@ -4927,7 +5173,8 @@ update_ref_and_keys(THD *thd, DYNAMIC_ARRAY *keyuse,JOIN_TAB *join_tab, /* set a barrier for the array of SARGABLE_PARAM */ (*sargables)[0].field= 0; - if (my_init_dynamic_array(keyuse,sizeof(KEYUSE),20,64)) + if (my_init_dynamic_array(keyuse,sizeof(KEYUSE),20,64, + MYF(MY_THREAD_SPECIFIC))) return TRUE; if (cond) @@ -5290,6 +5537,7 @@ void set_position(JOIN *join,uint idx,JOIN_TAB *table,KEYUSE *key) join->positions[idx].table= table; join->positions[idx].key=key; join->positions[idx].records_read=1.0; /* This is a const table */ + join->positions[idx].cond_selectivity= 1.0; join->positions[idx].ref_depend_map= 0; // join->positions[idx].loosescan_key= MAX_KEY; /* Not a LooseScan */ @@ -5564,7 +5812,7 @@ best_access_path(JOIN *join, else { uint key_parts= table->actual_n_key_parts(keyinfo); - if (!(records=keyinfo->rec_per_key[key_parts-1])) + if (!(records= keyinfo->actual_rec_per_key(key_parts-1))) { /* Prefer longer keys */ records= ((double) s->records / (double) rec * @@ -5665,7 +5913,7 @@ best_access_path(JOIN *join, else { /* Check if we have statistic about the distribution */ - if ((records= keyinfo->rec_per_key[max_key_part-1])) + if ((records= keyinfo->actual_rec_per_key(max_key_part-1))) { /* Fix for the case where the index statistics is too @@ -6011,6 +6259,7 @@ static void choose_initial_table_order(JOIN *join) TABLE_LIST *emb_subq; JOIN_TAB **tab= join->best_ref + join->const_tables; JOIN_TAB **tabs_end= tab + join->table_count - join->const_tables; + DBUG_ENTER("choose_initial_table_order"); /* Find where the top-level JOIN_TABs end and subquery JOIN_TABs start */ for (; tab != tabs_end; tab++) { @@ -6020,7 +6269,7 @@ static void choose_initial_table_order(JOIN *join) uint n_subquery_tabs= tabs_end - tab; if (!n_subquery_tabs) - return; + DBUG_VOID_RETURN; /* Copy the subquery JOIN_TABs to a separate array */ JOIN_TAB *subquery_tabs[MAX_TABLES]; @@ -6075,6 +6324,7 @@ static void choose_initial_table_order(JOIN *join) subq_tab += n_subquery_tables - 1; } } + DBUG_VOID_RETURN; } @@ -6106,6 +6356,8 @@ choose_plan(JOIN *join, table_map join_tables) { uint search_depth= join->thd->variables.optimizer_search_depth; uint prune_level= join->thd->variables.optimizer_prune_level; + uint use_cond_selectivity= + join->thd->variables.optimizer_use_condition_selectivity; bool straight_join= test(join->select_options & SELECT_STRAIGHT_JOIN); DBUG_ENTER("choose_plan"); @@ -6162,15 +6414,19 @@ choose_plan(JOIN *join, table_map join_tables) the greedy version. Will be removed when greedy_search is approved. */ join->best_read= DBL_MAX; - if (find_best(join, join_tables, join->const_tables, 1.0, 0.0)) + if (find_best(join, join_tables, join->const_tables, 1.0, 0.0, + use_cond_selectivity)) + { DBUG_RETURN(TRUE); + } } else { if (search_depth == 0) /* Automatically determine a reasonable value for 'search_depth' */ search_depth= determine_search_depth(join); - if (greedy_search(join, join_tables, search_depth, prune_level)) + if (greedy_search(join, join_tables, search_depth, prune_level, + use_cond_selectivity)) DBUG_RETURN(TRUE); } } @@ -6444,6 +6700,8 @@ optimize_straight_join(JOIN *join, table_map join_tables) bool disable_jbuf= join->thd->variables.join_cache_level == 0; double record_count= 1.0; double read_time= 0.0; + uint use_cond_selectivity= + join->thd->variables.optimizer_use_condition_selectivity; POSITION loose_scan_pos; for (JOIN_TAB **pos= join->best_ref + idx ; (s= *pos) ; pos++) @@ -6460,6 +6718,11 @@ optimize_straight_join(JOIN *join, table_map join_tables) &loose_scan_pos); join_tables&= ~(s->table->map); + double pushdown_cond_selectivity= 1.0; + if (use_cond_selectivity > 1) + pushdown_cond_selectivity= table_cond_selectivity(join, idx, s, + join_tables); + join->positions[idx].cond_selectivity= pushdown_cond_selectivity; ++idx; } @@ -6547,6 +6810,8 @@ optimize_straight_join(JOIN *join, table_map join_tables) @param search_depth controlls the exhaustiveness of the search @param prune_level the pruning heuristics that should be applied during search + @param use_cond_selectivity specifies how the selectivity of the conditions + pushed to a table should be taken into account @retval FALSE ok @@ -6558,7 +6823,8 @@ static bool greedy_search(JOIN *join, table_map remaining_tables, uint search_depth, - uint prune_level) + uint prune_level, + uint use_cond_selectivity) { double record_count= 1.0; double read_time= 0.0; @@ -6569,7 +6835,6 @@ greedy_search(JOIN *join, JOIN_TAB *best_table; // the next plan node to be added to the curr QEP // ==join->tables or # tables in the sj-mat nest we're optimizing uint n_tables __attribute__((unused)); - DBUG_ENTER("greedy_search"); /* number of tables that remain to be optimized */ @@ -6584,7 +6849,8 @@ greedy_search(JOIN *join, /* Find the extension of the current QEP with the lowest cost */ join->best_read= DBL_MAX; if (best_extension_by_limited_search(join, remaining_tables, idx, record_count, - read_time, search_depth, prune_level)) + read_time, search_depth, prune_level, + use_cond_selectivity)) DBUG_RETURN(TRUE); /* 'best_read < DBL_MAX' means that optimizer managed to find @@ -6824,6 +7090,240 @@ double JOIN::get_examined_rows() /** + @brief + Get the selectivity of equalities between columns when joining a table + + @param join The optimized join + @param idx The number of tables in the evaluated partual join + @param s The table to be joined for evaluation + @param rem_tables The bitmap of tables to be joined later + @param keyparts The number of key parts to used when joining s + @param ref_keyuse_steps Array of references to keyuses employed to join s +*/ + +static +double table_multi_eq_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, + table_map rem_tables, uint keyparts, + uint16 *ref_keyuse_steps) +{ + double sel= 1.0; + COND_EQUAL *cond_equal= join->cond_equal; + + if (!cond_equal || !cond_equal->current_level.elements) + return sel; + + if (!s->keyuse) + return sel; + + Item_equal *item_equal; + List_iterator_fast<Item_equal> it(cond_equal->current_level); + TABLE *table= s->table; + table_map table_bit= table->map; + POSITION *pos= &join->positions[idx]; + + while ((item_equal= it++)) + { + /* + Check whether we need to take into account the selectivity of + multiple equality item_equal. If this is the case multiply + the current value of sel by this selectivity + */ + table_map used_tables= item_equal->used_tables(); + if (!(used_tables & table_bit)) + continue; + if (item_equal->get_const()) + continue; + + Field *fld; + bool adjust_sel= FALSE; + Item_equal_fields_iterator fi(*item_equal); + while((fi++) && !adjust_sel) + { + Field *fld= fi.get_curr_field(); + if (fld->table->map != table_bit) + continue; + if (pos->key == 0) + adjust_sel= TRUE; + else + { + uint i; + KEYUSE *keyuse= pos->key; + uint key= keyuse->key; + + for (i= 0; i < keyparts; i++) + { + uint fldno; + if (is_hash_join_key_no(key)) + fldno= keyuse->keypart; + else + fldno= table->key_info[key].key_part[keyparts-1].fieldnr - 1; + if (fld->field_index == fldno) + break; + } + if (i == keyparts) + { + /* + Field fld is included in multiple equality item_equal + and is not a part of the ref key. + The selectivity of the multiple equality must be taken + into account unless one of the ref arguments is + equal to fld. + */ + adjust_sel= TRUE; + for (uint j= 0; j < keyparts && adjust_sel; j++) + { + if (j > 0) + keyuse+= ref_keyuse_steps[j-1]; + Item *ref_item= keyuse->val; + if (ref_item->real_item()->type() == Item::FIELD_ITEM) + { + Item_field *field_item= (Item_field *) (ref_item->real_item()); + if (item_equal->contains(field_item->field)) + adjust_sel= FALSE; + } + } + } + } + } + if (adjust_sel) + { + /* + If ref == 0 and there are no fields in the multiple equality + item_equal that belong to the tables joined prior to s + then the selectivity of multiple equality will be set to 1.0. + */ + double eq_fld_sel= 1.0; + fi.rewind(); + while ((fi++)) + { + double curr_eq_fld_sel; + fld= fi.get_curr_field(); + if (!fld->table->map & ~(table_bit | rem_tables)) + continue; + curr_eq_fld_sel= get_column_avg_frequency(fld) / + fld->table->stat_records(); + if (curr_eq_fld_sel < 1.0) + set_if_bigger(eq_fld_sel, curr_eq_fld_sel); + } + sel*= eq_fld_sel; + } + } + return sel; +} + + +/** + @brief + Get the selectivity of conditions when joining a table + + @param join The optimized join + @param s The table to be joined for evaluation + @param rem_tables The bitmap of tables to be joined later + + @retval + selectivity of the conditions imposed on the rows of s +*/ + +static +double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, + table_map rem_tables) +{ + uint16 ref_keyuse_steps[MAX_REF_PARTS - 1]; + Field *field; + TABLE *table= s->table; + MY_BITMAP *read_set= table->read_set; + double sel= s->table->cond_selectivity; + double table_records= table->stat_records(); + POSITION *pos= &join->positions[idx]; + uint keyparts= 0; + uint found_part_ref_or_null= 0; + + /* Discount the selectivity of the access method used to join table s */ + if (s->quick && s->quick->index != MAX_KEY) + { + if (pos->key == 0 && table_records > 0) + { + sel/= table->quick_rows[s->quick->index]/table_records; + } + } + else if (pos->key != 0) + { + /* A ref/ access or hash join is used to join table */ + KEYUSE *keyuse= pos->key; + KEYUSE *prev_ref_keyuse= keyuse; + uint key= keyuse->key; + do + { + if (!(keyuse->used_tables & (rem_tables | table->map))) + { + if (are_tables_local(s, keyuse->val->used_tables())) + { + if (is_hash_join_key_no(key)) + { + if (keyparts == keyuse->keypart) + keyparts++; + } + else + { + if (keyparts == keyuse->keypart && + !(~(keyuse->val->used_tables()) & pos->ref_depend_map) && + !(found_part_ref_or_null & keyuse->optimize)) + { + keyparts++; + found_part_ref_or_null|= keyuse->optimize & ~KEY_OPTIMIZE_EQ; + } + } + if (keyparts > keyuse->keypart) + { + uint fldno; + if (is_hash_join_key_no(key)) + fldno= keyuse->keypart; + else + fldno= table->key_info[key].key_part[keyparts-1].fieldnr - 1; + if (keyuse->val->const_item()) + sel*= table->field[fldno]->cond_selectivity; + if (keyparts > 1) + { + ref_keyuse_steps[keyparts-2]= keyuse - prev_ref_keyuse; + prev_ref_keyuse= keyuse; + } + } + } + } + keyuse++; + } while (keyuse->table == table && keyuse->key == key); + } + + /* + If the field f from the table is equal to a field from one the + earlier joined tables then the selectivity of the range conditions + over the field f must be discounted. + */ + for (Field **f_ptr=table->field ; (field= *f_ptr) ; f_ptr++) + { + if (!bitmap_is_set(read_set, field->field_index) || + !field->next_equal_field) + continue; + for (Field *next_field= field->next_equal_field; + next_field != field; + next_field= next_field->next_equal_field) + { + if (!(next_field->table->map & rem_tables) && next_field->table != table) + { + sel/= field->cond_selectivity; + break; + } + } + } + + sel*= table_multi_eq_cond_selectivity(join, idx, s, rem_tables, + keyparts, ref_keyuse_steps); + + return sel; +} + + +/** Find a good, possibly optimal, query execution plan (QEP) by a possibly exhaustive search. @@ -6933,6 +7433,8 @@ double JOIN::get_examined_rows() @param prune_level pruning heuristics that should be applied during optimization (values: 0 = EXHAUSTIVE, 1 = PRUNE_BY_TIME_OR_ROWS) + @param use_cond_selectivity specifies how the selectivity of the conditions + pushed to a table should be taken into account @retval FALSE ok @@ -6947,12 +7449,13 @@ best_extension_by_limited_search(JOIN *join, double record_count, double read_time, uint search_depth, - uint prune_level) + uint prune_level, + uint use_cond_selectivity) { DBUG_ENTER("best_extension_by_limited_search"); THD *thd= join->thd; - if (thd->killed) // Abort + if (thd->check_killed()) // Abort DBUG_RETURN(TRUE); DBUG_EXECUTE("opt", print_plan(join, idx, read_time, record_count, idx, @@ -7050,16 +7553,25 @@ best_extension_by_limited_search(JOIN *join, } } + double pushdown_cond_selectivity= 1.0; + if (use_cond_selectivity > 1) + pushdown_cond_selectivity= table_cond_selectivity(join, idx, s, + remaining_tables & + ~real_table_bit); + join->positions[idx].cond_selectivity= pushdown_cond_selectivity; + double partial_join_cardinality= current_record_count * + pushdown_cond_selectivity; if ( (search_depth > 1) && (remaining_tables & ~real_table_bit) & allowed_tables ) { /* Recursively expand the current partial plan */ swap_variables(JOIN_TAB*, join->best_ref[idx], *pos); if (best_extension_by_limited_search(join, remaining_tables & ~real_table_bit, idx + 1, - current_record_count, + partial_join_cardinality, current_read_time, search_depth - 1, - prune_level)) + prune_level, + use_cond_selectivity)) DBUG_RETURN(TRUE); swap_variables(JOIN_TAB*, join->best_ref[idx], *pos); } @@ -7077,7 +7589,7 @@ best_extension_by_limited_search(JOIN *join, { memcpy((uchar*) join->best_positions, (uchar*) join->positions, sizeof(POSITION) * (idx + 1)); - join->record_count= current_record_count; + join->record_count= partial_join_cardinality; join->best_read= current_read_time - 0.001; } DBUG_EXECUTE("opt", print_plan(join, idx+1, @@ -7105,11 +7617,11 @@ best_extension_by_limited_search(JOIN *join, */ static bool find_best(JOIN *join,table_map rest_tables,uint idx,double record_count, - double read_time) + double read_time, uint use_cond_selectivity) { DBUG_ENTER("find_best"); THD *thd= join->thd; - if (thd->killed) + if (thd->check_killed()) DBUG_RETURN(TRUE); if (!rest_tables) { @@ -7156,20 +7668,30 @@ find_best(JOIN *join,table_map rest_tables,uint idx,double record_count, advance_sj_state(join, rest_tables, idx, ¤t_record_count, ¤t_read_time, &loose_scan_pos); - if (best_record_count > current_record_count || + double pushdown_cond_selectivity= 1.0; + if (use_cond_selectivity > 1) + pushdown_cond_selectivity= table_cond_selectivity(join, idx, s, + rest_tables & + ~real_table_bit); + join->positions[idx].cond_selectivity= pushdown_cond_selectivity; + double partial_join_cardinality= current_record_count * + pushdown_cond_selectivity; + + if (best_record_count > partial_join_cardinality || best_read_time > current_read_time || (idx == join->const_tables && s->table == join->sort_by_table)) { - if (best_record_count >= current_record_count && + if (best_record_count >= partial_join_cardinality && best_read_time >= current_read_time && (!(s->key_dependent & rest_tables) || records < 2.0)) { - best_record_count=current_record_count; + best_record_count= partial_join_cardinality; best_read_time=current_read_time; } swap_variables(JOIN_TAB*, join->best_ref[idx], *pos); if (find_best(join,rest_tables & ~real_table_bit,idx+1, - current_record_count,current_read_time)) + partial_join_cardinality,current_read_time, + use_cond_selectivity)) DBUG_RETURN(TRUE); swap_variables(JOIN_TAB*, join->best_ref[idx], *pos); } @@ -7722,6 +8244,7 @@ get_best_combination(JOIN *join) */ SJ_MATERIALIZATION_INFO *sjm= cur_pos->table->emb_sj_nest->sj_mat_info; j->records= j->records_read= (ha_rows)(sjm->is_sj_scan? sjm->rows : 1); + j->cond_selectivity= 1.0; JOIN_TAB *jt; JOIN_TAB_RANGE *jt_range; if (!(jt= (JOIN_TAB*)join->thd->alloc(sizeof(JOIN_TAB)*sjm->tables)) || @@ -7785,6 +8308,7 @@ get_best_combination(JOIN *join) to access join->best_positions[]. */ j->records_read= (ha_rows)join->best_positions[tablenr].records_read; + j->cond_selectivity= join->best_positions[tablenr].cond_selectivity; join->map2table[j->table->tablenr]= j; /* If we've reached the end of sjm nest, switch back to main sequence */ @@ -7806,6 +8330,7 @@ get_best_combination(JOIN *join) join->table_access_tabs= join->join_tab; join->top_table_access_tabs_count= join->top_join_tab_count; + update_depend_map(join); DBUG_RETURN(0); } @@ -7882,6 +8407,7 @@ static bool create_hj_key_for_table(JOIN *join, JOIN_TAB *join_tab, keyinfo->key_length=0; keyinfo->algorithm= HA_KEY_ALG_UNDEF; keyinfo->flags= HA_GENERATED_KEY; + keyinfo->is_statistics_from_stat_tables= FALSE; keyinfo->name= (char *) "$hj"; keyinfo->rec_per_key= (ulong*) thd->calloc(sizeof(ulong)*key_parts); if (!keyinfo->rec_per_key) @@ -8051,6 +8577,7 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, j->ref.null_rejecting= 0; j->ref.disable_cache= FALSE; j->ref.null_ref_part= NO_REF_PART; + j->ref.const_ref_part_map= 0; keyuse=org_keyuse; store_key **ref_key= j->ref.key_copy; @@ -8088,6 +8615,13 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, if (keyuse->null_rejecting) j->ref.null_rejecting|= (key_part_map)1 << i; keyuse_uses_no_tables= keyuse_uses_no_tables && !keyuse->used_tables; + /* + Todo: we should remove this check for thd->lex->describe on the next + line. With SHOW EXPLAIN code, EXPLAIN printout code no longer depends + on it. However, removing the check caused change in lots of query + plans! Does the optimizer depend on the contents of + table_ref->key_copy ? If yes, do we produce incorrect EXPLAINs? + */ if (!keyuse->val->used_tables() && !thd->lex->describe) { // Compare against constant store_key_item tmp(thd, @@ -8100,6 +8634,7 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, if (thd->is_fatal_error) DBUG_RETURN(TRUE); tmp.copy(); + j->ref.const_ref_part_map |= key_part_map(1) << i ; } else *ref_key++= get_store_key(thd, @@ -8214,6 +8749,7 @@ JOIN::make_simple_join(JOIN *parent, TABLE *temp_table) !(parent->join_tab_reexec= (JOIN_TAB*) thd->alloc(sizeof(JOIN_TAB)))) DBUG_RETURN(TRUE); /* purecov: inspected */ + // psergey-todo: here, save the pointer for original join_tabs. join_tab= parent->join_tab_reexec; table= &parent->table_reexec[0]; parent->table_reexec[0]= temp_table; table_count= top_join_tab_count= 1; @@ -8616,10 +9152,6 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) { /* there may be a select without a cond. */ if (join->table_count > 1) cond->update_used_tables(); // Tablenr may have changed - if (join->const_tables == join->table_count && - thd->lex->current_select->master_unit() == - &thd->lex->unit) // not upper level SELECT - join->const_table_map|=RAND_TABLE_BIT; /* Extract expressions that depend on constant tables @@ -9585,7 +10117,7 @@ end_sj_materialize(JOIN *join, JOIN_TAB *join_tab, bool end_of_records) if (item->is_null()) DBUG_RETURN(NESTED_LOOP_OK); } - fill_record(thd, table->field, sjm->sjm_table_cols, TRUE, FALSE); + fill_record(thd, table, table->field, sjm->sjm_table_cols, TRUE, FALSE); if (thd->is_error()) DBUG_RETURN(NESTED_LOOP_ERROR); /* purecov: inspected */ if ((error= table->file->ha_write_tmp_row(table->record[0]))) @@ -10041,6 +10573,86 @@ restart: } } +/** + Remove pushdown conditions that are already checked by the scan phase + of BNL/BNLH joins. + + @note + If the single-table condition for this table will be used by a + blocked join to pre-filter this table's rows, there is no need + to re-check the same single-table condition for each joined record. + + This method removes from JOIN_TAB::select_cond and JOIN_TAB::select::cond + all top-level conjuncts that also appear in in JOIN_TAB::cache_select::cond. +*/ + +void JOIN_TAB::remove_redundant_bnl_scan_conds() +{ + if (!(select_cond && cache_select && cache && + (cache->get_join_alg() == JOIN_CACHE::BNL_JOIN_ALG || + cache->get_join_alg() == JOIN_CACHE::BNLH_JOIN_ALG))) + return; + + /* + select->cond is not processed separately. This method assumes it is always + the same as select_cond. + */ + DBUG_ASSERT(!select || !select->cond || + (select->cond == select_cond)); + + if (is_cond_and(select_cond)) + { + List_iterator<Item> pushed_cond_li(*((Item_cond*) select_cond)->argument_list()); + Item *pushed_item; + Item_cond_and *reduced_select_cond= new Item_cond_and; + + if (is_cond_and(cache_select->cond)) + { + List_iterator<Item> scan_cond_li(*((Item_cond*) cache_select->cond)->argument_list()); + Item *scan_item; + while ((pushed_item= pushed_cond_li++)) + { + bool found= false; + scan_cond_li.rewind(); + while ((scan_item= scan_cond_li++)) + { + if (pushed_item->eq(scan_item, 0)) + { + found= true; + break; + } + } + if (!found) + reduced_select_cond->add(pushed_item); + } + } + else + { + while ((pushed_item= pushed_cond_li++)) + { + if (!pushed_item->eq(cache_select->cond, 0)) + reduced_select_cond->add(pushed_item); + } + } + + /* + JOIN_CACHE::check_match uses JOIN_TAB::select->cond instead of + JOIN_TAB::select_cond. set_cond() sets both pointers. + */ + if (reduced_select_cond->argument_list()->is_empty()) + set_cond(NULL); + else if (reduced_select_cond->argument_list()->elements == 1) + set_cond(reduced_select_cond->argument_list()->head()); + else + { + reduced_select_cond->quick_fix_field(); + set_cond(reduced_select_cond); + } + } + else if (select_cond->eq(cache_select->cond, 0)) + set_cond(NULL); +} + /* Plan refinement stage: do various setup things for the executor @@ -10292,6 +10904,15 @@ make_join_readinfo(JOIN *join, ulonglong options, uint no_jbuf_after) abort(); /* purecov: end */ } + + tab->remove_redundant_bnl_scan_conds(); + DBUG_EXECUTE("where", + char buff[256]; + String str(buff,sizeof(buff),system_charset_info); + str.length(0); + str.append(tab->table? tab->table->alias.c_ptr() :"<no_table_name>"); + str.append(" final_pushdown_cond"); + print_where(tab->select_cond, str.c_ptr_safe(), QT_ORDINARY);); } uint n_top_tables= join->join_tab_ranges.head()->end - join->join_tab_ranges.head()->start; @@ -10465,7 +11086,7 @@ double JOIN_TAB::scan_time() } else { - found_records= records= table->file->stats.records; + found_records= records= table->stat_records(); read_time= table->file->scan_time(); /* table->quick_condition_rows has already been set to @@ -10476,7 +11097,7 @@ double JOIN_TAB::scan_time() } else { - found_records= records=table->file->stats.records; + found_records= records=table->stat_records(); read_time= found_records ? (double)found_records: 10.0;// TODO:fix this stub res= read_time; } @@ -10494,7 +11115,7 @@ ha_rows JOIN_TAB::get_examined_rows() { ha_rows examined_rows; - if (select && select->quick) + if (select && select->quick && use_quick != 2) examined_rows= select->quick->records; else if (type == JT_NEXT || type == JT_ALL || type == JT_HASH || type ==JT_HASH_NEXT) @@ -10517,7 +11138,7 @@ ha_rows JOIN_TAB::get_examined_rows() handler->info(HA_STATUS_VARIABLE) has been called in make_join_statistics() */ - examined_rows= table->file->stats.records; + examined_rows= table->stat_records(); } } } @@ -10548,10 +11169,18 @@ bool JOIN_TAB::preread_init() mysql_handle_single_derived(join->thd->lex, derived, DT_CREATE | DT_FILL)) return TRUE; + preread_init_done= TRUE; if (select && select->quick) select->quick->replace_handler(table->file); + DBUG_EXECUTE_IF("show_explain_probe_join_tab_preread", + if (dbug_user_var_equals_int(join->thd, + "show_explain_probe_select_id", + join->select_lex->select_number)) + dbug_serve_apcs(join->thd, 1); + ); + /* init ftfuns for just initialized derived table */ if (table->fulltext_searched) init_ftfuncs(join->thd, join->select_lex, test(join->order)); @@ -10762,6 +11391,9 @@ void JOIN::cleanup(bool full) { DBUG_ENTER("JOIN::cleanup"); DBUG_PRINT("enter", ("full %u", (uint) full)); + + if (full) + have_query_plan= QEP_DELETED; if (table) { @@ -11311,14 +11943,6 @@ public: COND_CMP(Item *a,Item_func *b) :and_level(a),cmp_func(b) {} }; -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class I_List<COND_CMP>; -template class I_List_iterator<COND_CMP>; -template class List<Item_func_match>; -template class List_iterator<Item_func_match>; -#endif - - /** Find the multiple equality predicate containing a field. @@ -11803,7 +12427,8 @@ static bool check_equality(THD *thd, Item *item, COND_EQUAL *cond_equal, */ static COND *build_equal_items_for_cond(THD *thd, COND *cond, - COND_EQUAL *inherited) + COND_EQUAL *inherited, + bool link_item_fields) { Item_equal *item_equal; COND_EQUAL cond_equal; @@ -11850,6 +12475,7 @@ static COND *build_equal_items_for_cond(THD *thd, COND *cond, List_iterator_fast<Item_equal> it(cond_equal.current_level); while ((item_equal= it++)) { + item_equal->set_link_equal_fields(link_item_fields); item_equal->fix_fields(thd, NULL); item_equal->update_used_tables(); set_if_bigger(thd->lex->current_select->max_equal_elems, @@ -11869,7 +12495,8 @@ static COND *build_equal_items_for_cond(THD *thd, COND *cond, while ((item= li++)) { Item *new_item; - if ((new_item= build_equal_items_for_cond(thd, item, inherited)) != item) + if ((new_item= build_equal_items_for_cond(thd, item, inherited, FALSE)) + != item) { /* This replacement happens only for standalone equalities */ /* @@ -12012,9 +12639,9 @@ static COND *build_equal_items_for_cond(THD *thd, COND *cond, @endcode Thus, applying equalities from the where condition we basically can get more freedom in performing join operations. - Althogh we don't use this property now, it probably makes sense to use + Although we don't use this property now, it probably makes sense to use it in the future. - @param thd Thread handler + @param thd Thread handler @param cond condition to build the multiple equalities for @param inherited path to all inherited multiple equality items @param join_list list of join tables to which the condition @@ -12023,6 +12650,7 @@ static COND *build_equal_items_for_cond(THD *thd, COND *cond, for on expressions @param[out] cond_equal_ref pointer to the structure to place built equalities in + @param link_equal_items equal fields are to be linked @return pointer to the transformed condition containing multiple equalities @@ -12032,14 +12660,15 @@ static COND *build_equal_items(JOIN *join, COND *cond, COND_EQUAL *inherited, List<TABLE_LIST> *join_list, bool ignore_on_conds, - COND_EQUAL **cond_equal_ref) + COND_EQUAL **cond_equal_ref, + bool link_equal_fields) { THD *thd= join->thd; COND_EQUAL *cond_equal= 0; if (cond) { - cond= build_equal_items_for_cond(thd, cond, inherited); + cond= build_equal_items_for_cond(thd, cond, inherited, link_equal_fields); cond->update_used_tables(); if (cond->type() == Item::COND_ITEM && ((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC) @@ -13558,9 +14187,10 @@ void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab, static COND * -optimize_cond(JOIN *join, COND *conds, +optimize_cond(JOIN *join, COND *conds, List<TABLE_LIST> *join_list, bool ignore_on_conds, - Item::cond_result *cond_value, COND_EQUAL **cond_equal) + Item::cond_result *cond_value, COND_EQUAL **cond_equal, + int flags) { THD *thd= join->thd; DBUG_ENTER("optimize_cond"); @@ -13583,9 +14213,10 @@ optimize_cond(JOIN *join, COND *conds, multiple equality contains a constant. */ DBUG_EXECUTE("where", print_where(conds, "original", QT_ORDINARY);); - conds= build_equal_items(join, conds, NULL, join_list, ignore_on_conds, - cond_equal); - DBUG_EXECUTE("where",print_where(conds,"after equal_items", QT_ORDINARY);); + conds= build_equal_items(join, conds, NULL, join_list, + ignore_on_conds, cond_equal, + test(flags & OPT_LINK_EQUAL_FIELDS)); + DBUG_EXECUTE("where",print_where(conds,"after equal_items", QT_ORDINARY);); /* change field = field to field = const for each found field = const */ propagate_cond_constants(thd, (I_List<COND_CMP> *) 0, conds, conds); @@ -14377,6 +15008,8 @@ Field *create_tmp_field_from_field(THD *thd, Field *org_field, ((Field_double *) new_field)->not_fixed= TRUE; new_field->vcol_info= 0; new_field->stored_in_db= TRUE; + new_field->cond_selectivity= 1.0; + new_field->next_equal_field= NULL; } return new_field; } @@ -14721,6 +15354,9 @@ void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps) bitmap_init(&table->eq_join_set, (my_bitmap_map*) (bitmaps+ 3*bitmap_buffer_size(field_count)), field_count, FALSE); + bitmap_init(&table->cond_set, + (my_bitmap_map*) (bitmaps+ 4*bitmap_buffer_size(field_count)), + field_count, FALSE); /* write_set and all_set are copies of read_set */ table->def_write_set= table->def_read_set; table->s->all_set= table->def_read_set; @@ -14868,7 +15504,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields, if (param->precomputed_group_by) copy_func_count+= param->sum_func_count; - init_sql_alloc(&own_root, TABLE_ALLOC_BLOCK_SIZE, 0); + init_sql_alloc(&own_root, TABLE_ALLOC_BLOCK_SIZE, 0, MYF(MY_THREAD_SPECIFIC)); if (!multi_alloc_root(&own_root, &table, sizeof(*table), @@ -14886,7 +15522,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields, &tmpname, (uint) strlen(path)+1, &group_buff, (group && ! using_unique_constraint ? param->group_length : 0), - &bitmaps, bitmap_buffer_size(field_count)*4, + &bitmaps, bitmap_buffer_size(field_count)*5, NullS)) { if (temp_pool_slot != MY_BIT_NONE) @@ -15349,8 +15985,11 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields, keyinfo->usable_key_parts=keyinfo->key_parts= param->group_parts; keyinfo->ext_key_parts= keyinfo->key_parts; keyinfo->key_length=0; - keyinfo->rec_per_key=0; + keyinfo->rec_per_key=NULL; + keyinfo->read_stats= NULL; + keyinfo->collected_stats= NULL; keyinfo->algorithm= HA_KEY_ALG_UNDEF; + keyinfo->is_statistics_from_stat_tables= FALSE; keyinfo->name= (char*) "group_key"; ORDER *cur_group= group; for (; cur_group ; cur_group= cur_group->next, key_part_info++) @@ -15463,7 +16102,10 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields, keyinfo->key_length= 0; // Will compute the sum of the parts below. keyinfo->name= (char*) "distinct_key"; keyinfo->algorithm= HA_KEY_ALG_UNDEF; + keyinfo->is_statistics_from_stat_tables= FALSE; keyinfo->rec_per_key=0; + keyinfo->read_stats= NULL; + keyinfo->collected_stats= NULL; /* Create an extra field to hold NULL bits so that unique indexes on @@ -15620,7 +16262,7 @@ TABLE *create_virtual_tmp_table(THD *thd, List<Create_field> &field_list) &share, sizeof(*share), &field, (field_count + 1) * sizeof(Field*), &blob_field, (field_count+1) *sizeof(uint), - &bitmaps, bitmap_buffer_size(field_count)*4, + &bitmaps, bitmap_buffer_size(field_count)*5, NullS)) return 0; @@ -16174,7 +16816,7 @@ create_internal_tmp_table_from_heap2(THD *thd, TABLE *table, DBUG_EXECUTE_IF("raise_error", write_err= HA_ERR_FOUND_DUPP_KEY ;); if (write_err) goto err; - if (thd->killed) + if (thd->check_killed()) { thd->send_kill_message(); goto err_killed; @@ -16441,6 +17083,14 @@ do_select(JOIN *join,List<Item> *fields,TABLE *table,Procedure *procedure) else { DBUG_ASSERT(join->table_count); + + DBUG_EXECUTE_IF("show_explain_probe_do_select", + if (dbug_user_var_equals_int(join->thd, + "show_explain_probe_select_id", + join->select_lex->select_number)) + dbug_serve_apcs(join->thd, 1); + ); + if (join->outer_ref_cond && !join->outer_ref_cond->val_int()) error= NESTED_LOOP_NO_MORE_ROWS; else @@ -16571,7 +17221,7 @@ sub_select_cache(JOIN *join, JOIN_TAB *join_tab, bool end_of_records) rc= sub_select(join, join_tab, end_of_records); DBUG_RETURN(rc); } - if (join->thd->killed) + if (join->thd->check_killed()) { /* The user has aborted the execution of the query */ join->thd->send_kill_message(); @@ -16861,12 +17511,14 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, DBUG_ENTER("evaluate_join_record"); DBUG_PRINT("enter", ("evaluate_join_record join: %p join_tab: %p" - " cond: %p error: %d", join, join_tab, select_cond, error)); + " cond: %p error: %d alias %s", + join, join_tab, select_cond, error, + join_tab->table->alias.ptr())); if (error > 0 || (join->thd->is_error())) // Fatal error DBUG_RETURN(NESTED_LOOP_ERROR); if (error < 0) DBUG_RETURN(NESTED_LOOP_NO_MORE_ROWS); - if (join->thd->killed) // Aborted by user + if (join->thd->check_killed()) // Aborted by user { join->thd->send_kill_message(); DBUG_RETURN(NESTED_LOOP_KILLED); /* purecov: inspected */ @@ -16974,6 +17626,7 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, if (join_tab->check_weed_out_table && found) { int res= join_tab->check_weed_out_table->sj_weedout_check_row(join->thd); + DBUG_PRINT("info", ("weedout_check: %d", res)); if (res == -1) DBUG_RETURN(NESTED_LOOP_ERROR); else if (res == 1) @@ -16994,8 +17647,8 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, (See above join->return_tab= tab). */ join->examined_rows++; - DBUG_PRINT("counts", ("join->examined_rows++: %lu", - (ulong) join->examined_rows)); + DBUG_PRINT("counts", ("join->examined_rows++: %lu found: %d", + (ulong) join->examined_rows, (int) found)); if (found) { @@ -17212,7 +17865,7 @@ join_read_const_table(JOIN_TAB *tab, POSITION *pos) { if ((error=join_read_system(tab))) { // Info for DESCRIBE - tab->info="const row not found"; + tab->info= ET_CONST_ROW_NOT_FOUND; /* Mark for EXPLAIN that the row was not found */ pos->records_read=0.0; pos->ref_depend_map= 0; @@ -17238,7 +17891,7 @@ join_read_const_table(JOIN_TAB *tab, POSITION *pos) table->disable_keyread(); if (error) { - tab->info="unique row not found"; + tab->info= ET_UNIQUE_ROW_NOT_FOUND; /* Mark for EXPLAIN that the row was not found */ pos->records_read=0.0; pos->ref_depend_map= 0; @@ -17643,6 +18296,14 @@ int read_first_record_seq(JOIN_TAB *tab) static int test_if_quick_select(JOIN_TAB *tab) { + DBUG_EXECUTE_IF("show_explain_probe_test_if_quick_select", + if (dbug_user_var_equals_int(tab->join->thd, + "show_explain_probe_select_id", + tab->join->select_lex->select_number)) + dbug_serve_apcs(tab->join->thd, 1); + ); + + delete tab->select->quick; tab->select->quick=0; return tab->select->test_quick_select(tab->join->thd, tab->keys, @@ -17893,7 +18554,25 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), if ((error= join->result->send_data(*join->fields))) DBUG_RETURN(error < 0 ? NESTED_LOOP_OK : NESTED_LOOP_ERROR); } - if (++join->send_records >= join->unit->select_limit_cnt && + + ++join->send_records; + if (join->send_records >= join->unit->select_limit_cnt && + !join->do_send_rows) + { + /* + If filesort is used for sorting, stop after select_limit_cnt+1 + records are read. Because of optimization in some cases it can + provide only select_limit_cnt+1 records. + */ + if (join->order && + join->sortorder && + join->select_options & OPTION_FOUND_ROWS) + { + DBUG_PRINT("info", ("filesort NESTED_LOOP_QUERY_LIMIT")); + DBUG_RETURN(NESTED_LOOP_QUERY_LIMIT); + } + } + if (join->send_records >= join->unit->select_limit_cnt && join->do_send_rows) { if (join->select_options & OPTION_FOUND_ROWS) @@ -18117,7 +18796,7 @@ end_write(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), } } end: - if (join->thd->killed) + if (join->thd->check_killed()) { join->thd->send_kill_message(); DBUG_RETURN(NESTED_LOOP_KILLED); /* purecov: inspected */ @@ -18188,7 +18867,7 @@ end_update(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), } join->send_records++; end: - if (join->thd->killed) + if (join->thd->check_killed()) { join->thd->send_kill_message(); DBUG_RETURN(NESTED_LOOP_KILLED); /* purecov: inspected */ @@ -18238,7 +18917,7 @@ end_unique_update(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), DBUG_RETURN(NESTED_LOOP_ERROR); /* purecov: inspected */ } } - if (join->thd->killed) + if (join->thd->check_killed()) { join->thd->send_kill_message(); DBUG_RETURN(NESTED_LOOP_KILLED); /* purecov: inspected */ @@ -18316,7 +18995,7 @@ end_write_group(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), if (join->procedure) join->procedure->add(); end: - if (join->thd->killed) + if (join->thd->check_killed()) { join->thd->send_kill_message(); DBUG_RETURN(NESTED_LOOP_KILLED); /* purecov: inspected */ @@ -19323,7 +20002,7 @@ test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order,ha_rows select_limit, uint saved_best_key_parts= 0; int best_key_direction= 0; JOIN *join= tab->join; - ha_rows table_records= table->file->stats.records; + ha_rows table_records= table->stat_records(); test_if_cheaper_ordering(tab, order, table, usable_keys, ref_key, select_limit, @@ -19439,7 +20118,7 @@ check_reverse_order: { tab->ref.key= -1; tab->ref.key_parts= 0; - if (select_limit < table->file->stats.records) + if (select_limit < table->stat_records()) tab->limit= select_limit; table->disable_keyread(); } @@ -19594,6 +20273,8 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order, { uint length= 0; ha_rows examined_rows; + ha_rows found_rows; + ha_rows filesort_retval= HA_POS_ERROR; TABLE *table; SQL_SELECT *select; JOIN_TAB *tab; @@ -19682,7 +20363,8 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order, goto err; /* purecov: inspected */ table->sort.io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE), - MYF(MY_WME | MY_ZEROFILL)); + MYF(MY_WME | MY_ZEROFILL| + MY_THREAD_SPECIFIC)); table->status=0; // May be wrong if quick_select if (!tab->preread_init_done && tab->preread_init()) @@ -19726,9 +20408,11 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order, if (table->s->tmp_table) table->file->info(HA_STATUS_VARIABLE); // Get record count - table->sort.found_records=filesort(thd, table,join->sortorder, length, - select, filesort_limit, 0, - &examined_rows); + filesort_retval= filesort(thd, table, join->sortorder, length, + select, filesort_limit, 0, + &examined_rows, &found_rows); + table->sort.found_records= filesort_retval; + tab->records= found_rows; // For SQL_CALC_ROWS if (quick_created) { @@ -19746,72 +20430,6 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order, *(join->pre_sort_join_tab)= *tab; - /*TODO: here, close the index scan, cancel index-only read. */ - tab->records= table->sort.found_records; // For SQL_CALC_ROWS -#if 0 - /* MariaDB doesn't need the following: */ - if (select) - { - /* - We need to preserve tablesort's output resultset here, because - QUICK_INDEX_MERGE_SELECT::~QUICK_INDEX_MERGE_SELECT (called by - SQL_SELECT::cleanup()) may free it assuming it's the result of the quick - select operation that we no longer need. Note that all the other parts of - this data structure are cleaned up when - QUICK_INDEX_MERGE_SELECT::get_next encounters end of data, so the next - SQL_SELECT::cleanup() call changes sort.io_cache alone. - */ - IO_CACHE *tablesort_result_cache; - - tablesort_result_cache= table->sort.io_cache; - table->sort.io_cache= NULL; - - if (select->quick && - select->quick->get_type() == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) - { - tab->filesort_used_loose_index_scan= true; - - QUICK_GROUP_MIN_MAX_SELECT *minmax_quick= - static_cast<QUICK_GROUP_MIN_MAX_SELECT*>(select->quick); - if (minmax_quick->is_agg_distinct()) - tab->filesort_used_loose_index_scan_agg_distinct= true; - } - - /* - If a quick object was created outside of create_sort_index() - that might be reused, then do not call select->cleanup() since - it will delete the quick object. - */ - if (!keep_quick) - { - select->cleanup(); - - // If we deleted the quick object we need to clear quick_keys - table->quick_keys.clear_all(); - table->intersect_keys.clear_all(); - } - else - { - // Need to close the index scan in order to re-use the handler - tab->select->quick->range_end(); - } - - /* - The select object is now ready for the next use. To avoid that - the select object is used when reading the records in sorted - order we set the pointer to it to NULL. The select pointer will - be restored from the saved_select pointer when this select - operation is completed (@see JOIN::exec). This ensures that it - will be re-used when filesort is used by subqueries that are - executed multiple times. - */ - tab->saved_select= tab->select; - tab->select= NULL; - - // Restore the output resultset - table->sort.io_cache= tablesort_result_cache; - } -#endif tab->select=NULL; tab->set_select_cond(NULL, __LINE__); tab->type=JT_ALL; // Read with normal read_record @@ -19822,12 +20440,11 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order, goto err; tab->join->examined_rows+=examined_rows; - DBUG_RETURN(table->sort.found_records == HA_POS_ERROR); + DBUG_RETURN(filesort_retval == HA_POS_ERROR); err: DBUG_RETURN(-1); } - void JOIN::clean_pre_sort_join_tab() { //TABLE *table= pre_sort_join_tab->table; @@ -19836,29 +20453,6 @@ void JOIN::clean_pre_sort_join_tab() the table already deleted by st_select_lex_unit::cleanup(). We rely on that fake_select_lex didn't have quick select. */ -#if 0 - if (pre_sort_join_tab->select && pre_sort_join_tab->select->quick) - { - /* - We need to preserve tablesort's output resultset here, because - QUICK_INDEX_MERGE_SELECT::~QUICK_INDEX_MERGE_SELECT (called by - SQL_SELECT::cleanup()) may free it assuming it's the result of the quick - select operation that we no longer need. Note that all the other parts of - this data structure are cleaned up when - QUICK_INDEX_MERGE_SELECT::get_next encounters end of data, so the next - SQL_SELECT::cleanup() call changes sort.io_cache alone. - */ - IO_CACHE *tablesort_result_cache; - - tablesort_result_cache= table->sort.io_cache; - table->sort.io_cache= NULL; - pre_sort_join_tab->select->cleanup(); - table->quick_keys.clear_all(); // as far as we cleanup select->quick - table->intersect_keys.clear_all(); - table->sort.io_cache= tablesort_result_cache; - } -#endif - //table->disable_keyread(); // Restore if we used indexes if (pre_sort_join_tab->select && pre_sort_join_tab->select->quick) { pre_sort_join_tab->select->cleanup(); @@ -19982,7 +20576,7 @@ static int remove_dup_with_compare(THD *thd, TABLE *table, Field **first_field, error= file->ha_rnd_next(record); for (;;) { - if (thd->killed) + if (thd->check_killed()) { thd->send_kill_message(); error=0; @@ -20103,7 +20697,7 @@ static int remove_dup_with_hash_index(THD *thd, TABLE *table, for (;;) { uchar *org_key_pos; - if (thd->killed) + if (thd->check_killed()) { thd->send_kill_message(); error=0; @@ -20179,7 +20773,7 @@ SORT_FIELD *make_unireg_sortorder(ORDER *order, uint *length, pos= sort= sortorder; if (!pos) - return 0; + DBUG_RETURN(0); for (;order;order=order->next,pos++) { @@ -20196,6 +20790,7 @@ SORT_FIELD *make_unireg_sortorder(ORDER *order, uint *length, else pos->item= *order->item; pos->reverse=! order->asc; + DBUG_ASSERT(pos->field != NULL || pos->item != NULL); } *length=count; DBUG_RETURN(sort); @@ -22166,162 +22761,372 @@ void JOIN::clear() } } -/** - EXPLAIN handling. - Send a description about what how the select will be done to stdout. +/* + Print an EXPLAIN line with all NULLs and given message in the 'Extra' column */ -static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, - bool distinct,const char *message) +int print_explain_message_line(select_result_sink *result, + uint8 options, + uint select_number, + const char *select_type, + ha_rows *rows, + const char *message) { - List<Item> field_list; - List<Item> item_list; - THD *thd=join->thd; - select_result *result=join->result; + const CHARSET_INFO *cs= system_charset_info; Item *item_null= new Item_null(); - CHARSET_INFO *cs= system_charset_info; - int quick_type; - DBUG_ENTER("select_describe"); - DBUG_PRINT("info", ("Select 0x%lx, type %s, message %s", - (ulong)join->select_lex, join->select_lex->type, - message ? message : "NULL")); - /* Don't log this into the slow query log */ - thd->server_status&= ~(SERVER_QUERY_NO_INDEX_USED | SERVER_QUERY_NO_GOOD_INDEX_USED); - join->unit->offset_limit_cnt= 0; + List<Item> item_list; - /* - NOTE: the number/types of items pushed into item_list must be in sync with - EXPLAIN column types as they're "defined" in THD::send_explain_fields() - */ - if (message) - { - item_list.push_back(new Item_int((int32) - join->select_lex->select_number)); - item_list.push_back(new Item_string(join->select_lex->type, - strlen(join->select_lex->type), cs)); - for (uint i=0 ; i < 7; i++) - item_list.push_back(item_null); - if (join->thd->lex->describe & DESCRIBE_PARTITIONS) - item_list.push_back(item_null); - if (join->thd->lex->describe & DESCRIBE_EXTENDED) - item_list.push_back(item_null); + item_list.push_back(new Item_int((int32) select_number)); + item_list.push_back(new Item_string(select_type, + strlen(select_type), cs)); + /* `table` */ + item_list.push_back(item_null); - item_list.push_back(new Item_string(message,strlen(message),cs)); - if (result->send_data(item_list)) - join->error= 1; + /* `partitions` */ + if (options & DESCRIBE_PARTITIONS) + item_list.push_back(item_null); + + /* type, possible_keys, key, key_len, ref */ + for (uint i=0 ; i < 5; i++) + item_list.push_back(item_null); + + /* `rows` */ + if (rows) + { + item_list.push_back(new Item_int(*rows, + MY_INT64_NUM_DECIMAL_DIGITS)); } - else if (join->select_lex == join->unit->fake_select_lex) + else + item_list.push_back(item_null); + + /* `filtered` */ + if (options & DESCRIBE_EXTENDED) + item_list.push_back(item_null); + + /* `Extra` */ + if (message) + item_list.push_back(new Item_string(message,strlen(message),cs)); + else + item_list.push_back(item_null); + + if (result->send_data(item_list)) + return 1; + return 0; +} + + +/* + Make a comma-separated list of possible_keys names and add it into the string +*/ + +void make_possible_keys_line(TABLE *table, key_map possible_keys, String *line) +{ + if (!possible_keys.is_clear_all()) { - /* - here we assume that the query will return at least two rows, so we - show "filesort" in EXPLAIN. Of course, sometimes we'll be wrong - and no filesort will be actually done, but executing all selects in - the UNION to provide precise EXPLAIN information will hardly be - appreciated :) - */ - char table_name_buffer[SAFE_NAME_LEN]; - item_list.empty(); - /* id */ - item_list.push_back(new Item_null); - /* select_type */ - item_list.push_back(new Item_string(join->select_lex->type, - strlen(join->select_lex->type), - cs)); - /* table */ - { - SELECT_LEX *sl= join->unit->first_select(); - uint len= 6, lastop= 0; - memcpy(table_name_buffer, STRING_WITH_LEN("<union")); - for (; sl && len + lastop + 5 < NAME_LEN; sl= sl->next_select()) - { - len+= lastop; - lastop= my_snprintf(table_name_buffer + len, NAME_LEN - len, - "%u,", sl->select_number); - } - if (sl || len + lastop >= NAME_LEN) - { - memcpy(table_name_buffer + len, STRING_WITH_LEN("...>") + 1); - len+= 4; - } - else + uint j; + for (j=0 ; j < table->s->keys ; j++) + { + if (possible_keys.is_set(j)) { - len+= lastop; - table_name_buffer[len - 1]= '>'; // change ',' to '>' + if (line->length()) + line->append(','); + line->append(table->key_info[j].name, + strlen(table->key_info[j].name), + system_charset_info); } - item_list.push_back(new Item_string(table_name_buffer, len, cs)); } - /* partitions */ - if (join->thd->lex->describe & DESCRIBE_PARTITIONS) + } +} + +/* + Print an EXPLAIN output row, based on information provided in the parameters + + @note + Parameters that may have NULL value in EXPLAIN output, should be passed + (char*)NULL. + + @return + 0 - OK + 1 - OOM Error +*/ + +int print_explain_row(select_result_sink *result, + uint8 options, + uint select_number, + const char *select_type, + const char *table_name, + const char *partitions, + enum join_type jtype, + const char *possible_keys, + const char *index, + const char *key_len, + const char *ref, + ha_rows *rows, + const char *extra) +{ + const CHARSET_INFO *cs= system_charset_info; + Item *item_null= new Item_null(); + List<Item> item_list; + Item *item; + + item_list.push_back(new Item_int((int32) select_number)); + item_list.push_back(new Item_string(select_type, + strlen(select_type), cs)); + item_list.push_back(new Item_string(table_name, + strlen(table_name), cs)); + if (options & DESCRIBE_PARTITIONS) + { + if (partitions) + { + item_list.push_back(new Item_string(partitions, + strlen(partitions), cs)); + } + else item_list.push_back(item_null); - /* type */ - item_list.push_back(new Item_string(join_type_str[JT_ALL], - strlen(join_type_str[JT_ALL]), - cs)); - /* possible_keys */ - item_list.push_back(item_null); - /* key*/ + } + + const char *jtype_str= join_type_str[jtype]; + item_list.push_back(new Item_string(jtype_str, + strlen(jtype_str), cs)); + + item= possible_keys? new Item_string(possible_keys, strlen(possible_keys), + cs) : item_null; + item_list.push_back(item); + + /* 'index */ + item= index ? new Item_string(index, strlen(index), cs) : item_null; + item_list.push_back(item); + + /* 'key_len */ + item= key_len ? new Item_string(key_len, strlen(key_len), cs) : item_null; + item_list.push_back(item); + + /* 'ref' */ + item= ref ? new Item_string(ref, strlen(ref), cs) : item_null; + item_list.push_back(item); + + /* 'rows' */ + if (rows) + { + item_list.push_back(new Item_int(*rows, + MY_INT64_NUM_DECIMAL_DIGITS)); + } + else item_list.push_back(item_null); - /* key_len */ + + /* 'filtered' */ + const double filtered=100.0; + if (options & DESCRIBE_EXTENDED) + item_list.push_back(new Item_float(filtered, 2)); + + /* 'Extra' */ + if (extra) + item_list.push_back(new Item_string(extra, strlen(extra), cs)); + else item_list.push_back(item_null); - /* ref */ + + if (result->send_data(item_list)) + return 1; + return 0; +} + + +int print_fake_select_lex_join(select_result_sink *result, bool on_the_fly, + SELECT_LEX *select_lex, uint8 explain_flags) +{ + const CHARSET_INFO *cs= system_charset_info; + Item *item_null= new Item_null(); + List<Item> item_list; + if (on_the_fly) + select_lex->set_explain_type(on_the_fly); + /* + here we assume that the query will return at least two rows, so we + show "filesort" in EXPLAIN. Of course, sometimes we'll be wrong + and no filesort will be actually done, but executing all selects in + the UNION to provide precise EXPLAIN information will hardly be + appreciated :) + */ + char table_name_buffer[SAFE_NAME_LEN]; + item_list.empty(); + /* id */ + item_list.push_back(new Item_null); + /* select_type */ + item_list.push_back(new Item_string(select_lex->type, + strlen(select_lex->type), + cs)); + /* table */ + { + SELECT_LEX *sl= select_lex->master_unit()->first_select(); + uint len= 6, lastop= 0; + memcpy(table_name_buffer, STRING_WITH_LEN("<union")); + for (; sl && len + lastop + 5 < NAME_LEN; sl= sl->next_select()) + { + len+= lastop; + lastop= my_snprintf(table_name_buffer + len, NAME_LEN - len, + "%u,", sl->select_number); + } + if (sl || len + lastop >= NAME_LEN) + { + memcpy(table_name_buffer + len, STRING_WITH_LEN("...>") + 1); + len+= 4; + } + else + { + len+= lastop; + table_name_buffer[len - 1]= '>'; // change ',' to '>' + } + item_list.push_back(new Item_string(table_name_buffer, len, cs)); + } + /* partitions */ + if (explain_flags & DESCRIBE_PARTITIONS) item_list.push_back(item_null); - /* in_rows */ - if (join->thd->lex->describe & DESCRIBE_EXTENDED) - item_list.push_back(item_null); - /* rows */ + /* type */ + item_list.push_back(new Item_string(join_type_str[JT_ALL], + strlen(join_type_str[JT_ALL]), + cs)); + /* possible_keys */ + item_list.push_back(item_null); + /* key*/ + item_list.push_back(item_null); + /* key_len */ + item_list.push_back(item_null); + /* ref */ + item_list.push_back(item_null); + /* in_rows */ + if (explain_flags & DESCRIBE_EXTENDED) item_list.push_back(item_null); - /* extra */ - if (join->unit->global_parameters->order_list.first) - item_list.push_back(new Item_string("Using filesort", - 14, cs)); - else - item_list.push_back(new Item_string("", 0, cs)); + /* rows */ + item_list.push_back(item_null); + /* extra */ + if (select_lex->master_unit()->global_parameters->order_list.first) + item_list.push_back(new Item_string("Using filesort", + 14, cs)); + else + item_list.push_back(new Item_string("", 0, cs)); + + if (result->send_data(item_list)) + return 1; + return 0; +} - if (result->send_data(item_list)) - join->error= 1; + +/* + Append MRR information from quick select to the given string +*/ + +void explain_append_mrr_info(QUICK_RANGE_SELECT *quick, String *res) +{ + char mrr_str_buf[128]; + mrr_str_buf[0]=0; + int len; + handler *h= quick->head->file; + len= h->multi_range_read_explain_info(quick->mrr_flags, mrr_str_buf, + sizeof(mrr_str_buf)); + if (len > 0) + { + //res->append(STRING_WITH_LEN("; ")); + res->append(mrr_str_buf, len); + } +} + + +/////////////////////////////////////////////////////////////////////////////// +// TODO: join with make_possible_keys_line ? +void append_possible_keys(String *str, TABLE *table, key_map possible_keys) +{ + uint j; + for (j=0 ; j < table->s->keys ; j++) + { + if (possible_keys.is_set(j)) + { + if (str->length()) + str->append(','); + str->append(table->key_info[j].name, + strlen(table->key_info[j].name), + system_charset_info); + } + } +} + + +/* + Save Query Plan Footprint + + @note + Currently, this function may be called multiple times +*/ + +int JOIN::save_explain_data_intern(Explain_query *output, bool need_tmp_table, + bool need_order, bool distinct, + const char *message) +{ + Explain_node *explain_node; + JOIN *join= this; /* Legacy: this code used to be a non-member function */ + THD *thd=join->thd; + const CHARSET_INFO *cs= system_charset_info; + int quick_type; + int error= 0; + DBUG_ENTER("JOIN::save_explain_data_intern"); + DBUG_PRINT("info", ("Select 0x%lx, type %s, message %s", + (ulong)join->select_lex, join->select_lex->type, + message ? message : "NULL")); + DBUG_ASSERT(have_query_plan == QEP_AVAILABLE); + /* Don't log this into the slow query log */ + + if (message) + { + Explain_select *xpl_sel; + explain_node= xpl_sel= new (output->mem_root) Explain_select; + join->select_lex->set_explain_type(true); + + xpl_sel->select_id= join->select_lex->select_number; + xpl_sel->select_type= join->select_lex->type; + xpl_sel->message= message; + /* Setting xpl_sel->message means that all other members are invalid */ + output->add_node(xpl_sel); + } + else if (join->select_lex == join->unit->fake_select_lex) + { + /* Do nothing, Explain_union will create and print fake_select_lex */ } else if (!join->select_lex->master_unit()->derived || join->select_lex->master_unit()->derived->is_materialized_derived()) { + Explain_select *xpl_sel; + explain_node= xpl_sel= new (output->mem_root) Explain_select; table_map used_tables=0; - bool printing_materialize_nest= FALSE; - uint select_id= join->select_lex->select_number; + join->select_lex->set_explain_type(true); + xpl_sel->select_id= join->select_lex->select_number; + xpl_sel->select_type= join->select_lex->type; + + JOIN_TAB* const first_top_tab= first_breadth_first_tab(join, WALK_OPTIMIZATION_TABS); for (JOIN_TAB *tab= first_breadth_first_tab(join, WALK_OPTIMIZATION_TABS); tab; tab= next_breadth_first_tab(join, WALK_OPTIMIZATION_TABS, tab)) { + uint select_id; if (tab->bush_root_tab) { JOIN_TAB *first_sibling= tab->bush_root_tab->bush_children->start; select_id= first_sibling->emb_sj_nest->sj_subq_pred->get_identifier(); - printing_materialize_nest= TRUE; } - + else + select_id= join->select_lex->select_number; + TABLE *table=tab->table; TABLE_LIST *table_list= tab->table->pos_in_table_list; - char buff[512]; - char buff1[512], buff2[512], buff3[512], buff4[512]; - char keylen_str_buf[64]; + char buff4[512]; my_bool key_read; - String extra(buff, sizeof(buff),cs); char table_name_buffer[SAFE_NAME_LEN]; - String tmp1(buff1,sizeof(buff1),cs); - String tmp2(buff2,sizeof(buff2),cs); - String tmp3(buff3,sizeof(buff3),cs); String tmp4(buff4,sizeof(buff4),cs); - char hash_key_prefix[]= "#hash#"; KEY *key_info= 0; uint key_len= 0; - bool is_hj= tab->type == JT_HASH || tab->type ==JT_HASH_NEXT; - - extra.length(0); - tmp1.length(0); - tmp2.length(0); - tmp3.length(0); tmp4.length(0); quick_type= -1; + QUICK_SELECT_I *quick= NULL; + JOIN_TAB *saved_join_tab= NULL; /* Don't show eliminated tables */ if (table->map & join->eliminated_tables) @@ -22330,27 +23135,27 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, continue; } - item_list.empty(); - /* id */ - item_list.push_back(new Item_uint((uint32)select_id)); - /* select_type */ - const char* stype= printing_materialize_nest? "MATERIALIZED" : - join->select_lex->type; - item_list.push_back(new Item_string(stype, strlen(stype), cs)); - - if ((tab->type == JT_ALL || tab->type == JT_HASH) && - tab->select && tab->select->quick) + if (join->table_access_tabs == join->join_tab && + tab == (first_top_tab + join->const_tables) && pre_sort_join_tab) { - quick_type= tab->select->quick->get_type(); - if ((quick_type == QUICK_SELECT_I::QS_TYPE_INDEX_MERGE) || - (quick_type == QUICK_SELECT_I::QS_TYPE_INDEX_INTERSECT) || - (quick_type == QUICK_SELECT_I::QS_TYPE_ROR_INTERSECT) || - (quick_type == QUICK_SELECT_I::QS_TYPE_ROR_UNION)) - tab->type= tab->type == JT_ALL ? JT_INDEX_MERGE : JT_HASH_INDEX_MERGE; - else - tab->type= tab->type == JT_ALL ? JT_RANGE : JT_HASH_RANGE; + saved_join_tab= tab; + tab= pre_sort_join_tab; } + Explain_table_access *eta= new (output->mem_root) Explain_table_access; + xpl_sel->add_table(eta); + eta->key.set(thd->mem_root, NULL, (uint)-1); + eta->quick_info= NULL; + + /* id */ + if (tab->bush_root_tab) + eta->sjm_nest_select_id= select_id; + else + eta->sjm_nest_select_id= 0; + + /* select_type */ + xpl_sel->select_type= join->select_lex->type; + /* table */ if (table->derived_select_number) { @@ -22358,7 +23163,7 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, int len= my_snprintf(table_name_buffer, sizeof(table_name_buffer)-1, "<derived%u>", table->derived_select_number); - item_list.push_back(new Item_string(table_name_buffer, len, cs)); + eta->table_name.copy(table_name_buffer, len, cs); } else if (tab->bush_children) { @@ -22368,60 +23173,54 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, sizeof(table_name_buffer)-1, "<subquery%d>", ctab->emb_sj_nest->sj_subq_pred->get_identifier()); - item_list.push_back(new Item_string(table_name_buffer, len, cs)); + eta->table_name.copy(table_name_buffer, len, cs); } else { TABLE_LIST *real_table= table->pos_in_table_list; - item_list.push_back(new Item_string(real_table->alias, - strlen(real_table->alias), cs)); + eta->table_name.copy(real_table->alias, strlen(real_table->alias), cs); } + /* "partitions" column */ - if (join->thd->lex->describe & DESCRIBE_PARTITIONS) { #ifdef WITH_PARTITION_STORAGE_ENGINE partition_info *part_info; if (!table->derived_select_number && (part_info= table->part_info)) { - Item_string *item_str= new Item_string(cs); - make_used_partitions_str(part_info, &item_str->str_value); - item_list.push_back(item_str); + make_used_partitions_str(part_info, &eta->used_partitions); + eta->used_partitions_set= true; } else - item_list.push_back(item_null); + eta->used_partitions_set= false; #else /* just produce empty column if partitioning is not compiled in */ - item_list.push_back(item_null); + eta->used_partitions_set= false; #endif } + /* "type" column */ - item_list.push_back(new Item_string(join_type_str[tab->type], - strlen(join_type_str[tab->type]), - cs)); - /* Build "possible_keys" value and add it to item_list */ - if (!tab->keys.is_clear_all()) - { - uint j; - for (j=0 ; j < table->s->keys ; j++) - { - if (tab->keys.is_set(j)) - { - if (tmp1.length()) - tmp1.append(','); - tmp1.append(table->key_info[j].name, - strlen(table->key_info[j].name), - system_charset_info); - } - } + enum join_type tab_type= tab->type; + if ((tab->type == JT_ALL || tab->type == JT_HASH) && + tab->select && tab->select->quick && tab->use_quick != 2) + { + quick= tab->select->quick; + quick_type= tab->select->quick->get_type(); + if ((quick_type == QUICK_SELECT_I::QS_TYPE_INDEX_MERGE) || + (quick_type == QUICK_SELECT_I::QS_TYPE_INDEX_INTERSECT) || + (quick_type == QUICK_SELECT_I::QS_TYPE_ROR_INTERSECT) || + (quick_type == QUICK_SELECT_I::QS_TYPE_ROR_UNION)) + tab_type= tab->type == JT_ALL ? JT_INDEX_MERGE : JT_HASH_INDEX_MERGE; + else + tab_type= tab->type == JT_ALL ? JT_RANGE : JT_HASH_RANGE; } - if (tmp1.length()) - item_list.push_back(new Item_string(tmp1.ptr(),tmp1.length(),cs)); - else - item_list.push_back(item_null); + eta->type= tab_type; - /* Build "key", "key_len", and "ref" values and add them to item_list */ - if (tab->type == JT_NEXT) + /* Build "possible_keys" value */ + append_possible_keys(&eta->possible_keys_str, table, tab->keys); + + /* Build "key", "key_len", and "ref" */ + if (tab_type == JT_NEXT) { key_info= table->key_info+tab->index; key_len= key_info->key_length; @@ -22431,56 +23230,55 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, key_info= tab->get_keyinfo_by_key_no(tab->ref.key); key_len= tab->ref.key_length; } - if (key_info) + + /* + In STRAIGHT_JOIN queries, there can be join tabs with JT_CONST type + that still have quick selects. + */ + if (tab->select && tab->select->quick && tab_type != JT_CONST) { - register uint length; - if (is_hj) - tmp2.append(hash_key_prefix, strlen(hash_key_prefix), cs); - tmp2.append(key_info->name, strlen(key_info->name), cs); - length= (longlong10_to_str(key_len, keylen_str_buf, 10) - - keylen_str_buf); - tmp3.append(keylen_str_buf, length, cs); - if (tab->ref.key_parts) + eta->quick_info= tab->select->quick->get_explain(thd->mem_root); + } + + if (key_info) /* 'index' or 'ref' access */ + { + eta->key.set(thd->mem_root, key_info->name, key_len); + + if (tab->ref.key_parts && tab_type != JT_FT) { - for (store_key **ref=tab->ref.key_copy ; *ref ; ref++) + store_key **ref=tab->ref.key_copy; + for (uint kp= 0; kp < tab->ref.key_parts; kp++) { if (tmp4.length()) tmp4.append(','); - tmp4.append((*ref)->name(), strlen((*ref)->name()), cs); + + if ((key_part_map(1) << kp) & tab->ref.const_ref_part_map) + tmp4.append("const"); + else + { + tmp4.append((*ref)->name(), strlen((*ref)->name()), cs); + ref++; + } } } } - if (is_hj && tab->type != JT_HASH) + + if (tab_type == JT_HASH_NEXT) /* full index scan + hash join */ { - tmp2.append(':'); - tmp3.append(':'); + eta->hash_next_key.set(thd->mem_root, + table->key_info[tab->index].name, + table->key_info[tab->index].key_length); } - if (tab->type == JT_HASH_NEXT) + + if (key_info) { - register uint length; - key_info= table->key_info+tab->index; - key_len= key_info->key_length; - tmp2.append(key_info->name, strlen(key_info->name), cs); - length= (longlong10_to_str(key_len, keylen_str_buf, 10) - - keylen_str_buf); - tmp3.append(keylen_str_buf, length, cs); - } - if (tab->type != JT_CONST && tab->select && tab->select->quick) - tab->select->quick->add_keys_and_lengths(&tmp2, &tmp3); - if (key_info || (tab->select && tab->select->quick)) - { - if (tmp2.length()) - item_list.push_back(new Item_string(tmp2.ptr(),tmp2.length(),cs)); - else - item_list.push_back(item_null); - if (tmp3.length()) - item_list.push_back(new Item_string(tmp3.ptr(),tmp3.length(),cs)); - else - item_list.push_back(item_null); - if (key_info && tab->type != JT_NEXT) - item_list.push_back(new Item_string(tmp4.ptr(),tmp4.length(),cs)); + if (key_info && tab_type != JT_NEXT) + { + eta->ref.copy(tmp4); + eta->ref_set= true; + } else - item_list.push_back(item_null); + eta->ref_set= false; } else { @@ -22490,122 +23288,112 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, { const char *tmp_buff; int f_idx; + StringBuffer<64> key_name_buf; if (table_list->has_db_lookup_value) { + /* The "key" has the name of the column referring to the database */ f_idx= table_list->schema_table->idx_field1; tmp_buff= table_list->schema_table->fields_info[f_idx].field_name; - tmp2.append(tmp_buff, strlen(tmp_buff), cs); + key_name_buf.append(tmp_buff, strlen(tmp_buff), cs); } if (table_list->has_table_lookup_value) { if (table_list->has_db_lookup_value) - tmp2.append(','); + key_name_buf.append(','); + f_idx= table_list->schema_table->idx_field2; tmp_buff= table_list->schema_table->fields_info[f_idx].field_name; - tmp2.append(tmp_buff, strlen(tmp_buff), cs); + key_name_buf.append(tmp_buff, strlen(tmp_buff), cs); } - if (tmp2.length()) - item_list.push_back(new Item_string(tmp2.ptr(),tmp2.length(),cs)); - else - item_list.push_back(item_null); + + if (key_name_buf.length()) + eta->key.set(thd->mem_root, key_name_buf.c_ptr_safe(), -1); } - else - item_list.push_back(item_null); - item_list.push_back(item_null); - item_list.push_back(item_null); + eta->ref_set= false; } - /* Add "rows" field to item_list. */ + /* "rows" */ if (table_list /* SJM bushes don't have table_list */ && table_list->schema_table) { - /* in_rows */ - if (join->thd->lex->describe & DESCRIBE_EXTENDED) - item_list.push_back(item_null); - /* rows */ - item_list.push_back(item_null); + /* I_S tables have rows=extra=NULL */ + eta->rows_set= false; + eta->filtered_set= false; } else { ha_rows examined_rows= tab->get_examined_rows(); - item_list.push_back(new Item_int((longlong) (ulonglong) examined_rows, - MY_INT64_NUM_DECIMAL_DIGITS)); + eta->rows_set= true; + eta->rows= examined_rows; - /* Add "filtered" field to item_list. */ - if (join->thd->lex->describe & DESCRIBE_EXTENDED) + /* "filtered" */ + float f= 0.0; + if (examined_rows) { - float f= 0.0; - if (examined_rows) + double pushdown_cond_selectivity= tab->cond_selectivity; + if (pushdown_cond_selectivity == 1.0) f= (float) (100.0 * tab->records_read / examined_rows); - set_if_smaller(f, 100.0); - item_list.push_back(new Item_float(f, 2)); + else + f= (float) (100.0 * pushdown_cond_selectivity); } + set_if_smaller(f, 100.0); + eta->filtered_set= true; + eta->filtered= f; } - /* Build "Extra" field and add it to item_list. */ + /* Build "Extra" field and save it */ key_read=table->key_read; - if ((tab->type == JT_NEXT || tab->type == JT_CONST) && + if ((tab_type == JT_NEXT || tab_type == JT_CONST) && table->covering_keys.is_set(tab->index)) key_read=1; if (quick_type == QUICK_SELECT_I::QS_TYPE_ROR_INTERSECT && - !((QUICK_ROR_INTERSECT_SELECT*)tab->select->quick)->need_to_fetch_row) + !((QUICK_ROR_INTERSECT_SELECT*)quick)->need_to_fetch_row) key_read=1; if (tab->info) - item_list.push_back(new Item_string(tab->info,strlen(tab->info),cs)); + { + eta->push_extra(tab->info); + } else if (tab->packed_info & TAB_INFO_HAVE_VALUE) { if (tab->packed_info & TAB_INFO_USING_INDEX) - extra.append(STRING_WITH_LEN("; Using index")); + eta->push_extra(ET_USING_INDEX); if (tab->packed_info & TAB_INFO_USING_WHERE) - extra.append(STRING_WITH_LEN("; Using where")); + eta->push_extra(ET_USING_WHERE); if (tab->packed_info & TAB_INFO_FULL_SCAN_ON_NULL) - extra.append(STRING_WITH_LEN("; Full scan on NULL key")); - /* Skip initial "; "*/ - const char *str= extra.ptr(); - uint32 len= extra.length(); - if (len) - { - str += 2; - len -= 2; - } - item_list.push_back(new Item_string(str, len, cs)); + eta->push_extra(ET_FULL_SCAN_ON_NULL_KEY); } else { uint keyno= MAX_KEY; if (tab->ref.key_parts) keyno= tab->ref.key; - else if (tab->select && tab->select->quick) - keyno = tab->select->quick->index; + else if (tab->select && quick) + keyno = quick->index; if (keyno != MAX_KEY && keyno == table->file->pushed_idx_cond_keyno && table->file->pushed_idx_cond) - extra.append(STRING_WITH_LEN("; Using index condition")); + eta->push_extra(ET_USING_INDEX_CONDITION); else if (tab->cache_idx_cond) - extra.append(STRING_WITH_LEN("; Using index condition(BKA)")); + eta->push_extra(ET_USING_INDEX_CONDITION_BKA); if (quick_type == QUICK_SELECT_I::QS_TYPE_ROR_UNION || quick_type == QUICK_SELECT_I::QS_TYPE_ROR_INTERSECT || quick_type == QUICK_SELECT_I::QS_TYPE_INDEX_INTERSECT || quick_type == QUICK_SELECT_I::QS_TYPE_INDEX_MERGE) { - extra.append(STRING_WITH_LEN("; Using ")); - tab->select->quick->add_info_string(&extra); + eta->push_extra(ET_USING); } if (tab->select) { if (tab->use_quick == 2) { - /* 4 bits per 1 hex digit + terminating '\0' */ - char buf[MAX_KEY / 4 + 1]; - extra.append(STRING_WITH_LEN("; Range checked for each " - "record (index map: 0x")); - extra.append(tab->keys.print(buf)); - extra.append(')'); + eta->push_extra(ET_RANGE_CHECKED_FOR_EACH_RECORD); + eta->range_checked_map= tab->keys; } - else if (tab->select->cond) + else if (tab->select->cond || + (tab->cache_select && tab->cache_select->cond)) { const COND *pushed_cond= tab->table->file->pushed_cond; @@ -22615,16 +23403,19 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, HA_MUST_USE_TABLE_CONDITION_PUSHDOWN)) && pushed_cond) { - extra.append(STRING_WITH_LEN("; Using where with pushed " - "condition")); - if (thd->lex->describe & DESCRIBE_EXTENDED) + eta->push_extra(ET_USING_WHERE_WITH_PUSHED_CONDITION); + /* + psergey-todo: what to do? This was useful with NDB only. + + if (explain_flags & DESCRIBE_EXTENDED) { extra.append(STRING_WITH_LEN(": ")); ((COND *)pushed_cond)->print(&extra, QT_ORDINARY); } + */ } else - extra.append(STRING_WITH_LEN("; Using where")); + eta->push_extra(ET_USING_WHERE); } } if (table_list /* SJM bushes don't have table_list */ && @@ -22632,19 +23423,20 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, table_list->schema_table->i_s_requested_object & OPTIMIZE_I_S_TABLE) { if (!table_list->table_open_method) - extra.append(STRING_WITH_LEN("; Skip_open_table")); + eta->push_extra(ET_SKIP_OPEN_TABLE); else if (table_list->table_open_method == OPEN_FRM_ONLY) - extra.append(STRING_WITH_LEN("; Open_frm_only")); + eta->push_extra(ET_OPEN_FRM_ONLY); else - extra.append(STRING_WITH_LEN("; Open_full_table")); + eta->push_extra(ET_OPEN_FULL_TABLE); + /* psergey-note: the following has a bug.*/ if (table_list->has_db_lookup_value && table_list->has_table_lookup_value) - extra.append(STRING_WITH_LEN("; Scanned 0 databases")); + eta->push_extra(ET_SCANNED_0_DATABASES); else if (table_list->has_db_lookup_value || table_list->has_table_lookup_value) - extra.append(STRING_WITH_LEN("; Scanned 1 database")); + eta->push_extra(ET_SCANNED_1_DATABASE); else - extra.append(STRING_WITH_LEN("; Scanned all databases")); + eta->push_extra(ET_SCANNED_ALL_DATABASES); } if (key_read) { @@ -22652,69 +23444,52 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, { QUICK_GROUP_MIN_MAX_SELECT *qgs= (QUICK_GROUP_MIN_MAX_SELECT *) tab->select->quick; - extra.append(STRING_WITH_LEN("; Using index for group-by")); - qgs->append_loose_scan_type(&extra); + eta->push_extra(ET_USING_INDEX_FOR_GROUP_BY); + eta->loose_scan_is_scanning= qgs->loose_scan_is_scanning(); } else - extra.append(STRING_WITH_LEN("; Using index")); + eta->push_extra(ET_USING_INDEX); } if (table->reginfo.not_exists_optimize) - extra.append(STRING_WITH_LEN("; Not exists")); + eta->push_extra(ET_NOT_EXISTS); - /* - if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE && - !(((QUICK_RANGE_SELECT*)(tab->select->quick))->mrr_flags & - HA_MRR_USE_DEFAULT_IMPL)) - { - extra.append(STRING_WITH_LEN("; Using MRR")); - } - */ if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE) { - char mrr_str_buf[128]; - mrr_str_buf[0]=0; - int len; - uint mrr_flags= - ((QUICK_RANGE_SELECT*)(tab->select->quick))->mrr_flags; - len= table->file->multi_range_read_explain_info(mrr_flags, - mrr_str_buf, - sizeof(mrr_str_buf)); - if (len > 0) - { - extra.append(STRING_WITH_LEN("; ")); - extra.append(mrr_str_buf, len); - } + explain_append_mrr_info((QUICK_RANGE_SELECT*)(tab->select->quick), + &eta->mrr_type); + if (eta->mrr_type.length() > 0) + eta->push_extra(ET_USING_MRR); } if (need_tmp_table) { need_tmp_table=0; - extra.append(STRING_WITH_LEN("; Using temporary")); + xpl_sel->using_temporary= true; } if (need_order) { need_order=0; - extra.append(STRING_WITH_LEN("; Using filesort")); + xpl_sel->using_filesort= true; } if (distinct & test_all_bits(used_tables, join->select_list_used_tables)) - extra.append(STRING_WITH_LEN("; Distinct")); + eta->push_extra(ET_DISTINCT); if (tab->loosescan_match_tab) { - extra.append(STRING_WITH_LEN("; LooseScan")); + eta->push_extra(ET_LOOSESCAN); } if (tab->first_weedout_table) - extra.append(STRING_WITH_LEN("; Start temporary")); + eta->push_extra(ET_START_TEMPORARY); if (tab->check_weed_out_table) - extra.append(STRING_WITH_LEN("; End temporary")); + eta->push_extra(ET_END_TEMPORARY); else if (tab->do_firstmatch) { - if (tab->do_firstmatch == join->join_tab - 1) - extra.append(STRING_WITH_LEN("; FirstMatch")); + if (tab->do_firstmatch == /*join->join_tab*/ first_top_tab - 1) + eta->push_extra(ET_FIRST_MATCH); else { - extra.append(STRING_WITH_LEN("; FirstMatch(")); + eta->push_extra(ET_FIRST_MATCH); TABLE *prev_table=tab->do_firstmatch->table; if (prev_table->derived_select_number) { @@ -22723,11 +23498,10 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, int len= my_snprintf(namebuf, sizeof(namebuf)-1, "<derived%u>", prev_table->derived_select_number); - extra.append(namebuf, len); + eta->firstmatch_table_name.append(namebuf, len); } else - extra.append(prev_table->pos_in_table_list->alias); - extra.append(STRING_WITH_LEN(")")); + eta->firstmatch_table_name.append(prev_table->pos_in_table_list->alias); } } @@ -22735,34 +23509,84 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, { if (tab->ref.cond_guards[part]) { - extra.append(STRING_WITH_LEN("; Full scan on NULL key")); + eta->push_extra(ET_FULL_SCAN_ON_NULL_KEY); break; } } if (tab->cache) { - extra.append(STRING_WITH_LEN("; Using join buffer")); - tab->cache->print_explain_comment(&extra); + eta->push_extra(ET_USING_JOIN_BUFFER); + tab->cache->save_explain_data(&eta->bka_type); } - - /* Skip initial "; "*/ - const char *str= extra.ptr(); - uint32 len= extra.length(); - if (len) - { - str += 2; - len -= 2; - } - item_list.push_back(new Item_string(str, len, cs)); } + if (saved_join_tab) + tab= saved_join_tab; + // For next iteration used_tables|=table->map; - if (result->send_data(item_list)) - join->error= 1; } + output->add_node(xpl_sel); + } + + for (SELECT_LEX_UNIT *unit= join->select_lex->first_inner_unit(); + unit; + unit= unit->next_unit()) + { + /* + Display subqueries only if + (1) they are not parts of ON clauses that were eliminated by table + elimination. + (2) they are not merged derived tables + */ + if (!(unit->item && unit->item->eliminated) && // (1) + (!unit->derived || unit->derived->is_materialized_derived())) // (2) + { + explain_node->add_child(unit->first_select()->select_number); + } + } + + if (!error && select_lex->is_top_level_node()) + output->query_plan_ready(); + + + DBUG_RETURN(error); +} + + +/* + This function serves as "shortcut point" for EXPLAIN queries. + + The EXPLAIN statement executes just like its SELECT counterpart would + execute, except that JOIN::exec() will call select_describe() instead of + actually executing the query. + + Inside select_describe(): + - Query plan is updated with latest QEP choices made at the start of + JOIN::exec(). + - the proces of "almost execution" is invoked for the children subqueries. + + Overall, select_describe() is a legacy of old EXPLAIN implementation and + should be removed. +*/ + +static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, + bool distinct,const char *message) +{ + THD *thd=join->thd; + select_result *result=join->result; + DBUG_ENTER("select_describe"); + + /* Update the QPF with latest values of using_temporary, using_filesort */ + Explain_select *explain_sel; + uint select_nr= join->select_lex->select_number; + if ((explain_sel= thd->lex->explain->get_select(select_nr))) + { + explain_sel->using_temporary= need_tmp_table; + explain_sel->using_filesort= need_order; } + for (SELECT_LEX_UNIT *unit= join->select_lex->first_inner_unit(); unit; unit= unit->next_unit()) @@ -22805,13 +23629,13 @@ bool mysql_explain_union(THD *thd, SELECT_LEX_UNIT *unit, select_result *result) for (SELECT_LEX *sl= first; sl; sl= sl->next_select()) { - sl->set_explain_type(); + sl->set_explain_type(FALSE); sl->options|= SELECT_DESCRIBE; } if (unit->is_union()) { - unit->fake_select_lex->select_number= UINT_MAX; // jost for initialization + unit->fake_select_lex->select_number= FAKE_SELECT_LEX_ID; // jost for initialization unit->fake_select_lex->type= "UNION RESULT"; unit->fake_select_lex->options|= SELECT_DESCRIBE; if (!(res= unit->prepare(thd, result, SELECT_NO_UNLOCK | SELECT_DESCRIBE))) @@ -22921,6 +23745,7 @@ static void print_join(THD *thd, List_iterator_fast<TABLE_LIST> ti(*tables); TABLE_LIST **table; uint non_const_tables= 0; + DBUG_ENTER("print_join"); for (TABLE_LIST *t= ti++; t ; t= ti++) { @@ -22934,13 +23759,13 @@ static void print_join(THD *thd, if (!non_const_tables) { str->append(STRING_WITH_LEN("dual")); - return; // all tables were optimized away + DBUG_VOID_RETURN; // all tables were optimized away } ti.rewind(); if (!(table= (TABLE_LIST **)thd->alloc(sizeof(TABLE_LIST*) * non_const_tables))) - return; // out of memory + DBUG_VOID_RETURN; // out of memory TABLE_LIST *tmp, **t= table + (non_const_tables - 1); while ((tmp= ti++)) @@ -22979,6 +23804,7 @@ static void print_join(THD *thd, } print_table_array(thd, eliminated_tables, str, table, table + non_const_tables, query_type); + DBUG_VOID_RETURN; } /** @@ -23487,7 +24313,8 @@ JOIN::reoptimize(Item *added_where, table_map join_tables, reset_query_plan(); if (!keyuse.buffer && - my_init_dynamic_array(&keyuse, sizeof(KEYUSE), 20, 64)) + my_init_dynamic_array(&keyuse, sizeof(KEYUSE), 20, 64, + MYF(MY_THREAD_SPECIFIC))) { delete_dynamic(&added_keyuse); return REOPT_ERROR; @@ -23620,7 +24447,7 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table, int best_key= -1; bool is_best_covering= FALSE; double fanout= 1; - ha_rows table_records= table->file->stats.records; + ha_rows table_records= table->stat_records(); bool group= join && join->group && order == join->group_list; ha_rows ref_key_quick_rows= HA_POS_ERROR; const bool has_limit= (select_limit_arg != HA_POS_ERROR); @@ -23712,7 +24539,7 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table, if (used_key_parts > used_index_parts) used_pk_parts= used_key_parts-used_index_parts; rec_per_key= used_key_parts ? - keyinfo->rec_per_key[used_key_parts-1] : 1; + keyinfo->actual_rec_per_key(used_key_parts-1) : 1; /* Take into account the selectivity of the used pk prefix */ if (used_pk_parts) { @@ -23727,14 +24554,14 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table, rec_per_key= 1; if (rec_per_key > 1) { - rec_per_key*= pkinfo->rec_per_key[used_pk_parts-1]; - rec_per_key/= pkinfo->rec_per_key[0]; + rec_per_key*= pkinfo->actual_rec_per_key(used_pk_parts-1); + rec_per_key/= pkinfo->actual_rec_per_key(0); /* The value of rec_per_key for the extended key has to be adjusted accordingly if some components of the secondary key are included in the primary key. */ - for(uint i= 0; i < used_pk_parts; i++) + for(uint i= 1; i < used_pk_parts; i++) { if (pkinfo->key_part[i].field->key_start.is_set(nr)) { @@ -23742,10 +24569,9 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table, We presume here that for any index rec_per_key[i] != 0 if rec_per_key[0] != 0. */ - DBUG_ASSERT(pkinfo->rec_per_key[i]); - DBUG_ASSERT(i > 0); - rec_per_key*= pkinfo->rec_per_key[i-1]; - rec_per_key/= pkinfo->rec_per_key[i]; + DBUG_ASSERT(pkinfo->actual_rec_per_key(i)); + rec_per_key*= pkinfo->actual_rec_per_key(i-1); + rec_per_key/= pkinfo->actual_rec_per_key(i); } } } @@ -23790,7 +24616,7 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table, select_limit= (ha_rows) (select_limit * (double) table_records / table->quick_condition_rows); - rec_per_key= keyinfo->rec_per_key[keyinfo->key_parts-1]; + rec_per_key= keyinfo->actual_rec_per_key(keyinfo->key_parts-1); set_if_bigger(rec_per_key, 1); /* Here we take into account the fact that rows are @@ -23854,6 +24680,8 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table, @param table Table to find a key @param select Pointer to access/update select->quick (if any) @param limit LIMIT clause parameter + @param [out] scanned_limit How many records we expect to scan + Valid if *need_sort=FALSE. @param [out] need_sort TRUE if filesort needed @param [out] reverse TRUE if the key is reversed again given ORDER (undefined if key == MAX_KEY) @@ -23871,7 +24699,8 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table, */ uint get_index_for_order(ORDER *order, TABLE *table, SQL_SELECT *select, - ha_rows limit, bool *need_sort, bool *reverse) + ha_rows limit, ha_rows *scanned_limit, + bool *need_sort, bool *reverse) { if (!order) { @@ -23913,6 +24742,7 @@ uint get_index_for_order(ORDER *order, TABLE *table, SQL_SELECT *select, { select->set_quick(reverse_quick); *need_sort= FALSE; + *scanned_limit= select->quick->records; return select->quick->index; } else @@ -23931,7 +24761,7 @@ uint get_index_for_order(ORDER *order, TABLE *table, SQL_SELECT *select, Update quick_condition_rows since single table UPDATE/DELETE procedures don't call make_join_statistics() and leave this variable uninitialized. */ - table->quick_condition_rows= table->file->stats.records; + table->quick_condition_rows= table->stat_records(); int key, direction; if (test_if_cheaper_ordering(NULL, order, table, @@ -23941,6 +24771,7 @@ uint get_index_for_order(ORDER *order, TABLE *table, SQL_SELECT *select, !is_key_used(table, key, table->write_set)) { *need_sort= FALSE; + *scanned_limit= limit; *reverse= (direction < 0); return key; } @@ -23949,6 +24780,78 @@ uint get_index_for_order(ORDER *order, TABLE *table, SQL_SELECT *select, return MAX_KEY; } +/* + Count how much times conditions are true for several first rows of the table + + @param thd thread handle + @param rows_to_read how much rows to check + @param table table which should be checked + @conds conds list of conditions and countars for them + + @return number of really checked rows or 0 in case of error or empty table +*/ + +ulong check_selectivity(THD *thd, + ulong rows_to_read, + TABLE *table, + List<COND_STATISTIC> *conds) +{ + ulong count= 0; + COND_STATISTIC *cond; + List_iterator_fast<COND_STATISTIC> it(*conds); + handler *file= table->file; + uchar *record= table->record[0]; + int error= 0; + DBUG_ENTER("check_selectivity"); + + DBUG_ASSERT(rows_to_read > 0); + while ((cond= it++)) + { + DBUG_ASSERT(cond->cond); + DBUG_ASSERT(cond->cond->used_tables() == table->map); + cond->positive= 0; + } + it.rewind(); + + if (file->ha_rnd_init_with_error(1)) + DBUG_RETURN(0); + do + { + error= file->ha_rnd_next(record); + + if (thd->killed) + { + thd->send_kill_message(); + count= 0; + goto err; + } + if (error) + { + if (error == HA_ERR_RECORD_DELETED) + continue; + if (error == HA_ERR_END_OF_FILE) + break; + goto err; + } + + count++; + while ((cond= it++)) + { + if (cond->cond->val_bool()) + cond->positive++; + } + it.rewind(); + + } while (count < rows_to_read); + + file->ha_rnd_end(); + DBUG_RETURN(count); + +err: + DBUG_PRINT("error", ("error %d", error)); + file->ha_rnd_end(); + DBUG_RETURN(0); +} /** @} (end of group Query_Optimizer) diff --git a/sql/sql_select.h b/sql/sql_select.h index 30d2f4abe56..5ecc3f0b18e 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -101,6 +101,13 @@ typedef struct st_table_ref uchar *key_buff; ///< value to look for with key uchar *key_buff2; ///< key_buff+key_length store_key **key_copy; // + + /* + Bitmap of key parts which refer to constants. key_copy only has copiers for + non-const key parts. + */ + key_part_map const_ref_part_map; + Item **items; ///< val()'s for each keypart /* Array of pointers to trigger variables. Some/all of the pointers may be @@ -191,6 +198,12 @@ int rr_sequential(READ_RECORD *info); int rr_sequential_and_unpack(READ_RECORD *info); +#include "sql_explain.h" + +/************************************************************************************** + * New EXPLAIN structures END + *************************************************************************************/ + class JOIN_CACHE; class SJ_TMP_TABLE; class JOIN_TAB_RANGE; @@ -245,7 +258,8 @@ typedef struct st_join_table { JOIN_TAB_RANGE *bush_children; /* Special content for EXPLAIN 'Extra' column or NULL if none */ - const char *info; + enum explain_extra_tag info; + /* Bitmap of TAB_INFO_* bits that encodes special line for EXPLAIN 'Extra' column, or 0 if there is no info. @@ -285,6 +299,9 @@ typedef struct st_join_table { /* psergey-todo: make the below have type double, like POSITION::records_read? */ ha_rows records_read; + /* The selectivity of the conditions that can be pushed to the table */ + double cond_selectivity; + /* Startup cost for execution */ double startup_cost; @@ -527,6 +544,7 @@ typedef struct st_join_table { !(used_sjm_lookup_tables & ~emb_sj_nest->sj_inner_tables)); } + void remove_redundant_bnl_scan_conds(); } JOIN_TAB; @@ -766,6 +784,9 @@ typedef struct st_position :public Sql_alloc */ double records_read; + /* The selectivity of the pushed down conditions */ + double cond_selectivity; + /* Cost accessing the table in course of the entire complete join execution, i.e. cost of one access method use (e.g. 'range' or 'ref' scan ) times @@ -928,7 +949,7 @@ public: */ JOIN_TAB *table_access_tabs; uint top_table_access_tabs_count; - + JOIN_TAB **map2table; ///< mapping between table indexes and JOIN_TABs JOIN_TAB *join_tab_save; ///< saved join_tab for subquery reexecution @@ -1207,8 +1228,14 @@ public: const char *zero_result_cause; ///< not 0 if exec must return zero result bool union_part; ///< this subselect is part of union + + enum join_optimization_state { NOT_OPTIMIZED=0, + OPTIMIZATION_IN_PROGRESS=1, + OPTIMIZATION_DONE=2}; bool optimized; ///< flag to avoid double optimization in EXPLAIN bool initialized; ///< flag to avoid double init_execution calls + + enum { QEP_NOT_PRESENT_YET, QEP_AVAILABLE, QEP_DELETED} have_query_plan; /* Additional WHERE and HAVING predicates to be considered for IN=>EXISTS @@ -1291,6 +1318,7 @@ public: ref_pointer_array_size= 0; zero_result_cause= 0; optimized= 0; + have_query_plan= QEP_NOT_PRESENT_YET; initialized= 0; cleaned= 0; cond_equal= 0; @@ -1315,6 +1343,8 @@ public: pre_sort_join_tab= NULL; emb_sjm_nest= NULL; sjm_lookup_tables= 0; + + exec_saved_explain= false; /* The following is needed because JOIN::cleanup(true) may be called for joins for which JOIN::optimize was aborted with an error before a proper @@ -1323,15 +1353,24 @@ public: table_access_tabs= NULL; } + /* + TRUE <=> There was a JOIN::exec() call, which saved this JOIN's EXPLAIN. + The idea is that we also save at the end of JOIN::optimize(), but that + might not be the final plan. + */ + bool exec_saved_explain; + int prepare(Item ***rref_pointer_array, TABLE_LIST *tables, uint wind_num, COND *conds, uint og_num, ORDER *order, bool skip_order_by, ORDER *group, Item *having, ORDER *proc_param, SELECT_LEX *select, SELECT_LEX_UNIT *unit); bool prepare_stage2(); int optimize(); + int optimize_inner(); int reinit(); int init_execution(); void exec(); + void exec_inner(); int destroy(); void restore_tmp(); bool alloc_func_list(); @@ -1445,6 +1484,11 @@ public: { return (unit->item && unit->item->is_in_predicate()); } + void save_explain_data(Explain_query *output, bool can_overwrite, + bool need_tmp_table, bool need_order, bool distinct); + int save_explain_data_intern(Explain_query *output, bool need_tmp_table, + bool need_order, bool distinct, + const char *message); private: /** TRUE if the query contains an aggregate function but has no GROUP @@ -1796,8 +1840,12 @@ inline bool optimizer_flag(THD *thd, uint flag) return (thd->variables.optimizer_switch & flag); } +int print_fake_select_lex_join(select_result_sink *result, bool on_the_fly, + SELECT_LEX *select_lex, uint8 select_options); + uint get_index_for_order(ORDER *order, TABLE *table, SQL_SELECT *select, - ha_rows limit, bool *need_sort, bool *reverse); + ha_rows limit, ha_rows *scanned_limit, + bool *need_sort, bool *reverse); ORDER *simple_remove_const(ORDER *order, COND *where); bool const_expression_in_where(COND *cond, Item *comp_item, Field *comp_field= NULL, @@ -1811,6 +1859,31 @@ void eliminate_tables(JOIN *join); /* Index Condition Pushdown entry point function */ void push_index_cond(JOIN_TAB *tab, uint keyno); +#define OPT_LINK_EQUAL_FIELDS 1 + +/* EXPLAIN-related utility functions */ +int print_explain_message_line(select_result_sink *result, + uint8 options, + uint select_number, + const char *select_type, + ha_rows *rows, + const char *message); +void explain_append_mrr_info(QUICK_RANGE_SELECT *quick, String *res); +int print_explain_row(select_result_sink *result, + uint8 options, + uint select_number, + const char *select_type, + const char *table_name, + const char *partitions, + enum join_type jtype, + const char *possible_keys, + const char *index, + const char *key_len, + const char *ref, + ha_rows *rows, + const char *extra); +void make_possible_keys_line(TABLE *table, key_map possible_keys, String *line); + /**************************************************************************** Temporary table support for SQL Runtime ***************************************************************************/ @@ -1840,4 +1913,17 @@ void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps); double prev_record_reads(POSITION *positions, uint idx, table_map found_ref); void fix_list_after_tbl_changes(SELECT_LEX *new_parent, List<TABLE_LIST> *tlist); +struct st_cond_statistic +{ + Item *cond; + Field *field_arg; + ulong positive; +}; +typedef struct st_cond_statistic COND_STATISTIC; + +ulong check_selectivity(THD *thd, + ulong rows_to_read, + TABLE *table, + List<COND_STATISTIC> *conds); + #endif /* SQL_SELECT_INCLUDED */ diff --git a/sql/sql_servers.cc b/sql/sql_servers.cc index dce679a883f..b5b7f9866c5 100644 --- a/sql/sql_servers.cc +++ b/sql/sql_servers.cc @@ -158,7 +158,7 @@ bool servers_init(bool dont_read_servers_table) } /* Initialize the mem root for data */ - init_sql_alloc(&mem, ACL_ALLOC_BLOCK_SIZE, 0); + init_sql_alloc(&mem, ACL_ALLOC_BLOCK_SIZE, 0, MYF(MY_THREAD_SPECIFIC)); if (dont_read_servers_table) goto end; @@ -178,7 +178,7 @@ bool servers_init(bool dont_read_servers_table) return_val= servers_reload(thd); delete thd; /* Remember that we don't have a THD */ - my_pthread_setspecific_ptr(THR_THD, 0); + set_current_thd(0); end: DBUG_RETURN(return_val); @@ -209,7 +209,7 @@ static bool servers_load(THD *thd, TABLE_LIST *tables) my_hash_reset(&servers_cache); free_root(&mem, MYF(0)); - init_sql_alloc(&mem, ACL_ALLOC_BLOCK_SIZE, 0); + init_sql_alloc(&mem, ACL_ALLOC_BLOCK_SIZE, 0, MYF(0)); if (init_read_record(&read_record_info,thd,table=tables[0].table,NULL,1,0, FALSE)) diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 91e1693fb94..27fe535a3ab 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2009, 2012, Monty Program Ab. + Copyright (c) 2009, 2013, Monty Program Ab This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -45,6 +45,7 @@ #include "set_var.h" #include "sql_trigger.h" #include "sql_derived.h" +#include "sql_statistics.h" #include "sql_connect.h" #include "authors.h" #include "contributors.h" @@ -56,11 +57,8 @@ #include <my_dir.h> #include "lock.h" // MYSQL_OPEN_IGNORE_FLUSH #include "debug_sync.h" -#include "datadict.h" // dd_frm_type() #include "keycaches.h" -#define STR_OR_NIL(S) ((S) ? (S) : "<nil>") - #ifdef WITH_PARTITION_STORAGE_ENGINE #include "ha_partition.h" #endif @@ -92,6 +90,8 @@ enum enum_i_s_events_fields ISE_DB_CL }; +#define USERNAME_WITH_HOST_CHAR_LENGTH (USERNAME_CHAR_LENGTH + HOSTNAME_LENGTH + 2) + #ifndef NO_EMBEDDED_ACCESS_CHECKS static const char *grant_names[]={ "select","insert","update","delete","create","drop","reload","shutdown", @@ -121,6 +121,14 @@ append_algorithm(TABLE_LIST *table, String *buff); static COND * make_cond_for_info_schema(COND *cond, TABLE_LIST *table); +typedef struct st_lookup_field_values +{ + LEX_STRING db_value, table_value; + bool wild_db_value, wild_table_value; +} LOOKUP_FIELD_VALUES; + +bool get_lookup_field_values(THD *, COND *, TABLE_LIST *, LOOKUP_FIELD_VALUES *); + /*************************************************************************** ** List all table types supported ***************************************************************************/ @@ -159,7 +167,6 @@ static my_bool show_plugins(THD *thd, plugin_ref plugin, cs); switch (plugin_state(plugin)) { - /* case PLUGIN_IS_FREED: does not happen */ case PLUGIN_IS_DELETED: table->field[2]->store(STRING_WITH_LEN("DELETED"), cs); break; @@ -172,6 +179,9 @@ static my_bool show_plugins(THD *thd, plugin_ref plugin, case PLUGIN_IS_DISABLED: table->field[2]->store(STRING_WITH_LEN("DISABLED"), cs); break; + case PLUGIN_IS_FREED: // filtered in fill_plugins, used in fill_all_plugins + table->field[2]->store(STRING_WITH_LEN("NOT INSTALLED"), cs); + break; default: DBUG_ASSERT(0); } @@ -269,6 +279,65 @@ int fill_plugins(THD *thd, TABLE_LIST *tables, COND *cond) } +int fill_all_plugins(THD *thd, TABLE_LIST *tables, COND *cond) +{ + DBUG_ENTER("fill_all_plugins"); + TABLE *table= tables->table; + LOOKUP_FIELD_VALUES lookup; + + if (get_lookup_field_values(thd, cond, tables, &lookup)) + DBUG_RETURN(0); + + if (lookup.db_value.str && !lookup.db_value.str[0]) + DBUG_RETURN(0); // empty string never matches a valid SONAME + + MY_DIR *dirp= my_dir(opt_plugin_dir, MY_THREAD_SPECIFIC); + if (!dirp) + { + my_error(ER_CANT_READ_DIR, MYF(0), opt_plugin_dir, my_errno); + DBUG_RETURN(1); + } + + if (!lookup.db_value.str) + plugin_dl_foreach(thd, 0, show_plugins, table); + + const char *wstr= lookup.db_value.str, *wend= wstr + lookup.db_value.length; + for (uint i=0; i < (uint) dirp->number_of_files; i++) + { + FILEINFO *file= dirp->dir_entry+i; + LEX_STRING dl= { file->name, strlen(file->name) }; + const char *dlend= dl.str + dl.length; + const size_t so_ext_len= sizeof(SO_EXT) - 1; + + if (strcasecmp(dlend - so_ext_len, SO_EXT)) + continue; + + if (lookup.db_value.str) + { + if (lookup.wild_db_value) + { + if (my_wildcmp(files_charset_info, dl.str, dlend, wstr, wend, + wild_prefix, wild_one, wild_many)) + continue; + } + else + { + if (my_strnncoll(files_charset_info, + (uchar*)dl.str, dl.length, + (uchar*)lookup.db_value.str, lookup.db_value.length)) + continue; + } + } + + plugin_dl_foreach(thd, &dl, show_plugins, table); + thd->clear_error(); + } + + my_dirend(dirp); + DBUG_RETURN(0); +} + + /*************************************************************************** ** List all Authors. ** If you can update it, you get to be in it :) @@ -457,7 +526,7 @@ bool ignore_db_dirs_init() { return my_init_dynamic_array(&ignore_db_dirs_array, sizeof(LEX_STRING *), - 0, 0); + 0, 0, MYF(0)); } @@ -688,6 +757,11 @@ db_name_is_in_ignore_db_dirs_list(const char *directory) return my_hash_search(&ignore_db_dirs_hash, (uchar *) buff, buff_len)!=NULL; } +enum find_files_result { + FIND_FILES_OK, + FIND_FILES_OOM, + FIND_FILES_DIR +}; /* find_files() - find files in a given directory. @@ -696,11 +770,10 @@ db_name_is_in_ignore_db_dirs_list(const char *directory) find_files() thd thread handler files put found files in this list - db database name to set in TABLE_LIST structure + db database name to search tables in + or NULL to search for databases path path to database wild filter for found files - dir read databases in path if TRUE, read .frm files in - database otherwise RETURN FIND_FILES_OK success @@ -709,64 +782,40 @@ db_name_is_in_ignore_db_dirs_list(const char *directory) */ -find_files_result -find_files(THD *thd, List<LEX_STRING> *files, const char *db, - const char *path, const char *wild, bool dir) +static find_files_result +find_files(THD *thd, Dynamic_array<LEX_STRING*> *files, LEX_STRING *db, + const char *path, const LEX_STRING *wild) { - uint i; - char *ext; MY_DIR *dirp; - FILEINFO *file; - LEX_STRING *file_name= 0; - uint file_name_len; -#ifndef NO_EMBEDDED_ACCESS_CHECKS - uint col_access=thd->col_access; -#endif - uint wild_length= 0; - TABLE_LIST table_list; + Discovered_table_list tl(thd, files, wild); DBUG_ENTER("find_files"); - if (wild) - { - if (!wild[0]) - wild= 0; - else - wild_length= strlen(wild); - } - - bzero((char*) &table_list,sizeof(table_list)); - - if (!(dirp = my_dir(path,MYF(dir ? MY_WANT_STAT : 0)))) + if (!(dirp = my_dir(path, MY_THREAD_SPECIFIC | (db ? 0 : MY_WANT_STAT)))) { if (my_errno == ENOENT) - my_error(ER_BAD_DB_ERROR, MYF(ME_BELL+ME_WAITTANG), db); + my_error(ER_BAD_DB_ERROR, MYF(ME_BELL | ME_WAITTANG), db->str); else - my_error(ER_CANT_READ_DIR, MYF(ME_BELL+ME_WAITTANG), path, my_errno); + my_error(ER_CANT_READ_DIR, MYF(ME_BELL | ME_WAITTANG), path, my_errno); DBUG_RETURN(FIND_FILES_DIR); } - for (i=0 ; i < (uint) dirp->number_off_files ; i++) + if (!db) /* Return databases */ { - char uname[SAFE_NAME_LEN + 1]; /* Unencoded name */ - file=dirp->dir_entry+i; - if (dir) - { /* Return databases */ - if ((file->name[0] == '.' && - ((file->name[1] == '.' && file->name[2] == '\0') || - file->name[1] == '\0'))) - continue; /* . or .. */ + for (uint i=0; i < (uint) dirp->number_of_files; i++) + { + FILEINFO *file= dirp->dir_entry+i; #ifdef USE_SYMDIR char *ext; char buff[FN_REFLEN]; if (my_use_symdir && !strcmp(ext=fn_ext(file->name), ".sym")) { - /* Only show the sym file if it points to a directory */ - char *end; + /* Only show the sym file if it points to a directory */ + char *end; *ext=0; /* Remove extension */ - unpack_dirname(buff, file->name); - end= strend(buff); - if (end != buff && end[-1] == FN_LIBCHAR) - end[-1]= 0; // Remove end FN_LIBCHAR + unpack_dirname(buff, file->name); + end= strend(buff); + if (end != buff && end[-1] == FN_LIBCHAR) + end[-1]= 0; // Remove end FN_LIBCHAR if (!mysql_file_stat(key_file_misc, buff, file->mystat, MYF(0))) continue; } @@ -777,70 +826,25 @@ find_files(THD *thd, List<LEX_STRING> *files, const char *db, if (is_in_ignore_db_dirs_list(file->name)) continue; - file_name_len= filename_to_tablename(file->name, uname, sizeof(uname)); - if (wild) - { - if (lower_case_table_names) - { - if (my_wildcmp(files_charset_info, - uname, uname + file_name_len, - wild, wild + wild_length, - wild_prefix, wild_one, wild_many)) - continue; - } - else if (wild_compare(uname, wild, 0)) - continue; - } - } - else - { - // Return only .frm files which aren't temp files. - if (my_strcasecmp(system_charset_info, ext=fn_rext(file->name),reg_ext) || - is_prefix(file->name, tmp_file_prefix)) - continue; - *ext=0; - file_name_len= filename_to_tablename(file->name, uname, sizeof(uname)); - if (wild) - { - if (lower_case_table_names) - { - if (my_wildcmp(files_charset_info, - uname, uname + file_name_len, - wild, wild + wild_length, - wild_prefix, wild_one,wild_many)) - continue; - } - else if (wild_compare(uname, wild, 0)) - continue; - } - } -#ifndef NO_EMBEDDED_ACCESS_CHECKS - /* Don't show tables where we don't have any privileges */ - if (db && !(col_access & TABLE_ACLS)) - { - table_list.db= (char*) db; - table_list.db_length= strlen(db); - table_list.table_name= uname; - table_list.table_name_length= file_name_len; - table_list.grant.privilege=col_access; - if (check_grant(thd, TABLE_ACLS, &table_list, TRUE, 1, TRUE)) - continue; - } -#endif - if (!(file_name= - thd->make_lex_string(file_name, uname, file_name_len, TRUE)) || - files->push_back(file_name)) - { - my_dirend(dirp); - DBUG_RETURN(FIND_FILES_OOM); + if (tl.add_file(file->name)) + goto err; } + tl.sort(); + } + else + { + if (ha_discover_table_names(thd, db, dirp, &tl, false)) + goto err; } - DBUG_PRINT("info",("found: %d files", files->elements)); - my_dirend(dirp); - (void) ha_find_files(thd, db, path, wild, dir, files); + DBUG_PRINT("info",("found: %zu files", files->elements())); + my_dirend(dirp); DBUG_RETURN(FIND_FILES_OK); + +err: + my_dirend(dirp); + DBUG_RETURN(FIND_FILES_OOM); } @@ -1386,8 +1390,35 @@ static void append_directory(THD *thd, String *packet, const char *dir_type, #define LIST_PROCESS_HOST_LEN 64 -static bool get_field_default_value(THD *thd, Field *timestamp_field, - Field *field, String *def_value, + +/** + Print "ON UPDATE" clause of a field into a string. + + @param timestamp_field Pointer to timestamp field of a table. + @param field The field to generate ON UPDATE clause for. + @bool lcase Whether to print in lower case. + @return false on success, true on error. +*/ +static bool print_on_update_clause(Field *field, String *val, bool lcase) +{ + DBUG_ASSERT(val->charset()->mbminlen == 1); + val->length(0); + if (field->has_update_default_function()) + { + if (lcase) + val->append(STRING_WITH_LEN("on update ")); + else + val->append(STRING_WITH_LEN("ON UPDATE ")); + val->append(STRING_WITH_LEN("CURRENT_TIMESTAMP")); + if (field->decimals() > 0) + val->append_parenthesized(field->decimals()); + return true; + } + return false; +} + + +static bool get_field_default_value(THD *thd, Field *field, String *def_value, bool quoted) { bool has_default; @@ -1398,8 +1429,7 @@ static bool get_field_default_value(THD *thd, Field *timestamp_field, We are using CURRENT_TIMESTAMP instead of NOW because it is more standard */ - has_now_default= (timestamp_field == field && - field->unireg_check != Field::TIMESTAMP_UN_FIELD); + has_now_default= field->has_insert_default_function(); has_default= (field_type != FIELD_TYPE_BLOB && !(field->flags & NO_DEFAULT_VALUE_FLAG) && @@ -1411,7 +1441,11 @@ static bool get_field_default_value(THD *thd, Field *timestamp_field, if (has_default) { if (has_now_default) + { def_value->append(STRING_WITH_LEN("CURRENT_TIMESTAMP")); + if (field->decimals() > 0) + def_value->append_parenthesized(field->decimals()); + } else if (!field->is_null()) { // Not null by default char tmp[MAX_FIELD_WIDTH]; @@ -1644,16 +1678,18 @@ int store_create_info(THD *thd, TABLE_LIST *table_list, String *packet, } if (!field->vcol_info && - get_field_default_value(thd, table->timestamp_field, - field, &def_value, 1)) + get_field_default_value(thd, field, &def_value, 1)) { packet->append(STRING_WITH_LEN(" DEFAULT ")); packet->append(def_value.ptr(), def_value.length(), system_charset_info); } - if (!limited_mysql_mode && table->timestamp_field == field && - field->unireg_check != Field::TIMESTAMP_DN_FIELD) - packet->append(STRING_WITH_LEN(" ON UPDATE CURRENT_TIMESTAMP")); + if (!limited_mysql_mode && print_on_update_clause(field, &def_value, false)) + { + packet->append(STRING_WITH_LEN(" ")); + packet->append(def_value); + } + if (field->unireg_check == Field::NEXT_NUMBER && !(thd->variables.sql_mode & MODE_NO_FIELD_OPTIONS)) @@ -2028,8 +2064,11 @@ void append_definer(THD *thd, String *buffer, const LEX_STRING *definer_user, { buffer->append(STRING_WITH_LEN("DEFINER=")); append_identifier(thd, buffer, definer_user->str, definer_user->length); - buffer->append('@'); - append_identifier(thd, buffer, definer_host->str, definer_host->length); + if (definer_host->str[0]) + { + buffer->append('@'); + append_identifier(thd, buffer, definer_host->str, definer_host->length); + } buffer->append(' '); } @@ -2124,10 +2163,6 @@ public: double progress; }; -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class I_List<thread_info>; -#endif - static const char *thread_state_info(THD *tmp) { #ifndef EMBEDDED_LIBRARY @@ -2292,6 +2327,259 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose) DBUG_VOID_RETURN; } + +/* + Produce EXPLAIN data. + + This function is APC-scheduled to be run in the context of the thread that + we're producing EXPLAIN for. +*/ + +void Show_explain_request::call_in_target_thread() +{ + Query_arena backup_arena; + bool printed_anything= FALSE; + + /* + Change the arena because JOIN::print_explain and co. are going to allocate + items. Let them allocate them on our arena. + */ + target_thd->set_n_backup_active_arena((Query_arena*)request_thd, + &backup_arena); + + query_str.copy(target_thd->query(), + target_thd->query_length(), + target_thd->query_charset()); + + DBUG_ASSERT(current_thd == target_thd); + set_current_thd(request_thd); + if (target_thd->lex->print_explain(explain_buf, 0 /* explain flags*/, + &printed_anything)) + { + failed_to_produce= TRUE; + } + set_current_thd(target_thd); + + if (!printed_anything) + failed_to_produce= TRUE; + + target_thd->restore_active_arena((Query_arena*)request_thd, &backup_arena); +} + + +int select_result_explain_buffer::send_data(List<Item> &items) +{ + int res; + THD *cur_thd= current_thd; + DBUG_ENTER("select_result_explain_buffer::send_data"); + + /* + Switch to the recieveing thread, so that we correctly count memory used + by it. This is needed as it's the receiving thread that will free the + memory. + */ + set_current_thd(thd); + fill_record(thd, dst_table, dst_table->field, items, TRUE, FALSE); + res= dst_table->file->ha_write_tmp_row(dst_table->record[0]); + set_current_thd(cur_thd); + DBUG_RETURN(test(res)); +} + +bool select_result_text_buffer::send_result_set_metadata(List<Item> &fields, uint flag) +{ + n_columns= fields.elements; + return append_row(fields, true /*send item names */); + return send_data(fields); +} + + +int select_result_text_buffer::send_data(List<Item> &items) +{ + return append_row(items, false /*send item values */); +} + +int select_result_text_buffer::append_row(List<Item> &items, bool send_names) +{ + List_iterator<Item> it(items); + Item *item; + char **row; + int column= 0; + + if (!(row= (char**) thd->alloc(sizeof(char*) * n_columns))) + return true; + rows.push_back(row); + + while ((item= it++)) + { + DBUG_ASSERT(column < n_columns); + StringBuffer<32> buf; + const char *data_ptr; + size_t data_len; + if (send_names) + { + data_ptr= item->name; + data_len= strlen(item->name); + } + else + { + String *res; + res= item->val_str(&buf); + if (item->null_value) + { + data_ptr= "NULL"; + data_len=4; + } + else + { + data_ptr= res->c_ptr_safe(); + data_len= res->length(); + } + } + + char *ptr= (char*)thd->alloc(data_len + 1); + memcpy(ptr, data_ptr, data_len + 1); + row[column]= ptr; + + column++; + } + return false; +} + + +void select_result_text_buffer::save_to(String *res) +{ + List_iterator<char*> it(rows); + char **row; + res->append("#\n"); + while ((row= it++)) + { + res->append("# explain: "); + for (int i=0; i < n_columns; i++) + { + if (i) + res->append('\t'); + res->append(row[i]); + } + res->append("\n"); + } + res->append("#\n"); +} + + +/* + Store the SHOW EXPLAIN output in the temporary table. +*/ + +int fill_show_explain(THD *thd, TABLE_LIST *table, COND *cond) +{ + const char *calling_user; + THD *tmp; + my_thread_id thread_id; + DBUG_ENTER("fill_show_explain"); + + DBUG_ASSERT(cond==NULL); + thread_id= thd->lex->value_list.head()->val_int(); + calling_user= (thd->security_ctx->master_access & PROCESS_ACL) ? NullS : + thd->security_ctx->priv_user; + + if ((tmp= find_thread_by_id(thread_id))) + { + Security_context *tmp_sctx= tmp->security_ctx; + /* + If calling_user==NULL, calling thread has SUPER or PROCESS + privilege, and so can do SHOW EXPLAIN on any user. + + if calling_user!=NULL, he's only allowed to view SHOW EXPLAIN on + his own threads. + */ + if (calling_user && (!tmp_sctx->user || strcmp(calling_user, + tmp_sctx->user))) + { + my_error(ER_SPECIFIC_ACCESS_DENIED_ERROR, MYF(0), "PROCESS"); + mysql_mutex_unlock(&tmp->LOCK_thd_data); + DBUG_RETURN(1); + } + + if (tmp == thd) + { + mysql_mutex_unlock(&tmp->LOCK_thd_data); + my_error(ER_TARGET_NOT_EXPLAINABLE, MYF(0)); + DBUG_RETURN(1); + } + + bool bres; + /* + Ok we've found the thread of interest and it won't go away because + we're holding its LOCK_thd data. Post it a SHOW EXPLAIN request. + */ + bool timed_out; + int timeout_sec= 30; + Show_explain_request explain_req; + select_result_explain_buffer *explain_buf; + + explain_buf= new select_result_explain_buffer(thd, table->table); + + explain_req.explain_buf= explain_buf; + explain_req.target_thd= tmp; + explain_req.request_thd= thd; + explain_req.failed_to_produce= FALSE; + + /* Ok, we have a lock on target->LOCK_thd_data, can call: */ + bres= tmp->apc_target.make_apc_call(thd, &explain_req, timeout_sec, &timed_out); + + if (bres || explain_req.failed_to_produce) + { + if (thd->killed) + thd->send_kill_message(); + else if (timed_out) + my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0)); + else + my_error(ER_TARGET_NOT_EXPLAINABLE, MYF(0)); + + bres= TRUE; + } + else + { + /* + Push the query string as a warning. The query may be in a different + charset than the charset that's used for error messages, so, convert it + if needed. + */ + CHARSET_INFO *fromcs= explain_req.query_str.charset(); + CHARSET_INFO *tocs= error_message_charset_info; + char *warning_text; + if (!my_charset_same(fromcs, tocs)) + { + uint conv_length= 1 + tocs->mbmaxlen * explain_req.query_str.length() / + fromcs->mbminlen; + uint dummy_errors; + char *to, *p; + if (!(to= (char*)thd->alloc(conv_length + 1))) + DBUG_RETURN(1); + p= to; + p+= copy_and_convert(to, conv_length, tocs, + explain_req.query_str.c_ptr(), + explain_req.query_str.length(), fromcs, + &dummy_errors); + *p= 0; + warning_text= to; + } + else + warning_text= explain_req.query_str.c_ptr_safe(); + + push_warning(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_YES, warning_text); + } + DBUG_RETURN(bres); + } + else + { + my_error(ER_NO_SUCH_THREAD, MYF(0), thread_id); + DBUG_RETURN(1); + } +} + + int fill_schema_processlist(THD* thd, TABLE_LIST* tables, COND* cond) { TABLE *table= tables->table; @@ -2402,6 +2690,21 @@ int fill_schema_processlist(THD* thd, TABLE_LIST* tables, COND* cond) } mysql_mutex_unlock(&tmp->LOCK_thd_data); + /* + This may become negative if we free a memory allocated by another + thread in this thread. However it's better that we notice it eventually + than hide it. + */ + table->field[12]->store((longlong) (tmp->status_var.memory_used + + sizeof(THD)), + FALSE); + table->field[12]->set_notnull(); + table->field[13]->store((longlong) tmp->examined_row_count, TRUE); + table->field[13]->set_notnull(); + + /* QUERY_ID */ + table->field[14]->store(tmp->query_id, TRUE); + if (schema_table_store_record(thd, table)) { mysql_mutex_unlock(&LOCK_thread_count); @@ -2424,7 +2727,7 @@ static bool status_vars_inited= 0; C_MODE_START static int show_var_cmp(const void *var1, const void *var2) { - return strcmp(((SHOW_VAR*)var1)->name, ((SHOW_VAR*)var2)->name); + return strcasecmp(((SHOW_VAR*)var1)->name, ((SHOW_VAR*)var2)->name); } C_MODE_END @@ -2474,7 +2777,7 @@ int add_status_vars(SHOW_VAR *list) if (status_vars_inited) mysql_mutex_lock(&LOCK_status); if (!all_status_vars.buffer && // array is not allocated yet - do it now - my_init_dynamic_array(&all_status_vars, sizeof(SHOW_VAR), 200, 20)) + my_init_dynamic_array(&all_status_vars, sizeof(SHOW_VAR), 200, 20, MYF(0))) { res= 1; goto err; @@ -2624,19 +2927,45 @@ static bool show_status_array(THD *thd, const char *wild, for (; variables->name; variables++) { + bool wild_checked; strnmov(prefix_end, variables->name, len); name_buffer[sizeof(name_buffer)-1]=0; /* Safety */ if (ucase_names) my_caseup_str(system_charset_info, name_buffer); + else + { + my_casedn_str(system_charset_info, name_buffer); + DBUG_ASSERT(name_buffer[0] >= 'a'); + DBUG_ASSERT(name_buffer[0] <= 'z'); + + /* traditionally status variables have a first letter uppercased */ + if (status_var) + name_buffer[0]-= 'a' - 'A'; + } + restore_record(table, s->default_values); table->field[0]->store(name_buffer, strlen(name_buffer), system_charset_info); + + /* + Compare name for types that can't return arrays. We do this to not + calculate the value for function variables that we will not access + */ + if ((variables->type != SHOW_FUNC && variables->type != SHOW_ARRAY)) + { + if (wild && wild[0] && wild_case_compare(system_charset_info, + name_buffer, wild)) + continue; + wild_checked= 1; // Avoid checking it again + } + /* - if var->type is SHOW_FUNC, call the function. - Repeat as necessary, if new var is again SHOW_FUNC + if var->type is SHOW_FUNC or SHOW_SIMPLE_FUNC, call the function. + Repeat as necessary, if new var is again one of the above */ - for (var=variables; var->type == SHOW_FUNC; var= &tmp) + for (var=variables; var->type == SHOW_FUNC || + var->type == SHOW_SIMPLE_FUNC; var= &tmp) ((mysql_show_var_func)(var->value))(thd, &tmp, buff); SHOW_TYPE show_type=var->type; @@ -2647,8 +2976,9 @@ static bool show_status_array(THD *thd, const char *wild, } else { - if (!(wild && wild[0] && wild_case_compare(system_charset_info, - name_buffer, wild)) && + if ((wild_checked || + (wild && wild[0] && wild_case_compare(system_charset_info, + name_buffer, wild))) && (!cond || cond->val_int())) { char *value=var->value; @@ -2813,7 +3143,8 @@ static int aggregate_user_stats(HASH *all_user_stats, HASH *agg_user_stats) { // First entry for this role. if (!(agg_user= (USER_STATS*) my_malloc(sizeof(USER_STATS), - MYF(MY_WME | MY_ZEROFILL)))) + MYF(MY_WME | MY_ZEROFILL| + MY_THREAD_SPECIFIC)))) { sql_print_error("Malloc in aggregate_user_stats failed"); DBUG_RETURN(1); @@ -3118,13 +3449,6 @@ void calc_sum_of_all_status(STATUS_VAR *to) /* This is only used internally, but we need it here as a forward reference */ extern ST_SCHEMA_TABLE schema_tables[]; -typedef struct st_lookup_field_values -{ - LEX_STRING db_value, table_value; - bool wild_db_value, wild_table_value; -} LOOKUP_FIELD_VALUES; - - /* Store record to I_S table, convert HEAP table to MyISAM if necessary @@ -3232,8 +3556,8 @@ bool get_lookup_value(THD *thd, Item_func *item_func, (uchar *) item_field->field_name, strlen(item_field->field_name), 0)) { - thd->make_lex_string(&lookup_field_vals->db_value, tmp_str->ptr(), - tmp_str->length(), FALSE); + thd->make_lex_string(&lookup_field_vals->db_value, + tmp_str->ptr(), tmp_str->length()); } /* Lookup value is table name */ else if (!cs->coll->strnncollsp(cs, (uchar *) field_name2, @@ -3241,8 +3565,8 @@ bool get_lookup_value(THD *thd, Item_func *item_func, (uchar *) item_field->field_name, strlen(item_field->field_name), 0)) { - thd->make_lex_string(&lookup_field_vals->table_value, tmp_str->ptr(), - tmp_str->length(), FALSE); + thd->make_lex_string(&lookup_field_vals->table_value, + tmp_str->ptr(), tmp_str->length()); } } return 0; @@ -3419,7 +3743,7 @@ bool get_lookup_field_values(THD *thd, COND *cond, TABLE_LIST *tables, LOOKUP_FIELD_VALUES *lookup_field_values) { LEX *lex= thd->lex; - const char *wild= lex->wild ? lex->wild->ptr() : NullS; + String *wild= lex->wild; bool rc= 0; bzero((char*) lookup_field_values, sizeof(LOOKUP_FIELD_VALUES)); @@ -3427,8 +3751,8 @@ bool get_lookup_field_values(THD *thd, COND *cond, TABLE_LIST *tables, case SQLCOM_SHOW_DATABASES: if (wild) { - thd->make_lex_string(&lookup_field_values->db_value, - wild, strlen(wild), 0); + thd->make_lex_string(&lookup_field_values->db_value, + wild->ptr(), wild->length()); lookup_field_values->wild_db_value= 1; } break; @@ -3437,14 +3761,25 @@ bool get_lookup_field_values(THD *thd, COND *cond, TABLE_LIST *tables, case SQLCOM_SHOW_TRIGGERS: case SQLCOM_SHOW_EVENTS: thd->make_lex_string(&lookup_field_values->db_value, - lex->select_lex.db, strlen(lex->select_lex.db), 0); + lex->select_lex.db, strlen(lex->select_lex.db)); if (wild) { thd->make_lex_string(&lookup_field_values->table_value, - wild, strlen(wild), 0); + wild->ptr(), wild->length()); lookup_field_values->wild_table_value= 1; } break; + case SQLCOM_SHOW_PLUGINS: + if (lex->ident.str) + thd->make_lex_string(&lookup_field_values->db_value, + lex->ident.str, lex->ident.length); + else if (lex->wild) + { + thd->make_lex_string(&lookup_field_values->db_value, + lex->wild->ptr(), lex->wild->length()); + lookup_field_values->wild_db_value= 1; + } + break; default: /* The "default" is for queries over I_S. @@ -3487,23 +3822,15 @@ enum enum_schema_tables get_schema_table_idx(ST_SCHEMA_TABLE *schema_table) wild wild string idx_field_vals idx_field_vals->db_name contains db name or wild string - with_i_schema returns 1 if we added 'IS' name to list - otherwise returns 0 RETURN zero success non-zero error */ -int make_db_list(THD *thd, List<LEX_STRING> *files, - LOOKUP_FIELD_VALUES *lookup_field_vals, - bool *with_i_schema) +int make_db_list(THD *thd, Dynamic_array<LEX_STRING*> *files, + LOOKUP_FIELD_VALUES *lookup_field_vals) { - LEX_STRING *i_s_name_copy= 0; - i_s_name_copy= thd->make_lex_string(i_s_name_copy, - INFORMATION_SCHEMA_NAME.str, - INFORMATION_SCHEMA_NAME.length, TRUE); - *with_i_schema= 0; if (lookup_field_vals->wild_db_value) { /* @@ -3516,12 +3843,11 @@ int make_db_list(THD *thd, List<LEX_STRING> *files, INFORMATION_SCHEMA_NAME.str, lookup_field_vals->db_value.str)) { - *with_i_schema= 1; - if (files->push_back(i_s_name_copy)) + if (files->append_val(&INFORMATION_SCHEMA_NAME)) return 1; } - return (find_files(thd, files, NullS, mysql_data_home, - lookup_field_vals->db_value.str, 1) != FIND_FILES_OK); + return find_files(thd, files, 0, mysql_data_home, + &lookup_field_vals->db_value); } @@ -3537,12 +3863,11 @@ int make_db_list(THD *thd, List<LEX_STRING> *files, if (is_infoschema_db(lookup_field_vals->db_value.str, lookup_field_vals->db_value.length)) { - *with_i_schema= 1; - if (files->push_back(i_s_name_copy)) + if (files->append_val(&INFORMATION_SCHEMA_NAME)) return 1; return 0; } - if (files->push_back(&lookup_field_vals->db_value)) + if (files->append_val(&lookup_field_vals->db_value)) return 1; return 0; } @@ -3551,17 +3876,15 @@ int make_db_list(THD *thd, List<LEX_STRING> *files, Create list of existing databases. It is used in case of select from information schema table */ - if (files->push_back(i_s_name_copy)) + if (files->append_val(&INFORMATION_SCHEMA_NAME)) return 1; - *with_i_schema= 1; - return (find_files(thd, files, NullS, - mysql_data_home, NullS, 1) != FIND_FILES_OK); + return find_files(thd, files, 0, mysql_data_home, &null_lex_str); } struct st_add_schema_table { - List<LEX_STRING> *files; + Dynamic_array<LEX_STRING*> *files; const char *wild; }; @@ -3571,7 +3894,7 @@ static my_bool add_schema_table(THD *thd, plugin_ref plugin, { LEX_STRING *file_name= 0; st_add_schema_table *data= (st_add_schema_table *)p_data; - List<LEX_STRING> *file_list= data->files; + Dynamic_array<LEX_STRING*> *file_list= data->files; const char *wild= data->wild; ST_SCHEMA_TABLE *schema_table= plugin_data(plugin, ST_SCHEMA_TABLE *); DBUG_ENTER("add_schema_table"); @@ -3591,16 +3914,16 @@ static my_bool add_schema_table(THD *thd, plugin_ref plugin, DBUG_RETURN(0); } - if ((file_name= thd->make_lex_string(file_name, schema_table->table_name, - strlen(schema_table->table_name), - TRUE)) && - !file_list->push_back(file_name)) + if ((file_name= thd->make_lex_string(schema_table->table_name, + strlen(schema_table->table_name))) && + !file_list->append(file_name)) DBUG_RETURN(0); DBUG_RETURN(1); } -int schema_tables_add(THD *thd, List<LEX_STRING> *files, const char *wild) +int schema_tables_add(THD *thd, Dynamic_array<LEX_STRING*> *files, + const char *wild) { LEX_STRING *file_name= 0; ST_SCHEMA_TABLE *tmp_schema_table= schema_tables; @@ -3624,9 +3947,9 @@ int schema_tables_add(THD *thd, List<LEX_STRING> *files, const char *wild) continue; } if ((file_name= - thd->make_lex_string(file_name, tmp_schema_table->table_name, - strlen(tmp_schema_table->table_name), TRUE)) && - !files->push_back(file_name)) + thd->make_lex_string(tmp_schema_table->table_name, + strlen(tmp_schema_table->table_name))) && + !files->append(file_name)) continue; DBUG_RETURN(1); } @@ -3651,7 +3974,6 @@ int schema_tables_add(THD *thd, List<LEX_STRING> *files, const char *wild) @param[in] table_names List of table names in database @param[in] lex pointer to LEX struct @param[in] lookup_field_vals pointer to LOOKUP_FIELD_VALUE struct - @param[in] with_i_schema TRUE means that we add I_S tables to list @param[in] db_name database name @return Operation status @@ -3661,40 +3983,32 @@ int schema_tables_add(THD *thd, List<LEX_STRING> *files, const char *wild) */ static int -make_table_name_list(THD *thd, List<LEX_STRING> *table_names, LEX *lex, - LOOKUP_FIELD_VALUES *lookup_field_vals, - bool with_i_schema, LEX_STRING *db_name) +make_table_name_list(THD *thd, Dynamic_array<LEX_STRING*> *table_names, + LEX *lex, LOOKUP_FIELD_VALUES *lookup_field_vals, + LEX_STRING *db_name) { char path[FN_REFLEN + 1]; build_table_filename(path, sizeof(path) - 1, db_name->str, "", "", 0); if (!lookup_field_vals->wild_table_value && lookup_field_vals->table_value.str) { - if (with_i_schema) + if (db_name == &INFORMATION_SCHEMA_NAME) { LEX_STRING *name; ST_SCHEMA_TABLE *schema_table= find_schema_table(thd, lookup_field_vals->table_value.str); if (schema_table && !schema_table->hidden) { - if (!(name= - thd->make_lex_string(NULL, schema_table->table_name, - strlen(schema_table->table_name), TRUE)) || - table_names->push_back(name)) + if (!(name= thd->make_lex_string(schema_table->table_name, + strlen(schema_table->table_name))) || + table_names->append(name)) return 1; } } else { - if (table_names->push_back(&lookup_field_vals->table_value)) + if (table_names->append_val(&lookup_field_vals->table_value)) return 1; - /* - Check that table is relevant in current transaction. - (used for ndb engine, see ndbcluster_find_files(), ha_ndbcluster.cc) - */ - (void) ha_find_files(thd, db_name->str, path, - lookup_field_vals->table_value.str, 0, - table_names); } return 0; } @@ -3703,12 +4017,12 @@ make_table_name_list(THD *thd, List<LEX_STRING> *table_names, LEX *lex, This call will add all matching the wildcards (if specified) IS tables to the list */ - if (with_i_schema) + if (db_name == &INFORMATION_SCHEMA_NAME) return (schema_tables_add(thd, table_names, lookup_field_vals->table_value.str)); - find_files_result res= find_files(thd, table_names, db_name->str, path, - lookup_field_vals->table_value.str, 0); + find_files_result res= find_files(thd, table_names, db_name, path, + &lookup_field_vals->table_value); if (res != FIND_FILES_OK) { /* @@ -3804,10 +4118,10 @@ fill_schema_table_by_open(THD *thd, bool is_show_fields_or_keys, These copies are used for make_table_list() while unaltered values are passed to process_table() functions. */ - if (!thd->make_lex_string(&db_name, orig_db_name->str, - orig_db_name->length, FALSE) || - !thd->make_lex_string(&table_name, orig_table_name->str, - orig_table_name->length, FALSE)) + if (!thd->make_lex_string(&db_name, + orig_db_name->str, orig_db_name->length) || + !thd->make_lex_string(&table_name, + orig_table_name->str, orig_table_name->length)) goto end; /* @@ -3874,12 +4188,14 @@ fill_schema_table_by_open(THD *thd, bool is_show_fields_or_keys, of backward compatibility. */ if (!is_show_fields_or_keys && result && thd->is_error() && - thd->stmt_da->sql_errno() == ER_NO_SUCH_TABLE) + (thd->stmt_da->sql_errno() == ER_NO_SUCH_TABLE || + thd->stmt_da->sql_errno() == ER_WRONG_OBJECT)) { /* Hide error for a non-existing table. For example, this error can occur when we use a where condition - with a db name and table, but the table does not exist. + with a db name and table, but the table does not exist or + there is a view with the same name. */ result= false; thd->clear_error(); @@ -3931,7 +4247,6 @@ end: @param[in] table TABLE struct for I_S table @param[in] db_name database name @param[in] table_name table name - @param[in] with_i_schema I_S table if TRUE @return Operation status @retval 0 success @@ -3939,37 +4254,28 @@ end: */ static int fill_schema_table_names(THD *thd, TABLE_LIST *tables, - LEX_STRING *db_name, LEX_STRING *table_name, - bool with_i_schema) + LEX_STRING *db_name, LEX_STRING *table_name) { TABLE *table= tables->table; - if (with_i_schema) + if (db_name == &INFORMATION_SCHEMA_NAME) { table->field[3]->store(STRING_WITH_LEN("SYSTEM VIEW"), system_charset_info); } else if (tables->table_open_method != SKIP_OPEN_TABLE) { - enum legacy_db_type not_used; - char path[FN_REFLEN + 1]; - (void) build_table_filename(path, sizeof(path) - 1, db_name->str, - table_name->str, reg_ext, 0); - switch (dd_frm_type(thd, path, ¬_used)) { - case FRMTYPE_ERROR: - table->field[3]->store(STRING_WITH_LEN("ERROR"), - system_charset_info); - break; - case FRMTYPE_TABLE: - table->field[3]->store(STRING_WITH_LEN("BASE TABLE"), - system_charset_info); - break; - case FRMTYPE_VIEW: - table->field[3]->store(STRING_WITH_LEN("VIEW"), - system_charset_info); - break; - default: - DBUG_ASSERT(0); + CHARSET_INFO *cs= system_charset_info; + handlerton *hton; + if (ha_table_exists(thd, db_name->str, table_name->str, &hton)) + { + if (hton == view_pseudo_hton) + table->field[3]->store(STRING_WITH_LEN("VIEW"), cs); + else + table->field[3]->store(STRING_WITH_LEN("BASE TABLE"), cs); } + else + table->field[3]->store(STRING_WITH_LEN("ERROR"), cs); + if (thd->is_error() && thd->stmt_da->sql_errno() == ER_NO_SUCH_TABLE) { thd->clear_error(); @@ -4124,10 +4430,6 @@ static int fill_schema_table_from_frm(THD *thd, TABLE_LIST *tables, TABLE tbl; TABLE_LIST table_list; uint res= 0; - int not_used; - my_hash_value_type hash_value; - char key[MAX_DBKEY_LENGTH]; - uint key_length; char db_name_buff[NAME_LEN + 1], table_name_buff[NAME_LEN + 1]; bzero((char*) &table_list, sizeof(TABLE_LIST)); @@ -4186,7 +4488,7 @@ static int fill_schema_table_from_frm(THD *thd, TABLE_LIST *tables, if (schema_table->i_s_requested_object & OPEN_TRIGGER_ONLY) { - init_sql_alloc(&tbl.mem_root, TABLE_ALLOC_BLOCK_SIZE, 0); + init_sql_alloc(&tbl.mem_root, TABLE_ALLOC_BLOCK_SIZE, 0, MYF(0)); if (!Table_triggers_list::check_n_load(thd, db_name->str, table_name->str, &tbl, 1)) { @@ -4199,15 +4501,12 @@ static int fill_schema_table_from_frm(THD *thd, TABLE_LIST *tables, goto end; } - key_length= create_table_def_key(thd, key, &table_list, 0); - hash_value= my_calc_hash(&table_def_cache, (uchar*) key, key_length); - mysql_mutex_lock(&LOCK_open); - share= get_table_share(thd, &table_list, key, - key_length, OPEN_VIEW, ¬_used, hash_value); + share= get_table_share(thd, table_list.db, table_list.table_name, + GTS_TABLE | GTS_VIEW); if (!share) { res= 0; - goto end_unlock; + goto end; } if (share->is_view) @@ -4227,10 +4526,7 @@ static int fill_schema_table_from_frm(THD *thd, TABLE_LIST *tables, res= 1; goto end_share; } - } - if (share->is_view) - { if (open_new_frm(thd, share, table_name->str, (uint) (HA_OPEN_KEYFILE | HA_OPEN_RNDFILE | HA_GET_INDEX | HA_TRY_READ_ONLY), @@ -4256,10 +4552,10 @@ static int fill_schema_table_from_frm(THD *thd, TABLE_LIST *tables, free_root(&tbl.mem_root, MYF(0)); } + end_share: + mysql_mutex_lock(&LOCK_open); release_table_share(share); - -end_unlock: mysql_mutex_unlock(&LOCK_open); end: @@ -4349,14 +4645,12 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) { LEX *lex= thd->lex; TABLE *table= tables->table; + TABLE_LIST table_acl_check; SELECT_LEX *lsel= tables->schema_select_lex; ST_SCHEMA_TABLE *schema_table= tables->schema_table; LOOKUP_FIELD_VALUES lookup_field_vals; - LEX_STRING *db_name, *table_name; - bool with_i_schema; enum enum_schema_tables schema_table_idx; - List<LEX_STRING> db_names; - List_iterator_fast<LEX_STRING> it(db_names); + Dynamic_array<LEX_STRING*> db_names; COND *partial_cond= 0; int error= 1; Open_tables_backup open_tables_state_backup; @@ -4419,9 +4713,9 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) goto err; } - DBUG_PRINT("INDEX VALUES",("db_name='%s', table_name='%s'", - STR_OR_NIL(lookup_field_vals.db_value.str), - STR_OR_NIL(lookup_field_vals.table_value.str))); + DBUG_PRINT("info",("db_name='%s', table_name='%s'", + lookup_field_vals.db_value.str, + lookup_field_vals.table_value.str)); if (!lookup_field_vals.wild_db_value && !lookup_field_vals.wild_table_value) { @@ -4458,11 +4752,13 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) goto err; } - if (make_db_list(thd, &db_names, &lookup_field_vals, &with_i_schema)) + bzero((char*) &table_acl_check, sizeof(table_acl_check)); + + if (make_db_list(thd, &db_names, &lookup_field_vals)) goto err; - it.rewind(); /* To get access to new elements in basis list */ - while ((db_name= it++)) + for (size_t i=0; i < db_names.elements(); i++) { + LEX_STRING *db_name= db_names.at(i); #ifndef NO_EMBEDDED_ACCESS_CHECKS if (!(check_access(thd, SELECT_ACL, db_name->str, &thd->col_access, NULL, 0, 1) || @@ -4471,18 +4767,30 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) acl_get(sctx->host, sctx->ip, sctx->priv_user, db_name->str, 0)) #endif { - List<LEX_STRING> table_names; + Dynamic_array<LEX_STRING*> table_names; int res= make_table_name_list(thd, &table_names, lex, - &lookup_field_vals, - with_i_schema, db_name); + &lookup_field_vals, db_name); if (res == 2) /* Not fatal error, continue */ continue; if (res) goto err; - List_iterator_fast<LEX_STRING> it_files(table_names); - while ((table_name= it_files++)) + for (size_t i=0; i < table_names.elements(); i++) { + LEX_STRING *table_name= table_names.at(i); + +#ifndef NO_EMBEDDED_ACCESS_CHECKS + if (!(thd->col_access & TABLE_ACLS)) + { + table_acl_check.db= db_name->str; + table_acl_check.db_length= db_name->length; + table_acl_check.table_name= table_name->str; + table_acl_check.table_name_length= table_name->length; + table_acl_check.grant.privilege= thd->col_access; + if (check_grant(thd, TABLE_ACLS, &table_acl_check, TRUE, 1, TRUE)) + continue; + } +#endif restore_record(table, s->default_values); table->field[schema_table->idx_field1]-> store(db_name->str, db_name->length, system_charset_info); @@ -4510,18 +4818,18 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) /* SHOW TABLE NAMES command */ if (schema_table_idx == SCH_TABLE_NAMES) { - if (fill_schema_table_names(thd, tables, db_name, - table_name, with_i_schema)) + if (fill_schema_table_names(thd, tables, db_name, table_name)) continue; } - else if (schema_table_idx == SCH_TRIGGERS && with_i_schema) + else if (schema_table_idx == SCH_TRIGGERS && + db_name == &INFORMATION_SCHEMA_NAME) { continue; } else { if (!(table_open_method & ~OPEN_FRM_ONLY) && - !with_i_schema) + db_name != &INFORMATION_SCHEMA_NAME) { /* Here we need to filter out warnings, which can happen @@ -4555,11 +4863,6 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) } } } - /* - If we have information schema its always the first table and only - the first table. Reset for other tables. - */ - with_i_schema= 0; } } @@ -4591,9 +4894,7 @@ int fill_schema_schemata(THD *thd, TABLE_LIST *tables, COND *cond) */ LOOKUP_FIELD_VALUES lookup_field_vals; - List<LEX_STRING> db_names; - LEX_STRING *db_name; - bool with_i_schema; + Dynamic_array<LEX_STRING*> db_names; HA_CREATE_INFO create; TABLE *table= tables->table; #ifndef NO_EMBEDDED_ACCESS_CHECKS @@ -4606,15 +4907,14 @@ int fill_schema_schemata(THD *thd, TABLE_LIST *tables, COND *cond) DBUG_PRINT("INDEX VALUES",("db_name: %s table_name: %s", lookup_field_vals.db_value.str, lookup_field_vals.table_value.str)); - if (make_db_list(thd, &db_names, &lookup_field_vals, - &with_i_schema)) + if (make_db_list(thd, &db_names, &lookup_field_vals)) DBUG_RETURN(1); /* If we have lookup db value we should check that the database exists */ if(lookup_field_vals.db_value.str && !lookup_field_vals.wild_db_value && - !with_i_schema) + db_names.at(0) != &INFORMATION_SCHEMA_NAME) { char path[FN_REFLEN+16]; uint path_len; @@ -4628,15 +4928,14 @@ int fill_schema_schemata(THD *thd, TABLE_LIST *tables, COND *cond) DBUG_RETURN(0); } - List_iterator_fast<LEX_STRING> it(db_names); - while ((db_name=it++)) + for (size_t i=0; i < db_names.elements(); i++) { - if (with_i_schema) // information schema name is always first in list + LEX_STRING *db_name= db_names.at(i); + if (db_name == &INFORMATION_SCHEMA_NAME) { if (store_schema_shemata(thd, table, db_name, system_charset_info)) DBUG_RETURN(1); - with_i_schema= 0; continue; } #ifndef NO_EMBEDDED_ACCESS_CHECKS @@ -4721,7 +5020,7 @@ static int get_schema_tables_record(THD *thd, TABLE_LIST *tables, if (share->db_type() == partition_hton && share->partition_info_str_len) { - tmp_db_type= share->default_part_db_type; + tmp_db_type= plugin_hton(share->default_part_plugin); is_partitioned= TRUE; } #endif @@ -5060,7 +5359,7 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables, const char *wild= lex->wild ? lex->wild->ptr() : NullS; CHARSET_INFO *cs= system_charset_info; TABLE *show_table; - Field **ptr, *field, *timestamp_field; + Field **ptr, *field; int count; DBUG_ENTER("get_schema_column_record"); @@ -5084,7 +5383,6 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables, show_table= tables->table; count= 0; ptr= show_table->field; - timestamp_field= show_table->timestamp_field; show_table->use_all_columns(); // Required for default restore_record(show_table, s->default_values); @@ -5132,7 +5430,7 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables, cs); table->field[4]->store((longlong) count, TRUE); - if (get_field_default_value(thd, timestamp_field, field, &type, 0)) + if (get_field_default_value(thd, field, &type, 0)) { table->field[5]->store(type.ptr(), type.length(), cs); table->field[5]->set_notnull(); @@ -5149,10 +5447,8 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables, if (field->unireg_check == Field::NEXT_NUMBER) table->field[17]->store(STRING_WITH_LEN("auto_increment"), cs); - if (timestamp_field == field && - field->unireg_check != Field::TIMESTAMP_DN_FIELD) - table->field[17]->store(STRING_WITH_LEN("on update CURRENT_TIMESTAMP"), - cs); + if (print_on_update_clause(field, &type, true)) + table->field[17]->store(type.ptr(), type.length(), cs); if (field->vcol_info) { if (field->stored_in_db) @@ -5205,7 +5501,7 @@ static my_bool iter_schema_engines(THD *thd, plugin_ref plugin, void *ptable) { TABLE *table= (TABLE *) ptable; - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); const char *wild= thd->lex->wild ? thd->lex->wild->ptr() : NullS; CHARSET_INFO *scs= system_charset_info; handlerton *default_type= ha_default_handlerton(thd); @@ -5759,9 +6055,12 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, TABLE *show_table= tables->table; KEY *key_info=show_table->s->key_info; if (show_table->file) + { show_table->file->info(HA_STATUS_VARIABLE | HA_STATUS_NO_LOCK | HA_STATUS_TIME); + set_statistics_for_table(thd, show_table); + } for (uint i=0 ; i < show_table->s->keys ; i++,key_info++) { KEY_PART_INFO *key_part= key_info->key_part; @@ -5792,8 +6091,8 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, KEY *key=show_table->key_info+i; if (key->rec_per_key[j]) { - ha_rows records=(show_table->file->stats.records / - key->rec_per_key[j]); + ha_rows records= (ha_rows) ((double) show_table->stat_records() / + key->actual_rec_per_key(j)); table->field[9]->store((longlong) records, TRUE); table->field[9]->set_notnull(); } @@ -7652,16 +7951,22 @@ int make_schema_select(THD *thd, SELECT_LEX *sel, We have to make non const db_name & table_name because of lower_case_table_names */ - thd->make_lex_string(&db, INFORMATION_SCHEMA_NAME.str, - INFORMATION_SCHEMA_NAME.length, 0); - thd->make_lex_string(&table, schema_table->table_name, - strlen(schema_table->table_name), 0); - if (schema_table->old_format(thd, schema_table) || /* Handle old syntax */ - !sel->add_table_to_list(thd, new Table_ident(thd, db, table, 0), + if (!thd->make_lex_string(&db, INFORMATION_SCHEMA_NAME.str, + INFORMATION_SCHEMA_NAME.length)) + DBUG_RETURN(1); + + if (!thd->make_lex_string(&table, schema_table->table_name, + strlen(schema_table->table_name))) + DBUG_RETURN(1); + + if (schema_table->old_format(thd, schema_table)) + + DBUG_RETURN(1); + + if (!sel->add_table_to_list(thd, new Table_ident(thd, db, table, 0), 0, 0, TL_READ, MDL_SHARED_READ)) - { DBUG_RETURN(1); - } + DBUG_RETURN(0); } @@ -7853,7 +8158,7 @@ static my_bool run_hton_fill_schema_table(THD *thd, plugin_ref plugin, { struct run_hton_fill_schema_table_args *args= (run_hton_fill_schema_table_args *) arg; - handlerton *hton= plugin_data(plugin, handlerton *); + handlerton *hton= plugin_hton(plugin); if (hton->fill_is_table && hton->state == SHOW_OPTION_YES) hton->fill_is_table(hton, thd, args->tables, args->cond, get_schema_table_idx(args->tables->schema_table)); @@ -8079,6 +8384,22 @@ ST_FIELD_INFO collation_fields_info[]= }; +ST_FIELD_INFO applicable_roles_fields_info[]= +{ + {"GRANTEE", USERNAME_WITH_HOST_CHAR_LENGTH, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, + {"ROLE_NAME", USERNAME_CHAR_LENGTH, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, + {"IS_GRANTABLE", 3, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, + {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} +}; + + +ST_FIELD_INFO enabled_roles_fields_info[]= +{ + {"ROLE_NAME", USERNAME_CHAR_LENGTH, MYSQL_TYPE_STRING, 0, MY_I_S_MAYBE_NULL, 0, SKIP_OPEN_TABLE}, + {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} +}; + + ST_FIELD_INFO engines_fields_info[]= { {"ENGINE", 64, MYSQL_TYPE_STRING, 0, 0, "Engine", SKIP_OPEN_TABLE}, @@ -8232,7 +8553,7 @@ ST_FIELD_INFO view_fields_info[]= ST_FIELD_INFO user_privileges_fields_info[]= { - {"GRANTEE", 81, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, + {"GRANTEE", USERNAME_WITH_HOST_CHAR_LENGTH, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"TABLE_CATALOG", FN_REFLEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"PRIVILEGE_TYPE", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"IS_GRANTABLE", 3, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, @@ -8242,7 +8563,7 @@ ST_FIELD_INFO user_privileges_fields_info[]= ST_FIELD_INFO schema_privileges_fields_info[]= { - {"GRANTEE", 81, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, + {"GRANTEE", USERNAME_WITH_HOST_CHAR_LENGTH, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"TABLE_CATALOG", FN_REFLEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"TABLE_SCHEMA", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"PRIVILEGE_TYPE", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, @@ -8253,7 +8574,7 @@ ST_FIELD_INFO schema_privileges_fields_info[]= ST_FIELD_INFO table_privileges_fields_info[]= { - {"GRANTEE", 81, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, + {"GRANTEE", USERNAME_WITH_HOST_CHAR_LENGTH, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"TABLE_CATALOG", FN_REFLEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"TABLE_SCHEMA", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"TABLE_NAME", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, @@ -8265,7 +8586,7 @@ ST_FIELD_INFO table_privileges_fields_info[]= ST_FIELD_INFO column_privileges_fields_info[]= { - {"GRANTEE", 81, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, + {"GRANTEE", USERNAME_WITH_HOST_CHAR_LENGTH, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"TABLE_CATALOG", FN_REFLEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"TABLE_SCHEMA", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"TABLE_NAME", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, @@ -8447,6 +8768,9 @@ ST_FIELD_INFO processlist_fields_info[]= {"MAX_STAGE", 2, MYSQL_TYPE_TINY, 0, 0, "Max_stage", SKIP_OPEN_TABLE}, {"PROGRESS", 703, MYSQL_TYPE_DECIMAL, 0, 0, "Progress", SKIP_OPEN_TABLE}, + {"MEMORY_USED", 7, MYSQL_TYPE_LONG, 0, 0, "Memory_used", SKIP_OPEN_TABLE}, + {"EXAMINED_ROWS", 7, MYSQL_TYPE_LONG, 0, 0, "Examined_rows", SKIP_OPEN_TABLE}, + {"QUERY_ID", 4, MYSQL_TYPE_LONGLONG, 0, 0, 0, SKIP_OPEN_TABLE}, {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} }; @@ -8456,7 +8780,7 @@ ST_FIELD_INFO plugin_fields_info[]= {"PLUGIN_NAME", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 0, "Name", SKIP_OPEN_TABLE}, {"PLUGIN_VERSION", 20, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, - {"PLUGIN_STATUS", 10, MYSQL_TYPE_STRING, 0, 0, "Status", SKIP_OPEN_TABLE}, + {"PLUGIN_STATUS", 16, MYSQL_TYPE_STRING, 0, 0, "Status", SKIP_OPEN_TABLE}, {"PLUGIN_TYPE", 80, MYSQL_TYPE_STRING, 0, 0, "Type", SKIP_OPEN_TABLE}, {"PLUGIN_TYPE_VERSION", 20, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}, {"PLUGIN_LIBRARY", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0, 1, "Library", @@ -8636,6 +8960,32 @@ ST_FIELD_INFO keycache_fields_info[]= }; +ST_FIELD_INFO show_explain_fields_info[]= +{ + /* field_name, length, type, value, field_flags, old_name*/ + {"id", 3, MYSQL_TYPE_LONGLONG, 0 /*value*/, MY_I_S_MAYBE_NULL, "id", + SKIP_OPEN_TABLE}, + {"select_type", 19, MYSQL_TYPE_STRING, 0 /*value*/, 0, "select_type", + SKIP_OPEN_TABLE}, + {"table", NAME_CHAR_LEN, MYSQL_TYPE_STRING, 0 /*value*/, MY_I_S_MAYBE_NULL, + "table", SKIP_OPEN_TABLE}, + {"type", 15, MYSQL_TYPE_STRING, 0, MY_I_S_MAYBE_NULL, "type", SKIP_OPEN_TABLE}, + {"possible_keys", NAME_CHAR_LEN*MAX_KEY, MYSQL_TYPE_STRING, 0/*value*/, + MY_I_S_MAYBE_NULL, "possible_keys", SKIP_OPEN_TABLE}, + {"key", NAME_CHAR_LEN*MAX_KEY, MYSQL_TYPE_STRING, 0/*value*/, + MY_I_S_MAYBE_NULL, "key", SKIP_OPEN_TABLE}, + {"key_len", NAME_CHAR_LEN*MAX_KEY, MYSQL_TYPE_STRING, 0/*value*/, + MY_I_S_MAYBE_NULL, "key_len", SKIP_OPEN_TABLE}, + {"ref", NAME_CHAR_LEN*MAX_REF_PARTS, MYSQL_TYPE_STRING, 0/*value*/, + MY_I_S_MAYBE_NULL, "ref", SKIP_OPEN_TABLE}, + {"rows", 10, MYSQL_TYPE_LONGLONG, 0/*value*/, MY_I_S_MAYBE_NULL, "rows", + SKIP_OPEN_TABLE}, + {"Extra", 255, MYSQL_TYPE_STRING, 0/*value*/, 0 /*flags*/, "Extra", + SKIP_OPEN_TABLE}, + {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} +}; + + /* Description of ST_FIELD_INFO in table.h @@ -8645,6 +8995,10 @@ ST_FIELD_INFO keycache_fields_info[]= ST_SCHEMA_TABLE schema_tables[]= { + {"ALL_PLUGINS", plugin_fields_info, create_schema_table, + fill_all_plugins, make_old_format, 0, 5, -1, 0, 0}, + {"APPLICABLE_ROLES", applicable_roles_fields_info, create_schema_table, + fill_schema_applicable_roles, 0, 0, -1, -1, 0, 0}, {"CHARACTER_SETS", charsets_fields_info, create_schema_table, fill_schema_charsets, make_character_sets_old_format, 0, -1, -1, 0, 0}, {"CLIENT_STATISTICS", client_stats_fields_info, create_schema_table, @@ -8658,6 +9012,8 @@ ST_SCHEMA_TABLE schema_tables[]= OPTIMIZE_I_S_TABLE|OPEN_VIEW_FULL}, {"COLUMN_PRIVILEGES", column_privileges_fields_info, create_schema_table, fill_schema_column_privileges, 0, 0, -1, -1, 0, 0}, + {"ENABLED_ROLES", enabled_roles_fields_info, create_schema_table, + fill_schema_enabled_roles, 0, 0, -1, -1, 0, 0}, {"ENGINES", engines_fields_info, create_schema_table, fill_schema_engines, make_old_format, 0, -1, -1, 0, 0}, #ifdef HAVE_EVENT_SCHEDULER @@ -8667,6 +9023,8 @@ ST_SCHEMA_TABLE schema_tables[]= {"EVENTS", events_fields_info, create_schema_table, 0, make_old_format, 0, -1, -1, 0, 0}, #endif + {"EXPLAIN", show_explain_fields_info, create_schema_table, fill_show_explain, + make_old_format, 0, -1, -1, TRUE /*hidden*/ , 0}, {"FILES", files_fields_info, create_schema_table, hton_fill_schema_table, 0, 0, -1, -1, 0, 0}, {"GLOBAL_STATUS", variables_fields_info, create_schema_table, @@ -8742,18 +9100,13 @@ ST_SCHEMA_TABLE schema_tables[]= }; -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class List_iterator_fast<char>; -template class List<char>; -#endif - int initialize_schema_table(st_plugin_int *plugin) { ST_SCHEMA_TABLE *schema_table; DBUG_ENTER("initialize_schema_table"); if (!(schema_table= (ST_SCHEMA_TABLE *)my_malloc(sizeof(ST_SCHEMA_TABLE), - MYF(MY_WME | MY_ZEROFILL)))) + MYF(MY_WME | MY_ZEROFILL)))) DBUG_RETURN(1); /* Historical Requirement */ plugin->data= schema_table; // shortcut for the future diff --git a/sql/sql_show.h b/sql/sql_show.h index 6e87f6097f0..10276e8b65e 100644 --- a/sql/sql_show.h +++ b/sql/sql_show.h @@ -19,6 +19,7 @@ #include "sql_list.h" /* List */ #include "handler.h" /* enum_schema_tables */ #include "table.h" /* enum_schema_table_state */ +#include "my_apc.h" /* Forward declarations */ class JOIN; @@ -26,21 +27,13 @@ class String; class THD; class sp_name; struct TABLE_LIST; -struct st_ha_create_information; typedef class st_select_lex SELECT_LEX; -typedef st_ha_create_information HA_CREATE_INFO; struct LEX; typedef struct st_mysql_show_var SHOW_VAR; typedef struct st_schema_table ST_SCHEMA_TABLE; struct TABLE; typedef struct system_status_var STATUS_VAR; -enum find_files_result { - FIND_FILES_OK, - FIND_FILES_OOM, - FIND_FILES_DIR -}; - /* Used by handlers to store things in schema tables */ #define IS_FILES_FILE_ID 0 #define IS_FILES_FILE_NAME 1 @@ -81,9 +74,6 @@ enum find_files_result { #define IS_FILES_STATUS 36 #define IS_FILES_EXTRA 37 -find_files_result find_files(THD *thd, List<LEX_STRING> *files, const char *db, - const char *path, const char *wild, bool dir); - int store_create_info(THD *thd, TABLE_LIST *table_list, String *packet, HA_CREATE_INFO *create_info_arg, bool show_database); int view_store_create_info(THD *thd, TABLE_LIST *table, String *buff); @@ -131,6 +121,31 @@ enum enum_schema_tables get_schema_table_idx(ST_SCHEMA_TABLE *schema_table); /* These functions were under INNODB_COMPATIBILITY_HOOKS */ int get_quote_char_for_identifier(THD *thd, const char *name, uint length); +THD *find_thread_by_id(longlong id, bool query_id= false); + +class select_result_explain_buffer; +/* + SHOW EXPLAIN request object. +*/ + +class Show_explain_request : public Apc_target::Apc_call +{ +public: + THD *target_thd; /* thd that we're running SHOW EXPLAIN for */ + THD *request_thd; /* thd that run SHOW EXPLAIN command */ + + /* If true, there was some error when producing EXPLAIN output. */ + bool failed_to_produce; + + /* SHOW EXPLAIN will be stored here */ + select_result_explain_buffer *explain_buf; + + /* Query that we've got SHOW EXPLAIN for */ + String query_str; + + /* Overloaded virtual function */ + void call_in_target_thread(); +}; /* Handle the ignored database directories list for SHOW/I_S. */ bool ignore_db_dirs_init(); diff --git a/sql/sql_sort.h b/sql/sql_sort.h index f1a3a2f9d8b..d30ddfb6eec 100644 --- a/sql/sql_sort.h +++ b/sql/sql_sort.h @@ -16,6 +16,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#include "m_string.h" /* memset */ #include "my_global.h" /* uchar */ #include "my_base.h" /* ha_rows */ #include "my_sys.h" /* qsort2_cmp */ @@ -27,7 +28,6 @@ typedef struct st_sort_field SORT_FIELD; class Field; struct TABLE; - /* Defines used by filesort and uniques */ #define MERGEBUFF 7 @@ -65,41 +65,51 @@ struct BUFFPEK_COMPARE_CONTEXT void *key_compare_arg; }; -typedef struct st_sort_param { - uint rec_length; /* Length of sorted records */ - uint sort_length; /* Length of sorted columns */ - uint ref_length; /* Length of record ref. */ - uint addon_length; /* Length of added packed fields */ - uint res_length; /* Length of records in final sorted file/buffer */ - uint keys; /* Max keys / buffer */ + +class Sort_param { +public: + uint rec_length; // Length of sorted records. + uint sort_length; // Length of sorted columns. + uint ref_length; // Length of record ref. + uint addon_length; // Length of added packed fields. + uint res_length; // Length of records in final sorted file/buffer. + uint max_keys_per_buffer; // Max keys / buffer. uint min_dupl_count; - ha_rows max_rows,examined_rows; - TABLE *sort_form; /* For quicker make_sortkey */ + ha_rows max_rows; // Select limit, or HA_POS_ERROR if unlimited. + ha_rows examined_rows; // Number of examined rows. + TABLE *sort_form; // For quicker make_sortkey. SORT_FIELD *local_sortorder; SORT_FIELD *end; - SORT_ADDON_FIELD *addon_field; /* Descriptors for companion fields */ + SORT_ADDON_FIELD *addon_field; // Descriptors for companion fields. uchar *unique_buff; bool not_killable; char* tmp_buffer; - /* The fields below are used only by Unique class */ + // The fields below are used only by Unique class. qsort2_cmp compare; BUFFPEK_COMPARE_CONTEXT cmp_context; -} SORTPARAM; + Sort_param() + { + memset(this, 0, sizeof(*this)); + } + void init_for_filesort(uint sortlen, TABLE *table, + ulong max_length_for_sort_data, + ha_rows maxrows, bool sort_positions); +}; -int merge_many_buff(SORTPARAM *param, uchar *sort_buffer, + +int merge_many_buff(Sort_param *param, uchar *sort_buffer, BUFFPEK *buffpek, uint *maxbuffer, IO_CACHE *t_file); uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, uint sort_length); -int merge_buffers(SORTPARAM *param,IO_CACHE *from_file, - IO_CACHE *to_file, uchar *sort_buffer, - BUFFPEK *lastbuff,BUFFPEK *Fb, - BUFFPEK *Tb,int flag); -int merge_index(SORTPARAM *param, uchar *sort_buffer, +int merge_buffers(Sort_param *param,IO_CACHE *from_file, + IO_CACHE *to_file, uchar *sort_buffer, + BUFFPEK *lastbuff,BUFFPEK *Fb, + BUFFPEK *Tb,int flag); +int merge_index(Sort_param *param, uchar *sort_buffer, BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile, IO_CACHE *outfile); - void reuse_freed_buff(QUEUE *queue, BUFFPEK *reuse, uint key_length); #endif /* SQL_SORT_INCLUDED */ diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc new file mode 100644 index 00000000000..2e2886a1d3f --- /dev/null +++ b/sql/sql_statistics.cc @@ -0,0 +1,3556 @@ +/* Copyright (C) 2009 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/** + @file + + @brief + functions to update persitent statistical tables and to read from them + + @defgroup Query_Optimizer Query Optimizer + @{ +*/ + +#include "sql_base.h" +#include "key.h" +#include "sql_statistics.h" +#include "opt_range.h" +#include "my_atomic.h" + +/* + The system variable 'use_stat_tables' can take one of the + following values: + "never", "complementary", "preferably". + If the values of the variable 'use_stat_tables' is set to + "never then any statistical data from the persistent statistical tables + is ignored by the optimizer. + If the value of the variable 'use_stat_tables' is set to + "complementary" then a particular statistical characteristic is used + by the optimizer only if the database engine does not provide similar + statistics. For example, 'nulls_ratio' for table columns currently + are not provided by any engine. So optimizer uses this statistical data + from the statistical tables. At the same time it does not use + 'avg_frequency' for any index prefix from the statistical tables since + the a similar statistical characteristic 'records_per_key' can be + requested from the database engine. + If the value the variable 'use_stat_tables' is set to + "preferably" the optimizer uses a particular statistical data only if + it can't be found in the statistical data. + If an ANALYZE command is executed then it results in collecting + statistical data for the tables specified by the command and storing + the collected statistics in the persistent statistical tables only + when the value of the variable 'use_stat_tables' is not + equal to "never". +*/ + +/* Currently there are only 3 persistent statistical tables */ +static const uint STATISTICS_TABLES= 3; + +/* + The names of the statistical tables in this array must correspond the + definitions of the tables in the file ../scripts/mysql_system_tables.sql +*/ +static const LEX_STRING stat_table_name[STATISTICS_TABLES]= +{ + { C_STRING_WITH_LEN("table_stats") }, + { C_STRING_WITH_LEN("column_stats") }, + { C_STRING_WITH_LEN("index_stats") } +}; + +/* Name of database to which the statistical tables belong */ +static const LEX_STRING stat_tables_db_name= { C_STRING_WITH_LEN("mysql") }; + + +/** + @details + The function builds a list of TABLE_LIST elements for system statistical + tables using array of TABLE_LIST passed as a parameter. + The lock type of each element is set to TL_READ if for_write = FALSE, + otherwise it is set to TL_WRITE. +*/ + +static +inline void init_table_list_for_stat_tables(TABLE_LIST *tables, bool for_write) +{ + uint i; + + memset((char *) &tables[0], 0, sizeof(TABLE_LIST) * STATISTICS_TABLES); + + for (i= 0; i < STATISTICS_TABLES; i++) + { + tables[i].db= stat_tables_db_name.str; + tables[i].db_length= stat_tables_db_name.length; + tables[i].alias= tables[i].table_name= stat_table_name[i].str; + tables[i].table_name_length= stat_table_name[i].length; + tables[i].lock_type= for_write ? TL_WRITE : TL_READ; + if (i < STATISTICS_TABLES - 1) + tables[i].next_global= tables[i].next_local= + tables[i].next_name_resolution_table= &tables[i+1]; + if (i != 0) + tables[i].prev_global= &tables[i-1].next_global; + } +} + + +/** + @details + The function builds a TABLE_LIST containing only one element 'tbl' for + the statistical table called 'stat_tab_name'. + The lock type of the element is set to TL_READ if for_write = FALSE, + otherwise it is set to TL_WRITE. +*/ + +static +inline void init_table_list_for_single_stat_table(TABLE_LIST *tbl, + const LEX_STRING *stat_tab_name, + bool for_write) +{ + memset((char *) tbl, 0, sizeof(TABLE_LIST)); + + tbl->db= stat_tables_db_name.str; + tbl->db_length= stat_tables_db_name.length; + tbl->alias= tbl->table_name= stat_tab_name->str; + tbl->table_name_length= stat_tab_name->length; + tbl->lock_type= for_write ? TL_WRITE : TL_READ; +} + + +/** + @brief + Open all statistical tables and lock them +*/ + +static +inline int open_stat_tables(THD *thd, TABLE_LIST *tables, + Open_tables_backup *backup, + bool for_write) +{ + init_table_list_for_stat_tables(tables, for_write); + init_mdl_requests(tables); + return open_system_tables_for_read(thd, tables, backup); +} + + +/** + @brief + Open a statistical table and lock it +*/ +static +inline int open_single_stat_table(THD *thd, TABLE_LIST *table, + const LEX_STRING *stat_tab_name, + Open_tables_backup *backup, + bool for_write) +{ + init_table_list_for_single_stat_table(table, stat_tab_name, for_write); + init_mdl_requests(table); + return open_system_tables_for_read(thd, table, backup); +} + + +/* + The class Column_statistics_collected is a helper class used to collect + statistics on a table column. The class is derived directly from + the class Column_statistics, and, additionally to the fields of the + latter, it contains the fields to accumulate the results of aggregation + for the number of nulls in the column and for the size of the column + values. There is also a container for distinct column values used + to calculate the average number of records per distinct column value. +*/ + +class Column_statistics_collected :public Column_statistics +{ + +private: + Field *column; /* The column to collect statistics on */ + ha_rows nulls; /* To accumulate the number of nulls in the column */ + ulonglong column_total_length; /* To accumulate the size of column values */ + Count_distinct_field *count_distinct; /* The container for distinct + column values */ + + bool is_single_pk_col; /* TRUE <-> the only column of the primary key */ + +public: + + inline void init(THD *thd, Field * table_field); + inline void add(ha_rows rowno); + inline void finish(ha_rows rows); + inline void cleanup(); +}; + + +/** + Stat_table is the base class for classes Table_stat, Column_stat and + Index_stat. The methods of these classes allow us to read statistical + data from statistical tables, write collected statistical data into + statistical tables and update statistical data in these tables + as well as update access fields belonging to the primary key and + delete records by prefixes of the primary key. + Objects of the classes Table_stat, Column_stat and Index stat are used + for reading/writing statistics from/into persistent tables table_stats, + column_stats and index_stats correspondingly. These tables are stored in + the system database 'mysql'. + + Statistics is read and written always for a given database table t. When + an object of any of these classes is created a pointer to the TABLE + structure for this database table is passed as a parameter to the constructor + of the object. The other parameter is a pointer to the TABLE structure for + the corresponding statistical table st. So construction of an object to + read/write statistical data on table t from/into statistical table st + requires both table t and st to be opened. + In some cases the TABLE structure for table t may be undefined. Then + the objects of the classes Table_stat, Column_stat and Index stat are + created by the alternative constructor that require only the name + of the table t and the name of the database it belongs to. Currently the + alternative constructors are used only in the cases when some records + belonging to the table are to be deleted, or its keys are to be updated + + Reading/writing statistical data from/into a statistical table is always + performed by a key. At the moment there is only one key defined for each + statistical table and this key is primary. + The primary key for the table table_stats is built as (db_name, table_name). + The primary key for the table column_stats is built as (db_name, table_name, + column_name). + The primary key for the table index_stats is built as (db_name, table_name, + index_name, prefix_arity). + + Reading statistical data from a statistical table is performed by the + following pattern. First a table dependent method sets the values of the + the fields that comprise the lookup key. Then an implementation of the + method get_stat_values() declared in Stat_table as a pure virtual method + finds the row from the statistical table by the set key. If the row is + found the values of statistical fields are read from this row and are + distributed in the internal structures. + + Let's assume the statistical data is read for table t from database db. + + When statistical data is searched in the table table_stats first + Table_stat::set_key_fields() should set the fields of db_name and + table_name. Then get_stat_values looks for a row by the set key value, + and, if the row is found, reads the value from the column + table_stats.cardinality into the field read_stat.cardinality of the TABLE + structure for table t and sets the value of read_stat.cardinality_is_null + from this structure to FALSE. If the value of the 'cardinality' column + in the row is null or if no row is found read_stat.cardinality_is_null + is set to TRUE. + + When statistical data is searched in the table column_stats first + Column_stat::set_key_fields() should set the fields of db_name, table_name + and column_name with column_name taken out of the only parameter f of the + Field* type passed to this method. After this get_stat_values looks + for a row by the set key value. If the row is found the values of statistical + data columns min_value, max_value, nulls_ratio, avg_length, avg_frequency, + hist_size, hist_type, histogram are read into internal structures. Values + of nulls_ratio, avg_length, avg_frequency, hist_size, hist_type, histogram + are read into the corresponding fields of the read_stat structure from + the Field object f, while values from min_value and max_value are copied + into the min_value and max_value record buffers attached to the TABLE + structure for table t. + If the value of a statistical column in the found row is null, then the + corresponding flag in the f->read_stat.column_stat_nulls bitmap is set off. + Otherwise the flag is set on. If no row is found for the column the all flags + in f->column_stat_nulls are set off. + + When statistical data is searched in the table index_stats first + Index_stat::set_key_fields() has to be called to set the fields of db_name, + table_name, index_name and prefix_arity. The value of index_name is extracted + from the first parameter key_info of the KEY* type passed to the method. + This parameter specifies the index of interest idx. The second parameter + passed to the method specifies the arity k of the index prefix for which + statistical data is to be read. E.g. if the index idx consists of 3 + components (p1,p2,p3) the table index_stats usually will contain 3 rows for + this index: the first - for the prefix (p1), the second - for the prefix + (p1,p2), and the third - for the the prefix (p1,p2,p3). After the key fields + has been set a call of get_stat_value looks for a row by the set key value. + If the row is found and the value of the avg_frequency column is not null + then this value is assigned to key_info->read_stat.avg_frequency[k]. + Otherwise 0 is assigned to this element. + + The method Stat_table::update_stat is used to write statistical data + collected in the internal structures into a statistical table st. + It is assumed that before any invocation of this method a call of the + function st.set_key_fields has set the values of the primary key fields + that serve to locate the row from the statistical table st where the + the collected statistical data from internal structures are to be written + to. The statistical data is written from the counterparts of the + statistical fields of internal structures into which it would be read + by the functions get_stat_values. The counterpart fields are used + only when statistics is collected + When updating/inserting a row from the statistical table st the method + Stat_table::update_stat calls the implementation of the pure virtual + method store_field_values to transfer statistical data from the fields + of internal structures to the fields of record buffer used for updates + of the statistical table st. +*/ + +class Stat_table +{ + +private: + + /* Handler used for the retrieval of the statistical table stat_table */ + handler *stat_file; + + uint stat_key_length; /* Length of the key to access stat_table */ + uchar *record[2]; /* Record buffers used to access/update stat_table */ + uint stat_key_idx; /* The number of the key to access stat_table */ + + /* This is a helper function used only by the Stat_table constructors */ + void common_init_stat_table() + { + stat_file= stat_table->file; + /* Currently any statistical table has only one key */ + stat_key_idx= 0; + stat_key_info= &stat_table->key_info[stat_key_idx]; + stat_key_length= stat_key_info->key_length; + record[0]= stat_table->record[0]; + record[1]= stat_table->record[1]; + } + +protected: + + /* Statistical table to read statistics from or to update/delete */ + TABLE *stat_table; + KEY *stat_key_info; /* Structure for the index to access stat_table */ + + /* Table for which statistical data is read / updated */ + TABLE *table; + TABLE_SHARE *table_share; /* Table share for 'table */ + LEX_STRING *db_name; /* Name of the database containing 'table' */ + LEX_STRING *table_name; /* Name of the table 'table' */ + + void store_record_for_update() + { + store_record(stat_table, record[1]); + } + + void store_record_for_lookup() + { + DBUG_ASSERT(record[0] == stat_table->record[0]); + } + + bool update_record() + { + int err; + if ((err= stat_file->ha_update_row(record[1], record[0])) && + err != HA_ERR_RECORD_IS_THE_SAME) + return TRUE; + return FALSE; + } + +public: + + + /** + @details + This constructor has to be called by any constructor of the derived + classes. The constructor 'tunes' the private and protected members of + the constructed object to the statistical table 'stat_table' with the + statistical data of our interest and to the table 'tab' for which this + statistics has been collected. + */ + + Stat_table(TABLE *stat, TABLE *tab) + :stat_table(stat), table(tab) + { + table_share= tab->s; + common_init_stat_table(); + db_name= &table_share->db; + table_name= &table_share->table_name; + } + + + /** + @details + This constructor has to be called by any constructor of the derived + classes. The constructor 'tunes' the private and protected members of + the constructed object to the statistical table 'stat_table' with the + statistical data of our interest and to the table t for which this + statistics has been collected. The table t is uniquely specified + by the database name 'db' and the table name 'tab'. + */ + + Stat_table(TABLE *stat, LEX_STRING *db, LEX_STRING *tab) + :stat_table(stat), table_share(NULL) + { + common_init_stat_table(); + db_name= db; + table_name= tab; + } + + + virtual ~Stat_table() {} + + /** + @brief + Store the given values of fields for database name and table name + + @details + This is a purely virtual method. + The implementation for any derived class shall store the given + values of the database name and table name in the corresponding + fields of stat_table. + + @note + The method is called by the update_table_name_key_parts function. + */ + + virtual void change_full_table_name(LEX_STRING *db, LEX_STRING *tab)= 0; + + + /** + @brief + Store statistical data into fields of the statistical table + + @details + This is a purely virtual method. + The implementation for any derived class shall put the appropriate + statistical data into the corresponding fields of stat_table. + + @note + The method is called by the update_stat function. + */ + + virtual void store_stat_fields()= 0; + + + /** + @brief + Read statistical data from fields of the statistical table + + @details + This is a purely virtual method. + The implementation for any derived read shall read the appropriate + statistical data from the corresponding fields of stat_table. + */ + + virtual void get_stat_values()= 0; + + + /** + @brief + Find a record in the statistical table by a primary key + + @details + The function looks for a record in stat_table by its primary key. + It assumes that the key fields have been already stored in the record + buffer of stat_table. + + @retval + FALSE the record is not found + @retval + TRUE the record is found + */ + + bool find_stat() + { + uchar key[MAX_KEY_LENGTH]; + key_copy(key, record[0], stat_key_info, stat_key_length); + return !stat_file->ha_index_read_idx_map(record[0], stat_key_idx, key, + HA_WHOLE_KEY, HA_READ_KEY_EXACT); + } + + + /** + @brief + Find a record in the statistical table by a key prefix value + + @details + The function looks for a record in stat_table by the key value consisting + of 'prefix_parts' major components for the primary index. + It assumes that the key prefix fields have been already stored in the record + buffer of stat_table. + + @retval + FALSE the record is not found + @retval + TRUE the record is found + */ + + bool find_next_stat_for_prefix(uint prefix_parts) + { + uchar key[MAX_KEY_LENGTH]; + uint prefix_key_length= 0; + for (uint i= 0; i < prefix_parts; i++) + prefix_key_length+= stat_key_info->key_part[i].store_length; + key_copy(key, record[0], stat_key_info, prefix_key_length); + key_part_map prefix_map= (key_part_map) ((1 << prefix_parts) - 1); + return !stat_file->ha_index_read_idx_map(record[0], stat_key_idx, key, + prefix_map, HA_READ_KEY_EXACT); + } + + + /** + @brief + Update/insert a record in the statistical table with new statistics + + @details + The function first looks for a record by its primary key in the statistical + table stat_table. If the record is found the function updates statistical + fields of the records. The data for these fields are taken from internal + structures containing info on the table 'table'. If the record is not + found the function inserts a new record with the primary key set to the + search key and the statistical data taken from the internal structures. + The function assumes that the key fields have been already stored in + the record buffer of stat_table. + + @retval + FALSE success with the update/insert of the record + @retval + TRUE failure with the update/insert of the record + + @note + The function calls the virtual method store_stat_fields to populate the + statistical fields of the updated/inserted row with new statistics. + */ + + bool update_stat() + { + if (find_stat()) + { + store_record_for_update(); + store_stat_fields(); + return update_record(); + } + else + { + int err; + store_stat_fields(); + if ((err= stat_file->ha_write_row(record[0]))) + return TRUE; + } + return FALSE; + } + + + /** + @brief + Update the table name fields in the current record of stat_table + + @details + The function updates the fields containing database name and table name + for the last found record in the statistical table stat_table. + The corresponding names for update is taken from the parameters + db and tab. + + @retval + FALSE success with the update of the record + @retval + TRUE failure with the update of the record + + @note + The function calls the virtual method change_full_table_name + to store the new names in the record buffer used for updates. + */ + + bool update_table_name_key_parts(LEX_STRING *db, LEX_STRING *tab) + { + store_record_for_update(); + change_full_table_name(db, tab); + bool rc= update_record(); + store_record_for_lookup(); + return rc; + } + + + /** + @brief + Delete the current record of the statistical table stat_table + + @details + The function deletes the last found record from the statistical + table stat_table. + + @retval + FALSE success with the deletion of the record + @retval + TRUE failure with the deletion of the record + */ + + bool delete_stat() + { + int err; + if ((err= stat_file->ha_delete_row(record[0]))) + return TRUE; + return FALSE; + } +}; + + +/* + An object of the class Table_stat is created to read statistical + data on tables from the statistical table table_stats, to update + table_stats with such statistical data, or to update columns + of the primary key, or to delete the record by its primary key or + its prefix. + Rows from the statistical table are read and updated always by + primary key. +*/ + +class Table_stat: public Stat_table +{ + +private: + + Field *db_name_field; /* Field for the column table_stats.db_name */ + Field *table_name_field; /* Field for the column table_stats.table_name */ + + void common_init_table_stat() + { + db_name_field= stat_table->field[TABLE_STAT_DB_NAME]; + table_name_field= stat_table->field[TABLE_STAT_TABLE_NAME]; + } + + void change_full_table_name(LEX_STRING *db, LEX_STRING *tab) + { + db_name_field->store(db->str, db->length, system_charset_info); + table_name_field->store(tab->str, tab->length, system_charset_info); + } + +public: + + /** + @details + The constructor 'tunes' the private and protected members of the + constructed object for the statistical table table_stats to read/update + statistics on table 'tab'. The TABLE structure for the table table_stat + must be passed as a value for the parameter 'stat'. + */ + + Table_stat(TABLE *stat, TABLE *tab) :Stat_table(stat, tab) + { + common_init_table_stat(); + } + + + /** + @details + The constructor 'tunes' the private and protected members of the + object constructed for the statistical table table_stat for + the future updates/deletes of the record concerning the table 'tab' + from the database 'db'. + */ + + Table_stat(TABLE *stat, LEX_STRING *db, LEX_STRING *tab) + :Stat_table(stat, db, tab) + { + common_init_table_stat(); + } + + + /** + @brief + Set the key fields for the statistical table table_stat + + @details + The function sets the values of the fields db_name and table_name + in the record buffer for the statistical table table_stat. + These fields comprise the primary key for the table. + + @note + The function is supposed to be called before any use of the + method find_stat for an object of the Table_stat class. + */ + + void set_key_fields() + { + db_name_field->store(db_name->str, db_name->length, system_charset_info); + table_name_field->store(table_name->str, table_name->length, + system_charset_info); + } + + + /** + @brief + Store statistical data into statistical fields of table_stat + + @details + This implementation of a purely virtual method sets the value of the + column 'cardinality' of the statistical table table_stat according to + the value of the flag write_stat.cardinality_is_null and the value of + the field write_stat.cardinality' from the TABLE structure for 'table'. + */ + + void store_stat_fields() + { + Field *stat_field= stat_table->field[TABLE_STAT_CARDINALITY]; + if (table->collected_stats->cardinality_is_null) + stat_field->set_null(); + else + { + stat_field->set_notnull(); + stat_field->store(table->collected_stats->cardinality); + } + } + + + /** + @brief + Read statistical data from statistical fields of table_stat + + @details + This implementation of a purely virtual method first looks for a record + the statistical table table_stat by its primary key set the record + buffer with the help of Table_stat::set_key_fields. Then, if the row is + found the function reads the value of the column 'cardinality' of the table + table_stat and sets the value of the flag read_stat.cardinality_is_null + and the value of the field read_stat.cardinality' from the TABLE structure + for 'table' accordingly. + */ + + void get_stat_values() + { + Table_statistics *read_stats= table_share->stats_cb.table_stats; + read_stats->cardinality_is_null= TRUE; + read_stats->cardinality= 0; + if (find_stat()) + { + Field *stat_field= stat_table->field[TABLE_STAT_CARDINALITY]; + if (!stat_field->is_null()) + { + read_stats->cardinality_is_null= FALSE; + read_stats->cardinality= stat_field->val_int(); + } + } + } + +}; + + +/* + An object of the class Column_stat is created to read statistical data + on table columns from the statistical table column_stats, to update + column_stats with such statistical data, or to update columns + of the primary key, or to delete the record by its primary key or + its prefix. + Rows from the statistical table are read and updated always by + primary key. +*/ + +class Column_stat: public Stat_table +{ + +private: + + Field *db_name_field; /* Field for the column column_stats.db_name */ + Field *table_name_field; /* Field for the column column_stats.table_name */ + Field *column_name_field; /* Field for the column column_stats.column_name */ + + Field *table_field; /* Field from 'table' to read /update statistics on */ + + void common_init_column_stat_table() + { + db_name_field= stat_table->field[COLUMN_STAT_DB_NAME]; + table_name_field= stat_table->field[COLUMN_STAT_TABLE_NAME]; + column_name_field= stat_table->field[COLUMN_STAT_COLUMN_NAME]; + } + + void change_full_table_name(LEX_STRING *db, LEX_STRING *tab) + { + db_name_field->store(db->str, db->length, system_charset_info); + table_name_field->store(tab->str, tab->length, system_charset_info); + } + +public: + + /** + @details + The constructor 'tunes' the private and protected members of the + constructed object for the statistical table column_stats to read/update + statistics on fields of the table 'tab'. The TABLE structure for the table + column_stats must be passed as a value for the parameter 'stat'. + */ + + Column_stat(TABLE *stat, TABLE *tab) :Stat_table(stat, tab) + { + common_init_column_stat_table(); + } + + + /** + @details + The constructor 'tunes' the private and protected members of the + object constructed for the statistical table column_stats for + the future updates/deletes of the record concerning the table 'tab' + from the database 'db'. + */ + + Column_stat(TABLE *stat, LEX_STRING *db, LEX_STRING *tab) + :Stat_table(stat, db, tab) + { + common_init_column_stat_table(); + } + + /** + @brief + Set table name fields for the statistical table column_stats + + @details + The function stores the values of the fields db_name and table_name + of the statistical table column_stats in the record buffer. + */ + + void set_full_table_name() + { + db_name_field->store(db_name->str, db_name->length, system_charset_info); + table_name_field->store(table_name->str, table_name->length, + system_charset_info); + } + + + /** + @brief + Set the key fields for the statistical table column_stats + + @param + col Field for the 'table' column to read/update statistics on + + @details + The function stores the values of the fields db_name, table_name and + column_name in the record buffer for the statistical table column_stats. + These fields comprise the primary key for the table. + It also sets table_field to the passed parameter. + + @note + The function is supposed to be called before any use of the + method find_stat for an object of the Column_stat class. + */ + + void set_key_fields(Field *col) + { + set_full_table_name(); + const char *column_name= col->field_name; + column_name_field->store(column_name, strlen(column_name), + system_charset_info); + table_field= col; + } + + + /** + @brief + Update the table name fields in the current record of stat_table + + @details + The function updates the primary key fields containing database name, + table name, and column name for the last found record in the statistical + table column_stats. + + @retval + FALSE success with the update of the record + @retval + TRUE failure with the update of the record + */ + + bool update_column_key_part(const char *col) + { + store_record_for_update(); + set_full_table_name(); + column_name_field->store(col, strlen(col), system_charset_info); + bool rc= update_record(); + store_record_for_lookup(); + return rc; + } + + + /** + @brief + Store statistical data into statistical fields of column_stats + + @details + This implementation of a purely virtual method sets the value of the + columns 'min_value', 'max_value', 'nulls_ratio', 'avg_length', + 'avg_frequency', 'hist_size', 'hist_type' and 'histogram' of the + stistical table columns_stat according to the contents of the bitmap + write_stat.column_stat_nulls and the values of the fields min_value, + max_value, nulls_ratio, avg_length, avg_frequency, hist_size, hist_type + and histogram of the structure write_stat from the Field structure + for the field 'table_field'. + The value of the k-th column in the table columns_stat is set to NULL + if the k-th bit in the bitmap 'column_stat_nulls' is set to 1. + + @note + A value from the field min_value/max_value is always converted + into a utf8 string. If the length of the column 'min_value'/'max_value' + is less than the length of the string the string is trimmed to fit the + length of the column. + */ + + void store_stat_fields() + { + char buff[MAX_FIELD_WIDTH]; + String val(buff, sizeof(buff), &my_charset_utf8_bin); + + for (uint i= COLUMN_STAT_MIN_VALUE; i <= COLUMN_STAT_HISTOGRAM; i++) + { + Field *stat_field= stat_table->field[i]; + if (table_field->collected_stats->is_null(i)) + stat_field->set_null(); + else + { + stat_field->set_notnull(); + switch (i) { + case COLUMN_STAT_MIN_VALUE: + if (table_field->type() == MYSQL_TYPE_BIT) + stat_field->store(table_field->collected_stats->min_value->val_int()); + else + { + table_field->collected_stats->min_value->val_str(&val); + stat_field->store(val.ptr(), val.length(), &my_charset_utf8_bin); + } + break; + case COLUMN_STAT_MAX_VALUE: + if (table_field->type() == MYSQL_TYPE_BIT) + stat_field->store(table_field->collected_stats->max_value->val_int()); + else + { + table_field->collected_stats->max_value->val_str(&val); + stat_field->store(val.ptr(), val.length(), &my_charset_utf8_bin); + } + break; + case COLUMN_STAT_NULLS_RATIO: + stat_field->store(table_field->collected_stats->get_nulls_ratio()); + break; + case COLUMN_STAT_AVG_LENGTH: + stat_field->store(table_field->collected_stats->get_avg_length()); + break; + case COLUMN_STAT_AVG_FREQUENCY: + stat_field->store(table_field->collected_stats->get_avg_frequency()); + break; + case COLUMN_STAT_HIST_SIZE: + stat_field->store(table_field->collected_stats->histogram.get_size()); + break; + case COLUMN_STAT_HIST_TYPE: + stat_field->store(table_field->collected_stats->histogram.get_type() + + 1); + break; + case COLUMN_STAT_HISTOGRAM: + const char * col_histogram= + (const char *) (table_field->collected_stats->histogram.get_values()); + stat_field->store(col_histogram, + table_field->collected_stats->histogram.get_size(), + &my_charset_bin); + break; + } + } + } + } + + + /** + @brief + Read statistical data from statistical fields of column_stats + + @details + This implementation of a purely virtual method first looks for a record + in the statistical table column_stats by its primary key set in the record + buffer with the help of Column_stat::set_key_fields. Then, if the row is + found, the function reads the values of the columns 'min_value', + 'max_value', 'nulls_ratio', 'avg_length', 'avg_frequency', 'hist_size' and + 'hist_type" of the table column_stat and sets accordingly the value of + the bitmap read_stat.column_stat_nulls' and the values of the fields + min_value, max_value, nulls_ratio, avg_length, avg_frequency, hist_size and + hist_type of the structure read_stat from the Field structure for the field + 'table_field'. + */ + + void get_stat_values() + { + table_field->read_stats->set_all_nulls(); + + if (table_field->read_stats->min_value) + table_field->read_stats->min_value->set_null(); + if (table_field->read_stats->max_value) + table_field->read_stats->max_value->set_null(); + + if (find_stat()) + { + char buff[MAX_FIELD_WIDTH]; + String val(buff, sizeof(buff), &my_charset_utf8_bin); + + for (uint i= COLUMN_STAT_MIN_VALUE; i <= COLUMN_STAT_HIST_TYPE; i++) + { + Field *stat_field= stat_table->field[i]; + + if (!stat_field->is_null() && + (i > COLUMN_STAT_MAX_VALUE || + (i == COLUMN_STAT_MIN_VALUE && + table_field->read_stats->min_value) || + (i == COLUMN_STAT_MAX_VALUE && + table_field->read_stats->max_value))) + { + table_field->read_stats->set_not_null(i); + + switch (i) { + case COLUMN_STAT_MIN_VALUE: + stat_field->val_str(&val); + table_field->read_stats->min_value->store(val.ptr(), val.length(), + &my_charset_utf8_bin); + break; + case COLUMN_STAT_MAX_VALUE: + stat_field->val_str(&val); + table_field->read_stats->max_value->store(val.ptr(), val.length(), + &my_charset_utf8_bin); + break; + case COLUMN_STAT_NULLS_RATIO: + table_field->read_stats->set_nulls_ratio(stat_field->val_real()); + break; + case COLUMN_STAT_AVG_LENGTH: + table_field->read_stats->set_avg_length(stat_field->val_real()); + break; + case COLUMN_STAT_AVG_FREQUENCY: + table_field->read_stats->set_avg_frequency(stat_field->val_real()); + break; + case COLUMN_STAT_HIST_SIZE: + table_field->read_stats->histogram.set_size(stat_field->val_int()); + break; + case COLUMN_STAT_HIST_TYPE: + Histogram_type hist_type= (Histogram_type) (stat_field->val_int() - + 1); + table_field->read_stats->histogram.set_type(hist_type); + break; + } + } + } + } + } + + + /** + @brief + Read histogram from of column_stats + + @details + This method first looks for a record in the statistical table column_stats + by its primary key set the record buffer with the help of + Column_stat::set_key_fields. Then, if the row is found, the function reads + the value of the column 'histogram' of the table column_stat and sets + accordingly the corresponding bit in the bitmap read_stat.column_stat_nulls. + The method assumes that the value of histogram size and the pointer to + the histogram location has been already set in the fields size and values + of read_stats->histogram. + */ + + void get_histogram_value() + { + if (find_stat()) + { + char buff[MAX_FIELD_WIDTH]; + String val(buff, sizeof(buff), &my_charset_utf8_bin); + uint fldno= COLUMN_STAT_HISTOGRAM; + Field *stat_field= stat_table->field[fldno]; + table_field->read_stats->set_not_null(fldno); + stat_field->val_str(&val); + memcpy(table_field->read_stats->histogram.get_values(), + val.ptr(), table_field->read_stats->histogram.get_size()); + } + } + +}; + + +/* + An object of the class Index_stat is created to read statistical + data on tables from the statistical table table_stat, to update + index_stats with such statistical data, or to update columns + of the primary key, or to delete the record by its primary key or + its prefix. + Rows from the statistical table are read and updated always by + primary key. +*/ + +class Index_stat: public Stat_table +{ + +private: + + Field *db_name_field; /* Field for the column index_stats.db_name */ + Field *table_name_field; /* Field for the column index_stats.table_name */ + Field *index_name_field; /* Field for the column index_stats.table_name */ + Field *prefix_arity_field; /* Field for the column index_stats.prefix_arity */ + + KEY *table_key_info; /* Info on the index to read/update statistics on */ + uint prefix_arity; /* Number of components of the index prefix of interest */ + + void common_init_index_stat_table() + { + db_name_field= stat_table->field[INDEX_STAT_DB_NAME]; + table_name_field= stat_table->field[INDEX_STAT_TABLE_NAME]; + index_name_field= stat_table->field[INDEX_STAT_INDEX_NAME]; + prefix_arity_field= stat_table->field[INDEX_STAT_PREFIX_ARITY]; + } + + void change_full_table_name(LEX_STRING *db, LEX_STRING *tab) + { + db_name_field->store(db->str, db->length, system_charset_info); + table_name_field->store(tab->str, tab->length, system_charset_info); + } + +public: + + + /** + @details + The constructor 'tunes' the private and protected members of the + constructed object for the statistical table index_stats to read/update + statistics on prefixes of different indexes of the table 'tab'. + The TABLE structure for the table index_stats must be passed as a value + for the parameter 'stat'. + */ + + Index_stat(TABLE *stat, TABLE*tab) :Stat_table(stat, tab) + { + common_init_index_stat_table(); + } + + + /** + @details + The constructor 'tunes' the private and protected members of the + object constructed for the statistical table index_stats for + the future updates/deletes of the record concerning the table 'tab' + from the database 'db'. + */ + + Index_stat(TABLE *stat, LEX_STRING *db, LEX_STRING *tab) + :Stat_table(stat, db, tab) + { + common_init_index_stat_table(); + } + + + /** + @brief + Set table name fields for the statistical table index_stats + + @details + The function stores the values of the fields db_name and table_name + of the statistical table index_stats in the record buffer. + */ + + void set_full_table_name() + { + db_name_field->store(db_name->str, db_name->length, system_charset_info); + table_name_field->store(table_name->str, table_name->length, + system_charset_info); + } + + /** + @brief + Set the key fields of index_stats used to access records for index prefixes + + @param + index_info Info for the index of 'table' to read/update statistics on + + @details + The function sets the values of the fields db_name, table_name and + index_name in the record buffer for the statistical table index_stats. + It also sets table_key_info to the passed parameter. + + @note + The function is supposed to be called before any use of the method + find_next_stat_for_prefix for an object of the Index_stat class. + */ + + void set_index_prefix_key_fields(KEY *index_info) + { + set_full_table_name(); + char *index_name= index_info->name; + index_name_field->store(index_name, strlen(index_name), + system_charset_info); + table_key_info= index_info; + } + + + /** + @brief + Set the key fields for the statistical table index_stats + + @param + index_info Info for the index of 'table' to read/update statistics on + @param + index_prefix_arity Number of components in the index prefix of interest + + @details + The function sets the values of the fields db_name, table_name and + index_name, prefix_arity in the record buffer for the statistical + table index_stats. These fields comprise the primary key for the table. + + @note + The function is supposed to be called before any use of the + method find_stat for an object of the Index_stat class. + */ + + void set_key_fields(KEY *index_info, uint index_prefix_arity) + { + set_index_prefix_key_fields(index_info); + prefix_arity= index_prefix_arity; + prefix_arity_field->store(index_prefix_arity, TRUE); + } + + + /** + @brief + Store statistical data into statistical fields of table index_stats + + @details + This implementation of a purely virtual method sets the value of the + column 'avg_frequency' of the statistical table index_stats according to + the value of write_stat.avg_frequency[Index_stat::prefix_arity] + from the KEY_INFO structure 'table_key_info'. + If the value of write_stat. avg_frequency[Index_stat::prefix_arity] is + equal to 0, the value of the column is set to NULL. + */ + + void store_stat_fields() + { + Field *stat_field= stat_table->field[INDEX_STAT_AVG_FREQUENCY]; + double avg_frequency= + table_key_info->collected_stats->get_avg_frequency(prefix_arity-1); + if (avg_frequency == 0) + stat_field->set_null(); + else + { + stat_field->set_notnull(); + stat_field->store(avg_frequency); + } + } + + + /** + @brief + Read statistical data from statistical fields of index_stats + + @details + This implementation of a purely virtual method first looks for a record the + statistical table index_stats by its primary key set the record buffer with + the help of Index_stat::set_key_fields. If the row is found the function + reads the value of the column 'avg_freguency' of the table index_stat and + sets the value of read_stat.avg_frequency[Index_stat::prefix_arity] + from the KEY_INFO structure 'table_key_info' accordingly. If the value of + the column is NULL, read_stat.avg_frequency[Index_stat::prefix_arity] is + set to 0. Otherwise, read_stat.avg_frequency[Index_stat::prefix_arity] is + set to the value of the column. + */ + + void get_stat_values() + { + double avg_frequency= 0; + if(find_stat()) + { + Field *stat_field= stat_table->field[INDEX_STAT_AVG_FREQUENCY]; + if (!stat_field->is_null()) + avg_frequency= stat_field->val_real(); + } + table_key_info->read_stats->set_avg_frequency(prefix_arity-1, avg_frequency); + } + +}; + +/* + Histogram_builder is a helper class that is used to build histograms + for columns +*/ + +class Histogram_builder +{ + Field *column; /* table field for which the histogram is built */ + uint col_length; /* size of this field */ + ha_rows records; /* number of records the histogram is built for */ + Field *min_value; /* pointer to the minimal value for the field */ + Field *max_value; /* pointer to the maximal value for the field */ + Histogram *histogram; /* the histogram location */ + uint hist_width; /* the number of points in the histogram */ + double bucket_capacity; /* number of rows in a bucket of the histogram */ + uint curr_bucket; /* number of the current bucket to be built */ + ulonglong count; /* number of values retrieved */ + ulonglong count_distinct; /* number of distinct values retrieved */ + +public: + Histogram_builder(Field *col, uint col_len, ha_rows rows) + : column(col), col_length(col_len), records(rows) + { + Column_statistics *col_stats= col->collected_stats; + min_value= col_stats->min_value; + max_value= col_stats->max_value; + histogram= &col_stats->histogram; + hist_width= histogram->get_width(); + bucket_capacity= (double) records / (hist_width + 1); + curr_bucket= 0; + count= 0; + count_distinct= 0; + } + + ulonglong get_count_distinct() { return count_distinct; } + + int next(void *elem, element_count elem_cnt) + { + count_distinct++; + count+= elem_cnt; + if (curr_bucket == hist_width) + return 0; + if (count > bucket_capacity * (curr_bucket + 1)) + { + column->store_field_value((uchar *) elem, col_length); + histogram->set_value(curr_bucket, + column->pos_in_interval(min_value, max_value)); + curr_bucket++; + while (curr_bucket != hist_width && + count > bucket_capacity * (curr_bucket + 1)) + { + histogram->set_prev_value(curr_bucket); + curr_bucket++; + } + } + return 0; + } +}; + + +C_MODE_START + +int histogram_build_walk(void *elem, element_count elem_cnt, void *arg) +{ + Histogram_builder *hist_builder= (Histogram_builder *) arg; + return hist_builder->next(elem, elem_cnt); +} + +C_MODE_END + + +/* + The class Count_distinct_field is a helper class used to calculate + the number of distinct values for a column. The class employs the + Unique class for this purpose. + The class Count_distinct_field is used only by the function + collect_statistics_for_table to calculate the values for + column avg_frequency of the statistical table column_stats. +*/ + +class Count_distinct_field: public Sql_alloc +{ +protected: + + /* Field for which the number of distinct values is to be find out */ + Field *table_field; + Unique *tree; /* The helper object to contain distinct values */ + uint tree_key_length; /* The length of the keys for the elements of 'tree */ + +public: + + Count_distinct_field() {} + + /** + @param + field Field for which the number of distinct values is + to be find out + @param + max_heap_table_size The limit for the memory used by the RB tree container + of the constructed Unique object 'tree' + + @details + The constructor sets the values of 'table_field' and 'tree_key_length', + and then calls the 'new' operation to create a Unique object for 'tree'. + The type of 'field' and the value max_heap_table_size of determine the set + of the parameters to be passed to the constructor of the Unique object. + */ + + Count_distinct_field(Field *field, uint max_heap_table_size) + { + table_field= field; + tree_key_length= field->pack_length(); + + tree= new Unique((qsort_cmp2) simple_str_key_cmp, (void*) field, + tree_key_length, max_heap_table_size, 1); + } + + virtual ~Count_distinct_field() + { + delete tree; + tree= NULL; + } + + /* + @brief + Check whether the Unique object tree has been successfully created + */ + bool exists() + { + return (tree != NULL); + } + + /* + @brief + Add the value of 'field' to the container of the Unique object 'tree' + */ + virtual bool add() + { + return tree->unique_add(table_field->ptr); + } + + /* + @brief + Calculate the number of elements accumulated in the container of 'tree' + */ + ulonglong get_value() + { + ulonglong count; + if (tree->elements == 0) + return (ulonglong) tree->elements_in_tree(); + count= 0; + tree->walk(table_field->table, count_distinct_walk, (void*) &count); + return count; + } + + /* + @brief + Build the histogram for the elements accumulated in the container of 'tree' + */ + ulonglong get_value_with_histogram(ha_rows rows) + { + Histogram_builder hist_builder(table_field, tree_key_length, rows); + tree->walk(table_field->table, histogram_build_walk, (void *) &hist_builder); + return hist_builder.get_count_distinct(); + } + + /* + @brief + Get the size of the histogram in bytes built for table_field + */ + uint get_hist_size() + { + return table_field->collected_stats->histogram.get_size(); + } + + /* + @brief + Get the pointer to the histogram built for table_field + */ + uchar *get_histogram() + { + return table_field->collected_stats->histogram.get_values(); + } + +}; + + +static +int simple_ulonglong_key_cmp(void* arg, uchar* key1, uchar* key2) +{ + ulonglong *val1= (ulonglong *) key1; + ulonglong *val2= (ulonglong *) key2; + return *val1 > *val2 ? 1 : *val1 == *val2 ? 0 : -1; +} + + +/* + The class Count_distinct_field_bit is derived from the class + Count_distinct_field to be used only for fields of the MYSQL_TYPE_BIT type. + The class provides a different implementation for the method add +*/ + +class Count_distinct_field_bit: public Count_distinct_field +{ +public: + + Count_distinct_field_bit(Field *field, uint max_heap_table_size) + { + table_field= field; + tree_key_length= sizeof(ulonglong); + + tree= new Unique((qsort_cmp2) simple_ulonglong_key_cmp, + (void*) &tree_key_length, + tree_key_length, max_heap_table_size, 1); + } + + bool add() + { + longlong val= table_field->val_int(); + return tree->unique_add(&val); + } +}; + + +/* + The class Index_prefix_calc is a helper class used to calculate the values + for the column 'avg_frequency' of the statistical table index_stats. + For any table t from the database db and any k-component prefix of the + index i for this table the row from index_stats with the primary key + (db,t,i,k) must contain in the column 'avg_frequency' either NULL or + the number that is the ratio of N and V, where N is the number of index + entries without NULL values in the first k components of the index i, + and V is the number of distinct tuples composed of the first k components + encountered among these index entries. + Currently the objects of this class are used only by the function + collect_statistics_for_index. +*/ + +class Index_prefix_calc: public Sql_alloc +{ + +private: + + /* Table containing index specified by index_info */ + TABLE *index_table; + /* Info for the index i for whose prefix 'avg_frequency' is calculated */ + KEY *index_info; + /* The maximum number of the components in the prefixes of interest */ + uint prefixes; + bool empty; + + /* This structure is created for every k components of the index i */ + class Prefix_calc_state + { + public: + /* + The number of the scanned index entries without nulls + in the first k components + */ + ulonglong entry_count; + /* + The number if the scanned index entries without nulls with + the last encountered k-component prefix + */ + ulonglong prefix_count; + /* The values of the last encountered k-component prefix */ + Cached_item *last_prefix; + }; + + /* + Array of structures used to calculate 'avg_frequency' for different + prefixes of the index i + */ + Prefix_calc_state *calc_state; + +public: + + bool is_single_comp_pk; + + Index_prefix_calc(TABLE *table, KEY *key_info) + : index_table(table), index_info(key_info) + { + uint i; + Prefix_calc_state *state; + uint key_parts= table->actual_n_key_parts(key_info); + empty= TRUE; + prefixes= 0; + + is_single_comp_pk= FALSE; + uint pk= table->s->primary_key; + if ((uint) (table->key_info - key_info) == pk && + table->key_info[pk].key_parts == 1) + { + prefixes= 1; + is_single_comp_pk= TRUE; + return; + } + + if ((calc_state= + (Prefix_calc_state *) sql_alloc(sizeof(Prefix_calc_state)*key_parts))) + { + uint keyno= key_info-table->key_info; + for (i= 0, state= calc_state; i < key_parts; i++, state++) + { + /* + Do not consider prefixes containing a component that is only part + of the field. This limitation is set to avoid fetching data when + calculating the values of 'avg_frequency' for prefixes. + */ + if (!key_info->key_part[i].field->part_of_key.is_set(keyno)) + break; + + if (!(state->last_prefix= + new Cached_item_field(key_info->key_part[i].field))) + break; + state->entry_count= state->prefix_count= 0; + prefixes++; + } + } + } + + + /** + @breif + Change the elements of calc_state after reading the next index entry + + @details + This function is to be called at the index scan each time the next + index entry has been read into the record buffer. + For each of the index prefixes the function checks whether nulls + are encountered in any of the k components of the prefix. + If this is not the case the value of calc_state[k-1].entry_count + is incremented by 1. Then the function checks whether the value of + any of these k components has changed. If so, the value of + calc_state[k-1].prefix_count is incremented by 1. + */ + + void add() + { + uint i; + Prefix_calc_state *state; + uint first_changed= prefixes; + for (i= prefixes, state= calc_state+prefixes-1; i; i--, state--) + { + if (state->last_prefix->cmp()) + first_changed= i-1; + } + if (empty) + { + first_changed= 0; + empty= FALSE; + } + for (i= 0, state= calc_state; i < prefixes; i++, state++) + { + if (state->last_prefix->null_value) + break; + if (i >= first_changed) + state->prefix_count++; + state->entry_count++; + } + } + + /** + @brief + Calculate the values of avg_frequency for all prefixes of an index + + @details + This function is to be called after the index scan to count the number + of distinct index prefixes has been done. The function calculates + the value of avg_frequency for the index prefix with k components + as calc_state[k-1].entry_count/calc_state[k-1].prefix_count. + If calc_state[k-1].prefix_count happens to be 0, the value of + avg_frequency[k-1] is set to 0, i.e. is considered as unknown. + */ + + void get_avg_frequency() + { + uint i; + Prefix_calc_state *state; + + if (is_single_comp_pk) + { + index_info->collected_stats->set_avg_frequency(0, 1.0); + return; + } + + for (i= 0, state= calc_state; i < prefixes; i++, state++) + { + if (i < prefixes) + { + double val= state->prefix_count == 0 ? + 0 : (double) state->entry_count / state->prefix_count; + index_info->collected_stats->set_avg_frequency(i, val); + } + } + } +}; + + +/** + @brief + Create fields for min/max values to collect column statistics + + @param + table Table the fields are created for + + @details + The function first allocates record buffers to store min/max values + for 'table's fields. Then for each table field f it creates Field structures + that points to these buffers rather that to the record buffer as the + Field object for f does. The pointers of the created fields are placed + in the collected_stats structure of the Field object for f. + The function allocates the buffers for min/max values in the table + memory. + + @note + The buffers allocated when min/max values are used to read statistics + from the persistent statistical tables differ from those buffers that + are used when statistics on min/max values for column is collected + as they are allocated in different mem_roots. + The same is true for the fields created for min/max values. +*/ + +static +void create_min_max_statistical_fields_for_table(TABLE *table) +{ + uint rec_buff_length= table->s->rec_buff_length; + + if ((table->collected_stats->min_max_record_buffers= + (uchar *) alloc_root(&table->mem_root, 2*rec_buff_length))) + { + uchar *record= table->collected_stats->min_max_record_buffers; + memset(record, 0, 2*rec_buff_length); + + for (uint i=0; i < 2; i++, record+= rec_buff_length) + { + for (Field **field_ptr= table->field; *field_ptr; field_ptr++) + { + Field *fld; + Field *table_field= *field_ptr; + my_ptrdiff_t diff= record-table->record[0]; + if (!bitmap_is_set(table->read_set, table_field->field_index)) + continue; + if (!(fld= table_field->clone(&table->mem_root, table, diff, TRUE))) + continue; + if (i == 0) + table_field->collected_stats->min_value= fld; + else + table_field->collected_stats->max_value= fld; + } + } + } +} + + +/** + @brief + Create fields for min/max values to read column statistics + + @param + thd Thread handler + @param + table_share Table share the fields are created for + @param + is_safe TRUE <-> at any time only one thread can perform the function + + @details + The function first allocates record buffers to store min/max values + for 'table_share's fields. Then for each field f it creates Field structures + that points to these buffers rather that to the record buffer as the + Field object for f does. The pointers of the created fields are placed + in the read_stats structure of the Field object for f. + The function allocates the buffers for min/max values in the table share + memory. + If the parameter is_safe is TRUE then it is guaranteed that at any given time + only one thread is executed the code of the function. + + @note + The buffers allocated when min/max values are used to collect statistics + from the persistent statistical tables differ from those buffers that + are used when statistics on min/max values for column is read as they + are allocated in different mem_roots. + The same is true for the fields created for min/max values. +*/ + +static +void create_min_max_statistical_fields_for_table_share(THD *thd, + TABLE_SHARE *table_share) +{ + TABLE_STATISTICS_CB *stats_cb= &table_share->stats_cb; + Table_statistics *stats= stats_cb->table_stats; + + if (stats->min_max_record_buffers) + return; + + uint rec_buff_length= table_share->rec_buff_length; + + if ((stats->min_max_record_buffers= + (uchar *) alloc_root(&stats_cb->mem_root, 2*rec_buff_length))) + { + uchar *record= stats->min_max_record_buffers; + memset(record, 0, 2*rec_buff_length); + + for (uint i=0; i < 2; i++, record+= rec_buff_length) + { + for (Field **field_ptr= table_share->field; *field_ptr; field_ptr++) + { + Field *fld; + Field *table_field= *field_ptr; + my_ptrdiff_t diff= record - table_share->default_values; + if (!(fld= table_field->clone(&stats_cb->mem_root, diff))) + continue; + if (i == 0) + table_field->read_stats->min_value= fld; + else + table_field->read_stats->max_value= fld; + } + } + } + +} + + +/** + @brief + Allocate memory for the table's statistical data to be collected + + @param + table Table for which the memory for statistical data is allocated + + @note + The function allocates the memory for the statistical data on 'table' with + the intention to collect the data there. The memory is allocated for + the statistics on the table, on the table's columns, and on the table's + indexes. The memory is allocated in the table's mem_root. + + @retval + 0 If the memory for all statistical data has been successfully allocated + @retval + 1 Otherwise + + @note + Each thread allocates its own memory to collect statistics on the table + It allows us, for example, to collect statistics on the different indexes + of the same table in parallel. +*/ + +int alloc_statistics_for_table(THD* thd, TABLE *table) +{ + Field **field_ptr; + uint fields; + + DBUG_ENTER("alloc_statistics_for_table"); + + + Table_statistics *table_stats= + (Table_statistics *) alloc_root(&table->mem_root, + sizeof(Table_statistics)); + + fields= table->s->fields ; + Column_statistics_collected *column_stats= + (Column_statistics_collected *) alloc_root(&table->mem_root, + sizeof(Column_statistics_collected) * + (fields+1)); + + uint keys= table->s->keys; + Index_statistics *index_stats= + (Index_statistics *) alloc_root(&table->mem_root, + sizeof(Index_statistics) * keys); + + uint key_parts= table->s->ext_key_parts; + ulong *idx_avg_frequency= (ulong*) alloc_root(&table->mem_root, + sizeof(ulong) * key_parts); + + uint columns= 0; + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + if (bitmap_is_set(table->read_set, (*field_ptr)->field_index)) + columns++; + } + uint hist_size= thd->variables.histogram_size; + Histogram_type hist_type= (Histogram_type) (thd->variables.histogram_type); + uchar *histogram= NULL; + if (hist_size > 0) + histogram= (uchar *) alloc_root(&table->mem_root, hist_size * columns); + + if (!table_stats || !column_stats || !index_stats || !idx_avg_frequency || + (hist_size && !histogram)) + DBUG_RETURN(1); + + table->collected_stats= table_stats; + table_stats->column_stats= column_stats; + table_stats->index_stats= index_stats; + table_stats->idx_avg_frequency= idx_avg_frequency; + table_stats->histograms= histogram; + + memset(column_stats, 0, sizeof(Column_statistics) * (fields+1)); + + for (field_ptr= table->field; *field_ptr; field_ptr++, column_stats++) + { + (*field_ptr)->collected_stats= column_stats; + (*field_ptr)->collected_stats->max_value= NULL; + (*field_ptr)->collected_stats->min_value= NULL; + if (bitmap_is_set(table->read_set, (*field_ptr)->field_index)) + { + column_stats->histogram.set_size(hist_size); + column_stats->histogram.set_type(hist_type); + column_stats->histogram.set_values(histogram); + histogram+= hist_size; + } + } + + memset(idx_avg_frequency, 0, sizeof(ulong) * key_parts); + + KEY *key_info, *end; + for (key_info= table->key_info, end= key_info + table->s->keys; + key_info < end; + key_info++, index_stats++) + { + key_info->collected_stats= index_stats; + key_info->collected_stats->init_avg_frequency(idx_avg_frequency); + idx_avg_frequency+= key_info->ext_key_parts; + } + + create_min_max_statistical_fields_for_table(table); + + DBUG_RETURN(0); +} + + +/** + @brief + Check whether any persistent statistics for the processed command is needed + + @param + thd The thread handle + + @details + The function checks whether any persitent statistics for the processed + command is needed to be read. + + @retval + TRUE statistics is needed to be read + @retval + FALSE Otherwise +*/ + +static +inline bool statistics_for_command_is_needed(THD *thd) +{ + if (thd->bootstrap || thd->variables.use_stat_tables == NEVER) + return FALSE; + + switch(thd->lex->sql_command) { + case SQLCOM_SELECT: + case SQLCOM_INSERT: + case SQLCOM_INSERT_SELECT: + case SQLCOM_UPDATE: + case SQLCOM_UPDATE_MULTI: + case SQLCOM_DELETE: + case SQLCOM_DELETE_MULTI: + case SQLCOM_REPLACE: + case SQLCOM_REPLACE_SELECT: + break; + default: + return FALSE; + } + + return TRUE; +} + + +/** + @brief + Allocate memory for the statistical data used by a table share + + @param + thd Thread handler + @param + table_share Table share for which the memory for statistical data is allocated + @param + is_safe TRUE <-> at any time only one thread can perform the function + + @note + The function allocates the memory for the statistical data on a table in the + table's share memory with the intention to read the statistics there from + the system persistent statistical tables mysql.table_stat, mysql.column_stats, + mysql.index_stats. The memory is allocated for the statistics on the table, + on the tables's columns, and on the table's indexes. The memory is allocated + in the table_share's mem_root. + If the parameter is_safe is TRUE then it is guaranteed that at any given time + only one thread is executed the code of the function. + + @retval + 0 If the memory for all statistical data has been successfully allocated + @retval + 1 Otherwise + + @note + The situation when more than one thread try to allocate memory for + statistical data is rare. It happens under the following scenario: + 1. One thread executes a query over table t with the system variable + 'use_stat_tables' set to 'never'. + 2. After this the second thread sets 'use_stat_tables' to 'preferably' + and executes a query over table t. + 3. Simultaneously the third thread sets 'use_stat_tables' to 'preferably' + and executes a query over table t. + Here the second and the third threads try to allocate the memory for + statistical data at the same time. The precautions are taken to + guarantee the correctness of the allocation. + + @note + Currently the function always is called with the parameter is_safe set + to FALSE. +*/ + +int alloc_statistics_for_table_share(THD* thd, TABLE_SHARE *table_share, + bool is_safe) +{ + + Field **field_ptr; + KEY *key_info, *end; + TABLE_STATISTICS_CB *stats_cb= &table_share->stats_cb; + + DBUG_ENTER("alloc_statistics_for_table_share"); + + DEBUG_SYNC(thd, "statistics_mem_alloc_start1"); + DEBUG_SYNC(thd, "statistics_mem_alloc_start2"); + + if (!statistics_for_command_is_needed(thd)) + DBUG_RETURN(1); + + if (!is_safe) + mysql_mutex_lock(&table_share->LOCK_ha_data); + + if (stats_cb->stats_can_be_read) + { + if (!is_safe) + mysql_mutex_unlock(&table_share->LOCK_ha_data); + DBUG_RETURN(0); + } + + Table_statistics *table_stats= stats_cb->table_stats; + if (!table_stats) + { + table_stats= (Table_statistics *) alloc_root(&stats_cb->mem_root, + sizeof(Table_statistics)); + if (!table_stats) + { + if (!is_safe) + mysql_mutex_unlock(&table_share->LOCK_ha_data); + DBUG_RETURN(1); + } + memset(table_stats, 0, sizeof(Table_statistics)); + stats_cb->table_stats= table_stats; + } + + uint fields= table_share->fields; + Column_statistics *column_stats= table_stats->column_stats; + if (!column_stats) + { + column_stats= (Column_statistics *) alloc_root(&stats_cb->mem_root, + sizeof(Column_statistics) * + (fields+1)); + if (column_stats) + { + memset(column_stats, 0, sizeof(Column_statistics) * (fields+1)); + table_stats->column_stats= column_stats; + for (field_ptr= table_share->field; + *field_ptr; + field_ptr++, column_stats++) + { + (*field_ptr)->read_stats= column_stats; + (*field_ptr)->read_stats->min_value= NULL; + (*field_ptr)->read_stats->max_value= NULL; + } + create_min_max_statistical_fields_for_table_share(thd, table_share); + } + } + + uint keys= table_share->keys; + Index_statistics *index_stats= table_stats->index_stats; + if (!index_stats) + { + index_stats= (Index_statistics *) alloc_root(&stats_cb->mem_root, + sizeof(Index_statistics) * + keys); + if (index_stats) + { + table_stats->index_stats= index_stats; + for (key_info= table_share->key_info, end= key_info + keys; + key_info < end; + key_info++, index_stats++) + { + key_info->read_stats= index_stats; + } + } + } + + uint key_parts= table_share->ext_key_parts; + ulong *idx_avg_frequency= table_stats->idx_avg_frequency; + if (!idx_avg_frequency) + { + idx_avg_frequency= (ulong*) alloc_root(&stats_cb->mem_root, + sizeof(ulong) * key_parts); + if (idx_avg_frequency) + { + memset(idx_avg_frequency, 0, sizeof(ulong) * key_parts); + table_stats->idx_avg_frequency= idx_avg_frequency; + for (key_info= table_share->key_info, end= key_info + keys; + key_info < end; + key_info++) + { + key_info->read_stats->init_avg_frequency(idx_avg_frequency); + idx_avg_frequency+= key_info->ext_key_parts; + } + } + } + + if (column_stats && index_stats && idx_avg_frequency) + stats_cb->stats_can_be_read= TRUE; + + if (!is_safe) + mysql_mutex_unlock(&table_share->LOCK_ha_data); + + DBUG_RETURN(0); +} + + +/** + @brief + Allocate memory for the histogram used by a table share + + @param + thd Thread handler + @param + table_share Table share for which the memory for histogram data is allocated + @param + is_safe TRUE <-> at any time only one thread can perform the function + + @note + The function allocates the memory for the histogram built for a table in the + table's share memory with the intention to read the data there from the + system persistent statistical table mysql.column_stats, + The memory is allocated in the table_share's mem_root. + If the parameter is_safe is TRUE then it is guaranteed that at any given time + only one thread is executed the code of the function. + + @retval + 0 If the memory for all statistical data has been successfully allocated + @retval + 1 Otherwise + + @note + Currently the function always is called with the parameter is_safe set + to FALSE. +*/ + +static +int alloc_histograms_for_table_share(THD* thd, TABLE_SHARE *table_share, + bool is_safe) +{ + TABLE_STATISTICS_CB *stats_cb= &table_share->stats_cb; + + DBUG_ENTER("alloc_histograms_for_table_share"); + + if (!is_safe) + mysql_mutex_lock(&table_share->LOCK_ha_data); + + if (stats_cb->histograms_can_be_read) + { + if (!is_safe) + mysql_mutex_unlock(&table_share->LOCK_ha_data); + DBUG_RETURN(0); + } + + Table_statistics *table_stats= stats_cb->table_stats; + ulong total_hist_size= table_stats->total_hist_size; + + if (total_hist_size && !table_stats->histograms) + { + uchar *histograms= (uchar *) alloc_root(&stats_cb->mem_root, + total_hist_size); + if (!histograms) + { + if (!is_safe) + mysql_mutex_unlock(&table_share->LOCK_ha_data); + DBUG_RETURN(1); + } + memset(histograms, 0, total_hist_size); + table_stats->histograms= histograms; + stats_cb->histograms_can_be_read= TRUE; + } + + if (!is_safe) + mysql_mutex_unlock(&table_share->LOCK_ha_data); + + DBUG_RETURN(0); + +} + +/** + @brief + Initialize the aggregation fields to collect statistics on a column + + @param + thd Thread handler + @param + table_field Column to collect statistics for +*/ + +inline +void Column_statistics_collected::init(THD *thd, Field *table_field) +{ + uint max_heap_table_size= thd->variables.max_heap_table_size; + TABLE *table= table_field->table; + uint pk= table->s->primary_key; + + is_single_pk_col= FALSE; + + if (pk != MAX_KEY && table->key_info[pk].key_parts == 1 && + table->key_info[pk].key_part[0].fieldnr == table_field->field_index + 1) + is_single_pk_col= TRUE; + + column= table_field; + + set_all_nulls(); + + nulls= 0; + column_total_length= 0; + if (is_single_pk_col) + count_distinct= NULL; + if (table_field->flags & BLOB_FLAG) + count_distinct= NULL; + else + { + count_distinct= + table_field->type() == MYSQL_TYPE_BIT ? + new Count_distinct_field_bit(table_field, max_heap_table_size) : + new Count_distinct_field(table_field, max_heap_table_size); + } + if (count_distinct && !count_distinct->exists()) + count_distinct= NULL; +} + + +/** + @brief + Perform aggregation for a row when collecting statistics on a column + + @param + rowno The order number of the row +*/ + +inline +void Column_statistics_collected::add(ha_rows rowno) +{ + + if (column->is_null()) + nulls++; + else + { + column_total_length+= column->value_length(); + if (min_value && column->update_min(min_value, rowno == nulls)) + set_not_null(COLUMN_STAT_MIN_VALUE); + if (max_value && column->update_max(max_value, rowno == nulls)) + set_not_null(COLUMN_STAT_MAX_VALUE); + if (count_distinct) + count_distinct->add(); + } +} + + +/** + @brief + Get the results of aggregation when collecting the statistics on a column + + @param + rows The total number of rows in the table +*/ + +inline +void Column_statistics_collected::finish(ha_rows rows) +{ + double val; + + if (rows) + { + val= (double) nulls / rows; + set_nulls_ratio(val); + set_not_null(COLUMN_STAT_NULLS_RATIO); + } + if (rows - nulls) + { + val= (double) column_total_length / (rows - nulls); + set_avg_length(val); + set_not_null(COLUMN_STAT_AVG_LENGTH); + } + if (count_distinct) + { + ulonglong distincts; + uint hist_size= count_distinct->get_hist_size(); + if (hist_size == 0) + distincts= count_distinct->get_value(); + else + distincts= count_distinct->get_value_with_histogram(rows - nulls); + if (distincts) + { + val= (double) (rows - nulls) / distincts; + set_avg_frequency(val); + set_not_null(COLUMN_STAT_AVG_FREQUENCY); + } + else + hist_size= 0; + histogram.set_size(hist_size); + set_not_null(COLUMN_STAT_HIST_SIZE); + if (hist_size && distincts) + { + set_not_null(COLUMN_STAT_HIST_TYPE); + histogram.set_values(count_distinct->get_histogram()); + set_not_null(COLUMN_STAT_HISTOGRAM); + } + delete count_distinct; + count_distinct= NULL; + } + else if (is_single_pk_col) + { + val= 1.0; + set_avg_frequency(val); + set_not_null(COLUMN_STAT_AVG_FREQUENCY); + } +} + + +/** + @brief + Clean up auxiliary structures used for aggregation +*/ + +inline +void Column_statistics_collected::cleanup() +{ + if (count_distinct) + { + delete count_distinct; + count_distinct= NULL; + } +} + + +/** + @brief + Collect statistical data on an index + + @param + table The table the index belongs to + index The number of this index in the table + + @details + The function collects the value of 'avg_frequency' for the prefixes + on an index from 'table'. The index is specified by its number. + If the scan is successful the calculated statistics is saved in the + elements of the array write_stat.avg_frequency of the KEY_INFO structure + for the index. The statistics for the prefix with k components is saved + in the element number k-1. + + @retval + 0 If the statistics has been successfully collected + @retval + 1 Otherwise + + @note + The function collects statistics for the index prefixes for one index + scan during which no data is fetched from the table records. That's why + statistical data for prefixes that contain part of a field is not + collected. + The function employs an object of the helper class Index_prefix_calc to + count for each index prefix the number of index entries without nulls and + the number of distinct entries among them. + +*/ + +static +int collect_statistics_for_index(THD *thd, TABLE *table, uint index) +{ + int rc= 0; + KEY *key_info= &table->key_info[index]; + ha_rows rows= 0; + Index_prefix_calc index_prefix_calc(table, key_info); + DBUG_ENTER("collect_statistics_for_index"); + + DEBUG_SYNC(table->in_use, "statistics_collection_start1"); + DEBUG_SYNC(table->in_use, "statistics_collection_start2"); + + if (index_prefix_calc.is_single_comp_pk) + { + index_prefix_calc.get_avg_frequency(); + DBUG_RETURN(rc); + } + + table->key_read= 1; + table->file->extra(HA_EXTRA_KEYREAD); + + table->file->ha_index_init(index, TRUE); + rc= table->file->ha_index_first(table->record[0]); + while (rc != HA_ERR_END_OF_FILE) + { + if (thd->killed) + break; + + if (rc) + break; + rows++; + index_prefix_calc.add(); + rc= table->file->ha_index_next(table->record[0]); + } + table->key_read= 0; + table->file->ha_index_end(); + + rc= (rc == HA_ERR_END_OF_FILE && !thd->killed) ? 0 : 1; + + if (!rc) + index_prefix_calc.get_avg_frequency(); + + DBUG_RETURN(rc); +} + + +/** + @brief + Collect statistical data for a table + + @param + thd The thread handle + @param + table The table to collect statistics on + + @details + The function collects data for various statistical characteristics on + the table 'table'. These data is saved in the internal fields that could + be reached from 'table'. The data is prepared to be saved in the persistent + statistical table by the function update_statistics_for_table. + The collected statistical values are not placed in the same fields that + keep the statistical data used by the optimizer. Therefore, at any time, + there is no collision between the statistics being collected and the one + used by the optimizer to look for optimal query execution plans for other + clients. + + @retval + 0 If the statistics has been successfully collected + @retval + 1 Otherwise + + @note + The function first collects statistical data for statistical characteristics + to be saved in the statistical tables table_stat and column_stats. To do this + it performs a full table scan of 'table'. At this scan the function collects + statistics on each column of the table and count the total number of the + scanned rows. To calculate the value of 'avg_frequency' for a column the + function constructs an object of the helper class Count_distinct_field + (or its derivation). Currently this class cannot count the number of + distinct values for blob columns. So the value of 'avg_frequency' for + blob columns is always null. + After the full table scan the function calls collect_statistics_for_index + for each table index. The latter performs full index scan for each index. + + @note + Currently the statistical data is collected indiscriminately for all + columns/indexes of 'table', for all statistical characteristics. + TODO. Collect only specified statistical characteristics for specified + columns/indexes. + + @note + Currently the process of collecting statistical data is not optimized. + For example, 'avg_frequency' for a column could be copied from the + 'avg_frequency' collected for an index if this column is used as the + first component of the index. Min and min values for this column could + be extracted from the index as well. +*/ + +int collect_statistics_for_table(THD *thd, TABLE *table) +{ + int rc; + Field **field_ptr; + Field *table_field; + ha_rows rows= 0; + handler *file=table->file; + + DBUG_ENTER("collect_statistics_for_table"); + + table->collected_stats->cardinality_is_null= TRUE; + table->collected_stats->cardinality= 0; + + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + table_field= *field_ptr; + if (!bitmap_is_set(table->read_set, table_field->field_index)) + continue; + table_field->collected_stats->init(thd, table_field); + } + + /* Perform a full table scan to collect statistics on 'table's columns */ + if (!(rc= file->ha_rnd_init(TRUE))) + { + DEBUG_SYNC(table->in_use, "statistics_collection_start"); + + while ((rc= file->ha_rnd_next(table->record[0])) != HA_ERR_END_OF_FILE) + { + if (thd->killed) + break; + + if (rc) + { + if (rc == HA_ERR_RECORD_DELETED) + continue; + break; + } + + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + table_field= *field_ptr; + if (!bitmap_is_set(table->read_set, table_field->field_index)) + continue; + table_field->collected_stats->add(rows); + } + rows++; + } + file->ha_rnd_end(); + } + rc= (rc == HA_ERR_END_OF_FILE && !thd->killed) ? 0 : 1; + + /* + Calculate values for all statistical characteristics on columns and + and for each field f of 'table' save them in the write_stat structure + from the Field object for f. + */ + if (!rc) + { + table->collected_stats->cardinality_is_null= FALSE; + table->collected_stats->cardinality= rows; + } + + bitmap_clear_all(table->write_set); + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + table_field= *field_ptr; + if (!bitmap_is_set(table->read_set, table_field->field_index)) + continue; + bitmap_set_bit(table->write_set, table_field->field_index); + if (!rc) + table_field->collected_stats->finish(rows); + else + table_field->collected_stats->cleanup(); + } +bitmap_clear_all(table->write_set); + + if (!rc) + { + uint key; + key_map::Iterator it(table->keys_in_use_for_query); + + MY_BITMAP *save_read_set= table->read_set; + table->read_set= &table->tmp_set; + bitmap_set_all(table->read_set); + + /* Collect statistics for indexes */ + while ((key= it++) != key_map::Iterator::BITMAP_END) + { + if ((rc= collect_statistics_for_index(thd, table, key))) + break; + } + + table->read_set= save_read_set; + } + + DBUG_RETURN(rc); +} + + +/** + @brief + Update statistics for a table in the persistent statistical tables + + @param + thd The thread handle + @param + table The table to collect statistics on + + @details + For each statistical table st the function looks for the rows from this + table that contain statistical data on 'table'. If rows with given + statistical characteristics exist they are updated with the new statistical + values taken from internal structures for 'table'. Otherwise new rows + with these statistical characteristics are added into st. + It is assumed that values stored in the statistical tables are found and + saved by the function collect_statistics_for_table. + + @retval + 0 If all statistical tables has been successfully updated + @retval + 1 Otherwise + + @note + The function is called when executing the ANALYZE actions for 'table'. + The function first unlocks the opened table the statistics on which has + been collected, but does not closes it, so all collected statistical data + remains in internal structures for 'table'. Then the function opens the + statistical tables and writes the statistical data for 'table'into them. + It is not allowed just to open statistical tables for writing when some + other tables are locked for reading. + After the statistical tables have been opened they are updated one by one + with the new statistics on 'table'. Objects of the helper classes + Table_stat, Column_stat and Index_stat are employed for this. + After having been updated the statistical system tables are closed. +*/ + +int update_statistics_for_table(THD *thd, TABLE *table) +{ + TABLE_LIST tables[STATISTICS_TABLES]; + Open_tables_backup open_tables_backup; + uint i; + int err; + enum_binlog_format save_binlog_format; + int rc= 0; + TABLE *stat_table; + + DBUG_ENTER("update_statistics_for_table"); + + DEBUG_SYNC(thd, "statistics_update_start"); + + if (open_stat_tables(thd, tables, &open_tables_backup, TRUE)) + { + thd->clear_error(); + DBUG_RETURN(rc); + } + + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); + + /* Update the statistical table table_stats */ + stat_table= tables[TABLE_STAT].table; + Table_stat table_stat(stat_table, table); + restore_record(stat_table, s->default_values); + table_stat.set_key_fields(); + err= table_stat.update_stat(); + if (err) + rc= 1; + + /* Update the statistical table colum_stats */ + stat_table= tables[COLUMN_STAT].table; + Column_stat column_stat(stat_table, table); + for (Field **field_ptr= table->field; *field_ptr; field_ptr++) + { + Field *table_field= *field_ptr; + if (!bitmap_is_set(table->read_set, table_field->field_index)) + continue; + restore_record(stat_table, s->default_values); + column_stat.set_key_fields(table_field); + err= column_stat.update_stat(); + if (err && !rc) + rc= 1; + } + + /* Update the statistical table index_stats */ + stat_table= tables[INDEX_STAT].table; + uint key; + key_map::Iterator it(table->keys_in_use_for_query); + Index_stat index_stat(stat_table, table); + + while ((key= it++) != key_map::Iterator::BITMAP_END) + { + KEY *key_info= table->key_info+key; + uint key_parts= table->actual_n_key_parts(key_info); + for (i= 0; i < key_parts; i++) + { + restore_record(stat_table, s->default_values); + index_stat.set_key_fields(key_info, i+1); + err= index_stat.update_stat(); + if (err && !rc) + rc= 1; + } + } + + thd->restore_stmt_binlog_format(save_binlog_format); + + close_system_tables(thd, &open_tables_backup); + + DBUG_RETURN(rc); +} + + +/** + @brief + Read statistics for a table from the persistent statistical tables + + @param + thd The thread handle + @param + table The table to read statistics on + @param + stat_tables The array of TABLE_LIST objects for statistical tables + + @details + For each statistical table the function looks for the rows from this + table that contain statistical data on 'table'. If such rows is found + the data from statistical columns of it is read into the appropriate + fields of internal structures for 'table'. Later at the query processing + this data are supposed to be used by the optimizer. + The parameter stat_tables should point to an array of TABLE_LIST + objects for all statistical tables linked into a list. All statistical + tables are supposed to be opened. + The function is called by read_statistics_for_tables_if_needed(). + + @retval + 0 If data has been successfully read for the table + @retval + 1 Otherwise + + @note + Objects of the helper classes Table_stat, Column_stat and Index_stat + are employed to read statistical data from the statistical tables. + now. +*/ + +static +int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) +{ + uint i; + TABLE *stat_table; + Field *table_field; + Field **field_ptr; + KEY *key_info, *key_info_end; + TABLE_SHARE *table_share= table->s; + Table_statistics *read_stats= table_share->stats_cb.table_stats; + + DBUG_ENTER("read_statistics_for_table"); + + /* Read statistics from the statistical table table_stats */ + stat_table= stat_tables[TABLE_STAT].table; + Table_stat table_stat(stat_table, table); + table_stat.set_key_fields(); + table_stat.get_stat_values(); + + /* Read statistics from the statistical table column_stats */ + stat_table= stat_tables[COLUMN_STAT].table; + ulong total_hist_size= 0; + Column_stat column_stat(stat_table, table); + for (field_ptr= table_share->field; *field_ptr; field_ptr++) + { + table_field= *field_ptr; + column_stat.set_key_fields(table_field); + column_stat.get_stat_values(); + total_hist_size+= table_field->read_stats->histogram.get_size(); + } + read_stats->total_hist_size= total_hist_size; + + /* Read statistics from the statistical table index_stats */ + stat_table= stat_tables[INDEX_STAT].table; + Index_stat index_stat(stat_table, table); + for (key_info= table_share->key_info, + key_info_end= key_info + table_share->keys; + key_info < key_info_end; key_info++) + { + uint key_parts= key_info->ext_key_parts; + for (i= 0; i < key_parts; i++) + { + index_stat.set_key_fields(key_info, i+1); + index_stat.get_stat_values(); + } + + key_part_map ext_key_part_map= key_info->ext_key_part_map; + if (key_info->key_parts != key_info->ext_key_parts && + key_info->read_stats->get_avg_frequency(key_info->key_parts) == 0) + { + KEY *pk_key_info= table_share->key_info + table_share->primary_key; + uint k= key_info->key_parts; + uint pk_parts= pk_key_info->key_parts; + ha_rows n_rows= read_stats->cardinality; + double k_dist= n_rows / key_info->read_stats->get_avg_frequency(k-1); + uint m= 0; + for (uint j= 0; j < pk_parts; j++) + { + if (!(ext_key_part_map & 1 << j)) + { + for (uint l= k; l < k + m; l++) + { + double avg_frequency= + pk_key_info->read_stats->get_avg_frequency(j-1); + set_if_smaller(avg_frequency, 1); + double val= pk_key_info->read_stats->get_avg_frequency(j) / + avg_frequency; + key_info->read_stats->set_avg_frequency (l, val); + } + } + else + { + double avg_frequency= pk_key_info->read_stats->get_avg_frequency(j); + key_info->read_stats->set_avg_frequency(k + m, avg_frequency); + m++; + } + } + for (uint l= k; l < k + m; l++) + { + double avg_frequency= key_info->read_stats->get_avg_frequency(l); + if (avg_frequency == 0 || read_stats->cardinality_is_null) + avg_frequency= 1; + else if (avg_frequency > 1) + { + avg_frequency/= k_dist; + set_if_bigger(avg_frequency, 1); + } + key_info->read_stats->set_avg_frequency(l, avg_frequency); + } + } + } + + table->stats_is_read= TRUE; + + DBUG_RETURN(0); +} + + +/** + @brief + Check whether any statistics is to be read for tables from a table list + + @param + thd The thread handle + @param + tables The tables list for whose tables the check is to be done + + @details + The function checks whether for any of the tables opened and locked for + a statement statistics from statistical tables is needed to be read. + + @retval + TRUE statistics for any of the tables is needed to be read + @retval + FALSE Otherwise +*/ + +static +bool statistics_for_tables_is_needed(THD *thd, TABLE_LIST *tables) +{ + if (!tables) + return FALSE; + + if (!statistics_for_command_is_needed(thd)) + return FALSE; + + /* + Do not read statistics for any query over non-user tables. + If the query references some statistical tables, but not all + of them, reading the statistics may lead to a deadlock + */ + for (TABLE_LIST *tl= tables; tl; tl= tl->next_global) + { + if (!tl->is_view_or_derived() && tl->table) + { + TABLE_SHARE *table_share= tl->table->s; + if (table_share && + (table_share->table_category != TABLE_CATEGORY_USER || + table_share->tmp_table != NO_TMP_TABLE)) + return FALSE; + } + } + + for (TABLE_LIST *tl= tables; tl; tl= tl->next_global) + { + if (!tl->is_view_or_derived() && tl->table) + { + TABLE_SHARE *table_share= tl->table->s; + if (table_share && + table_share->stats_cb.stats_can_be_read && + (!table_share->stats_cb.stats_is_read || + (!table_share->stats_cb.histograms_are_read && + thd->variables.optimizer_use_condition_selectivity > 3))) + return TRUE; + if (table_share->stats_cb.stats_is_read) + tl->table->stats_is_read= TRUE; + if (table_share->stats_cb.histograms_are_read) + tl->table->histograms_are_read= TRUE; + } + } + + return FALSE; +} + + +/** + @brief + Read histogram for a table from the persistent statistical tables + + @param + thd The thread handle + @param + table The table to read histograms for + @param + stat_tables The array of TABLE_LIST objects for statistical tables + + @details + For the statistical table columns_stats the function looks for the rows + from this table that contain statistical data on 'table'. If such rows + are found the histograms from them are read into the memory allocated + for histograms of 'table'. Later at the query processing these histogram + are supposed to be used by the optimizer. + The parameter stat_tables should point to an array of TABLE_LIST + objects for all statistical tables linked into a list. All statistical + tables are supposed to be opened. + The function is called by read_statistics_for_tables_if_needed(). + + @retval + 0 If data has been successfully read for the table + @retval + 1 Otherwise + + @note + Objects of the helper Column_stat are employed read histogram + from the statistical table column_stats now. +*/ + +static +int read_histograms_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) +{ + TABLE_SHARE *table_share= table->s; + + DBUG_ENTER("read_histograms_for_table"); + + if (!table_share->stats_cb.histograms_can_be_read) + { + (void) alloc_histograms_for_table_share(thd, table_share, FALSE); + } + if (table_share->stats_cb.histograms_can_be_read && + !table_share->stats_cb.histograms_are_read) + { + Field **field_ptr; + uchar *histogram= table_share->stats_cb.table_stats->histograms; + TABLE *stat_table= stat_tables[COLUMN_STAT].table; + Column_stat column_stat(stat_table, table); + for (field_ptr= table_share->field; *field_ptr; field_ptr++) + { + Field *table_field= *field_ptr; + uint hist_size= table_field->read_stats->histogram.get_size(); + if (hist_size) + { + column_stat.set_key_fields(table_field); + table_field->read_stats->histogram.set_values(histogram); + column_stat.get_histogram_value(); + histogram+= hist_size; + } + } + } + + DBUG_RETURN(0); +} + +/** + @brief + Read statistics for tables from a table list if it is needed + + @param + thd The thread handle + @param + tables The tables list for whose tables to read statistics + + @details + The function first checks whether for any of the tables opened and locked + for a statement statistics from statistical tables is needed to be read. + Then, if so, it opens system statistical tables for read and reads + the statistical data from them for those tables from the list for which it + makes sense. Then the function closes system statistical tables. + + @retval + 0 Statistics for tables was successfully read + @retval + 1 Otherwise +*/ + +int read_statistics_for_tables_if_needed(THD *thd, TABLE_LIST *tables) +{ + TABLE_LIST stat_tables[STATISTICS_TABLES]; + Open_tables_backup open_tables_backup; + + DBUG_ENTER("read_statistics_for_tables_if_needed"); + + DEBUG_SYNC(thd, "statistics_read_start"); + + if (!statistics_for_tables_is_needed(thd, tables)) + DBUG_RETURN(0); + + if (open_stat_tables(thd, stat_tables, &open_tables_backup, FALSE)) + { + thd->clear_error(); + DBUG_RETURN(1); + } + + for (TABLE_LIST *tl= tables; tl; tl= tl->next_global) + { + if (!tl->is_view_or_derived() && tl->table) + { + TABLE_SHARE *table_share= tl->table->s; + if (table_share && + table_share->stats_cb.stats_can_be_read && + !table_share->stats_cb.stats_is_read) + { + (void) read_statistics_for_table(thd, tl->table, stat_tables); + table_share->stats_cb.stats_is_read= TRUE; + } + if (table_share->stats_cb.stats_is_read) + tl->table->stats_is_read= TRUE; + if (thd->variables.optimizer_use_condition_selectivity > 3 && + table_share && !table_share->stats_cb.histograms_are_read) + { + (void) read_histograms_for_table(thd, tl->table, stat_tables); + table_share->stats_cb.histograms_are_read= TRUE; + } + if (table_share->stats_cb.stats_is_read) + tl->table->histograms_are_read= TRUE; + } + } + + close_system_tables(thd, &open_tables_backup); + + DBUG_RETURN(0); +} + + +/** + @brief + Delete statistics on a table from all statistical tables + + @param + thd The thread handle + @param + db The name of the database the table belongs to + @param + tab The name of the table whose statistics is to be deleted + + @details + The function delete statistics on the table called 'tab' of the database + 'db' from all statistical tables: table_stats, column_stats, index_stats. + + @retval + 0 If all deletions are successful + @retval + 1 Otherwise + + @note + The function is called when executing the statement DROP TABLE 'tab'. +*/ + +int delete_statistics_for_table(THD *thd, LEX_STRING *db, LEX_STRING *tab) +{ + int err; + enum_binlog_format save_binlog_format; + TABLE *stat_table; + TABLE_LIST tables[STATISTICS_TABLES]; + Open_tables_backup open_tables_backup; + int rc= 0; + + DBUG_ENTER("delete_statistics_for_table"); + + if (open_stat_tables(thd, tables, &open_tables_backup, TRUE)) + { + thd->clear_error(); + DBUG_RETURN(rc); + } + + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); + + /* Delete statistics on table from the statistical table index_stats */ + stat_table= tables[INDEX_STAT].table; + Index_stat index_stat(stat_table, db, tab); + index_stat.set_full_table_name(); + while (index_stat.find_next_stat_for_prefix(2)) + { + err= index_stat.delete_stat(); + if (err & !rc) + rc= 1; + } + + /* Delete statistics on table from the statistical table column_stats */ + stat_table= tables[COLUMN_STAT].table; + Column_stat column_stat(stat_table, db, tab); + column_stat.set_full_table_name(); + while (column_stat.find_next_stat_for_prefix(2)) + { + err= column_stat.delete_stat(); + if (err & !rc) + rc= 1; + } + + /* Delete statistics on table from the statistical table table_stats */ + stat_table= tables[TABLE_STAT].table; + Table_stat table_stat(stat_table, db, tab); + table_stat.set_key_fields(); + if (table_stat.find_stat()) + { + err= table_stat.delete_stat(); + if (err & !rc) + rc= 1; + } + + thd->restore_stmt_binlog_format(save_binlog_format); + + close_system_tables(thd, &open_tables_backup); + + DBUG_RETURN(rc); +} + + +/** + @brief + Delete statistics on a column of the specified table + + @param + thd The thread handle + @param + tab The table the column belongs to + @param + col The field of the column whose statistics is to be deleted + + @details + The function delete statistics on the column 'col' belonging to the table + 'tab' from the statistical table column_stats. + + @retval + 0 If the deletion is successful + @retval + 1 Otherwise + + @note + The function is called when dropping a table column or when changing + the definition of this column. +*/ + +int delete_statistics_for_column(THD *thd, TABLE *tab, Field *col) +{ + int err; + enum_binlog_format save_binlog_format; + TABLE *stat_table; + TABLE_LIST tables; + Open_tables_backup open_tables_backup; + int rc= 0; + + DBUG_ENTER("delete_statistics_for_column"); + + if (open_single_stat_table(thd, &tables, &stat_table_name[1], + &open_tables_backup, TRUE)) + { + thd->clear_error(); + DBUG_RETURN(rc); + } + + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); + + stat_table= tables.table; + Column_stat column_stat(stat_table, tab); + column_stat.set_key_fields(col); + if (column_stat.find_stat()) + { + err= column_stat.delete_stat(); + if (err) + rc= 1; + } + + thd->restore_stmt_binlog_format(save_binlog_format); + + close_system_tables(thd, &open_tables_backup); + + DBUG_RETURN(rc); +} + + +/** + @brief + Delete statistics on an index of the specified table + + @param + thd The thread handle + @param + tab The table the index belongs to + @param + key_info The descriptor of the index whose statistics is to be deleted + @param + ext_prefixes_only Delete statistics only on the index prefixes extended by + the components of the primary key + + @details + The function delete statistics on the index specified by 'key_info' + defined on the table 'tab' from the statistical table index_stats. + + @retval + 0 If the deletion is successful + @retval + 1 Otherwise + + @note + The function is called when dropping an index, or dropping/changing the + definition of a column used in the definition of the index. +*/ + +int delete_statistics_for_index(THD *thd, TABLE *tab, KEY *key_info, + bool ext_prefixes_only) +{ + int err; + enum_binlog_format save_binlog_format; + TABLE *stat_table; + TABLE_LIST tables; + Open_tables_backup open_tables_backup; + int rc= 0; + + DBUG_ENTER("delete_statistics_for_index"); + + if (open_single_stat_table(thd, &tables, &stat_table_name[2], + &open_tables_backup, TRUE)) + { + thd->clear_error(); + DBUG_RETURN(rc); + } + + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); + + stat_table= tables.table; + Index_stat index_stat(stat_table, tab); + if (!ext_prefixes_only) + { + index_stat.set_index_prefix_key_fields(key_info); + while (index_stat.find_next_stat_for_prefix(3)) + { + err= index_stat.delete_stat(); + if (err && !rc) + rc= 1; + } + } + else + { + for (uint i= key_info->key_parts; i < key_info->ext_key_parts; i++) + { + index_stat.set_key_fields(key_info, i+1); + if (index_stat.find_next_stat_for_prefix(4)) + { + err= index_stat.delete_stat(); + if (err && !rc) + rc= 1; + } + } + } + + thd->restore_stmt_binlog_format(save_binlog_format); + + close_system_tables(thd, &open_tables_backup); + + DBUG_RETURN(rc); +} + + +/** + @brief + Rename a table in all statistical tables + + @param + thd The thread handle + @param + db The name of the database the table belongs to + @param + tab The name of the table to be renamed in statistical tables + @param + new_tab The new name of the table + + @details + The function replaces the name of the table 'tab' from the database 'db' + for 'new_tab' in all all statistical tables: table_stats, column_stats, + index_stats. + + @retval + 0 If all updates of the table name are successful + @retval + 1 Otherwise + + @note + The function is called when executing any statement that renames a table +*/ + +int rename_table_in_stat_tables(THD *thd, LEX_STRING *db, LEX_STRING *tab, + LEX_STRING *new_db, LEX_STRING *new_tab) +{ + int err; + enum_binlog_format save_binlog_format; + TABLE *stat_table; + TABLE_LIST tables[STATISTICS_TABLES]; + Open_tables_backup open_tables_backup; + int rc= 0; + + DBUG_ENTER("rename_table_in_stat_tables"); + + if (open_stat_tables(thd, tables, &open_tables_backup, TRUE)) + { + thd->clear_error(); + DBUG_RETURN(rc); + } + + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); + + /* Rename table in the statistical table index_stats */ + stat_table= tables[INDEX_STAT].table; + Index_stat index_stat(stat_table, db, tab); + index_stat.set_full_table_name(); + while (index_stat.find_next_stat_for_prefix(2)) + { + err= index_stat.update_table_name_key_parts(new_db, new_tab); + if (err & !rc) + rc= 1; + index_stat.set_full_table_name(); + } + + /* Rename table in the statistical table column_stats */ + stat_table= tables[COLUMN_STAT].table; + Column_stat column_stat(stat_table, db, tab); + column_stat.set_full_table_name(); + while (column_stat.find_next_stat_for_prefix(2)) + { + err= column_stat.update_table_name_key_parts(new_db, new_tab); + if (err & !rc) + rc= 1; + column_stat.set_full_table_name(); + } + + /* Rename table in the statistical table table_stats */ + stat_table= tables[TABLE_STAT].table; + Table_stat table_stat(stat_table, db, tab); + table_stat.set_key_fields(); + if (table_stat.find_stat()) + { + err= table_stat.update_table_name_key_parts(new_db, new_tab); + if (err & !rc) + rc= 1; + } + + thd->restore_stmt_binlog_format(save_binlog_format); + + close_system_tables(thd, &open_tables_backup); + + DBUG_RETURN(rc); +} + + +/** + @brief + Rename a column in the statistical table column_stats + + @param + thd The thread handle + @param + tab The table the column belongs to + @param + col The column to be renamed + @param + new_name The new column name + + @details + The function replaces the name of the column 'col' belonging to the table + 'tab' for 'new_name' in the statistical table column_stats. + + @retval + 0 If all updates of the table name are successful + @retval + 1 Otherwise + + @note + The function is called when executing any statement that renames a column, + but does not change the column definition. +*/ + +int rename_column_in_stat_tables(THD *thd, TABLE *tab, Field *col, + const char *new_name) +{ + int err; + enum_binlog_format save_binlog_format; + TABLE *stat_table; + TABLE_LIST tables; + Open_tables_backup open_tables_backup; + int rc= 0; + + DBUG_ENTER("rename_column_in_stat_tables"); + + if (open_single_stat_table(thd, &tables, &stat_table_name[1], + &open_tables_backup, TRUE)) + { + thd->clear_error(); + DBUG_RETURN(rc); + } + + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); + + /* Rename column in the statistical table table_stat */ + stat_table= tables.table; + Column_stat column_stat(stat_table, tab); + column_stat.set_key_fields(col); + if (column_stat.find_stat()) + { + err= column_stat.update_column_key_part(new_name); + if (err & !rc) + rc= 1; + } + + thd->restore_stmt_binlog_format(save_binlog_format); + + close_system_tables(thd, &open_tables_backup); + + DBUG_RETURN(rc); +} + + +/** + @brief + Set statistics for a table that will be used by the optimizer + + @param + thd The thread handle + @param + table The table to set statistics for + + @details + Depending on the value of thd->variables.use_stat_tables + the function performs the settings for the table that will control + from where the statistical data used by the optimizer will be taken. +*/ + +void set_statistics_for_table(THD *thd, TABLE *table) +{ + TABLE_STATISTICS_CB *stats_cb= &table->s->stats_cb; + Table_statistics *read_stats= stats_cb->table_stats; + Use_stat_tables_mode use_stat_table_mode= get_use_stat_tables_mode(thd); + table->used_stat_records= + (use_stat_table_mode <= COMPLEMENTARY || + !table->stats_is_read || read_stats->cardinality_is_null) ? + table->file->stats.records : read_stats->cardinality; + KEY *key_info, *key_info_end; + for (key_info= table->key_info, key_info_end= key_info+table->s->keys; + key_info < key_info_end; key_info++) + { + key_info->is_statistics_from_stat_tables= + (use_stat_table_mode > COMPLEMENTARY && + table->stats_is_read && + key_info->read_stats->avg_frequency_is_inited() && + key_info->read_stats->get_avg_frequency(0) > 0.5); + } +} + + +/** + @brief + Get the average frequency for a column + + @param + field The column whose average frequency is required + + @retval + The required average frequency +*/ + +double get_column_avg_frequency(Field * field) +{ + double res; + TABLE *table= field->table; + + /* + Statistics is shared by table instances and is accessed through + the table share. If table->s->field is not set for 'table', then + no column statistics is available for the table . + */ + if (!table->s->field) + { + res= table->stat_records(); + return res; + } + + Column_statistics *col_stats= table->s->field[field->field_index]->read_stats; + + if (!col_stats) + res= table->stat_records(); + else + res= col_stats->get_avg_frequency(); + return res; +} + + +/** + @brief + Estimate the number of rows in a column range using data from stat tables + + @param + field The column whose range cardinality is to be estimated + @param + min_endp The left end of the range whose cardinality is required + @param + max_endp The right end of the range whose cardinality is required + @param + range_flag The range flags + + @details + The function gets an estimate of the number of rows in a column range + using the statistical data from the table column_stats. + + @retval + The required estimate of the rows in the column range +*/ + +double get_column_range_cardinality(Field *field, + key_range *min_endp, + key_range *max_endp, + uint range_flag) +{ + double res; + TABLE *table= field->table; + Column_statistics *col_stats= table->field[field->field_index]->read_stats; + double tab_records= table->stat_records(); + + if (!col_stats) + return tab_records; + + double col_nulls= tab_records * col_stats->get_nulls_ratio(); + + double col_non_nulls= tab_records - col_nulls; + + bool nulls_incl= field->null_ptr && min_endp && min_endp->key[0] && + !(range_flag & NEAR_MIN); + + if (col_non_nulls < 1) + res= 0; + else if (min_endp && max_endp && min_endp->length == max_endp->length && + !memcmp(min_endp->key, max_endp->key, min_endp->length)) + { + if (nulls_incl) + { + /* This is null single point range */ + res= col_nulls; + } + else + { + double avg_frequency= col_stats->get_avg_frequency(); + res= avg_frequency; + if (avg_frequency > 1.0 + 0.000001 && + col_stats->min_value && col_stats->max_value) + { + Histogram *hist= &col_stats->histogram; + if (hist->is_available()) + { + double pos= field->pos_in_interval(col_stats->min_value, + col_stats->max_value); + res= col_non_nulls * + hist->point_selectivity(pos, + avg_frequency / col_non_nulls); + } + } + } + } + else + { + if (col_stats->min_value && col_stats->max_value) + { + double sel, min_mp_pos, max_mp_pos; + + if (min_endp && !(field->null_ptr && min_endp->key[0])) + { + store_key_image_to_rec(field, (uchar *) min_endp->key, + min_endp->length); + min_mp_pos= field->pos_in_interval(col_stats->min_value, + col_stats->max_value); + } + else + min_mp_pos= 0.0; + if (max_endp) + { + store_key_image_to_rec(field, (uchar *) max_endp->key, + max_endp->length); + max_mp_pos= field->pos_in_interval(col_stats->min_value, + col_stats->max_value); + } + else + max_mp_pos= 1.0; + + Histogram *hist= &col_stats->histogram; + if (!hist->is_available()) + sel= (max_mp_pos - min_mp_pos); + else + sel= hist->range_selectivity(min_mp_pos, max_mp_pos); + res= col_non_nulls * sel; + set_if_bigger(res, col_stats->get_avg_frequency()); + } + else + res= col_non_nulls; + if (nulls_incl) + res+= col_nulls; + } + return res; +} diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h new file mode 100644 index 00000000000..c6a72478c34 --- /dev/null +++ b/sql/sql_statistics.h @@ -0,0 +1,416 @@ +/* Copyright 2006-2008 MySQL AB, 2008 Sun Microsystems, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef SQL_STATISTICS_H +#define SQL_STATISTICS_H + +typedef +enum enum_use_stat_tables_mode +{ + NEVER, + COMPLEMENTARY, + PEFERABLY, +} Use_stat_tables_mode; + +typedef +enum enum_histogram_type +{ + SINGLE_PREC_HB, + DOUBLE_PREC_HB +} Histogram_type; + +enum enum_stat_tables +{ + TABLE_STAT, + COLUMN_STAT, + INDEX_STAT, +}; + + +/* + These enumeration types comprise the dictionary of three + statistical tables table_stat, column_stat and index_stat + as they defined in ../scripts/mysql_system_tables.sql. + + It would be nice if the declarations of these types were + generated automatically by the table definitions. +*/ + +enum enum_table_stat_col +{ + TABLE_STAT_DB_NAME, + TABLE_STAT_TABLE_NAME, + TABLE_STAT_CARDINALITY +}; + +enum enum_column_stat_col +{ + COLUMN_STAT_DB_NAME, + COLUMN_STAT_TABLE_NAME, + COLUMN_STAT_COLUMN_NAME, + COLUMN_STAT_MIN_VALUE, + COLUMN_STAT_MAX_VALUE, + COLUMN_STAT_NULLS_RATIO, + COLUMN_STAT_AVG_LENGTH, + COLUMN_STAT_AVG_FREQUENCY, + COLUMN_STAT_HIST_SIZE, + COLUMN_STAT_HIST_TYPE, + COLUMN_STAT_HISTOGRAM +}; + +enum enum_index_stat_col +{ + INDEX_STAT_DB_NAME, + INDEX_STAT_TABLE_NAME, + INDEX_STAT_INDEX_NAME, + INDEX_STAT_PREFIX_ARITY, + INDEX_STAT_AVG_FREQUENCY +}; + +inline +Use_stat_tables_mode get_use_stat_tables_mode(THD *thd) +{ + return (Use_stat_tables_mode) (thd->variables.use_stat_tables); +} + +int read_statistics_for_tables_if_needed(THD *thd, TABLE_LIST *tables); +int collect_statistics_for_table(THD *thd, TABLE *table); +int alloc_statistics_for_table_share(THD* thd, TABLE_SHARE *share, + bool is_safe); +int alloc_statistics_for_table(THD *thd, TABLE *table); +int update_statistics_for_table(THD *thd, TABLE *table); +int delete_statistics_for_table(THD *thd, LEX_STRING *db, LEX_STRING *tab); +int delete_statistics_for_column(THD *thd, TABLE *tab, Field *col); +int delete_statistics_for_index(THD *thd, TABLE *tab, KEY *key_info, + bool ext_prefixes_only); +int rename_table_in_stat_tables(THD *thd, LEX_STRING *db, LEX_STRING *tab, + LEX_STRING *new_db, LEX_STRING *new_tab); +int rename_column_in_stat_tables(THD *thd, TABLE *tab, Field *col, + const char *new_name); +void set_statistics_for_table(THD *thd, TABLE *table); + +double get_column_avg_frequency(Field * field); + +double get_column_range_cardinality(Field *field, + key_range *min_endp, + key_range *max_endp, + uint range_flag); + +class Histogram +{ + +private: + Histogram_type type; + uint8 size; + uchar *values; + + uint prec_factor() + { + switch (type) { + case SINGLE_PREC_HB: + return ((uint) (1 << 8) - 1); + case DOUBLE_PREC_HB: + return ((uint) (1 << 16) - 1); + } + return 1; + } + +public: + uint get_width() + { + switch (type) { + case SINGLE_PREC_HB: + return size; + case DOUBLE_PREC_HB: + return size / 2; + } + return 0; + } + +private: + uint get_value(uint i) + { + switch (type) { + case SINGLE_PREC_HB: + return (uint) (((uint8 *) values)[i]); + case DOUBLE_PREC_HB: + return (uint) (((uint16 *) values)[i]); + } + return 0; + } + + uint find_bucket(double pos, bool first) + { + uint val= (uint) (pos * prec_factor()); + int lp= 0; + int rp= get_width() - 1; + int d= get_width() / 2; + uint i= lp + d; + for ( ; d; d= (rp - lp) / 2, i= lp + d) + { + if (val == get_value(i)) + break; + if (val < get_value(i)) + rp= i; + else if (val > get_value(i + 1)) + lp= i + 1; + else + break; + } + if (val == get_value(i)) + { + if (first) + { + while(i && val == get_value(i - 1)) + i--; + } + else + { + while(i + 1 < get_width() && val == get_value(i + 1)) + i++; + } + } + return i; + } + +public: + + uint get_size() { return (uint) size; } + + Histogram_type get_type() { return type; } + + uchar *get_values() { return (uchar *) values; } + + void set_size (ulonglong sz) { size= (uint8) sz; } + + void set_type (Histogram_type t) { type= t; } + + void set_values (uchar *vals) { values= (uchar *) vals; } + + bool is_available() { return get_size() > 0 && get_values(); } + + void set_value(uint i, double val) + { + switch (type) { + case SINGLE_PREC_HB: + ((uint8 *) values)[i]= (uint8) (val * prec_factor()); + return; + case DOUBLE_PREC_HB: + ((uint16 *) values)[i]= (uint16) (val * prec_factor()); + return; + } + } + + void set_prev_value(uint i) + { + switch (type) { + case SINGLE_PREC_HB: + ((uint8 *) values)[i]= ((uint8 *) values)[i-1]; + return; + case DOUBLE_PREC_HB: + ((uint16 *) values)[i]= ((uint16 *) values)[i-1]; + return; + } + } + + double range_selectivity(double min_pos, double max_pos) + { + double sel; + double bucket_sel= 1.0/(get_width() + 1); + uint min= find_bucket(min_pos, TRUE); + uint max= find_bucket(max_pos, FALSE); + sel= bucket_sel * (max - min + 1); + return sel; + } + + double point_selectivity(double pos, double avg_sel) + { + double sel; + double bucket_sel= 1.0/(get_width() + 1); + uint min= find_bucket(pos, TRUE); + uint max= min; + while (max + 1 < get_width() && get_value(max + 1) == get_value(max)) + max++; + double inv_prec_factor= (double) 1.0 / prec_factor(); + double width= (max + 1 == get_width() ? + 1.0 : get_value(max) * inv_prec_factor) - + (min == 0 ? + 0.0 : get_value(min-1) * inv_prec_factor); + sel= avg_sel * (bucket_sel * (max + 1 - min)) / width; + return sel; + } + +}; + + +class Columns_statistics; +class Index_statistics; + + +/* Statistical data on a table */ + +class Table_statistics +{ + +public: + my_bool cardinality_is_null; /* TRUE if the cardinality is unknown */ + ha_rows cardinality; /* Number of rows in the table */ + uchar *min_max_record_buffers; /* Record buffers for min/max values */ + Column_statistics *column_stats; /* Array of statistical data for columns */ + Index_statistics *index_stats; /* Array of statistical data for indexes */ + ulong *idx_avg_frequency; /* Array of records per key for index prefixes */ + ulong total_hist_size; /* Total size of all histograms */ + uchar *histograms; /* Sequence of histograms */ +}; + + +/* Statistical data on a column */ + +class Column_statistics +{ + +private: + static const uint Scale_factor_nulls_ratio= 100000; + static const uint Scale_factor_avg_length= 100000; + static const uint Scale_factor_avg_frequency= 100000; + +public: + /* + Bitmap indicating what statistical characteristics + are available for the column + */ + uint32 column_stat_nulls; + + /* Minimum value for the column */ + Field *min_value; + /* Maximum value for the column */ + Field *max_value; + +private: + + /* + The ratio Z/N multiplied by the scale factor Scale_factor_nulls_ratio, + where + N is the total number of rows, + Z is the number of nulls in the column + */ + ulong nulls_ratio; + + /* + Average number of bytes occupied by the representation of a + value of the column in memory buffers such as join buffer + multiplied by the scale factor Scale_factor_avg_length. + CHAR values are stripped of trailing spaces. + Flexible values are stripped of their length prefixes. + */ + ulong avg_length; + + /* + The ratio N/D multiplied by the scale factor Scale_factor_avg_frequency, + where + N is the number of rows with not null value in the column, + D the number of distinct values among them + */ + ulong avg_frequency; + +public: + + Histogram histogram; + + void set_all_nulls() + { + column_stat_nulls= + ((1 << (COLUMN_STAT_HISTOGRAM-COLUMN_STAT_COLUMN_NAME))-1) << + (COLUMN_STAT_COLUMN_NAME+1); + } + + void set_not_null(uint stat_field_no) + { + column_stat_nulls&= ~(1 << stat_field_no); + } + + bool is_null(uint stat_field_no) + { + return test(column_stat_nulls & (1 << stat_field_no)); + } + + double get_nulls_ratio() + { + return (double) nulls_ratio / Scale_factor_nulls_ratio; + } + + double get_avg_length() + { + return (double) avg_length / Scale_factor_avg_length; + } + + double get_avg_frequency() + { + return (double) avg_frequency / Scale_factor_avg_frequency; + } + + void set_nulls_ratio (double val) + { + nulls_ratio= (ulong) (val * Scale_factor_nulls_ratio); + } + + void set_avg_length (double val) + { + avg_length= (ulong) (val * Scale_factor_avg_length); + } + + void set_avg_frequency (double val) + { + avg_frequency= (ulong) (val * Scale_factor_avg_frequency); + } + +}; + + +/* Statistical data on an index prefixes */ + +class Index_statistics +{ + +private: + static const uint Scale_factor_avg_frequency= 100000; + /* + The k-th element of this array contains the ratio N/D + multiplied by the scale factor Scale_factor_avg_frequency, + where N is the number of index entries without nulls + in the first k components, and D is the number of distinct + k-component prefixes among them + */ + ulong *avg_frequency; + +public: + + void init_avg_frequency(ulong *ptr) { avg_frequency= ptr; } + + bool avg_frequency_is_inited() { return avg_frequency != NULL; } + + double get_avg_frequency(uint i) + { + return (double) avg_frequency[i] / Scale_factor_avg_frequency; + } + + void set_avg_frequency(uint i, double val) + { + avg_frequency[i]= (ulong) (val * Scale_factor_avg_frequency); + } + +}; + +#endif /* SQL_STATISTICS_H */ diff --git a/sql/sql_string.cc b/sql/sql_string.cc index 126ecebcc24..36415d2cb50 100644 --- a/sql/sql_string.cc +++ b/sql/sql_string.cc @@ -41,7 +41,9 @@ bool String::real_alloc(uint32 length) if (Alloced_length < arg_length) { free(); - if (!(Ptr=(char*) my_malloc(arg_length,MYF(MY_WME)))) + if (!(Ptr=(char*) my_malloc(arg_length,MYF(MY_WME | + (thread_specific ? + MY_THREAD_SPECIFIC : 0))))) return TRUE; Alloced_length=arg_length; alloced=1; @@ -89,10 +91,16 @@ bool String::realloc_raw(uint32 alloc_length) return TRUE; /* Overflow */ if (alloced) { - if (!(new_ptr= (char*) my_realloc(Ptr,len,MYF(MY_WME)))) + if (!(new_ptr= (char*) my_realloc(Ptr,len, + MYF(MY_WME | + (thread_specific ? + MY_THREAD_SPECIFIC : 0))))) return TRUE; // Signal error } - else if ((new_ptr= (char*) my_malloc(len,MYF(MY_WME)))) + else if ((new_ptr= (char*) my_malloc(len, + MYF(MY_WME | + (thread_specific ? + MY_THREAD_SPECIFIC : 0))))) { if (str_length > len - 1) str_length= 0; @@ -501,6 +509,24 @@ bool String::append(IO_CACHE* file, uint32 arg_length) return FALSE; } + +/** + Append a parenthesized number to String. + Used in various pieces of SHOW related code. + + @param nr Number + @param radix Radix, optional parameter, 10 by default. +*/ +bool String::append_parenthesized(long nr, int radix) +{ + char buff[64], *end; + buff[0]= '('; + end= int10_to_str(nr, buff + 1, radix); + *end++ = ')'; + return append(buff, (uint) (end - buff)); +} + + bool String::append_with_prefill(const char *s,uint32 arg_length, uint32 full_length, char fill_char) { @@ -749,79 +775,6 @@ String *copy_if_not_alloced(String *to,String *from,uint32 from_length) Help functions ****************************************************************************/ -/* - copy a string from one character set to another - - SYNOPSIS - copy_and_convert() - to Store result here - to_cs Character set of result string - from Copy from here - from_length Length of from string - from_cs From character set - - NOTES - 'to' must be big enough as form_length * to_cs->mbmaxlen - - RETURN - length of bytes copied to 'to' -*/ - - -static uint32 -copy_and_convert_extended(char *to, uint32 to_length, CHARSET_INFO *to_cs, - const char *from, uint32 from_length, - CHARSET_INFO *from_cs, - uint *errors) -{ - int cnvres; - my_wc_t wc; - const uchar *from_end= (const uchar*) from+from_length; - char *to_start= to; - uchar *to_end= (uchar*) to+to_length; - my_charset_conv_mb_wc mb_wc= from_cs->cset->mb_wc; - my_charset_conv_wc_mb wc_mb= to_cs->cset->wc_mb; - uint error_count= 0; - - while (1) - { - if ((cnvres= (*mb_wc)(from_cs, &wc, (uchar*) from, - from_end)) > 0) - from+= cnvres; - else if (cnvres == MY_CS_ILSEQ) - { - error_count++; - from++; - wc= '?'; - } - else if (cnvres > MY_CS_TOOSMALL) - { - /* - A correct multibyte sequence detected - But it doesn't have Unicode mapping. - */ - error_count++; - from+= (-cnvres); - wc= '?'; - } - else - break; // Not enough characters - -outp: - if ((cnvres= (*wc_mb)(to_cs, wc, (uchar*) to, to_end)) > 0) - to+= cnvres; - else if (cnvres == MY_CS_ILUNI && wc != '?') - { - error_count++; - wc= '?'; - goto outp; - } - else - break; - } - *errors= error_count; - return (uint32) (to - to_start); -} /* diff --git a/sql/sql_string.h b/sql/sql_string.h index 020628100cb..a6429266e55 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -56,23 +56,26 @@ class String { char *Ptr; uint32 str_length,Alloced_length, extra_alloc; - bool alloced; + bool alloced,thread_specific; CHARSET_INFO *str_charset; public: String() { - Ptr=0; str_length=Alloced_length=extra_alloc=0; alloced=0; + Ptr=0; str_length=Alloced_length=extra_alloc=0; + alloced= thread_specific= 0; str_charset= &my_charset_bin; } String(uint32 length_arg) { - alloced=0; Alloced_length= extra_alloc= 0; (void) real_alloc(length_arg); + alloced= thread_specific= 0; + Alloced_length= extra_alloc= 0; (void) real_alloc(length_arg); str_charset= &my_charset_bin; } String(const char *str, CHARSET_INFO *cs) { Ptr=(char*) str; str_length= (uint32) strlen(str); - Alloced_length= extra_alloc= 0; alloced=0; + Alloced_length= extra_alloc= 0; + alloced= thread_specific= 0; str_charset=cs; } /* @@ -82,18 +85,21 @@ public: */ String(const char *str,uint32 len, CHARSET_INFO *cs) { - Ptr=(char*) str; str_length=len; Alloced_length= extra_alloc=0; alloced=0; + Ptr=(char*) str; str_length=len; Alloced_length= extra_alloc=0; + alloced= thread_specific= 0; str_charset=cs; } String(char *str,uint32 len, CHARSET_INFO *cs) { - Ptr=(char*) str; Alloced_length=str_length=len; extra_alloc= 0; alloced=0; + Ptr=(char*) str; Alloced_length=str_length=len; extra_alloc= 0; + alloced= thread_specific= 0; str_charset=cs; } String(const String &str) { Ptr=str.Ptr ; str_length=str.str_length ; - Alloced_length=str.Alloced_length; extra_alloc= 0; alloced=0; + Alloced_length=str.Alloced_length; extra_alloc= 0; + alloced= thread_specific= 0; str_charset=str.str_charset; } static void *operator new(size_t size, MEM_ROOT *mem_root) throw () @@ -108,6 +114,12 @@ public: { /* never called */ } ~String() { free(); } + /* Mark variable thread specific it it's not allocated already */ + inline void set_thread_specific() + { + if (!alloced) + thread_specific= 1; + } inline void set_charset(CHARSET_INFO *charset_arg) { str_charset= charset_arg; } inline CHARSET_INFO *charset() const { return str_charset; } @@ -148,6 +160,11 @@ public: LEX_STRING lex_string = { (char*) ptr(), length() }; return lex_string; } + LEX_CSTRING lex_cstring() const + { + LEX_CSTRING lex_cstring = { ptr(), length() }; + return lex_cstring; + } void set(String &str,uint32 offset,uint32 arg_length) { @@ -333,6 +350,7 @@ public: Ptr=s.Ptr ; str_length=s.str_length ; Alloced_length=s.Alloced_length; extra_alloc= s.extra_alloc; alloced= s.alloced; + thread_specific= s.thread_specific; s.alloced= 0; } bool append(const String &s); @@ -347,6 +365,7 @@ public: bool append(IO_CACHE* file, uint32 arg_length); bool append_with_prefill(const char *s, uint32 arg_length, uint32 full_length, char fill_char); + bool append_parenthesized(long nr, int radix= 10); int strstr(const String &search,uint32 offset=0); // Returns offset to substring or -1 int strrstr(const String &search,uint32 offset=0); // Returns offset to substring or -1 bool replace(uint32 offset,uint32 arg_length,const char *to,uint32 length); @@ -497,6 +516,38 @@ public: } }; + +// The following class is a backport from MySQL 5.6: +/** + String class wrapper with a preallocated buffer of size buff_sz + + This class allows to replace sequences of: + char buff[12345]; + String str(buff, sizeof(buff)); + str.length(0); + with a simple equivalent declaration: + StringBuffer<12345> str; +*/ + +template<size_t buff_sz> +class StringBuffer : public String +{ + char buff[buff_sz]; + +public: + StringBuffer() : String(buff, buff_sz, &my_charset_bin) { length(0); } + explicit StringBuffer(const CHARSET_INFO *cs) : String(buff, buff_sz, cs) + { + length(0); + } + StringBuffer(const char *str, size_t length, const CHARSET_INFO *cs) + : String(buff, buff_sz, cs) + { + set(str, length, cs); + } +}; + + static inline bool check_if_only_end_space(CHARSET_INFO *cs, const char *str, const char *end) diff --git a/sql/sql_table.cc b/sql/sql_table.cc index c9f754a204f..d63ce1c8707 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -22,7 +22,6 @@ #include "unireg.h" #include "debug_sync.h" #include "sql_table.h" -#include "sql_rename.h" // do_rename #include "sql_parse.h" // test_if_data_home_dir #include "sql_cache.h" // query_cache_* #include "sql_base.h" // open_table_uncached, lock_table_names @@ -43,6 +42,7 @@ #include "discover.h" // readfrm #include "my_pthread.h" // pthread_mutex_t #include "log_event.h" // Query_log_event +#include "sql_statistics.h" #include <hash.h> #include <myisam.h> #include <my_dir.h> @@ -53,7 +53,6 @@ #include "sql_parse.h" #include "sql_show.h" #include "transaction.h" -#include "datadict.h" // dd_frm_type() #include "sql_audit.h" #ifdef __WIN__ @@ -72,8 +71,7 @@ static int copy_data_between_tables(THD *thd, TABLE *,TABLE *, static bool prepare_blob_field(THD *thd, Create_field *sql_field); static bool check_engine(THD *, const char *, const char *, HA_CREATE_INFO *); static int mysql_prepare_create_table(THD *, HA_CREATE_INFO *, Alter_info *, - bool, uint *, handler *, KEY **, uint *, - int); + uint *, handler *, KEY **, uint *, int); /** @brief Helper function for explain_filename @@ -367,42 +365,22 @@ uint explain_filename(THD* thd, Table name length. */ -uint filename_to_tablename(const char *from, char *to, uint to_length -#ifndef DBUG_OFF - , bool stay_quiet -#endif /* DBUG_OFF */ - ) +uint filename_to_tablename(const char *from, char *to, uint to_length, + bool stay_quiet) { uint errors; size_t res; DBUG_ENTER("filename_to_tablename"); DBUG_PRINT("enter", ("from '%s'", from)); - if (!strncmp(from, tmp_file_prefix, tmp_file_prefix_length)) + res= strconvert(&my_charset_filename, from, + system_charset_info, to, to_length, &errors); + if (errors) // Old 5.0 name { - /* Temporary table name. */ - res= (strnmov(to, from, to_length) - to); - } - else - { - res= strconvert(&my_charset_filename, from, - system_charset_info, to, to_length, &errors); - if (errors) // Old 5.0 name - { - res= (strxnmov(to, to_length, MYSQL50_TABLE_NAME_PREFIX, from, NullS) - - to); -#ifndef DBUG_OFF - if (!stay_quiet) { -#endif /* DBUG_OFF */ - sql_print_error("Invalid (old?) table or database name '%s'", from); -#ifndef DBUG_OFF - } -#endif /* DBUG_OFF */ - /* - TODO: add a stored procedure for fix table and database names, - and mention its name in error log. - */ - } + res= (strxnmov(to, to_length, MYSQL50_TABLE_NAME_PREFIX, from, NullS) - + to); + if (!stay_quiet) + sql_print_error("Invalid (old?) table or database name '%s'", from); } DBUG_PRINT("exit", ("to '%s'", to)); @@ -943,7 +921,7 @@ static int execute_ddl_log_action(THD *thd, DDL_LOG_ENTRY *ddl_log_entry) ddl_log_entry->handler_name)); handler_name.str= (char*)ddl_log_entry->handler_name; handler_name.length= strlen(ddl_log_entry->handler_name); - init_sql_alloc(&mem_root, TABLE_ALLOC_BLOCK_SIZE, 0); + init_sql_alloc(&mem_root, TABLE_ALLOC_BLOCK_SIZE, 0, MYF(MY_THREAD_SPECIFIC)); if (!strcmp(ddl_log_entry->handler_name, reg_ext)) frm_action= TRUE; else @@ -951,10 +929,10 @@ static int execute_ddl_log_action(THD *thd, DDL_LOG_ENTRY *ddl_log_entry) plugin_ref plugin= ha_resolve_by_name(thd, &handler_name); if (!plugin) { - my_error(ER_ILLEGAL_HA, MYF(0), ddl_log_entry->handler_name); + my_error(ER_UNKNOWN_STORAGE_ENGINE, MYF(0), ddl_log_entry->handler_name); goto error; } - hton= plugin_data(plugin, handlerton*); + hton= plugin_hton(plugin); file= get_new_handler((TABLE_SHARE*)0, &mem_root, hton); if (!file) { @@ -1525,7 +1503,7 @@ void execute_ddl_log_recovery() delete thd; my_free(file_entry_buf); /* Remember that we don't have a THD */ - my_pthread_setspecific_ptr(THR_THD, 0); + set_current_thd(0); DBUG_VOID_RETURN; } @@ -1658,14 +1636,10 @@ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) strxmov(shadow_frm_name, shadow_path, reg_ext, NullS); if (flags & WFRM_WRITE_SHADOW) { - if (mysql_prepare_create_table(lpt->thd, lpt->create_info, - lpt->alter_info, - /*tmp_table*/ 1, - &lpt->db_options, - lpt->table->file, - &lpt->key_info_buffer, - &lpt->key_count, - /*select_field_count*/ 0)) + if (mysql_prepare_create_table(lpt->thd, lpt->create_info, lpt->alter_info, + &lpt->db_options, lpt->table->file, + &lpt->key_info_buffer, &lpt->key_count, + C_ALTER_TABLE)) { DBUG_RETURN(TRUE); } @@ -1690,13 +1664,23 @@ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) #endif /* Write shadow frm file */ lpt->create_info->table_options= lpt->db_options; - if ((mysql_create_frm(lpt->thd, shadow_frm_name, lpt->db, - lpt->table_name, lpt->create_info, - lpt->alter_info->create_list, lpt->key_count, - lpt->key_info_buffer, lpt->table->file)) || - lpt->table->file->ha_create_handler_files(shadow_path, NULL, - CHF_CREATE_FLAG, - lpt->create_info)) + LEX_CUSTRING frm= build_frm_image(lpt->thd, lpt->table_name, + lpt->create_info, + lpt->alter_info->create_list, + lpt->key_count, lpt->key_info_buffer, + lpt->table->file); + if (!frm.str) + { + error= 1; + goto end; + } + + int error= writefrm(shadow_path, lpt->db, lpt->table_name, + lpt->create_info->tmp_table(), frm.str, frm.length); + my_free(const_cast<uchar*>(frm.str)); + + if (error || lpt->table->file->ha_create_partitioning_metadata(shadow_path, + NULL, CHF_CREATE_FLAG)) { mysql_file_delete(key_file_frm, shadow_frm_name, MYF(0)); error= 1; @@ -1711,12 +1695,12 @@ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) handlers that have the main version of the frm file stored in the handler. */ - uchar *data; + const uchar *data; size_t length; if (readfrm(shadow_path, &data, &length) || packfrm(data, length, &lpt->pack_frm_data, &lpt->pack_frm_len)) { - my_free(data); + my_free(const_cast<uchar*>(data)); my_free(lpt->pack_frm_data); mem_alloc_error(length); error= 1; @@ -1734,7 +1718,7 @@ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) */ build_table_filename(path, sizeof(path) - 1, lpt->db, lpt->table_name, "", 0); - strxmov(frm_name, path, reg_ext, NullS); + strxnmov(frm_name, sizeof(frm_name), path, reg_ext, NullS); /* When we are changing to use new frm file we need to ensure that we don't collide with another thread in process to open the frm file. @@ -1747,14 +1731,14 @@ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) */ if (mysql_file_delete(key_file_frm, frm_name, MYF(MY_WME)) || #ifdef WITH_PARTITION_STORAGE_ENGINE - lpt->table->file->ha_create_handler_files(path, shadow_path, - CHF_DELETE_FLAG, NULL) || + lpt->table->file->ha_create_partitioning_metadata(path, shadow_path, + CHF_DELETE_FLAG) || deactivate_ddl_log_entry(part_info->frm_log_entry->entry_pos) || (sync_ddl_log(), FALSE) || mysql_file_rename(key_file_frm, shadow_frm_name, frm_name, MYF(MY_WME)) || - lpt->table->file->ha_create_handler_files(path, shadow_path, - CHF_RENAME_FLAG, NULL)) + lpt->table->file->ha_create_partitioning_metadata(path, shadow_path, + CHF_RENAME_FLAG)) #else mysql_file_rename(key_file_frm, shadow_frm_name, frm_name, MYF(MY_WME))) @@ -1895,6 +1879,17 @@ bool mysql_rm_table(THD *thd,TABLE_LIST *tables, my_bool if_exists, } } + if (!in_bootstrap) + { + for (table= tables; table; table= table->next_local) + { + LEX_STRING db_name= { table->db, table->db_length }; + LEX_STRING table_name= { table->table_name, table->table_name_length }; + if (table->open_type == OT_BASE_ONLY || !find_temporary_table(thd, table)) + (void) delete_statistics_for_table(thd, &db_name, &table_name); + } + } + mysql_ha_rm_tables(thd, tables); if (!drop_temporary) @@ -1904,9 +1899,6 @@ bool mysql_rm_table(THD *thd,TABLE_LIST *tables, my_bool if_exists, if (lock_table_names(thd, tables, NULL, thd->variables.lock_wait_timeout, MYSQL_OPEN_SKIP_TEMPORARY)) DBUG_RETURN(true); - for (table= tables; table; table= table->next_local) - tdc_remove_table(thd, TDC_RT_REMOVE_ALL, table->db, table->table_name, - false); } else { @@ -2127,8 +2119,7 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, bool is_trans; char *db=table->db; size_t db_length= table->db_length; - handlerton *table_type; - enum legacy_db_type frm_db_type= DB_TYPE_UNKNOWN; + handlerton *table_type= 0; DBUG_PRINT("table", ("table_l: '%s'.'%s' table: 0x%lx s: 0x%lx", table->db, table->table_name, (long) table->table, @@ -2210,30 +2201,14 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, { non_temp_tables_count++; - if (thd->locked_tables_mode) - { - if (wait_while_table_is_used(thd, table->table, HA_EXTRA_NOT_USED, - TDC_RT_REMOVE_NOT_OWN_AND_MARK_NOT_USABLE)) - { - error= -1; - goto err; - } - close_all_tables_for_name(thd, table->table->s, - HA_EXTRA_PREPARE_FOR_DROP); - table->table= 0; - } - - /* Check that we have an exclusive lock on the table to be dropped. */ DBUG_ASSERT(thd->mdl_context.is_lock_owner(MDL_key::TABLE, table->db, table->table_name, - MDL_EXCLUSIVE)); + MDL_SHARED)); alias= (lower_case_table_names == 2) ? table->alias : table->table_name; /* remove .frm file and engine files */ path_length= build_table_filename(path, sizeof(path) - 1, db, alias, - reg_ext, - table->internal_tmp_table ? - FN_IS_TMP : 0); + reg_ext, 0); /* This handles the case where a "DROP" was executed and a regular @@ -2266,14 +2241,9 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, } } DEBUG_SYNC(thd, "rm_table_no_locks_before_delete_table"); - DBUG_EXECUTE_IF("sleep_before_no_locks_delete_table", - my_sleep(100000);); error= 0; - if (drop_temporary || - ((access(path, F_OK) && - ha_create_table_from_engine(thd, db, alias)) || - (!drop_view && - dd_frm_type(thd, path, &frm_db_type) != FRMTYPE_TABLE))) + if ((drop_temporary || !ha_table_exists(thd, db, alias, &table_type) || + (!drop_view && table_type == view_pseudo_hton))) { /* One of the following cases happened: @@ -2296,29 +2266,46 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, { char *end; /* - Cannot use the db_type from the table, since that might have changed - while waiting for the exclusive name lock. + It could happen that table's share in the table_def_cache + is the only thing that keeps the engine plugin loaded + (if it is uninstalled and waits for the ref counter to drop to 0). + + In this case, the tdc_remove_table() below will release and unload + the plugin. And ha_delete_table() will get a dangling pointer. + + Let's lock the plugin till the end of the statement. */ - if (frm_db_type == DB_TYPE_UNKNOWN) + if (table_type && table_type != view_pseudo_hton) + ha_lock_engine(thd, table_type); + + if (thd->locked_tables_mode) { - dd_frm_type(thd, path, &frm_db_type); - DBUG_PRINT("info", ("frm_db_type %d from %s", frm_db_type, path)); + if (wait_while_table_is_used(thd, table->table, HA_EXTRA_NOT_USED, + TDC_RT_REMOVE_NOT_OWN_AND_MARK_NOT_USABLE)) + { + error= -1; + goto err; + } + /* the following internally does TDC_RT_REMOVE_ALL */ + close_all_tables_for_name(thd, table->table->s, + HA_EXTRA_PREPARE_FOR_DROP); + table->table= 0; } - table_type= ha_resolve_by_legacy_type(thd, frm_db_type); + else + tdc_remove_table(thd, TDC_RT_REMOVE_ALL, table->db, table->table_name, + false); + + /* Check that we have an exclusive lock on the table to be dropped. */ + DBUG_ASSERT(thd->mdl_context.is_lock_owner(MDL_key::TABLE, table->db, + table->table_name, + MDL_EXCLUSIVE)); + // Remove extension for delete *(end= path + path_length - reg_ext_length)= '\0'; - DBUG_PRINT("info", ("deleting table of type %d", - (table_type ? table_type->db_type : 0))); + error= ha_delete_table(thd, table_type, path, db, table->table_name, !dont_log_query); - /* No error if non existent table and 'IF EXIST' clause or view */ - if ((error == ENOENT || error == HA_ERR_NO_SUCH_TABLE) && - (if_exists || table_type == NULL)) - { - error= 0; - thd->clear_error(); - } if (error == HA_ERR_ROW_IS_REFERENCED) { /* the table is referenced by a foreign key constraint */ @@ -2326,18 +2313,29 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, } if (!error || error == ENOENT || error == HA_ERR_NO_SUCH_TABLE) { - int new_error; + int frm_delete_error, trigger_drop_error= 0; /* Delete the table definition file */ strmov(end,reg_ext); - if (!(new_error= mysql_file_delete(key_file_frm, path, MYF(MY_WME)))) + frm_delete_error= mysql_file_delete(key_file_frm, path, MYF(MY_WME)); + if (frm_delete_error) + frm_delete_error= my_errno; + else { non_tmp_table_deleted= TRUE; - new_error= Table_triggers_list::drop_all_triggers(thd, db, - table->table_name); + trigger_drop_error= + Table_triggers_list::drop_all_triggers(thd, db, table->table_name); + } + + if (trigger_drop_error || + (frm_delete_error && frm_delete_error != ENOENT)) + error= 1; + else if (!frm_delete_error || !error || if_exists) + { + error= 0; + thd->clear_error(); } - error|= new_error; } - non_tmp_error= error ? TRUE : non_tmp_error; + non_tmp_error= error ? TRUE : non_tmp_error; } if (error) { @@ -2638,7 +2636,6 @@ void calculate_interval_lengths(CHARSET_INFO *cs, TYPELIB *interval, prepare_create_field() sql_field field to prepare for packing blob_columns count for BLOBs - timestamps count for timestamps table_flags table flags DESCRIPTION @@ -2652,7 +2649,6 @@ void calculate_interval_lengths(CHARSET_INFO *cs, TYPELIB *interval, int prepare_create_field(Create_field *sql_field, uint *blob_columns, - int *timestamps, int *timestamps_with_niladic, longlong table_flags) { unsigned int dup_val_count; @@ -2774,21 +2770,6 @@ int prepare_create_field(Create_field *sql_field, (sql_field->decimals << FIELDFLAG_DEC_SHIFT)); break; case MYSQL_TYPE_TIMESTAMP: - /* We should replace old TIMESTAMP fields with their newer analogs */ - if (sql_field->unireg_check == Field::TIMESTAMP_OLD_FIELD) - { - if (!*timestamps) - { - sql_field->unireg_check= Field::TIMESTAMP_DNUN_FIELD; - (*timestamps_with_niladic)++; - } - else - sql_field->unireg_check= Field::NONE; - } - else if (sql_field->unireg_check != Field::NONE) - (*timestamps_with_niladic)++; - - (*timestamps)++; /* fall-through */ default: sql_field->pack_flag=(FIELDFLAG_NUMBER | @@ -2860,6 +2841,40 @@ bool check_duplicate_warning(THD *thd, char *msg, ulong length) } +/** + Modifies the first column definition whose SQL type is TIMESTAMP + by adding the features DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP. + + @param column_definitions The list of column definitions, in the physical + order in which they appear in the table. + */ +void promote_first_timestamp_column(List<Create_field> *column_definitions) +{ + List_iterator<Create_field> it(*column_definitions); + Create_field *column_definition; + + while ((column_definition= it++) != NULL) + { + if (column_definition->sql_type == MYSQL_TYPE_TIMESTAMP || // TIMESTAMP + column_definition->unireg_check == Field::TIMESTAMP_OLD_FIELD) // Legacy + { + if ((column_definition->flags & NOT_NULL_FLAG) != 0 && // NOT NULL, + column_definition->def == NULL && // no constant default, + column_definition->unireg_check == Field::NONE) // no function default + { + DBUG_PRINT("info", ("First TIMESTAMP column '%s' was promoted to " + "DEFAULT CURRENT_TIMESTAMP ON UPDATE " + "CURRENT_TIMESTAMP", + column_definition->field_name + )); + column_definition->unireg_check= Field::TIMESTAMP_DNUN_FIELD; + } + return; + } + } +} + + /* Preparation for table creation @@ -2868,12 +2883,12 @@ bool check_duplicate_warning(THD *thd, char *msg, ulong length) thd Thread object. create_info Create information (like MAX_ROWS). alter_info List of columns and indexes to create - tmp_table If a temporary table is to be created. db_options INOUT Table options (like HA_OPTION_PACK_RECORD). file The handler for the new table. key_info_buffer OUT An array of KEY structs for the indexes. key_count OUT The number of elements in the array. - select_field_count The number of fields coming from a select table. + create_table_mode C_ORDINARY_CREATE, C_ALTER_TABLE, + C_CREATE_SELECT, C_ASSISTED_DISCOVERY DESCRIPTION Prepares the table and key structures for table creation. @@ -2888,11 +2903,9 @@ bool check_duplicate_warning(THD *thd, char *msg, ulong length) static int mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, - Alter_info *alter_info, - bool tmp_table, - uint *db_options, + Alter_info *alter_info, uint *db_options, handler *file, KEY **key_info_buffer, - uint *key_count, int select_field_count) + uint *key_count, int create_table_mode) { const char *key_name; Create_field *sql_field,*dup_field; @@ -2900,12 +2913,13 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, ulong record_offset= 0; KEY *key_info; KEY_PART_INFO *key_part_info; - int timestamps= 0, timestamps_with_niladic= 0; int field_no,dup_no; int select_field_pos,auto_increment=0; List_iterator<Create_field> it(alter_info->create_list); List_iterator<Create_field> it2(alter_info->create_list); uint total_uneven_bit_length= 0; + int select_field_count= C_CREATE_SELECT(create_table_mode); + bool tmp_table= create_table_mode == C_ALTER_TABLE; DBUG_ENTER("mysql_prepare_create_table"); select_field_pos= alter_info->create_list.elements - select_field_count; @@ -3184,7 +3198,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, DBUG_ASSERT(sql_field->charset != 0); if (prepare_create_field(sql_field, &blob_columns, - ×tamps, ×tamps_with_niladic, file->ha_table_flags())) DBUG_RETURN(TRUE); if (sql_field->sql_type == MYSQL_TYPE_VARCHAR) @@ -3192,8 +3205,8 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, sql_field->offset= record_offset; if (MTYP_TYPENR(sql_field->unireg_check) == Field::NEXT_NUMBER) auto_increment++; - if (parse_option_list(thd, &sql_field->option_struct, - sql_field->option_list, + if (parse_option_list(thd, create_info->db_type, &sql_field->option_struct, + &sql_field->option_list, create_info->db_type->field_options, FALSE, thd->mem_root)) DBUG_RETURN(TRUE); @@ -3215,12 +3228,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, record_offset+= sql_field->pack_length; } } - if (timestamps_with_niladic > 1) - { - my_message(ER_TOO_MUCH_AUTO_TIMESTAMP_COLS, - ER(ER_TOO_MUCH_AUTO_TIMESTAMP_COLS), MYF(0)); - DBUG_RETURN(TRUE); - } if (auto_increment > 1) { my_message(ER_WRONG_AUTO_KEY, ER(ER_WRONG_AUTO_KEY), MYF(0)); @@ -3229,15 +3236,13 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, if (auto_increment && (file->ha_table_flags() & HA_NO_AUTO_INCREMENT)) { - my_message(ER_TABLE_CANT_HANDLE_AUTO_INCREMENT, - ER(ER_TABLE_CANT_HANDLE_AUTO_INCREMENT), MYF(0)); + my_error(ER_TABLE_CANT_HANDLE_AUTO_INCREMENT, MYF(0), file->table_type()); DBUG_RETURN(TRUE); } if (blob_columns && (file->ha_table_flags() & HA_NO_BLOBS)) { - my_message(ER_TABLE_CANT_HANDLE_BLOB, ER(ER_TABLE_CANT_HANDLE_BLOB), - MYF(0)); + my_error(ER_TABLE_CANT_HANDLE_BLOB, MYF(0), file->table_type()); DBUG_RETURN(TRUE); } @@ -3403,8 +3408,8 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, key_info->usable_key_parts= key_number; key_info->algorithm= key->key_create_info.algorithm; key_info->option_list= key->option_list; - if (parse_option_list(thd, &key_info->option_struct, - key_info->option_list, + if (parse_option_list(thd, create_info->db_type, &key_info->option_struct, + &key_info->option_list, create_info->db_type->index_options, FALSE, thd->mem_root)) DBUG_RETURN(TRUE); @@ -3413,8 +3418,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, { if (!(file->ha_table_flags() & HA_CAN_FULLTEXT)) { - my_message(ER_TABLE_CANT_HANDLE_FT, ER(ER_TABLE_CANT_HANDLE_FT), - MYF(0)); + my_error(ER_TABLE_CANT_HANDLE_FT, MYF(0), file->table_type()); DBUG_RETURN(TRUE); } } @@ -3431,8 +3435,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, { if (!(file->ha_table_flags() & HA_CAN_RTREEKEYS)) { - my_message(ER_TABLE_CANT_HANDLE_SPKEYS, ER(ER_TABLE_CANT_HANDLE_SPKEYS), - MYF(0)); + my_error(ER_TABLE_CANT_HANDLE_SPKEYS, MYF(0), file->table_type()); DBUG_RETURN(TRUE); } if (key_info->key_parts != 1) @@ -3547,7 +3550,8 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, { if (!(file->ha_table_flags() & HA_CAN_INDEX_BLOBS)) { - my_error(ER_BLOB_USED_AS_KEY, MYF(0), column->field_name.str); + my_error(ER_BLOB_USED_AS_KEY, MYF(0), column->field_name.str, + file->table_type()); DBUG_RETURN(TRUE); } if (f_is_geom(sql_field->pack_flag) && sql_field->geom_type == @@ -3669,7 +3673,8 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, } else if (length == 0 && (sql_field->flags & NOT_NULL_FLAG)) { - my_error(ER_WRONG_KEY_COLUMN, MYF(0), column->field_name.str); + my_error(ER_WRONG_KEY_COLUMN, MYF(0), file->table_type(), + column->field_name.str); DBUG_RETURN(TRUE); } if (length > file->max_key_part_length() && key->type != Key::FULLTEXT) @@ -3835,8 +3840,23 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, } } - if (parse_option_list(thd, &create_info->option_struct, - create_info->option_list, + if (create_info->tmp_table()) + create_info->table_options|=HA_CREATE_DELAY_KEY_WRITE; + + /* Give warnings for not supported table options */ +#if defined(WITH_ARIA_STORAGE_ENGINE) + extern handlerton *maria_hton; + if (file->ht != maria_hton) +#endif + if (create_info->transactional) + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, + ER_ILLEGAL_HA_CREATE_OPTION, + ER(ER_ILLEGAL_HA_CREATE_OPTION), + file->engine_name()->str, + "TRANSACTIONAL=1"); + + if (parse_option_list(thd, file->partition_ht(), &create_info->option_struct, + &create_info->option_list, file->partition_ht()->table_options, FALSE, thd->mem_root)) DBUG_RETURN(TRUE); @@ -4007,164 +4027,58 @@ static bool check_if_created_table_can_be_opened(THD *thd, /* It is impossible to open definition of partitioned table without .par file. */ - if (file->ha_create_handler_files(path, NULL, CHF_CREATE_FLAG, create_info)) + if (file->ha_create_partitioning_metadata(path, NULL, CHF_CREATE_FLAG)) return TRUE; init_tmp_table_share(thd, &share, db, 0, table_name, path); + share.db_plugin= ha_lock_engine(thd, file->ht); - result= (open_table_def(thd, &share, 0) || + result= (open_table_def(thd, &share) || open_table_from_share(thd, &share, "", 0, (uint) READ_ALL, 0, &table, TRUE)); if (! result) (void) closefrm(&table, 0); free_table_share(&share); - (void) file->ha_create_handler_files(path, NULL, CHF_DELETE_FLAG, create_info); + (void) file->ha_create_partitioning_metadata(path, NULL, CHF_DELETE_FLAG); return result; } #endif -/** - Check that there is no frm file for given table - - @param old_path path to the old frm file - @param path path to the frm file in new encoding - @param db database name - @param table_name table name - @param alias table name for error message (for new encoding) - @param issue_error should we issue error messages - - @retval FALSE there is no frm file - @retval TRUE there is frm file -*/ - -bool check_table_file_presence(char *old_path, - char *path, - const char *db, - const char *table_name, - const char *alias, - bool issue_error) -{ - if (!access(path,F_OK)) - { - if (issue_error) - my_error(ER_TABLE_EXISTS_ERROR,MYF(0),alias); - return TRUE; - } - { - /* - Check if file of the table in 5.0 file name encoding exists. - - Except case when it is the same table. - */ - char tbl50[FN_REFLEN]; -#ifdef _WIN32 - if (check_if_legal_tablename(table_name) != 0) - { - /* - Check for reserved device names for which access() returns 0 - (CON, AUX etc). - */ - return FALSE; - } -#endif - strxmov(tbl50, mysql_data_home, "/", db, "/", table_name, NullS); - fn_format(tbl50, tbl50, "", reg_ext, MY_UNPACK_FILENAME); - if (!access(tbl50, F_OK) && - (old_path == NULL || - strcmp(old_path, tbl50) != 0)) - { - if (issue_error) - { - strxmov(tbl50, MYSQL50_TABLE_NAME_PREFIX, table_name, NullS); - my_error(ER_TABLE_EXISTS_ERROR, MYF(0), tbl50); - } - return TRUE; - } - } - return FALSE; -} - - -/* - Create a table - - SYNOPSIS - mysql_create_table_no_lock() - thd Thread object - db Database - table_name Table name - create_info Create information (like MAX_ROWS) - fields List of fields to create - keys List of keys to create - internal_tmp_table Set to 1 if this is an internal temporary table - (From ALTER TABLE) - select_field_count - is_trans identifies the type of engine where the table - was created: either trans or non-trans. - - DESCRIPTION - If one creates a temporary table, this is automatically opened - - Note that this function assumes that caller already have taken - exclusive metadata lock on table being created or used some other - way to ensure that concurrent operations won't intervene. - mysql_create_table() is a wrapper that can be used for this. - - no_log is needed for the case of CREATE ... SELECT, - as the logging will be done later in sql_insert.cc - select_field_count is also used for CREATE ... SELECT, - and must be zero for standard create of table. - - RETURN VALUES - FALSE OK - TRUE error -*/ - -bool mysql_create_table_no_lock(THD *thd, +handler *mysql_create_frm_image(THD *thd, const char *db, const char *table_name, HA_CREATE_INFO *create_info, - Alter_info *alter_info, - bool internal_tmp_table, - uint select_field_count, - bool *is_trans) + Alter_info *alter_info, int create_table_mode, + LEX_CUSTRING *frm) { - char path[FN_REFLEN + 1]; - uint path_length; - const char *alias; uint db_options, key_count; KEY *key_info_buffer; - handler *file; - bool error= TRUE; - DBUG_ENTER("mysql_create_table_no_lock"); - DBUG_PRINT("enter", ("db: '%s' table: '%s' tmp: %d", - db, table_name, internal_tmp_table)); + handler *file; + DBUG_ENTER("mysql_create_frm_image"); - - /* Check for duplicate fields and check type of table to create */ if (!alter_info->create_list.elements) { my_message(ER_TABLE_MUST_HAVE_COLUMNS, ER(ER_TABLE_MUST_HAVE_COLUMNS), MYF(0)); - DBUG_RETURN(TRUE); + DBUG_RETURN(NULL); } + if (check_engine(thd, db, table_name, create_info)) - DBUG_RETURN(TRUE); + DBUG_RETURN(NULL); set_table_default_charset(thd, create_info, (char*) db); db_options= create_info->table_options; - if (!create_info->frm_only && + if (create_table_mode != C_ALTER_TABLE_FRM_ONLY && create_info->row_type != ROW_TYPE_FIXED && create_info->row_type != ROW_TYPE_DEFAULT) db_options|= HA_OPTION_PACK_RECORD; - alias= table_case_name(create_info, table_name); if (!(file= get_new_handler((TABLE_SHARE*) 0, thd->mem_root, create_info->db_type))) { mem_alloc_error(sizeof(handler)); - DBUG_RETURN(TRUE); + DBUG_RETURN(NULL); } #ifdef WITH_PARTITION_STORAGE_ENGINE partition_info *part_info= thd->work_part_info; @@ -4181,7 +4095,7 @@ bool mysql_create_table_no_lock(THD *thd, if (!part_info) { mem_alloc_error(sizeof(partition_info)); - DBUG_RETURN(TRUE); + goto err; } file->set_auto_partitions(part_info); part_info->default_engine_type= create_info->db_type; @@ -4206,7 +4120,7 @@ bool mysql_create_table_no_lock(THD *thd, char *part_syntax_buf; uint syntax_len; handlerton *engine_type; - if (create_info->options & HA_LEX_CREATE_TMP_TABLE) + if (create_info->tmp_table()) { my_error(ER_PARTITION_NO_TEMPORARY, MYF(0)); goto err; @@ -4279,9 +4193,8 @@ bool mysql_create_table_no_lock(THD *thd, delete file; create_info->db_type= partition_hton; if (!(file= get_ha_partition(part_info))) - { - DBUG_RETURN(TRUE); - } + DBUG_RETURN(NULL); + /* If we have default number of partitions or subpartitions we might require to set-up the part_info object such that it @@ -4323,193 +4236,188 @@ bool mysql_create_table_no_lock(THD *thd, engine_type))) { mem_alloc_error(sizeof(handler)); - DBUG_RETURN(TRUE); + DBUG_RETURN(NULL); } } } #endif - if (mysql_prepare_create_table(thd, create_info, alter_info, - internal_tmp_table, - &db_options, file, - &key_info_buffer, &key_count, - select_field_count)) + if (mysql_prepare_create_table(thd, create_info, alter_info, &db_options, + file, &key_info_buffer, &key_count, + create_table_mode)) goto err; + create_info->table_options=db_options; - /* Check if table exists */ - if (create_info->options & HA_LEX_CREATE_TMP_TABLE) - { - path_length= build_tmptable_filename(thd, path, sizeof(path)); - create_info->table_options|=HA_CREATE_DELAY_KEY_WRITE; - } - else - { - path_length= build_table_filename(path, sizeof(path) - 1, db, alias, reg_ext, - internal_tmp_table ? FN_IS_TMP : 0); - } + *frm= build_frm_image(thd, table_name, create_info, + alter_info->create_list, key_count, + key_info_buffer, file); - /* Check if table already exists */ - if ((create_info->options & HA_LEX_CREATE_TMP_TABLE) && - find_temporary_table(thd, db, table_name)) + if (frm->str) + DBUG_RETURN(file); + +err: + delete file; + DBUG_RETURN(NULL); +} + + +/* + Create a table + + SYNOPSIS + mysql_create_table_no_lock() + thd Thread object + db Database + table_name Table name + create_info Create information (like MAX_ROWS) + fields List of fields to create + keys List of keys to create + is_trans identifies the type of engine where the table + was created: either trans or non-trans. + create_table_mode C_ORDINARY_CREATE, C_ALTER_TABLE, C_ASSISTED_DISCOVERY + or any positive number (for C_CREATE_SELECT). + + DESCRIPTION + If one creates a temporary table, this is automatically opened + + Note that this function assumes that caller already have taken + exclusive metadata lock on table being created or used some other + way to ensure that concurrent operations won't intervene. + mysql_create_table() is a wrapper that can be used for this. + + select_field_count is also used for CREATE ... SELECT, + and must be zero for standard create of table. + + RETURN VALUES + FALSE OK + TRUE error +*/ + +bool mysql_create_table_no_lock(THD *thd, + const char *db, const char *table_name, + HA_CREATE_INFO *create_info, + Alter_info *alter_info, bool *is_trans, + int create_table_mode) +{ + char path[FN_REFLEN + 1]; + uint path_length; + const char *alias; + handler *file= 0; + LEX_CUSTRING frm= {0,0}; + bool error= TRUE; + bool internal_tmp_table= create_table_mode == C_ALTER_TABLE || + create_table_mode == C_ALTER_TABLE_FRM_ONLY; + DBUG_ENTER("mysql_create_table_no_lock"); + DBUG_PRINT("enter", ("db: '%s' table: '%s' tmp: %d", + db, table_name, internal_tmp_table)); + + if (!my_use_symdir || (thd->variables.sql_mode & MODE_NO_DIR_IN_CREATE)) { - if (create_info->options & HA_LEX_CREATE_IF_NOT_EXISTS) - goto warn; - my_error(ER_TABLE_EXISTS_ERROR, MYF(0), alias); - goto err; + if (create_info->data_file_name) + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, + WARN_OPTION_IGNORED, ER(WARN_OPTION_IGNORED), + "DATA DIRECTORY"); + if (create_info->index_file_name) + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, + WARN_OPTION_IGNORED, ER(WARN_OPTION_IGNORED), + "INDEX DIRECTORY"); + create_info->data_file_name= create_info->index_file_name= 0; } + else + if (error_if_data_home_dir(create_info->data_file_name, "DATA DIRECTORY") || + error_if_data_home_dir(create_info->index_file_name, "INDEX DIRECTORY")|| + check_partition_dirs(thd->lex->part_info)) + goto err; - /* Give warnings for not supported table options */ -#if defined(WITH_ARIA_STORAGE_ENGINE) - extern handlerton *maria_hton; - if (file->ht != maria_hton) -#endif - if (create_info->transactional) - push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, - ER_ILLEGAL_HA_CREATE_OPTION, - ER(ER_ILLEGAL_HA_CREATE_OPTION), - file->engine_name()->str, - "TRANSACTIONAL=1"); + alias= table_case_name(create_info, table_name); - if (!internal_tmp_table && !(create_info->options & HA_LEX_CREATE_TMP_TABLE)) + /* Check if table exists */ + if (create_info->tmp_table()) { - if (check_table_file_presence(NULL, path, db, table_name, table_name, - !(create_info->options & - HA_LEX_CREATE_IF_NOT_EXISTS))) + path_length= build_tmptable_filename(thd, path, sizeof(path)); + path[path_length - reg_ext_length]= '\0'; // Remove .frm extension + + if (find_temporary_table(thd, db, table_name)) { if (create_info->options & HA_LEX_CREATE_IF_NOT_EXISTS) goto warn; + my_error(ER_TABLE_EXISTS_ERROR, MYF(0), alias); goto err; } - /* - We don't assert here, but check the result, because the table could be - in the table definition cache and in the same time the .frm could be - missing from the disk, in case of manual intervention which deletes - the .frm file. The user has to use FLUSH TABLES; to clear the cache. - Then she could create the table. This case is pretty obscure and - therefore we don't introduce a new error message only for it. - */ - mysql_mutex_lock(&LOCK_open); - if (get_cached_table_share(db, table_name)) - { - mysql_mutex_unlock(&LOCK_open); - my_error(ER_TABLE_EXISTS_ERROR, MYF(0), table_name); - goto err; - } - mysql_mutex_unlock(&LOCK_open); } - - /* - Check that table with given name does not already - exist in any storage engine. In such a case it should - be discovered and the error ER_TABLE_EXISTS_ERROR be returned - unless user specified CREATE TABLE IF EXISTS - An exclusive metadata lock ensures that no - one else is attempting to discover the table. Since - it's not on disk as a frm file, no one could be using it! - */ - if (!(create_info->options & HA_LEX_CREATE_TMP_TABLE)) + else { - bool create_if_not_exists = - create_info->options & HA_LEX_CREATE_IF_NOT_EXISTS; - int retcode = ha_table_exists_in_engine(thd, db, table_name); - DBUG_PRINT("info", ("exists_in_engine: %u",retcode)); - switch (retcode) - { - case HA_ERR_NO_SUCH_TABLE: - /* Normal case, no table exists. we can go and create it */ - break; - case HA_ERR_TABLE_EXIST: - DBUG_PRINT("info", ("Table existed in handler")); + path_length= build_table_filename(path, sizeof(path) - 1, db, alias, "", + internal_tmp_table ? FN_IS_TMP : 0); - if (create_if_not_exists) - goto warn; - my_error(ER_TABLE_EXISTS_ERROR,MYF(0),table_name); - goto err; - default: - DBUG_PRINT("info", ("error: %u from storage engine", retcode)); - my_error(retcode, MYF(0),table_name); - goto err; + if (!internal_tmp_table && ha_table_exists(thd, db, table_name)) + { + if (create_info->options & HA_LEX_CREATE_IF_NOT_EXISTS) + goto warn; + my_error(ER_TABLE_EXISTS_ERROR, MYF(0), table_name); + goto err; } } thd_proc_info(thd, "creating table"); -#ifdef HAVE_READLINK + if (create_table_mode == C_ASSISTED_DISCOVERY) { - size_t dirlen; - char dirpath[FN_REFLEN]; + /* check that it's used correctly */ + DBUG_ASSERT(alter_info->create_list.elements == 0); + DBUG_ASSERT(alter_info->key_list.elements == 0); - /* - data_file_name and index_file_name include the table name without - extension. Mostly this does not refer to an existing file. When - comparing data_file_name or index_file_name against the data - directory, we try to resolve all symbolic links. On some systems, - we use realpath(3) for the resolution. This returns ENOENT if the - resolved path does not refer to an existing file. my_realpath() - does then copy the requested path verbatim, without symlink - resolution. Thereafter the comparison can fail even if the - requested path is within the data directory. E.g. if symlinks to - another file system are used. To make realpath(3) return the - resolved path, we strip the table name and compare the directory - path only. If the directory doesn't exist either, table creation - will fail anyway. - */ - if (create_info->data_file_name) + TABLE_SHARE share; + handlerton *hton= create_info->db_type; + int ha_err; + Field *no_fields= 0; + + if (!hton->discover_table_structure) { - dirname_part(dirpath, create_info->data_file_name, &dirlen); - if (test_if_data_home_dir(dirpath)) - { - my_error(ER_WRONG_ARGUMENTS, MYF(0), "DATA DIRECTORY"); - goto err; - } + my_error(ER_ILLEGAL_HA, MYF(0), hton_name(hton)->str, db, table_name); + goto err; } - if (create_info->index_file_name) + + init_tmp_table_share(thd, &share, db, 0, table_name, path); + + /* prepare everything for discovery */ + share.field= &no_fields; + share.db_plugin= ha_lock_engine(thd, hton); + share.option_list= create_info->option_list; + share.connect_string= create_info->connect_string; + + if (parse_engine_table_options(thd, hton, &share)) + goto err; + + ha_err= hton->discover_table_structure(hton, thd, &share, create_info); + free_table_share(&share); + + if (ha_err) { - dirname_part(dirpath, create_info->index_file_name, &dirlen); - if (test_if_data_home_dir(dirpath)) - { - my_error(ER_WRONG_ARGUMENTS, MYF(0), "INDEX DIRECTORY"); - goto err; - } + my_error(ER_GET_ERRNO, MYF(0), ha_err, hton_name(hton)->str); + goto err; } } - -#ifdef WITH_PARTITION_STORAGE_ENGINE - if (check_partition_dirs(thd->lex->part_info)) - { - goto err; - } -#endif /* WITH_PARTITION_STORAGE_ENGINE */ - - if (!my_use_symdir || (thd->variables.sql_mode & MODE_NO_DIR_IN_CREATE)) -#endif /* HAVE_READLINK */ + else { - if (create_info->data_file_name) - push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, - WARN_OPTION_IGNORED, ER(WARN_OPTION_IGNORED), - "DATA DIRECTORY"); - if (create_info->index_file_name) - push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, - WARN_OPTION_IGNORED, ER(WARN_OPTION_IGNORED), - "INDEX DIRECTORY"); - create_info->data_file_name= create_info->index_file_name= 0; + file= mysql_create_frm_image(thd, db, table_name, create_info, alter_info, + create_table_mode, &frm); + if (!file) + goto err; + if (rea_create_table(thd, &frm, path, db, table_name, create_info, + create_table_mode == C_ALTER_TABLE_FRM_ONLY ? 0 : file)) + goto err; } - create_info->table_options=db_options; - - path[path_length - reg_ext_length]= '\0'; // Remove .frm extension - if (rea_create_table(thd, path, db, table_name, - create_info, alter_info->create_list, - key_count, key_info_buffer, file)) - goto err; - if (create_info->options & HA_LEX_CREATE_TMP_TABLE) + if (create_info->tmp_table()) { /* Open a table (skipping table cache) and add it into THD::temporary_tables list. */ - TABLE *table= open_table_uncached(thd, path, db, table_name, TRUE); + TABLE *table= open_table_uncached(thd, create_info->db_type, path, + db, table_name, TRUE); if (!table) { @@ -4523,7 +4431,7 @@ bool mysql_create_table_no_lock(THD *thd, thd->thread_specific_used= TRUE; } #ifdef WITH_PARTITION_STORAGE_ENGINE - else if (part_info && create_info->frm_only) + else if (thd->work_part_info && create_table_mode == C_ALTER_TABLE_FRM_ONLY) { /* For partitioned tables we can't find some problems with table @@ -4539,7 +4447,7 @@ bool mysql_create_table_no_lock(THD *thd, create_info, file)) { char frm_name[FN_REFLEN]; - strxmov(frm_name, path, reg_ext, NullS); + strxnmov(frm_name, sizeof(frm_name), path, reg_ext, NullS); (void) mysql_file_delete(key_file_frm, frm_name, MYF(0)); goto err; } @@ -4549,6 +4457,7 @@ bool mysql_create_table_no_lock(THD *thd, error= FALSE; err: thd_proc_info(thd, "After create"); + my_free(const_cast<uchar*>(frm.str)); delete file; DBUG_RETURN(error); @@ -4560,7 +4469,6 @@ warn: goto err; } - /** Implementation of SQLCOM_CREATE_TABLE. @@ -4575,40 +4483,38 @@ bool mysql_create_table(THD *thd, TABLE_LIST *create_table, HA_CREATE_INFO *create_info, Alter_info *alter_info) { - bool result; + const char *db= create_table->db; + const char *table_name= create_table->table_name; bool is_trans= FALSE; + int create_table_mode; DBUG_ENTER("mysql_create_table"); - /* - Open or obtain an exclusive metadata lock on table being created. - */ + /* Open or obtain an exclusive metadata lock on table being created */ if (open_and_lock_tables(thd, thd->lex->query_tables, FALSE, 0)) { /* is_error() may be 0 if table existed and we generated a warning */ - result= thd->is_error(); - goto end; + DBUG_RETURN(thd->is_error()); } /* Got lock. */ DEBUG_SYNC(thd, "locked_table_name"); - result= mysql_create_table_no_lock(thd, create_table->db, - create_table->table_name, create_info, - alter_info, FALSE, 0, &is_trans); + if (alter_info->create_list.elements || alter_info->key_list.elements) + create_table_mode= C_ORDINARY_CREATE; + else + create_table_mode= C_ASSISTED_DISCOVERY; - /* - Don't write statement if: - - Table creation has failed - - Row-based logging is used and we are creating a temporary table - Otherwise, the statement shall be binlogged. - */ - if (!result && - (!thd->is_current_stmt_binlog_format_row() || - (thd->is_current_stmt_binlog_format_row() && - !(create_info->options & HA_LEX_CREATE_TMP_TABLE)))) - result= write_bin_log(thd, TRUE, thd->query(), thd->query_length(), is_trans); + promote_first_timestamp_column(&alter_info->create_list); + if (mysql_create_table_no_lock(thd, db, table_name, create_info, alter_info, + &is_trans, create_table_mode)) + DBUG_RETURN(1); -end: + /* In RBR we don't need to log CREATE TEMPORARY TABLE */ + if (thd->is_current_stmt_binlog_format_row() && create_info->tmp_table()) + DBUG_RETURN(0); + + bool result; + result= write_bin_log(thd, TRUE, thd->query(), thd->query_length(), is_trans); DBUG_RETURN(result); } @@ -4735,9 +4641,13 @@ mysql_rename_table(handlerton *base, const char *old_db, if (!(flags & NO_FRM_RENAME) && rename_file_ext(from,to,reg_ext)) { error=my_errno; - /* Restore old file name */ if (file) - file->ha_rename_table(to_base, from_base); + { + if (error == ENOENT) + error= 0; // this is ok if file->ha_rename_table() succeeded + else + file->ha_rename_table(to_base, from_base); // Restore old file name + } } } delete file; @@ -4827,7 +4737,7 @@ bool mysql_create_like_table(THD* thd, TABLE_LIST* table, TABLE_LIST* src_table, local_create_info.options|= create_info->options&HA_LEX_CREATE_IF_NOT_EXISTS; /* Replace type of source table with one specified in the statement. */ local_create_info.options&= ~HA_LEX_CREATE_TMP_TABLE; - local_create_info.options|= create_info->options & HA_LEX_CREATE_TMP_TABLE; + local_create_info.options|= create_info->tmp_table(); /* Reset auto-increment counter for the new table. */ local_create_info.auto_increment_value= 0; /* @@ -4838,14 +4748,14 @@ bool mysql_create_like_table(THD* thd, TABLE_LIST* table, TABLE_LIST* src_table, if ((res= mysql_create_table_no_lock(thd, table->db, table->table_name, &local_create_info, &local_alter_info, - FALSE, 0, &is_trans))) + &is_trans, C_ORDINARY_CREATE))) goto err; /* Ensure that we have an exclusive lock on target table if we are creating non-temporary table. */ - DBUG_ASSERT((create_info->options & HA_LEX_CREATE_TMP_TABLE) || + DBUG_ASSERT((create_info->tmp_table()) || thd->mdl_context.is_lock_owner(MDL_key::TABLE, table->db, table->table_name, MDL_EXCLUSIVE)); @@ -4872,7 +4782,7 @@ bool mysql_create_like_table(THD* thd, TABLE_LIST* table, TABLE_LIST* src_table, 4 temporary temporary Nothing ==== ========= ========= ============================== */ - if (!(create_info->options & HA_LEX_CREATE_TMP_TABLE)) + if (!(create_info->tmp_table())) { if (src_table->table->s->tmp_table) // Case 2 { @@ -5038,6 +4948,246 @@ is_index_maintenance_unique (TABLE *table, Alter_info *alter_info) /* + Preparation for table creation + + SYNOPSIS + handle_if_exists_option() + thd Thread object. + table The altered table. + alter_info List of columns and indexes to create + + DESCRIPTION + Looks for the IF [NOT] EXISTS options, checks the states and remove items + from the list if existing found. + + RETURN VALUES + NONE +*/ + +static void +handle_if_exists_options(THD *thd, TABLE *table, Alter_info *alter_info) +{ + Field **f_ptr; + DBUG_ENTER("handle_if_exists_option"); + + /* Handle ADD COLUMN IF NOT EXISTS. */ + { + List_iterator<Create_field> it(alter_info->create_list); + Create_field *sql_field; + + while ((sql_field=it++)) + { + if (!sql_field->create_if_not_exists || sql_field->change) + continue; + /* + If there is a field with the same name in the table already, + remove the sql_field from the list. + */ + for (f_ptr=table->field; *f_ptr; f_ptr++) + { + if (my_strcasecmp(system_charset_info, + sql_field->field_name, (*f_ptr)->field_name) == 0) + { + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_DUP_FIELDNAME, ER(ER_DUP_FIELDNAME), + sql_field->field_name); + it.remove(); + if (alter_info->create_list.is_empty()) + { + alter_info->flags&= ~ALTER_ADD_COLUMN; + if (alter_info->key_list.is_empty()) + alter_info->flags&= ~ALTER_ADD_INDEX; + } + break; + } + } + } + } + + /* Handle MODIFY COLUMN IF EXISTS. */ + { + List_iterator<Create_field> it(alter_info->create_list); + Create_field *sql_field; + + while ((sql_field=it++)) + { + if (!sql_field->create_if_not_exists || !sql_field->change) + continue; + /* + If there is NO field with the same name in the table already, + remove the sql_field from the list. + */ + for (f_ptr=table->field; *f_ptr; f_ptr++) + { + if (my_strcasecmp(system_charset_info, + sql_field->field_name, (*f_ptr)->field_name) == 0) + { + break; + } + } + if (*f_ptr == NULL) + { + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_BAD_FIELD_ERROR, ER(ER_BAD_FIELD_ERROR), + sql_field->change, table->s->table_name.str); + it.remove(); + if (alter_info->create_list.is_empty()) + { + alter_info->flags&= ~(ALTER_ADD_COLUMN | ALTER_CHANGE_COLUMN); + if (alter_info->key_list.is_empty()) + alter_info->flags&= ~ALTER_ADD_INDEX; + } + } + } + } + + /* Handle DROP COLUMN/KEY IF EXISTS. */ + { + List_iterator<Alter_drop> drop_it(alter_info->drop_list); + Alter_drop *drop; + bool remove_drop; + while ((drop= drop_it++)) + { + if (!drop->drop_if_exists) + continue; + remove_drop= TRUE; + if (drop->type == Alter_drop::COLUMN) + { + /* + If there is NO field with that name in the table, + remove the 'drop' from the list. + */ + for (f_ptr=table->field; *f_ptr; f_ptr++) + { + if (my_strcasecmp(system_charset_info, + drop->name, (*f_ptr)->field_name) == 0) + { + remove_drop= FALSE; + break; + } + } + } + else /* Alter_drop::KEY */ + { + uint n_key; + for (n_key=0; n_key < table->s->keys; n_key++) + { + if (my_strcasecmp(system_charset_info, + drop->name, table->key_info[n_key].name) == 0) + { + remove_drop= FALSE; + break; + } + } + } + if (remove_drop) + { + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_CANT_DROP_FIELD_OR_KEY, ER(ER_CANT_DROP_FIELD_OR_KEY), + drop->name); + drop_it.remove(); + if (alter_info->drop_list.is_empty()) + alter_info->flags&= ~(ALTER_DROP_COLUMN | ALTER_DROP_INDEX); + } + } + } + + /* ALTER TABLE ADD KEY IF NOT EXISTS */ + /* ALTER TABLE ADD FOREIGN KEY IF NOT EXISTS */ + { + Key *key; + List_iterator<Key> key_it(alter_info->key_list); + uint n_key; + while ((key=key_it++)) + { + if (!key->create_if_not_exists) + continue; + for (n_key=0; n_key < table->s->keys; n_key++) + { + if (my_strcasecmp(system_charset_info, + key->name.str, table->key_info[n_key].name) == 0) + { + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_DUP_KEYNAME, ER(ER_DUP_KEYNAME), key->name.str); + key_it.remove(); + if (key->type == Key::FOREIGN_KEY) + { + /* ADD FOREIGN KEY appends two items. */ + key_it.remove(); + } + if (alter_info->key_list.is_empty()) + alter_info->flags&= ~ALTER_ADD_INDEX; + break; + } + } + } + } + +#ifdef WITH_PARTITION_STORAGE_ENGINE + partition_info *tab_part_info= table->part_info; + if (tab_part_info && thd->lex->check_exists) + { + /* ALTER TABLE ADD PARTITION IF NOT EXISTS */ + if (alter_info->flags & ALTER_ADD_PARTITION) + { + partition_info *alt_part_info= thd->lex->part_info; + if (alt_part_info) + { + List_iterator<partition_element> new_part_it(alt_part_info->partitions); + partition_element *pe; + while ((pe= new_part_it++)) + { + if (!tab_part_info->has_unique_name(pe)) + { + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_SAME_NAME_PARTITION, ER(ER_SAME_NAME_PARTITION), + pe->partition_name); + alter_info->flags&= ~ALTER_ADD_PARTITION; + thd->lex->part_info= NULL; + break; + } + } + } + } + /* ALTER TABLE DROP PARTITION IF EXISTS */ + if (alter_info->flags & ALTER_DROP_PARTITION) + { + List_iterator<char> names_it(alter_info->partition_names); + char *name; + + while ((name= names_it++)) + { + List_iterator<partition_element> part_it(tab_part_info->partitions); + partition_element *part_elem; + while ((part_elem= part_it++)) + { + if (my_strcasecmp(system_charset_info, + part_elem->partition_name, name) == 0) + break; + } + if (!part_elem) + { + push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, + ER_DROP_PARTITION_NON_EXISTENT, + ER(ER_DROP_PARTITION_NON_EXISTENT), "DROP"); + names_it.remove(); + } + } + if (alter_info->partition_names.elements == 0) + alter_info->flags&= ~ALTER_DROP_PARTITION; + } + } +#endif /*WITH_PARTITION_STORAGE_ENGINE*/ + + /* Clear the ALTER_FOREIGN_KEY flag if nothing other than that set. */ + if (alter_info->flags == ALTER_FOREIGN_KEY) + alter_info->flags= 0; + + DBUG_VOID_RETURN; +} + + +/* SYNOPSIS mysql_compare_tables() table The original table. @@ -5129,12 +5279,11 @@ mysql_compare_tables(TABLE *table, *need_copy_table= ALTER_TABLE_DATA_CHANGED; /* Create the prepared information. */ - if (mysql_prepare_create_table(thd, create_info, - &tmp_alter_info, - (table->s->tmp_table != NO_TMP_TABLE), - &db_options, - table->file, key_info_buffer, - &key_count, 0)) + int create_table_mode= table->s->tmp_table == NO_TMP_TABLE ? + C_ORDINARY_CREATE : C_ALTER_TABLE; + if (mysql_prepare_create_table(thd, create_info, &tmp_alter_info, + &db_options, table->file, key_info_buffer, + &key_count, create_table_mode)) DBUG_RETURN(1); /* Allocate result buffers. */ if (! (*index_drop_buffer= @@ -5195,6 +5344,21 @@ mysql_compare_tables(TABLE *table, thd->calloc(sizeof(void*) * table->s->keys)) == NULL) DBUG_RETURN(1); + tmp_new_field_it.init(tmp_alter_info.create_list); + for (i= 0, f_ptr= table->field, tmp_new_field= tmp_new_field_it++; + (field= *f_ptr); + i++, f_ptr++, tmp_new_field= tmp_new_field_it++) + { + if (field->is_equal(tmp_new_field) == IS_EQUAL_NO && + table->s->tmp_table == NO_TMP_TABLE) + (void) delete_statistics_for_column(thd, table, field); + else if (my_strcasecmp(system_charset_info, + field->field_name, + tmp_new_field->field_name)) + (void) rename_column_in_stat_tables(thd, table, field, + tmp_new_field->field_name); + } + /* Use transformed info to evaluate possibility of in-place ALTER TABLE but use the preserved field to persist modifications. @@ -5255,11 +5419,36 @@ mysql_compare_tables(TABLE *table, if (my_strcasecmp(system_charset_info, field->field_name, tmp_new_field->field_name)) - field->flags|= FIELD_IS_RENAMED; + { + field->flags|= FIELD_IS_RENAMED; + if (table->s->tmp_table == NO_TMP_TABLE) + rename_column_in_stat_tables(thd, table, field, + tmp_new_field->field_name); + } /* Evaluate changes bitmap and send to check_if_incompatible_data() */ if (!(tmp= field->is_equal(tmp_new_field))) { + if (table->s->tmp_table == NO_TMP_TABLE) + { + KEY *key_info= table->key_info; + for (uint i=0; i < table->s->keys; i++, key_info++) + { + if (field->part_of_key.is_set(i)) + { + uint key_parts= table->actual_n_key_parts(key_info); + for (uint j= 0; j < key_parts; j++) + { + if (key_info->key_part[j].fieldnr-1 == field->field_index) + { + (void) delete_statistics_for_index(thd, table, key_info, + j >= key_info->key_parts); + break; + } + } + } + } + } DBUG_PRINT("info", ("!field_is_equal('%s') -> ALTER_TABLE_DATA_CHANGED", new_field->field_name)); DBUG_RETURN(0); @@ -5358,6 +5547,21 @@ mysql_compare_tables(TABLE *table, field= table->field[key_part->fieldnr]; field->flags|= FIELD_IN_ADD_INDEX; } + if (table->s->tmp_table == NO_TMP_TABLE) + { + (void) delete_statistics_for_index(thd, table, table_key, FALSE); + if ((uint) (table_key - table->key_info) == table->s->primary_key) + { + KEY *tab_key_info= table->key_info; + for (uint j=0; j < table->s->keys; j++, tab_key_info++) + { + if (tab_key_info->key_parts != tab_key_info->ext_key_parts) + (void) delete_statistics_for_index(thd, table, tab_key_info, + TRUE); + } + } + } + DBUG_PRINT("info", ("index changed: '%s'", table_key->name)); } /*end of for (; table_key < table_key_end;) */ @@ -5445,6 +5649,7 @@ bool alter_table_manage_keys(TABLE *table, int indexes_were_disabled, switch (keys_onoff) { case ENABLE: + DEBUG_SYNC(table->in_use, "alter_table_enable_indexes"); error= table->file->ha_enable_indexes(HA_KEY_SWITCH_NONUNIQ_SAVE); break; case LEAVE_AS_IS: @@ -5459,7 +5664,8 @@ bool alter_table_manage_keys(TABLE *table, int indexes_were_disabled, { push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_NOTE, ER_ILLEGAL_HA, ER(ER_ILLEGAL_HA), - table->s->table_name.str); + table->file->table_type(), + table->s->db.str, table->s->table_name.str); error= 0; } else if (error) table->file->print_error(error, MYF(0)); @@ -5559,6 +5765,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, uint used_fields= create_info->used_fields; KEY *key_info=table->key_info; bool rc= TRUE; + bool modified_primary_key= FALSE; Create_field *def; Field **f_ptr,*field; DBUG_ENTER("mysql_prepare_alter_table"); @@ -5615,6 +5822,8 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, } if (drop) { + if (table->s->tmp_table == NO_TMP_TABLE) + (void) delete_statistics_for_column(thd, table, field); drop_it.remove(); /* ALTER TABLE DROP COLUMN always changes table data even in cases @@ -5753,7 +5962,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, Collect all keys which isn't in drop list. Add only those for which some fields exists. */ - + for (uint i=0 ; i < table->s->keys ; i++,key_info++) { char *key_name= key_info->name; @@ -5767,12 +5976,27 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, } if (drop) { + if (table->s->tmp_table == NO_TMP_TABLE) + { + (void) delete_statistics_for_index(thd, table, key_info, FALSE); + if (i == table->s->primary_key) + { + KEY *tab_key_info= table->key_info; + for (uint j=0; j < table->s->keys; j++, tab_key_info++) + { + if (tab_key_info->key_parts != tab_key_info->ext_key_parts) + (void) delete_statistics_for_index(thd, table, tab_key_info, + TRUE); + } + } + } drop_it.remove(); continue; } KEY_PART_INFO *key_part= key_info->key_part; key_parts.empty(); + bool delete_index_stat= FALSE; for (uint j=0 ; j < key_info->key_parts ; j++,key_part++) { if (!key_part->field) @@ -5795,7 +6019,12 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, break; } if (!cfield) + { + if (table->s->primary_key == i) + modified_primary_key= TRUE; + delete_index_stat= TRUE; continue; // Field is removed + } key_part_length= key_part->length; if (cfield->field) // Not new field { @@ -5837,6 +6066,15 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, strlen(cfield->field_name), key_part_length)); } + if (table->s->tmp_table == NO_TMP_TABLE) + { + if (delete_index_stat) + (void) delete_statistics_for_index(thd, table, key_info, FALSE); + else if (modified_primary_key && + key_info->key_parts != key_info->ext_key_parts) + (void) delete_statistics_for_index(thd, table, key_info, TRUE); + } + if (key_parts.elements) { KEY_CREATE_INFO key_create_info; @@ -5869,7 +6107,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, key= new Key(key_type, key_name, strlen(key_name), &key_create_info, test(key_info->flags & HA_GENERATED_KEY), - key_parts, key_info->option_list); + key_parts, key_info->option_list, FALSE); new_key_list.push_back(key); } } @@ -5988,13 +6226,11 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, TABLE *table, *new_table= 0; MDL_ticket *mdl_ticket; MDL_request target_mdl_request; - int error= 0; + int error= 0, create_table_mode= C_ALTER_TABLE; char tmp_name[80],old_name[32],new_name_buff[FN_REFLEN + 1]; - char old_name_buff[FN_REFLEN + 1]; char new_alias_buff[FN_REFLEN], *table_name, *db, *new_alias, *alias; char index_file[FN_REFLEN], data_file[FN_REFLEN]; char path[FN_REFLEN + 1]; - char reg_path[FN_REFLEN+1]; ha_rows copied,deleted; handlerton *old_db_type, *new_db_type, *save_old_db_type; enum_alter_table_change_level need_copy_table= ALTER_TABLE_METADATA_ONLY; @@ -6016,6 +6252,9 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, enum ha_extra_function extra_func= thd->locked_tables_mode ? HA_EXTRA_NOT_USED : HA_EXTRA_FORCE_REOPEN; + LEX_STRING old_db_name= { table_list->db, table_list->db_length }; + LEX_STRING old_table_name= { table_list->table_name, + table_list->table_name_length }; DBUG_ENTER("mysql_alter_table"); /* @@ -6046,7 +6285,8 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, (!create_info->db_type || /* unknown engine */ !(create_info->db_type->flags & HTON_SUPPORT_LOG_TABLES))) { - my_error(ER_UNSUPORTED_LOG_ENGINE, MYF(0)); + my_error(ER_UNSUPORTED_LOG_ENGINE, MYF(0), + hton_name(create_info->db_type)->str); DBUG_RETURN(TRUE); } @@ -6061,7 +6301,7 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, } /* - Assign variables table_name, new_name, db, new_db, path, reg_path + Assign variables table_name, new_name, db, new_db, path, to simplify further comparisions: we want to see if it's a RENAME later just by comparing the pointers, avoiding the need for strcmp. */ @@ -6071,7 +6311,6 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, db=table_list->db; if (!new_db || !my_strcasecmp(table_alias_charset, new_db, db)) new_db= db; - build_table_filename(reg_path, sizeof(reg_path) - 1, db, table_name, reg_ext, 0); build_table_filename(path, sizeof(path) - 1, db, table_name, "", 0); mysql_ha_rm_tables(thd, table_list); @@ -6199,12 +6438,10 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, */ build_table_filename(new_name_buff, sizeof(new_name_buff) - 1, new_db, new_name_buff, reg_ext, 0); - build_table_filename(old_name_buff, sizeof(old_name_buff) - 1, - db, table_name, reg_ext, 0); - if (check_table_file_presence(old_name_buff, new_name_buff, new_db, - new_name, new_alias, TRUE)) + if (!access(new_name_buff, F_OK)) { /* Table will be closed in do_command() */ + my_error(ER_TABLE_EXISTS_ERROR, MYF(0), new_alias); goto err; } } @@ -6270,11 +6507,19 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, DBUG_PRINT("info", ("old type: %s new type: %s", ha_resolve_storage_engine_name(old_db_type), ha_resolve_storage_engine_name(new_db_type))); - if (ha_check_storage_engine_flag(old_db_type, HTON_ALTER_NOT_SUPPORTED) || - ha_check_storage_engine_flag(new_db_type, HTON_ALTER_NOT_SUPPORTED)) + if (ha_check_storage_engine_flag(old_db_type, HTON_ALTER_NOT_SUPPORTED)) + { + DBUG_PRINT("info", ("doesn't support alter")); + my_error(ER_ILLEGAL_HA, MYF(0), hton_name(old_db_type)->str, + db, table_name); + goto err; + } + + if (ha_check_storage_engine_flag(new_db_type, HTON_ALTER_NOT_SUPPORTED)) { DBUG_PRINT("info", ("doesn't support alter")); - my_error(ER_ILLEGAL_HA, MYF(0), table_name); + my_error(ER_ILLEGAL_HA, MYF(0), hton_name(new_db_type)->str, + new_db, new_name); goto err; } @@ -6285,35 +6530,13 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, if (!(alter_info->flags & ~(ALTER_RENAME | ALTER_KEYS_ONOFF)) && !table->s->tmp_table) // no need to touch frm { - switch (alter_info->keys_onoff) { - case LEAVE_AS_IS: - break; - case ENABLE: - if (wait_while_table_is_used(thd, table, extra_func, - TDC_RT_REMOVE_NOT_OWN_AND_MARK_NOT_USABLE)) - goto err; - DEBUG_SYNC(thd,"alter_table_enable_indexes"); - error= table->file->ha_enable_indexes(HA_KEY_SWITCH_NONUNIQ_SAVE); - table->s->allow_access_to_protected_table(); - break; - case DISABLE: + if (alter_info->keys_onoff != LEAVE_AS_IS) + { if (wait_while_table_is_used(thd, table, extra_func, TDC_RT_REMOVE_NOT_OWN_AND_MARK_NOT_USABLE)) goto err; - error=table->file->ha_disable_indexes(HA_KEY_SWITCH_NONUNIQ_SAVE); + error= alter_table_manage_keys(table, 0, alter_info->keys_onoff); table->s->allow_access_to_protected_table(); - break; - default: - DBUG_ASSERT(FALSE); - error= 0; - break; - } - if (error == HA_ERR_WRONG_COMMAND) - { - error= 0; - push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, - ER_ILLEGAL_HA, ER(ER_ILLEGAL_HA), - table->alias.c_ptr()); } if (!error && (new_name != table_name || new_db != db)) @@ -6347,6 +6570,12 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, else { *fn_ext(new_name)=0; + + LEX_STRING new_db_name= { new_db, strlen(new_db) }; + LEX_STRING new_table_name= { new_alias, strlen(new_alias) }; + (void) rename_table_in_stat_tables(thd, &old_db_name, &old_table_name, + &new_db_name, &new_table_name); + if (mysql_rename_table(old_db_type,db,table_name,new_db,new_alias, 0)) error= -1; else if (Table_triggers_list::change_table_name(thd, db, @@ -6365,7 +6594,8 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, error= 0; push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, ER_ILLEGAL_HA, ER(ER_ILLEGAL_HA), - table->alias.c_ptr()); + table->file->table_type(), + table->s->db.str, table->s->table_name.str); } if (!error) @@ -6398,6 +6628,17 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, DBUG_RETURN(error); } + handle_if_exists_options(thd, table, alter_info); + + /* Look if we have to do anything at all. */ + /* Normally ALTER can become NOOP only after handling */ + /* the IF (NOT) EXISTS options. */ + if (alter_info->flags == 0) + { + copied= deleted= 0; + goto end_temporary; + } + /* We have to do full alter table. */ #ifdef WITH_PARTITION_STORAGE_ENGINE @@ -6433,6 +6674,7 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, need_copy_table= alter_info->change_level; set_table_default_charset(thd, create_info, db); + promote_first_timestamp_column(&alter_info->create_list); if (thd->variables.old_alter_table || (table->s->db_type() != create_info->db_type) @@ -6657,13 +6899,17 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, need_copy_table, need_lock_for_indexes)); } - /* - better have a negative test here, instead of positive, like - alter_info->flags & ALTER_ADD_COLUMN|ALTER_ADD_INDEX|... - so that ALTER TABLE won't break when somebody will add new flag - */ if (need_copy_table == ALTER_TABLE_METADATA_ONLY) - create_info->frm_only= 1; + { + char frm_name[FN_REFLEN+1]; + strxnmov(frm_name, sizeof(frm_name), path, reg_ext, NullS); + /* + C_ALTER_TABLE_FRM_ONLY can only be used if old frm exists. + discovering frm-less engines cannot enjoy this optimization. + */ + if (!my_access(frm_name, F_OK)) + create_table_mode= C_ALTER_TABLE_FRM_ONLY; + } #ifdef WITH_PARTITION_STORAGE_ENGINE if (table_for_fast_alter_partition) @@ -6732,13 +6978,13 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, my_sleep(100000);); /* Create a table with a temporary name. - With create_info->frm_only == 1 this creates a .frm file only and + With C_ALTER_TABLE_FRM_ONLY this creates a .frm file only and we keep the original row format. We don't log the statement, it will be logged later. */ if (need_copy_table == ALTER_TABLE_METADATA_ONLY) { - DBUG_ASSERT(create_info->frm_only); + DBUG_ASSERT(create_table_mode == C_ALTER_TABLE_FRM_ONLY); /* Ensure we keep the original table format */ create_info->table_options= ((create_info->table_options & ~HA_OPTION_PACK_RECORD) | @@ -6747,10 +6993,8 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, } tmp_disable_binlog(thd); create_info->options|=HA_CREATE_TMP_ALTER; - error= mysql_create_table_no_lock(thd, new_db, tmp_name, - create_info, - alter_info, - 1, 0, NULL); + error= mysql_create_table_no_lock(thd, new_db, tmp_name, create_info, + alter_info, NULL, create_table_mode); reenable_binlog(thd); if (error) goto err; @@ -6779,7 +7023,8 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, build_table_filename(path, sizeof(path) - 1, new_db, tmp_name, "", FN_IS_TMP); /* Open our intermediate table. */ - new_table= open_table_uncached(thd, path, new_db, tmp_name, TRUE); + new_table= open_table_uncached(thd, new_db_type, path, + new_db, tmp_name, TRUE); } if (!new_table) goto err_new_table_cleanup; @@ -6812,8 +7057,6 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, */ if (new_table && !(new_table->file->ha_table_flags() & HA_NO_COPY_ON_ALTER)) { - /* We don't want update TIMESTAMP fields during ALTER TABLE. */ - new_table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; new_table->next_number_field=new_table->found_next_number_field; DBUG_EXECUTE_IF("abort_copy_table", { my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0)); @@ -7131,6 +7374,15 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, table is renamed and the SE is also changed, then an intermediate table is created and the additional call will not take place. */ + + if (new_name != table_name || new_db != db) + { + LEX_STRING new_db_name= { new_db, strlen(new_db) }; + LEX_STRING new_table_name= { new_name, strlen(new_name) }; + (void) rename_table_in_stat_tables(thd, &old_db_name, &old_table_name, + &new_db_name, &new_table_name); + } + if (need_copy_table == ALTER_TABLE_METADATA_ONLY) { DBUG_ASSERT(new_db_type == old_db_type); @@ -7230,9 +7482,8 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, } /* Tell the handler that a new frm file is in place. */ - error= t_table_list->table->file->ha_create_handler_files(path, NULL, - CHF_INDEX_FLAG, - create_info); + error= t_table_list->table->file->ha_create_partitioning_metadata(path, NULL, + CHF_INDEX_FLAG); DBUG_ASSERT(thd->open_tables == t_table_list->table); close_thread_table(thd, &thd->open_tables); @@ -7255,7 +7506,7 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, DBUG_ASSERT(!(mysql_bin_log.is_open() && thd->is_current_stmt_binlog_format_row() && - (create_info->options & HA_LEX_CREATE_TMP_TABLE))); + (create_info->tmp_table()))); if (write_bin_log(thd, TRUE, thd->query(), thd->query_length())) DBUG_RETURN(TRUE); @@ -7286,7 +7537,8 @@ err_new_table_cleanup: } else (void) quick_rm_table(new_db_type, new_db, tmp_name, - create_info->frm_only ? FN_IS_TMP | FRM_ONLY : FN_IS_TMP); + create_table_mode == C_ALTER_TABLE_FRM_ONLY ? + FN_IS_TMP | FRM_ONLY : FN_IS_TMP); err: #ifdef WITH_PARTITION_STORAGE_ENGINE @@ -7413,11 +7665,13 @@ copy_data_between_tables(THD *thd, TABLE *from,TABLE *to, List<Item> fields; List<Item> all_fields; ha_rows examined_rows; + ha_rows found_rows; bool auto_increment_field_copied= 0; ulonglong save_sql_mode= thd->variables.sql_mode; ulonglong prev_insert_id, time_to_report_progress; List_iterator<Create_field> it(create); Create_field *def; + Field **dfield_ptr= to->default_field; DBUG_ENTER("copy_data_between_tables"); /* Two or 3 stages; Sorting, copying data and update indexes */ @@ -7430,11 +7684,11 @@ copy_data_between_tables(THD *thd, TABLE *from,TABLE *to, if (!(copy= new Copy_field[to->s->fields])) goto err; /* purecov: inspected */ + /* We need external lock before we can disable/enable keys */ if (to->file->ha_external_lock(thd, F_WRLCK)) goto err; errpos= 2; - /* We need external lock before we can disable/enable keys */ alter_table_manage_keys(to, from->file->indexes_are_disabled(), keys_onoff); /* We can abort alter table for any table type */ @@ -7443,10 +7697,12 @@ copy_data_between_tables(THD *thd, TABLE *from,TABLE *to, MODE_STRICT_ALL_TABLES)); from->file->info(HA_STATUS_VARIABLE); - to->file->ha_start_bulk_insert(from->file->stats.records); + to->file->ha_start_bulk_insert(from->file->stats.records, + ignore ? 0 : HA_CREATE_UNIQUE_INDEX_BY_SORT); errpos= 3; copy_end=copy; + to->s->default_fields= 0; for (Field **ptr=to->field ; *ptr ; ptr++) { def=it++; @@ -7466,8 +7722,23 @@ copy_data_between_tables(THD *thd, TABLE *from,TABLE *to, } (copy_end++)->set(*ptr,def->field,0); } - + else + { + /* + Update the set of auto-update fields to contain only the new fields + added to the table. Only these fields should be updated automatically. + Old fields keep their current values, and therefore should not be + present in the set of autoupdate fields. + */ + if ((*ptr)->has_insert_default_function()) + { + *(dfield_ptr++)= *ptr; + ++to->s->default_fields; + } + } } + if (dfield_ptr) + *dfield_ptr= NULL; if (order) { @@ -7484,7 +7755,8 @@ copy_data_between_tables(THD *thd, TABLE *from,TABLE *to, else { from->sort.io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE), - MYF(MY_FAE | MY_ZEROFILL)); + MYF(MY_FAE | MY_ZEROFILL | + MY_THREAD_SPECIFIC)); bzero((char *) &tables, sizeof(tables)); tables.table= from; tables.alias= tables.table_name= from->s->table_name.str; @@ -7496,8 +7768,9 @@ copy_data_between_tables(THD *thd, TABLE *from,TABLE *to, &tables, fields, all_fields, order) || !(sortorder= make_unireg_sortorder(order, &length, NULL)) || (from->sort.found_records= filesort(thd, from, sortorder, length, - (SQL_SELECT *) 0, HA_POS_ERROR, - 1, &examined_rows)) == + NULL, HA_POS_ERROR, + true, + &examined_rows, &found_rows)) == HA_POS_ERROR) goto err; } @@ -7557,6 +7830,11 @@ copy_data_between_tables(THD *thd, TABLE *from,TABLE *to, prev_insert_id= to->file->next_insert_id; if (to->vfield) update_virtual_fields(thd, to, VCOL_UPDATE_FOR_WRITE); + if (to->default_field && to->update_default_fields()) + { + error= 1; + break; + } if (thd->is_error()) { error= 1; @@ -7873,7 +8151,7 @@ static bool check_engine(THD *thd, const char *db_name, ha_resolve_storage_engine_name(*new_engine), table_name); } - if (create_info->options & HA_LEX_CREATE_TMP_TABLE && + if (create_info->tmp_table() && ha_check_storage_engine_flag(*new_engine, HTON_TEMPORARY_NOT_SUPPORTED)) { if (create_info->used_fields & HA_CREATE_USED_ENGINE) diff --git a/sql/sql_table.h b/sql/sql_table.h index c8ecb3ced4f..837bcdcbbfb 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -1,4 +1,5 @@ -/* Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2006, 2013, Oracle and/or its affiliates. + Copyright (c) 2011, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -18,6 +19,7 @@ #include "my_global.h" /* my_bool */ #include "my_sys.h" // pthread_mutex_t +#include "m_string.h" // LEX_CUSTRING class Alter_info; class Create_field; @@ -25,8 +27,9 @@ struct TABLE_LIST; class THD; struct TABLE; struct handlerton; +class handler; typedef struct st_ha_check_opt HA_CHECK_OPT; -typedef struct st_ha_create_information HA_CREATE_INFO; +struct HA_CREATE_INFO; typedef struct st_key KEY; typedef struct st_key_cache KEY_CACHE; typedef struct st_lock_param_type ALTER_PARTITION_PARAM_TYPE; @@ -121,11 +124,8 @@ enum enum_explain_filename_mode /** Don't check foreign key constraints while renaming table */ #define NO_FK_CHECKS (1 << 4) -uint filename_to_tablename(const char *from, char *to, uint to_length -#ifndef DBUG_OFF - , bool stay_quiet = false -#endif /* DBUG_OFF */ - ); +uint filename_to_tablename(const char *from, char *to, uint to_length, + bool stay_quiet = false); uint tablename_to_filename(const char *from, char *to, uint to_length); uint check_n_cut_mysql50_prefix(const char *from, char *to, uint to_length); bool check_mysql50_prefix(const char *name); @@ -133,18 +133,57 @@ uint build_table_filename(char *buff, size_t bufflen, const char *db, const char *table, const char *ext, uint flags); uint build_table_shadow_filename(char *buff, size_t bufflen, ALTER_PARTITION_PARAM_TYPE *lpt); -bool check_table_file_presence(char *old_path, char *path, const char *db, - const char *table_name, const char *alias, - bool issue_error); bool mysql_create_table(THD *thd, TABLE_LIST *create_table, HA_CREATE_INFO *create_info, Alter_info *alter_info); + +/* + mysql_create_table_no_lock can be called in one of the following + mutually exclusive situations: + + - Just a normal ordinary CREATE TABLE statement that explicitly + defines the table structure. + + - CREATE TABLE ... SELECT. It is special, because only in this case, + the list of fields is allowed to have duplicates, as long as one of the + duplicates comes from the select list, and the other doesn't. For + example in + + CREATE TABLE t1 (a int(5) NOT NUL) SELECT b+10 as a FROM t2; + + the list in alter_info->create_list will have two fields `a`. + + - ALTER TABLE, that creates a temporary table #sql-xxx, which will be later + renamed to replace the original table. + + - ALTER TABLE as above, but which only modifies the frm file, it only + creates an frm file for the #sql-xxx, the table in the engine is not + created. + + - Assisted discovery, CREATE TABLE statement without the table structure. + + These situations are distinguished by the following "create table mode" + values, where a CREATE ... SELECT is denoted by any non-negative number + (which should be the number of fields in the SELECT ... part), and other + cases use constants as defined below. +*/ +#define C_CREATE_SELECT(X) ((X) > 0 ? (X) : 0) +#define C_ORDINARY_CREATE 0 +#define C_ALTER_TABLE -1 +#define C_ALTER_TABLE_FRM_ONLY -2 +#define C_ASSISTED_DISCOVERY -3 + bool mysql_create_table_no_lock(THD *thd, const char *db, const char *table_name, HA_CREATE_INFO *create_info, + Alter_info *alter_info, bool *is_trans, + int create_table_mode); + +handler *mysql_create_frm_image(THD *thd, + const char *db, const char *table_name, + HA_CREATE_INFO *create_info, Alter_info *alter_info, - bool tmp_table, uint select_field_count, - bool *is_trans); + int create_table_mode, LEX_CUSTRING *frm); bool mysql_prepare_alter_table(THD *thd, TABLE *table, HA_CREATE_INFO *create_info, Alter_info *alter_info); @@ -189,7 +228,6 @@ void close_cached_table(THD *thd, TABLE *table); void sp_prepare_create_field(THD *thd, Create_field *sql_field); int prepare_create_field(Create_field *sql_field, uint *blob_columns, - int *timestamps, int *timestamps_with_niladic, longlong table_flags); CHARSET_INFO* get_sql_field_charset(Create_field *sql_field, HA_CREATE_INFO *create_info); @@ -210,6 +248,9 @@ void execute_ddl_log_recovery(); bool execute_ddl_log_entry(THD *thd, uint first_entry); bool check_duplicate_warning(THD *thd, char *msg, ulong length); +template<typename T> class List; +void promote_first_timestamp_column(List<Create_field> *column_definitions); + /* These prototypes where under INNODB_COMPATIBILITY_HOOKS. */ diff --git a/sql/sql_test.cc b/sql/sql_test.cc index 311400c0c6c..93b35b4918f 100644 --- a/sql/sql_test.cc +++ b/sql/sql_test.cc @@ -484,7 +484,9 @@ static void display_table_locks(void) void *saved_base; DYNAMIC_ARRAY saved_table_locks; - (void) my_init_dynamic_array(&saved_table_locks,sizeof(TABLE_LOCK_INFO), cached_open_tables() + 20,50); + (void) my_init_dynamic_array(&saved_table_locks,sizeof(TABLE_LOCK_INFO), + cached_open_tables() + 20, 50, + MYF(MY_THREAD_SPECIFIC)); mysql_mutex_lock(&THR_LOCK_lock); for (list= thr_lock_thread_list; list; list= list_rest(list)) { diff --git a/sql/sql_time.cc b/sql/sql_time.cc index efcde017671..507e77e7b2d 100644 --- a/sql/sql_time.cc +++ b/sql/sql_time.cc @@ -1048,13 +1048,13 @@ calc_time_diff(MYSQL_TIME *l_time1, MYSQL_TIME *l_time2, int l_sign, longlong *s (uint) l_time2->day); } - microseconds= ((longlong)days*LL(86400) + + microseconds= ((longlong)days*86400LL + (longlong)(l_time1->hour*3600L + l_time1->minute*60L + l_time1->second) - l_sign*(longlong)(l_time2->hour*3600L + l_time2->minute*60L + - l_time2->second)) * LL(1000000) + + l_time2->second)) * 1000000LL + (longlong)l_time1->second_part - l_sign*(longlong)l_time2->second_part; diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index dfb6ace57f4..d0c5bcf03c0 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -443,7 +443,7 @@ bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create) if (!create) { - bool if_exists= thd->lex->drop_if_exists; + bool if_exists= thd->lex->check_exists; /* Protect the query table list from the temporary and potentially @@ -663,46 +663,8 @@ bool Table_triggers_list::create_trigger(THD *thd, TABLE_LIST *tables, return 1; } - if (!lex->definer) - { - /* - DEFINER-clause is missing. - - If we are in slave thread, this means that we received CREATE TRIGGER - from the master, that does not support definer in triggers. So, we - should mark this trigger as non-SUID. Note that this does not happen - when we parse triggers' definitions during opening .TRG file. - LEX::definer is ignored in that case. - - Otherwise, we should use CURRENT_USER() as definer. - - NOTE: when CREATE TRIGGER statement is allowed to be executed in PS/SP, - it will be required to create the definer below in persistent MEM_ROOT - of PS/SP. - */ - - if (!thd->slave_thread) - { - if (!(lex->definer= create_default_definer(thd))) - return 1; - } - } - - /* - If the specified definer differs from the current user, we should check - that the current user has SUPER privilege (in order to create trigger - under another user one must have SUPER privilege). - */ - - if (lex->definer && - (strcmp(lex->definer->user.str, thd->security_ctx->priv_user) || - my_strcasecmp(system_charset_info, - lex->definer->host.str, - thd->security_ctx->priv_host))) - { - if (check_global_access(thd, SUPER_ACL)) - return TRUE; - } + if (sp_process_definer(thd)) + return 1; /* Let us check if all references to fields in old/new versions of row in @@ -794,29 +756,14 @@ bool Table_triggers_list::create_trigger(THD *thd, TABLE_LIST *tables, *trg_sql_mode= thd->variables.sql_mode; -#ifndef NO_EMBEDDED_ACCESS_CHECKS - if (lex->definer && !is_acl_user(lex->definer->host.str, - lex->definer->user.str)) - { - push_warning_printf(thd, - MYSQL_ERROR::WARN_LEVEL_NOTE, - ER_NO_SUCH_USER, - ER(ER_NO_SUCH_USER), - lex->definer->user.str, - lex->definer->host.str); - } -#endif /* NO_EMBEDDED_ACCESS_CHECKS */ - - if (lex->definer) + if (lex->sphead->m_chistics->suid != SP_IS_NOT_SUID) { /* SUID trigger. */ definer_user= lex->definer->user; definer_host= lex->definer->host; - trg_definer->str= trg_definer_holder; - trg_definer->length= strxmov(trg_definer->str, definer_user.str, "@", - definer_host.str, NullS) - trg_definer->str; + lex->definer->set_lex_string(trg_definer, trg_definer_holder); } else { @@ -854,7 +801,7 @@ bool Table_triggers_list::create_trigger(THD *thd, TABLE_LIST *tables, stmt_query->append(STRING_WITH_LEN("CREATE ")); - if (trg_definer) + if (lex->sphead->m_chistics->suid != SP_IS_NOT_SUID) { /* Append definer-clause if the trigger is SUID (a usual trigger in @@ -1776,7 +1723,7 @@ bool Table_triggers_list::drop_all_triggers(THD *thd, char *db, char *name) DBUG_ENTER("drop_all_triggers"); bzero(&table, sizeof(table)); - init_sql_alloc(&table.mem_root, 8192, 0); + init_sql_alloc(&table.mem_root, 8192, 0, MYF(0)); if (Table_triggers_list::check_n_load(thd, db, name, &table, 1)) { @@ -1996,7 +1943,7 @@ bool Table_triggers_list::change_table_name(THD *thd, const char *db, DBUG_ENTER("change_table_name"); bzero(&table, sizeof(table)); - init_sql_alloc(&table.mem_root, 8192, 0); + init_sql_alloc(&table.mem_root, 8192, 0, MYF(0)); /* This method interfaces the mysql server code protected by diff --git a/sql/sql_truncate.cc b/sql/sql_truncate.cc index d47fb24eaba..19ce553f5ce 100644 --- a/sql/sql_truncate.cc +++ b/sql/sql_truncate.cc @@ -258,27 +258,18 @@ static bool recreate_temporary_table(THD *thd, TABLE *table) { bool error= TRUE; TABLE_SHARE *share= table->s; - HA_CREATE_INFO create_info; handlerton *table_type= table->s->db_type(); DBUG_ENTER("recreate_temporary_table"); - memset(&create_info, 0, sizeof(create_info)); - create_info.options|= HA_LEX_CREATE_TMP_TABLE; - table->file->info(HA_STATUS_AUTO | HA_STATUS_NO_LOCK); /* Don't free share. */ close_temporary_table(thd, table, FALSE, FALSE); - /* - We must use share->normalized_path.str since for temporary tables it - differs from what dd_recreate_table() would generate based - on table and schema names. - */ - ha_create_table(thd, share->normalized_path.str, share->db.str, - share->table_name.str, &create_info, 1); + dd_recreate_table(thd, share->db.str, share->table_name.str, + share->normalized_path.str); - if (open_table_uncached(thd, share->path.str, share->db.str, + if (open_table_uncached(thd, table_type, share->path.str, share->db.str, share->table_name.str, TRUE)) { error= FALSE; @@ -350,9 +341,27 @@ bool Truncate_statement::lock_table(THD *thd, TABLE_LIST *table_ref, MYSQL_OPEN_SKIP_TEMPORARY)) DBUG_RETURN(TRUE); - if (dd_check_storage_engine_flag(thd, table_ref->db, table_ref->table_name, - HTON_CAN_RECREATE, hton_can_recreate)) + handlerton *hton; + if (!ha_table_exists(thd, table_ref->db, table_ref->table_name, &hton) || + hton == view_pseudo_hton) + { + my_error(ER_NO_SUCH_TABLE, MYF(0), table_ref->db, table_ref->table_name); DBUG_RETURN(TRUE); + } + + if (!hton) + { + /* + The table exists, but its storage engine is unknown, perhaps not + loaded at the moment. We need to open and parse the frm to know the + storage engine in question, so let's proceed with the truncation and + try to open the table. This will produce the correct error message + about unknown engine. + */ + *hton_can_recreate= false; + } + else + *hton_can_recreate= hton->flags & HTON_CAN_RECREATE; } /* diff --git a/sql/sql_udf.cc b/sql/sql_udf.cc index e5fac48a750..fdc932957b2 100644 --- a/sql/sql_udf.cc +++ b/sql/sql_udf.cc @@ -151,7 +151,7 @@ void udf_init() mysql_rwlock_init(key_rwlock_THR_LOCK_udf, &THR_LOCK_udf); - init_sql_alloc(&mem, UDF_ALLOC_BLOCK_SIZE, 0); + init_sql_alloc(&mem, UDF_ALLOC_BLOCK_SIZE, 0, MYF(0)); THD *new_thd = new THD; if (!new_thd || my_hash_init(&udf_hash,system_charset_info,32,0,0,get_hash_key, NULL, 0)) @@ -258,7 +258,7 @@ end: close_mysql_tables(new_thd); delete new_thd; /* Remember that we don't have a THD */ - my_pthread_setspecific_ptr(THR_THD, 0); + set_current_thd(0); DBUG_VOID_RETURN; } @@ -429,7 +429,6 @@ int mysql_create_function(THD *thd,udf_func *udf) TABLE *table; TABLE_LIST tables; udf_func *u_d; - bool save_binlog_row_based; DBUG_ENTER("mysql_create_function"); if (!initialized) @@ -460,13 +459,6 @@ int mysql_create_function(THD *thd,udf_func *udf) DBUG_RETURN(1); } - /* - Turn off row binlogging of this statement and use statement-based - so that all supporting tables are updated for CREATE FUNCTION command. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - tables.init_one_table(STRING_WITH_LEN("mysql"), STRING_WITH_LEN("func"), "func", TL_WRITE); table= open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT); @@ -538,27 +530,14 @@ int mysql_create_function(THD *thd,udf_func *udf) /* Binlog the create function. */ if (write_bin_log(thd, TRUE, thd->query(), thd->query_length())) - { - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(1); - } - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); + DBUG_RETURN(0); err: if (new_dl) dlclose(dl); mysql_rwlock_unlock(&THR_LOCK_udf); - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(1); } @@ -570,7 +549,6 @@ int mysql_drop_function(THD *thd,const LEX_STRING *udf_name) udf_func *udf; char *exact_name_str; uint exact_name_len; - bool save_binlog_row_based; DBUG_ENTER("mysql_drop_function"); if (!initialized) @@ -582,13 +560,6 @@ int mysql_drop_function(THD *thd,const LEX_STRING *udf_name) DBUG_RETURN(1); } - /* - Turn off row binlogging of this statement and use statement-based - so that all supporting tables are updated for DROP FUNCTION command. - */ - if ((save_binlog_row_based= thd->is_current_stmt_binlog_format_row())) - thd->clear_current_stmt_binlog_format_row(); - tables.init_one_table(STRING_WITH_LEN("mysql"), STRING_WITH_LEN("func"), "func", TL_WRITE); table= open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT); @@ -631,24 +602,12 @@ int mysql_drop_function(THD *thd,const LEX_STRING *udf_name) while binlogging, to avoid binlog inconsistency. */ if (write_bin_log(thd, TRUE, thd->query(), thd->query_length())) - { - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(1); - } - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); + DBUG_RETURN(0); + err: mysql_rwlock_unlock(&THR_LOCK_udf); - /* Restore the state of binlog format */ - DBUG_ASSERT(!thd->is_current_stmt_binlog_format_row()); - if (save_binlog_row_based) - thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(1); } diff --git a/sql/sql_udf.h b/sql/sql_udf.h index cdb15b9e0f5..4aa055b9858 100644 --- a/sql/sql_udf.h +++ b/sql/sql_udf.h @@ -103,14 +103,14 @@ class udf_handler :public Sql_alloc if (get_arguments()) { *null_value=1; - return LL(0); + return 0; } Udf_func_longlong func= (Udf_func_longlong) u_d->func; longlong tmp=func(&initid, &f_args, &is_null, &error); if (is_null || error) { *null_value=1; - return LL(0); + return 0; } *null_value=0; return tmp; diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 1d4ceb6245d..6cfdd9ebee8 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -63,7 +63,7 @@ int select_union::send_data(List<Item> &values) return 0; if (table->no_rows_with_nulls) table->null_catch_flags= CHECK_ROW_FOR_NULLS_TO_REJECT; - fill_record(thd, table->field, values, TRUE, FALSE); + fill_record(thd, table, table->field, values, TRUE, FALSE); if (thd->is_error()) return 1; if (table->no_rows_with_nulls) @@ -632,6 +632,7 @@ bool st_select_lex_unit::exec() ha_rows examined_rows= 0; bool first_execution= !executed; DBUG_ENTER("st_select_lex_unit::exec"); + bool was_executed= executed; if (executed && !uncacheable && !describe) DBUG_RETURN(FALSE); @@ -639,7 +640,14 @@ bool st_select_lex_unit::exec() if (!(uncacheable & ~UNCACHEABLE_EXPLAIN) && item) item->make_const(); - if ((saved_error= optimize())) + saved_error= optimize(); + + create_explain_query_if_not_exists(thd->lex, thd->mem_root); + + if (!saved_error && !was_executed) + save_union_explain(thd->lex->explain); + + if (saved_error) DBUG_RETURN(saved_error); if (uncacheable || !item || !item->assigned() || describe) @@ -742,6 +750,8 @@ bool st_select_lex_unit::exec() } } + DBUG_EXECUTE_IF("show_explain_probe_union_read", + dbug_serve_apcs(thd, 1);); /* Send result to 'result' */ saved_error= TRUE; { @@ -790,6 +800,9 @@ bool st_select_lex_unit::exec() */ if (!fake_select_lex->ref_pointer_array) fake_select_lex->n_child_sum_items+= global_parameters->n_sum_items; + + if (!was_executed) + save_union_explain_part2(thd->lex->explain); saved_error= mysql_select(thd, &fake_select_lex->ref_pointer_array, &result_table_list, diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 0cedf50e4a2..a97ad727d14 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -32,6 +32,7 @@ #include "sql_view.h" // check_key_in_view #include "sp_head.h" #include "sql_trigger.h" +#include "sql_statistics.h" #include "probes_mysql.h" #include "debug_sync.h" #include "key.h" // is_key_used @@ -259,7 +260,7 @@ int mysql_update(THD *thd, bool can_compare_record; int res; int error, loc_error; - uint used_index, dup_key_found; + uint dup_key_found; bool need_sort= TRUE; bool reverse= FALSE; #ifndef NO_EMBEDDED_ACCESS_CHECKS @@ -269,14 +270,18 @@ int mysql_update(THD *thd, ha_rows updated, found; key_map old_covering_keys; TABLE *table; - SQL_SELECT *select; + SQL_SELECT *select= NULL; READ_RECORD info; SELECT_LEX *select_lex= &thd->lex->select_lex; ulonglong id; List<Item> all_fields; killed_state killed_status= NOT_KILLED; + Update_plan query_plan(thd->mem_root); + query_plan.index= MAX_KEY; + query_plan.using_filesort= FALSE; DBUG_ENTER("mysql_update"); + create_explain_query(thd->lex, thd->mem_root); if (open_tables(thd, &table_list, &table_count, 0)) DBUG_RETURN(1); @@ -309,10 +314,14 @@ int mysql_update(THD *thd, my_error(ER_NON_UPDATABLE_TABLE, MYF(0), table_list->alias, "UPDATE"); DBUG_RETURN(1); } + query_plan.updating_a_view= test(table_list->view); + /* Calculate "table->covering_keys" based on the WHERE */ table->covering_keys= table->s->keys_in_use; table->quick_keys.clear_all(); + query_plan.select_lex= &thd->lex->select_lex; + query_plan.table= table; #ifndef NO_EMBEDDED_ACCESS_CHECKS /* Force privilege re-checking for views after they have been opened. */ want_privilege= (table_list->view ? UPDATE_ACL : @@ -342,19 +351,8 @@ int mysql_update(THD *thd, my_error(ER_NON_UPDATABLE_TABLE, MYF(0), table_list->alias, "UPDATE"); DBUG_RETURN(1); } - if (table->timestamp_field) - { - // Don't set timestamp column if this is modified - if (bitmap_is_set(table->write_set, - table->timestamp_field->field_index)) - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; - else - { - if (((uint) table->timestamp_field_type) & TIMESTAMP_AUTO_SET_ON_UPDATE) - bitmap_set_bit(table->write_set, - table->timestamp_field->field_index); - } - } + if (table->default_field) + table->mark_default_fields_for_write(); #ifndef NO_EMBEDDED_ACCESS_CHECKS /* Check values */ @@ -380,7 +378,12 @@ int mysql_update(THD *thd, Item::cond_result cond_value; conds= remove_eq_conds(thd, conds, &cond_value); if (cond_value == Item::COND_FALSE) + { limit= 0; // Impossible WHERE + query_plan.set_impossible_where(); + if (thd->lex->describe) + goto exit_without_my_ok; + } } /* @@ -389,7 +392,7 @@ int mysql_update(THD *thd, to compare records and detect data change. */ if ((table->file->ha_table_flags() & HA_PARTIAL_COLUMN_READ) && - (((uint) table->timestamp_field_type) & TIMESTAMP_AUTO_SET_ON_UPDATE)) + table->default_field && table->has_default_function(true)) bitmap_union(table->read_set, table->write_set); // Don't count on usage of 'only index' when calculating which key to use table->covering_keys.clear_all(); @@ -398,17 +401,27 @@ int mysql_update(THD *thd, if (prune_partitions(thd, table, conds)) { free_underlaid_joins(thd, select_lex); + + query_plan.set_no_partitions(); + if (thd->lex->describe) + goto exit_without_my_ok; + my_ok(thd); // No matching records DBUG_RETURN(0); } #endif /* Update the table->file->stats.records number */ table->file->info(HA_STATUS_VARIABLE | HA_STATUS_NO_LOCK); + set_statistics_for_table(thd, table); select= make_select(table, 0, 0, conds, 0, &error); if (error || !limit || thd->is_error() || (select && select->check_quick(thd, safe_update, limit))) { + query_plan.set_impossible_where(); + if (thd->lex->describe) + goto exit_without_my_ok; + delete select; free_underlaid_joins(thd, select_lex); /* @@ -443,20 +456,25 @@ int mysql_update(THD *thd, table->update_const_key_parts(conds); order= simple_remove_const(order, conds); + query_plan.scanned_rows= select? select->records: table->file->stats.records; if (select && select->quick && select->quick->unique_key_range()) { // Single row select (always "ordered"): Ok to use with key field UPDATE need_sort= FALSE; - used_index= MAX_KEY; + query_plan.index= MAX_KEY; used_key_is_modified= FALSE; } else { - used_index= get_index_for_order(order, table, select, limit, - &need_sort, &reverse); + ha_rows scanned_limit= query_plan.scanned_rows; + query_plan.index= get_index_for_order(order, table, select, limit, + &scanned_limit, &need_sort, &reverse); + if (!need_sort) + query_plan.scanned_rows= scanned_limit; + if (select && select->quick) { - DBUG_ASSERT(need_sort || used_index == select->quick->index); + DBUG_ASSERT(need_sort || query_plan.index == select->quick->index); used_key_is_modified= (!select->quick->unique_key_range() && select->quick->is_keys_used(table->write_set)); } @@ -464,18 +482,47 @@ int mysql_update(THD *thd, { if (need_sort) { // Assign table scan index to check below for modified key fields: - used_index= table->file->key_used_on_scan; + query_plan.index= table->file->key_used_on_scan; } - if (used_index != MAX_KEY) + if (query_plan.index != MAX_KEY) { // Check if we are modifying a key that we are used to search with: - used_key_is_modified= is_key_used(table, used_index, table->write_set); + used_key_is_modified= is_key_used(table, query_plan.index, table->write_set); } } } - + + /* + Query optimization is finished at this point. + - Save the decisions in the query plan + - if we're running EXPLAIN UPDATE, get out + */ + query_plan.select= select; + query_plan.possible_keys= select? select->possible_keys: key_map(0); + if (used_key_is_modified || order || partition_key_modified(table, table->write_set)) { + if (order && (need_sort || used_key_is_modified)) + query_plan.using_filesort= true; + else + query_plan.using_io_buffer= true; + } + + + /* + Ok, we have generated a query plan for the UPDATE. + - if we're running EXPLAIN UPDATE, goto produce explain output + - otherwise, execute the query plan + */ + if (thd->lex->describe) + goto exit_without_my_ok; + query_plan.save_explain_data(thd->lex->explain); + + DBUG_EXECUTE_IF("show_explain_probe_update_exec_start", + dbug_serve_apcs(thd, 1);); + + if (query_plan.using_filesort || query_plan.using_io_buffer) + { /* We can't update table directly; We must first search after all matching rows before updating the table! @@ -485,13 +532,13 @@ int mysql_update(THD *thd, DBUG_ASSERT(table->read_set == &table->def_read_set); DBUG_ASSERT(table->write_set == &table->def_write_set); - if (used_index < MAX_KEY && old_covering_keys.is_set(used_index)) - table->add_read_columns_used_by_index(used_index); + if (query_plan.index < MAX_KEY && old_covering_keys.is_set(query_plan.index)) + table->add_read_columns_used_by_index(query_plan.index); else table->use_all_columns(); /* note: We avoid sorting if we sort on the used index */ - if (order && (need_sort || used_key_is_modified)) + if (query_plan.using_filesort) { /* Doing an ORDER BY; Let filesort find and sort the rows we are going @@ -501,13 +548,16 @@ int mysql_update(THD *thd, uint length= 0; SORT_FIELD *sortorder; ha_rows examined_rows; + ha_rows found_rows; table->sort.io_cache = (IO_CACHE *) my_malloc(sizeof(IO_CACHE), - MYF(MY_FAE | MY_ZEROFILL)); + MYF(MY_FAE | MY_ZEROFILL | + MY_THREAD_SPECIFIC)); if (!(sortorder=make_unireg_sortorder(order, &length, NULL)) || (table->sort.found_records= filesort(thd, table, sortorder, length, - select, limit, 1, - &examined_rows)) + select, limit, + true, + &examined_rows, &found_rows)) == HA_POS_ERROR) { goto err; @@ -543,22 +593,22 @@ int mysql_update(THD *thd, /* When we get here, we have one of the following options: - A. used_index == MAX_KEY + A. query_plan.index == MAX_KEY This means we should use full table scan, and start it with init_read_record call - B. used_index != MAX_KEY + B. query_plan.index != MAX_KEY B.1 quick select is used, start the scan with init_read_record B.2 quick select is not used, this is full index scan (with LIMIT) Full index scan must be started with init_read_record_idx */ - if (used_index == MAX_KEY || (select && select->quick)) + if (query_plan.index == MAX_KEY || (select && select->quick)) { if (init_read_record(&info, thd, table, select, 0, 1, FALSE)) goto err; } else - init_read_record_idx(&info, thd, table, 1, used_index, reverse); + init_read_record_idx(&info, thd, table, 1, query_plan.index, reverse); thd_proc_info(thd, "Searching rows for update"); ha_rows tmp_limit= limit; @@ -624,6 +674,7 @@ int mysql_update(THD *thd, select= new SQL_SELECT; select->head=table; } + //psergey-todo: disable SHOW EXPLAIN because the plan was deleted? if (reinit_io_cache(&tempfile,READ_CACHE,0L,0,0)) error=1; /* purecov: inspected */ select->file=tempfile; // Read row ptrs from this file @@ -702,8 +753,7 @@ int mysql_update(THD *thd, continue; /* repeat the read of the same row if it still exists */ store_record(table,record[1]); - if (fill_record_n_invoke_before_triggers(thd, fields, values, 0, - table->triggers, + if (fill_record_n_invoke_before_triggers(thd, table, fields, values, 0, TRG_EVENT_UPDATE)) break; /* purecov: inspected */ @@ -711,6 +761,11 @@ int mysql_update(THD *thd, if (!can_compare_record || compare_record(table)) { + if (table->default_field && table->update_default_fields()) + { + error= 1; + break; + } if ((res= table_list->view_check_option(thd, ignore)) != VIEW_CHECK_OK) { @@ -976,11 +1031,21 @@ int mysql_update(THD *thd, DBUG_RETURN((error >= 0 || thd->is_error()) ? 1 : 0); err: + delete select; free_underlaid_joins(thd, select_lex); table->disable_keyread(); thd->abort_on_warning= 0; DBUG_RETURN(1); + +exit_without_my_ok: + query_plan.save_explain_data(thd->lex->explain); + + int err2= thd->lex->explain->send_explain(thd); + + delete select; + free_underlaid_joins(thd, select_lex); + DBUG_RETURN((err2 || thd->is_error()) ? 1 : 0); } /* @@ -1266,11 +1331,6 @@ int mysql_multi_update_prepare(THD *thd) while ((tl= ti++)) { TABLE *table= tl->table; - /* Only set timestamp column if this is not modified */ - if (table->timestamp_field && - bitmap_is_set(table->write_set, - table->timestamp_field->field_index)) - table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET; /* if table will be updated then check that it is unique */ if (table->map & tables_for_update) @@ -1406,11 +1466,11 @@ bool mysql_multi_update(THD *thd, { bool res; DBUG_ENTER("mysql_multi_update"); - + if (!(*result= new multi_update(table_list, - &thd->lex->select_lex.leaf_tables, - fields, values, - handle_duplicates, ignore))) + &thd->lex->select_lex.leaf_tables, + fields, values, + handle_duplicates, ignore))) { DBUG_RETURN(TRUE); } @@ -1418,7 +1478,6 @@ bool mysql_multi_update(THD *thd, thd->abort_on_warning= test(thd->variables.sql_mode & (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES)); - List<Item> total_list; res= mysql_select(thd, &select_lex->ref_pointer_array, @@ -1434,6 +1493,11 @@ bool mysql_multi_update(THD *thd, res|= thd->is_error(); if (unlikely(res)) (*result)->abort_result_set(); + else + { + if (thd->lex->describe) + res= thd->lex->explain->send_explain(thd); + } thd->abort_on_warning= 0; DBUG_RETURN(res); } @@ -1448,7 +1512,7 @@ multi_update::multi_update(TABLE_LIST *table_list, tmp_tables(0), updated(0), found(0), fields(field_list), values(value_list), table_count(0), copy_field(0), handle_duplicates(handle_duplicates_arg), do_update(1), trans_safe(1), - transactional_tables(0), ignore(ignore_arg), error_handled(0) + transactional_tables(0), ignore(ignore_arg), error_handled(0), prepared(0) {} @@ -1471,6 +1535,10 @@ int multi_update::prepare(List<Item> ¬_used_values, List_iterator<TABLE_LIST> ti(*leaves); DBUG_ENTER("multi_update::prepare"); + if (prepared) + DBUG_RETURN(0); + prepared= true; + thd->count_cuted_fields= CHECK_FIELD_WARN; thd->cuted_fields=0L; thd_proc_info(thd, "updating main table"); @@ -1520,8 +1588,7 @@ int multi_update::prepare(List<Item> ¬_used_values, to compare records and detect data change. */ if ((table->file->ha_table_flags() & HA_PARTIAL_COLUMN_READ) && - (((uint) table->timestamp_field_type) & - TIMESTAMP_AUTO_SET_ON_UPDATE)) + table->default_field && table->has_default_function(true)) bitmap_union(table->read_set, table->write_set); } } @@ -1909,10 +1976,10 @@ int multi_update::send_data(List<Item> ¬_used_values) table->status|= STATUS_UPDATED; store_record(table,record[1]); - if (fill_record_n_invoke_before_triggers(thd, *fields_for_table[offset], + if (fill_record_n_invoke_before_triggers(thd, table, *fields_for_table[offset], *values_for_table[offset], 0, - table->triggers, - TRG_EVENT_UPDATE)) + TRG_EVENT_UPDATE) || + (table->default_field && table->update_default_fields())) DBUG_RETURN(1); /* @@ -2013,7 +2080,7 @@ int multi_update::send_data(List<Item> ¬_used_values) } while ((tbl= tbl_it++)); /* Store regular updated fields in the row. */ - fill_record(thd, + fill_record(thd, tmp_table, tmp_table->field + 1 + unupdated_check_opt_tables.elements, *values_for_table[offset], TRUE, FALSE); @@ -2203,7 +2270,10 @@ int multi_update::do_updates() for (copy_field_ptr=copy_field; copy_field_ptr != copy_field_end; copy_field_ptr++) + { (*copy_field_ptr->do_copy)(copy_field_ptr); + copy_field_ptr->to_field->set_has_explicit_value(); + } if (table->triggers && table->triggers->process_triggers(thd, TRG_EVENT_UPDATE, @@ -2213,6 +2283,8 @@ int multi_update::do_updates() if (!can_compare_record || compare_record(table)) { int error; + if (table->default_field && (error= table->update_default_fields())) + goto err2; if ((error= cur_table->view_check_option(thd, ignore)) != VIEW_CHECK_OK) { diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 7be370b8e63..0b9dd782956 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -33,14 +33,13 @@ #include "sp_head.h" #include "sp.h" #include "sp_cache.h" -#include "datadict.h" // dd_frm_type() +#include "datadict.h" // dd_frm_is_view() #define MD5_BUFF_LENGTH 33 const LEX_STRING view_type= { C_STRING_WITH_LEN("VIEW") }; -static int mysql_register_view(THD *thd, TABLE_LIST *view, - enum_view_create_mode mode); +static int mysql_register_view(THD *, TABLE_LIST *, enum_view_create_mode); /* Make a unique name for an anonymous view column @@ -211,16 +210,12 @@ static void make_valid_column_names(List<Item> &item_list) static bool fill_defined_view_parts (THD *thd, TABLE_LIST *view) { - char key[MAX_DBKEY_LENGTH]; - uint key_length; LEX *lex= thd->lex; TABLE_LIST decoy; memcpy (&decoy, view, sizeof (TABLE_LIST)); - key_length= create_table_def_key(thd, key, view, 0); - - if (tdc_open_view(thd, &decoy, decoy.alias, key, key_length, - thd->mem_root, OPEN_VIEW_NO_PARSE)) + if (tdc_open_view(thd, &decoy, decoy.alias, thd->mem_root, + OPEN_VIEW_NO_PARSE)) return TRUE; if (!lex->definer) @@ -470,60 +465,9 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, } sp_cache_invalidate(); + if (sp_process_definer(thd)) + goto err; - if (!lex->definer) - { - /* - DEFINER-clause is missing; we have to create default definer in - persistent arena to be PS/SP friendly. - If this is an ALTER VIEW then the current user should be set as - the definer. - */ - Query_arena original_arena; - Query_arena *ps_arena = thd->activate_stmt_arena_if_needed(&original_arena); - - if (!(lex->definer= create_default_definer(thd))) - res= TRUE; - - if (ps_arena) - thd->restore_active_arena(ps_arena, &original_arena); - - if (res) - goto err; - } - -#ifndef NO_EMBEDDED_ACCESS_CHECKS - /* - check definer of view: - - same as current user - - current user has SUPER_ACL - */ - if (lex->definer && - (strcmp(lex->definer->user.str, thd->security_ctx->priv_user) != 0 || - my_strcasecmp(system_charset_info, - lex->definer->host.str, - thd->security_ctx->priv_host) != 0)) - { - if (!(thd->security_ctx->master_access & SUPER_ACL)) - { - my_error(ER_SPECIFIC_ACCESS_DENIED_ERROR, MYF(0), "SUPER"); - res= TRUE; - goto err; - } - else - { - if (!is_acl_user(lex->definer->host.str, - lex->definer->user.str)) - { - push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, - ER_NO_SUCH_USER, - ER(ER_NO_SUCH_USER), - lex->definer->user.str, - lex->definer->host.str); - } - } - } -#endif /* check that tables are not temporary and this VIEW do not used in query (it is possible with ALTERing VIEW). @@ -871,14 +815,18 @@ static int mysql_register_view(THD *thd, TABLE_LIST *view, view->source= thd->lex->create_view_select; if (!thd->make_lex_string(&view->select_stmt, view_query.ptr(), - view_query.length(), false)) + view_query.length())) { my_error(ER_OUT_OF_RESOURCES, MYF(0)); error= -1; goto err; } - view->file_version= 1; + /* + version 1 - before 10.0.5 + version 2 - empty definer_host means a role + */ + view->file_version= 2; view->calc_md5(md5); if (!(view->md5.str= (char*) thd->memdup(md5, 32))) { @@ -1004,7 +952,7 @@ loop_out: view->view_creation_ctx->get_connection_cl()->name); if (!thd->make_lex_string(&view->view_body_utf8, is_query.ptr(), - is_query.length(), false)) + is_query.length())) { my_error(ER_OUT_OF_RESOURCES, MYF(0)); error= -1; @@ -1073,19 +1021,16 @@ err: bool mysql_make_view(THD *thd, File_parser *parser, TABLE_LIST *table, uint flags) { - SELECT_LEX *end, *view_select; + SELECT_LEX *end, *UNINIT_VAR(view_select); LEX *old_lex, *lex; Query_arena *arena, backup; TABLE_LIST *top_view= table->top_table(); - bool parse_status; + bool UNINIT_VAR(parse_status); bool result, view_is_mergeable; TABLE_LIST *UNINIT_VAR(view_main_select_tables); DBUG_ENTER("mysql_make_view"); DBUG_PRINT("info", ("table: 0x%lx (%s)", (ulong) table, table->table_name)); - LINT_INIT(parse_status); - LINT_INIT(view_select); - if (table->view) { /* @@ -1168,8 +1113,16 @@ bool mysql_make_view(THD *thd, File_parser *parser, TABLE_LIST *table, push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, ER_VIEW_FRM_NO_USER, ER(ER_VIEW_FRM_NO_USER), table->db, table->table_name); - get_default_definer(thd, &table->definer); + get_default_definer(thd, &table->definer, false); } + + /* + since 10.0.5 definer.host can never be "" for a User, but it's + always "" for a Role. Before 10.0.5 it could be "" for a User, + but roles didn't exist. file_version helps. + */ + if (!table->definer.host.str[0] && table->file_version < 2) + table->definer.host= host_not_specified; // User, not Role /* Initialize view definition context by character set names loaded from @@ -1293,15 +1246,14 @@ bool mysql_make_view(THD *thd, File_parser *parser, TABLE_LIST *table, TABLE_LIST *view_tables= lex->query_tables; TABLE_LIST *view_tables_tail= 0; TABLE_LIST *tbl; - Security_context *security_ctx; + Security_context *security_ctx= 0; /* Check rights to run commands (EXPLAIN SELECT & SHOW CREATE) which show underlying tables. Skip this step if we are opening view for prelocking only. */ - if (!table->prelocking_placeholder && - (old_lex->sql_command == SQLCOM_SELECT && old_lex->describe)) + if (!table->prelocking_placeholder && (old_lex->describe)) { /* The user we run EXPLAIN as (either the connected user who issued @@ -1469,6 +1421,7 @@ bool mysql_make_view(THD *thd, File_parser *parser, TABLE_LIST *table, if (view_select->options & OPTION_TO_QUERY_CACHE) old_lex->select_lex.options|= OPTION_TO_QUERY_CACHE; +#ifndef NO_EMBEDDED_ACCESS_CHECKS if (table->view_suid) { /* @@ -1489,6 +1442,7 @@ bool mysql_make_view(THD *thd, File_parser *parser, TABLE_LIST *table, */ security_ctx= table->security_ctx; } +#endif /* Assign the context to the tables referenced in the view */ if (view_tables) @@ -1648,7 +1602,6 @@ bool mysql_drop_view(THD *thd, TABLE_LIST *views, enum_drop_mode drop_mode) String non_existant_views; char *wrong_object_db= NULL, *wrong_object_name= NULL; bool error= FALSE; - enum legacy_db_type not_used; bool some_views_deleted= FALSE; bool something_wrong= FALSE; DBUG_ENTER("mysql_drop_view"); @@ -1671,23 +1624,28 @@ bool mysql_drop_view(THD *thd, TABLE_LIST *views, enum_drop_mode drop_mode) for (view= views; view; view= view->next_local) { - frm_type_enum type= FRMTYPE_ERROR; + bool not_exist; build_table_filename(path, sizeof(path) - 1, view->db, view->table_name, reg_ext, 0); - if (access(path, F_OK) || - FRMTYPE_VIEW != (type= dd_frm_type(thd, path, ¬_used))) + if ((not_exist= my_access(path, F_OK)) || !dd_frm_is_view(thd, path)) { char name[FN_REFLEN]; my_snprintf(name, sizeof(name), "%s.%s", view->db, view->table_name); - if (thd->lex->drop_if_exists) + if (thd->lex->check_exists) { push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE, ER_BAD_TABLE_ERROR, ER(ER_BAD_TABLE_ERROR), name); continue; } - if (type == FRMTYPE_TABLE) + if (not_exist) + { + if (non_existant_views.length()) + non_existant_views.append(','); + non_existant_views.append(String(view->table_name,system_charset_info)); + } + else { if (!wrong_object_name) { @@ -1695,12 +1653,6 @@ bool mysql_drop_view(THD *thd, TABLE_LIST *views, enum_drop_mode drop_mode) wrong_object_name= view->table_name; } } - else - { - if (non_existant_views.length()) - non_existant_views.append(','); - non_existant_views.append(String(view->table_name,system_charset_info)); - } continue; } if (mysql_file_delete(key_file_frm, path, MYF(MY_WME))) @@ -1709,9 +1661,8 @@ bool mysql_drop_view(THD *thd, TABLE_LIST *views, enum_drop_mode drop_mode) some_views_deleted= TRUE; /* - For a view, there is a TABLE_SHARE object, but its - ref_count never goes above 1. Remove it from the table - definition cache, in case the view was cached. + For a view, there is a TABLE_SHARE object. + Remove it from the table definition cache, in case the view was cached. */ tdc_remove_table(thd, TDC_RT_REMOVE_ALL, view->db, view->table_name, FALSE); diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index de6bd59cf7e..8e9b2cc8115 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -58,6 +58,7 @@ #include <myisammrg.h> #include "keycaches.h" #include "set_var.h" +#include "rpl_mi.h" /* this is to get the bison compilation windows warnings out */ #ifdef _MSC_VER @@ -712,7 +713,7 @@ static bool add_create_index (LEX *lex, Key::Keytype type, { Key *key; key= new Key(type, name, info ? info : &lex->key_create_info, generated, - lex->col_list, lex->option_list); + lex->col_list, lex->option_list, lex->check_exists); if (key == NULL) return TRUE; @@ -782,10 +783,10 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %parse-param { THD *thd } %lex-param { THD *thd } /* - Currently there are 175 shift/reduce conflicts. + Currently there are 198 shift/reduce conflicts. We should not introduce new conflicts any more. */ -%expect 175 +%expect 198 /* Comments for TOKENS. @@ -799,7 +800,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); MYSQL-FUNC : MySQL extention, function INTERNAL : Not a real token, lex optimization OPERATOR : SQL operator - FUTURE-USE : Reserved for futur use + FUTURE-USE : Reserved for future use This makes the code grep-able, and helps maintenance. */ @@ -808,6 +809,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token ACCESSIBLE_SYM %token ACTION /* SQL-2003-N */ %token ADD /* SQL-2003-R */ +%token ADMIN_SYM /* SQL-2003-N */ %token ADDDATE_SYM /* MYSQL-FUNC */ %token AFTER_SYM /* SQL-2003-N */ %token AGAINST @@ -828,6 +830,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token AUTHORS_SYM %token AUTOEXTEND_SIZE_SYM %token AUTO_INC +%token AUTO_SYM %token AVG_ROW_LENGTH %token AVG_SYM /* SQL-2003-N */ %token BACKUP_SYM @@ -876,11 +879,10 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token COLLATION_SYM /* SQL-2003-N */ %token COLUMNS %token COLUMN_ADD_SYM +%token COLUMN_CHECK_SYM %token COLUMN_CREATE_SYM %token COLUMN_DELETE_SYM -%token COLUMN_EXISTS_SYM %token COLUMN_GET_SYM -%token COLUMN_LIST_SYM %token COLUMN_SYM /* SQL-2003-R */ %token COLUMN_NAME_SYM /* SQL-2003-N */ %token COMMENT_SYM @@ -909,6 +911,8 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token CUBE_SYM /* SQL-2003-R */ %token CURDATE /* MYSQL-FUNC */ %token CURRENT_USER /* SQL-2003-R */ +%token CURRENT_ROLE /* SQL-2003-R */ +%token CURRENT_POS_SYM %token CURSOR_SYM /* SQL-2003-R */ %token CURSOR_NAME_SYM /* SQL-2003-N */ %token CURTIME /* MYSQL-FUNC */ @@ -1023,6 +1027,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token HOUR_MINUTE_SYM %token HOUR_SECOND_SYM %token HOUR_SYM /* SQL-2003-R */ +%token ID_SYM /* MYSQL */ %token IDENT %token IDENTIFIED_SYM %token IDENT_QUOTED @@ -1089,6 +1094,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token LOW_PRIORITY %token LT /* OPERATOR */ %token MASTER_CONNECT_RETRY_SYM +%token MASTER_GTID_POS_SYM %token MASTER_HOST_SYM %token MASTER_LOG_FILE_SYM %token MASTER_LOG_POS_SYM @@ -1104,6 +1110,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token MASTER_SSL_VERIFY_SERVER_CERT_SYM %token MASTER_SYM %token MASTER_USER_SYM +%token MASTER_USE_GTID_SYM %token MASTER_HEARTBEAT_PERIOD_SYM %token MATCH /* SQL-2003-R */ %token MAX_CONNECTIONS_PER_HOUR @@ -1250,10 +1257,12 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token RESTORE_SYM %token RESTRICT %token RESUME_SYM +%token RETURNING_SYM %token RETURNS_SYM /* SQL-2003-R */ %token RETURN_SYM /* SQL-2003-R */ %token REVOKE /* SQL-2003-R */ %token RIGHT /* SQL-2003-R */ +%token ROLE_SYM %token ROLLBACK_SYM /* SQL-2003-R */ %token ROLLUP_SYM /* SQL-2003-R */ %token ROUTINE_SYM /* SQL-2003-N */ @@ -1286,6 +1295,8 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token SIGNED_SYM %token SIMPLE_SYM /* SQL-2003-N */ %token SLAVE +%token SLAVES +%token SLAVE_POS_SYM %token SLOW %token SMALLINT /* SQL-2003-R */ %token SNAPSHOT_SYM @@ -1449,14 +1460,14 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); IDENT_sys TEXT_STRING_sys TEXT_STRING_literal NCHAR_STRING opt_component key_cache_name sp_opt_label BIN_NUM label_ident TEXT_STRING_filesystem ident_or_empty - opt_constraint constraint opt_ident + opt_constraint constraint opt_ident opt_if_not_exists_ident %type <lex_str_ptr> opt_table_alias %type <table> table_ident table_ident_nodb references xid - table_ident_opt_wild + table_ident_opt_wild create_like %type <simple_string> remember_name remember_end opt_db text_or_password @@ -1466,7 +1477,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %type <num> type type_with_opt_collate int_type real_type order_dir lock_option - udf_type if_exists opt_local opt_table_options table_options + udf_type opt_if_exists opt_local opt_table_options table_options table_option opt_if_not_exists opt_no_write_to_binlog opt_temporary all_or_any opt_distinct opt_ignore_leaves fulltext_options spatial_type union_option @@ -1477,6 +1488,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); ev_alter_on_schedule_completion opt_ev_rename_to opt_ev_sql_stmt optional_flush_tables_arguments opt_dyncol_type dyncol_type opt_time_precision kill_type kill_option int_num + opt_default_time_precision %type <m_yes_no_unk> opt_chain opt_release @@ -1559,7 +1571,8 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %type <symbol> keyword keyword_sp -%type <lex_user> user grant_user +%type <lex_user> user grant_user grant_role user_or_role current_role + admin_option_for_role %type <charset> opt_collate @@ -1590,7 +1603,12 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); show describe load alter optimize keycache preload flush reset purge begin commit rollback savepoint release slave master_def master_defs master_file_def slave_until_opts - repair analyze check start checksum + repair analyze opt_with_admin opt_with_admin_option + analyze_table_list analyze_table_elem_spec + opt_persistent_stat_clause persistent_stat_spec + persistent_column_stat_spec persistent_index_stat_spec + table_column_list table_index_list table_index_name + check start checksum field_list field_list_item field_spec kill column_def key_def keycache_list keycache_list_or_parts assign_to_keycache assign_to_keycache_parts @@ -1608,11 +1626,12 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); opt_option opt_place opt_attribute opt_attribute_list attribute column_list column_list_id opt_column_list grant_privileges grant_ident grant_list grant_option - object_privilege object_privilege_list user_list rename_list + object_privilege object_privilege_list user_list user_and_role_list + rename_list clear_privileges flush_options flush_option opt_with_read_lock flush_options_list equal optional_braces - opt_mi_check_type opt_to mi_check_types normal_join + opt_mi_check_type opt_to mi_check_types table_to_table_list table_to_table opt_table_list opt_as handler_rkey_function handler_read_or_scan single_multi table_wild_list table_wild_one opt_wild @@ -1635,12 +1654,13 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); spatial_key_options fulltext_key_options normal_key_opt fulltext_key_opt spatial_key_opt fulltext_key_opts spatial_key_opts keep_gcc_happy - key_using_alg + key_using_alg shutdown part_column_list server_def server_options_list server_option definer_opt no_definer definer parse_vcol_expr vcol_opt_specifier vcol_opt_attribute vcol_opt_attribute_list vcol_attribute + explainable_command END_OF_INPUT %type <NONE> call sp_proc_stmts sp_proc_stmts1 sp_proc_stmt @@ -1659,7 +1679,7 @@ END_OF_INPUT %type <lex> sp_cursor_stmt %type <spname> sp_name %type <index_hint> index_hint_type -%type <num> index_hint_clause +%type <num> index_hint_clause normal_join inner_join %type <filetype> data_or_xml %type <NONE> signal_stmt resignal_stmt @@ -1672,8 +1692,11 @@ END_OF_INPUT %type <is_not_empty> opt_union_order_or_limit +%type <NONE> ROLE_SYM + %% + /* Indentation of grammar rules: @@ -1796,6 +1819,7 @@ statement: | set | signal_stmt | show + | shutdown | slave | start | truncate @@ -1897,7 +1921,7 @@ help: /* change master */ change: - CHANGE MASTER_SYM TO_SYM + CHANGE MASTER_SYM optional_connection_name TO_SYM { Lex->sql_command = SQLCOM_CHANGE_MASTER; } @@ -2044,8 +2068,59 @@ master_file_def: /* Adjust if < BIN_LOG_HEADER_SIZE (same comment as Lex->mi.pos) */ Lex->mi.relay_log_pos = max(BIN_LOG_HEADER_SIZE, Lex->mi.relay_log_pos); } + | MASTER_USE_GTID_SYM EQ CURRENT_POS_SYM + { + if (Lex->mi.use_gtid_opt != LEX_MASTER_INFO::LEX_GTID_UNCHANGED) + { + my_error(ER_DUP_ARGUMENT, MYF(0), "MASTER_use_gtid"); + MYSQL_YYABORT; + } + Lex->mi.use_gtid_opt= LEX_MASTER_INFO::LEX_GTID_CURRENT_POS; + } + ; + | MASTER_USE_GTID_SYM EQ SLAVE_POS_SYM + { + if (Lex->mi.use_gtid_opt != LEX_MASTER_INFO::LEX_GTID_UNCHANGED) + { + my_error(ER_DUP_ARGUMENT, MYF(0), "MASTER_use_gtid"); + MYSQL_YYABORT; + } + Lex->mi.use_gtid_opt= LEX_MASTER_INFO::LEX_GTID_SLAVE_POS; + } + ; + | MASTER_USE_GTID_SYM EQ NO_SYM + { + if (Lex->mi.use_gtid_opt != LEX_MASTER_INFO::LEX_GTID_UNCHANGED) + { + my_error(ER_DUP_ARGUMENT, MYF(0), "MASTER_use_gtid"); + MYSQL_YYABORT; + } + Lex->mi.use_gtid_opt= LEX_MASTER_INFO::LEX_GTID_NO; + } ; +optional_connection_name: + /* empty */ + { + LEX *lex= thd->lex; + lex->mi.connection_name= thd->variables.default_master_connection; + } + | connection_name; + ; + +connection_name: + TEXT_STRING_sys + { + Lex->mi.connection_name= $1; +#ifdef HAVE_REPLICATION + if (check_master_connection_name(&$1)) + { + my_error(ER_WRONG_ARGUMENTS, MYF(0), "MASTER_CONNECTION_NAME"); + MYSQL_YYABORT; + } +#endif + } + /* create a table */ create: @@ -2073,7 +2148,7 @@ create: lex->name.length= 0; lex->create_last_non_select_table= lex->last_table(); } - create2 + create_body { LEX *lex= thd->lex; lex->current_select= &lex->select_lex; @@ -2089,36 +2164,36 @@ create: } create_table_set_open_action_and_adjust_tables(lex); } - | CREATE opt_unique INDEX_SYM ident key_alg ON table_ident + | CREATE opt_unique INDEX_SYM opt_if_not_exists ident key_alg ON table_ident { - if (add_create_index_prepare(Lex, $7)) + if (add_create_index_prepare(Lex, $8)) MYSQL_YYABORT; } '(' key_list ')' normal_key_options { - if (add_create_index(Lex, $2, $4)) + if (add_create_index(Lex, $2, $5)) MYSQL_YYABORT; } - | CREATE fulltext INDEX_SYM ident init_key_options ON + | CREATE fulltext INDEX_SYM opt_if_not_exists ident init_key_options ON table_ident { - if (add_create_index_prepare(Lex, $7)) + if (add_create_index_prepare(Lex, $8)) MYSQL_YYABORT; } '(' key_list ')' fulltext_key_options { - if (add_create_index(Lex, $2, $4)) + if (add_create_index(Lex, $2, $5)) MYSQL_YYABORT; } - | CREATE spatial INDEX_SYM ident init_key_options ON + | CREATE spatial INDEX_SYM opt_if_not_exists ident init_key_options ON table_ident { - if (add_create_index_prepare(Lex, $7)) + if (add_create_index_prepare(Lex, $8)) MYSQL_YYABORT; } '(' key_list ')' spatial_key_options { - if (add_create_index(Lex, $2, $4)) + if (add_create_index(Lex, $2, $5)) MYSQL_YYABORT; } | CREATE DATABASE opt_if_not_exists ident @@ -2145,6 +2220,10 @@ create: { Lex->sql_command = SQLCOM_CREATE_USER; } + | CREATE ROLE_SYM clear_privileges role_list opt_with_admin + { + Lex->sql_command = SQLCOM_CREATE_ROLE; + } | CREATE LOGFILE_SYM GROUP_SYM logfile_group_info { Lex->alter_tablespace_info->ts_cmd_type= CREATE_LOGFILE_GROUP; @@ -4239,34 +4318,23 @@ size_number: End tablespace part */ -create2: - '(' create2a {} - | opt_create_table_options - opt_create_partitioning - create3 {} - | LIKE table_ident - { - TABLE_LIST *src_table; - LEX *lex= thd->lex; - - lex->create_info.options|= HA_LEX_CREATE_TABLE_LIKE; - src_table= lex->select_lex.add_table_to_list(thd, $2, NULL, 0, - TL_READ, - MDL_SHARED_READ); - if (! src_table) - MYSQL_YYABORT; - /* CREATE TABLE ... LIKE is not allowed for views. */ - src_table->required_type= FRMTYPE_TABLE; - } - | '(' LIKE table_ident ')' +create_body: + '(' create_field_list ')' + { Lex->create_info.option_list= NULL; } + opt_create_table_options opt_create_partitioning opt_create_select {} + | opt_create_table_options opt_create_partitioning opt_create_select {} + /* + the following rule is redundant, but there's a shift/reduce + conflict that prevents the rule above from parsing a syntax like + CREATE TABLE t1 (SELECT 1); + */ + | '(' create_select ')' { Select->set_braces(1);} union_opt {} + | create_like { - TABLE_LIST *src_table; - LEX *lex= thd->lex; - lex->create_info.options|= HA_LEX_CREATE_TABLE_LIKE; - src_table= lex->select_lex.add_table_to_list(thd, $3, NULL, 0, - TL_READ, - MDL_SHARED_READ); + Lex->create_info.options|= HA_LEX_CREATE_TABLE_LIKE; + TABLE_LIST *src_table= Lex->select_lex.add_table_to_list(thd, + $1, NULL, 0, TL_READ, MDL_SHARED_READ); if (! src_table) MYSQL_YYABORT; /* CREATE TABLE ... LIKE is not allowed for views. */ @@ -4274,21 +4342,12 @@ create2: } ; -create2a: - create_field_list ')' - { - Lex->create_info.option_list= NULL; - } - opt_create_table_options - opt_create_partitioning - create3 {} - | opt_create_partitioning - create_select ')' - { Select->set_braces(1);} - union_opt {} +create_like: + LIKE table_ident { $$= $2; } + | '(' LIKE table_ident ')' { $$= $3; } ; -create3: +opt_create_select: /* empty */ {} | opt_duplicate opt_as create_select { Select->set_braces(0);} @@ -5067,9 +5126,17 @@ table_option: ; opt_if_not_exists: - /* empty */ { $$= 0; } - | IF not EXISTS { $$=HA_LEX_CREATE_IF_NOT_EXISTS; } - ; + /* empty */ + { + Lex->check_exists= FALSE; + $$= 0; + } + | IF not EXISTS + { + Lex->check_exists= TRUE; + $$=HA_LEX_CREATE_IF_NOT_EXISTS; + } + ; opt_create_table_options: /* empty */ @@ -5301,7 +5368,7 @@ storage_engines: plugin_ref plugin= ha_resolve_by_name(thd, &$1); if (plugin) - $$= plugin_data(plugin, handlerton*); + $$= plugin_hton(plugin); else { if (thd->variables.sql_mode & MODE_NO_ENGINE_SUBSTITUTION) @@ -5323,7 +5390,7 @@ known_storage_engines: { plugin_ref plugin; if ((plugin= ha_resolve_by_name(thd, &$1))) - $$= plugin_data(plugin, handlerton*); + $$= plugin_hton(plugin); else { my_error(ER_UNKNOWN_STORAGE_ENGINE, MYF(0), $1.str); @@ -5387,14 +5454,14 @@ column_def: ; key_def: - normal_key_type opt_ident key_alg '(' key_list ')' + normal_key_type opt_if_not_exists_ident key_alg '(' key_list ')' { Lex->option_list= NULL; } normal_key_options { if (add_create_index (Lex, $1, $2)) MYSQL_YYABORT; } - | fulltext opt_key_or_index opt_ident init_key_options + | fulltext opt_key_or_index opt_if_not_exists_ident init_key_options '(' key_list ')' { Lex->option_list= NULL; } fulltext_key_options @@ -5402,7 +5469,7 @@ key_def: if (add_create_index (Lex, $1, $3)) MYSQL_YYABORT; } - | spatial opt_key_or_index opt_ident init_key_options + | spatial opt_key_or_index opt_if_not_exists_ident init_key_options '(' key_list ')' { Lex->option_list= NULL; } spatial_key_options @@ -5418,7 +5485,7 @@ key_def: if (add_create_index (Lex, $2, $3.str ? $3 : $1)) MYSQL_YYABORT; } - | opt_constraint FOREIGN KEY_SYM opt_ident '(' key_list ')' references + | opt_constraint FOREIGN KEY_SYM opt_if_not_exists_ident '(' key_list ')' references { LEX *lex=Lex; Key *key= new Foreign_key($4.str ? $4 : $1, lex->col_list, @@ -5426,7 +5493,8 @@ key_def: lex->ref_list, lex->fk_delete_opt, lex->fk_update_opt, - lex->fk_match_option); + lex->fk_match_option, + lex->check_exists); if (key == NULL) MYSQL_YYABORT; lex->alter_info.key_list.push_back(key); @@ -5873,9 +5941,9 @@ attribute: NULL_SYM { Lex->type&= ~ NOT_NULL_FLAG; } | not NULL_SYM { Lex->type|= NOT_NULL_FLAG; } | DEFAULT now_or_signed_literal { Lex->default_value=$2; } - | ON UPDATE_SYM NOW_SYM optional_braces + | ON UPDATE_SYM NOW_SYM opt_default_time_precision { - Item *item= new (thd->mem_root) Item_func_now_local(6); + Item *item= new (thd->mem_root) Item_func_now_local($4); if (item == NULL) MYSQL_YYABORT; Lex->on_update_value= item; @@ -5967,9 +6035,9 @@ type_with_opt_collate: now_or_signed_literal: - NOW_SYM optional_braces + NOW_SYM opt_default_time_precision { - $$= new (thd->mem_root) Item_func_now_local(6); + $$= new (thd->mem_root) Item_func_now_local($2); if ($$ == NULL) MYSQL_YYABORT; } @@ -6400,6 +6468,18 @@ opt_ident: | field_ident { $$= $1; } ; +opt_if_not_exists_ident: + opt_if_not_exists opt_ident + { + LEX *lex= Lex; + if (lex->check_exists && lex->sql_command != SQLCOM_ALTER_TABLE) + { + my_parse_error(ER(ER_SYNTAX_ERROR)); + MYSQL_YYABORT; + } + $$= $2; + }; + opt_component: /* empty */ { $$= null_lex_str; } | '.' ident { $$= $2; } @@ -6654,7 +6734,7 @@ alter_commands: new table and so forth. */ | add_partition_rule - | DROP PARTITION_SYM alt_part_name_list + | DROP PARTITION_SYM opt_if_exists alt_part_name_list { Lex->alter_info.flags|= ALTER_DROP_PARTITION; } @@ -6750,7 +6830,7 @@ all_or_alt_part_name_list: ; add_partition_rule: - ADD PARTITION_SYM opt_no_write_to_binlog + ADD PARTITION_SYM opt_if_not_exists opt_no_write_to_binlog { LEX *lex= Lex; lex->part_info= new partition_info(); @@ -6760,7 +6840,7 @@ add_partition_rule: MYSQL_YYABORT; } lex->alter_info.flags|= ALTER_ADD_PARTITION; - lex->no_write_to_binlog= $3; + lex->no_write_to_binlog= $4; } add_part_extra {} @@ -6836,7 +6916,7 @@ alter_list: ; add_column: - ADD opt_column + ADD opt_column opt_if_not_exists { LEX *lex=Lex; lex->change=0; @@ -6858,10 +6938,10 @@ alter_list_item: { Lex->alter_info.flags|= ALTER_ADD_COLUMN | ALTER_ADD_INDEX; } - | CHANGE opt_column field_ident + | CHANGE opt_column opt_if_exists field_ident { LEX *lex=Lex; - lex->change= $3.str; + lex->change= $4.str; lex->alter_info.flags|= ALTER_CHANGE_COLUMN; lex->option_list= NULL; } @@ -6869,7 +6949,7 @@ alter_list_item: { Lex->create_last_non_select_table= Lex->last_table(); } - | MODIFY_SYM opt_column field_ident + | MODIFY_SYM opt_column opt_if_exists field_ident { LEX *lex=Lex; lex->length=lex->dec=0; lex->type=0; @@ -6883,12 +6963,12 @@ alter_list_item: field_def { LEX *lex=Lex; - if (add_field_to_list(lex->thd,&$3, - (enum enum_field_types) $5, + if (add_field_to_list(lex->thd,&$4, + (enum enum_field_types) $6, lex->length,lex->dec,lex->type, lex->default_value, lex->on_update_value, &lex->comment, - $3.str, &lex->interval_list, lex->charset, + $4.str, &lex->interval_list, lex->charset, lex->uint_geom_type, lex->vcol_info, lex->option_list)) MYSQL_YYABORT; @@ -6897,32 +6977,33 @@ alter_list_item: { Lex->create_last_non_select_table= Lex->last_table(); } - | DROP opt_column field_ident opt_restrict + | DROP opt_column opt_if_exists field_ident opt_restrict { LEX *lex=Lex; - Alter_drop *ad= new Alter_drop(Alter_drop::COLUMN, $3.str); + Alter_drop *ad= new Alter_drop(Alter_drop::COLUMN, $4.str, $3); if (ad == NULL) MYSQL_YYABORT; lex->alter_info.drop_list.push_back(ad); lex->alter_info.flags|= ALTER_DROP_COLUMN; } - | DROP FOREIGN KEY_SYM opt_ident + | DROP FOREIGN KEY_SYM opt_if_exists opt_ident { Lex->alter_info.flags|= ALTER_DROP_INDEX | ALTER_FOREIGN_KEY; } | DROP PRIMARY_SYM KEY_SYM { LEX *lex=Lex; - Alter_drop *ad= new Alter_drop(Alter_drop::KEY, primary_key_name); + Alter_drop *ad= new Alter_drop(Alter_drop::KEY, primary_key_name, + FALSE); if (ad == NULL) MYSQL_YYABORT; lex->alter_info.drop_list.push_back(ad); lex->alter_info.flags|= ALTER_DROP_INDEX; } - | DROP key_or_index field_ident + | DROP key_or_index opt_if_exists field_ident { LEX *lex=Lex; - Alter_drop *ad= new Alter_drop(Alter_drop::KEY, $3.str); + Alter_drop *ad= new Alter_drop(Alter_drop::KEY, $4.str, $3); if (ad == NULL) MYSQL_YYABORT; lex->alter_info.drop_list.push_back(ad); @@ -7071,7 +7152,7 @@ opt_to: */ slave: - START_SYM SLAVE slave_thread_opts + START_SYM SLAVE optional_connection_name slave_thread_opts { LEX *lex=Lex; lex->sql_command = SQLCOM_SLAVE_START; @@ -7080,14 +7161,28 @@ slave: } slave_until {} - | STOP_SYM SLAVE slave_thread_opts + | START_SYM ALL SLAVES slave_thread_opts + { + LEX *lex=Lex; + lex->sql_command = SQLCOM_SLAVE_ALL_START; + lex->type = 0; + } + {} + | STOP_SYM SLAVE optional_connection_name slave_thread_opts { LEX *lex=Lex; lex->sql_command = SQLCOM_SLAVE_STOP; lex->type = 0; /* If you change this code don't forget to update SLAVE STOP too */ } - | SLAVE START_SYM slave_thread_opts + | STOP_SYM ALL SLAVES slave_thread_opts + { + LEX *lex=Lex; + lex->sql_command = SQLCOM_SLAVE_ALL_STOP; + lex->type = 0; + /* If you change this code don't forget to update SLAVE STOP too */ + } + | SLAVE optional_connection_name START_SYM slave_thread_opts { LEX *lex=Lex; lex->sql_command = SQLCOM_SLAVE_START; @@ -7095,7 +7190,7 @@ slave: } slave_until {} - | SLAVE STOP_SYM slave_thread_opts + | SLAVE optional_connection_name STOP_SYM slave_thread_opts { LEX *lex=Lex; lex->sql_command = SQLCOM_SLAVE_STOP; @@ -7152,6 +7247,10 @@ slave_until: MYSQL_YYABORT; } } + | UNTIL_SYM MASTER_GTID_POS_SYM EQ TEXT_STRING_sys + { + Lex->mi.gtid_pos_str = $4; + } ; slave_until_opts: @@ -7225,7 +7324,7 @@ analyze: /* Will be overriden during execution. */ YYPS->m_lock_type= TL_UNLOCK; } - table_list + analyze_table_list { LEX* lex= thd->lex; DBUG_ASSERT(!lex->m_stmt); @@ -7235,6 +7334,93 @@ analyze: } ; +analyze_table_list: + analyze_table_elem_spec + | analyze_table_list ',' analyze_table_elem_spec + ; + +analyze_table_elem_spec: + table_name opt_persistent_stat_clause + ; + +opt_persistent_stat_clause: + /* empty */ + {} + | PERSISTENT_SYM FOR_SYM persistent_stat_spec + { + thd->lex->with_persistent_for_clause= TRUE; + } + ; + +persistent_stat_spec: + ALL + {} + | COLUMNS persistent_column_stat_spec INDEXES persistent_index_stat_spec + {} + +persistent_column_stat_spec: + ALL {} + | '(' + { + LEX* lex= thd->lex; + lex->column_list= new List<LEX_STRING>; + if (lex->column_list == NULL) + MYSQL_YYABORT; + } + table_column_list + ')' + ; + +persistent_index_stat_spec: + ALL {} + | '(' + { + LEX* lex= thd->lex; + lex->index_list= new List<LEX_STRING>; + if (lex->index_list == NULL) + MYSQL_YYABORT; + } + table_index_list + ')' + ; + +table_column_list: + /* empty */ + {} + | ident + { + Lex->column_list->push_back((LEX_STRING*) + sql_memdup(&$1, sizeof(LEX_STRING))); + } + | table_column_list ',' ident + { + Lex->column_list->push_back((LEX_STRING*) + sql_memdup(&$3, sizeof(LEX_STRING))); + } + ; + +table_index_list: + /* empty */ + {} + | table_index_name + | table_index_list ',' table_index_name + ; + +table_index_name: + ident + { + Lex->index_list->push_back( + (LEX_STRING*) sql_memdup(&$1, sizeof(LEX_STRING))); + } + | + PRIMARY_SYM + { + LEX_STRING str= {(char*) "PRIMARY", 7}; + Lex->index_list->push_back( + (LEX_STRING*) sql_memdup(&str, sizeof(LEX_STRING))); + } + ; + binlog_base64_event: BINLOG_SYM TEXT_STRING_sys { @@ -7732,6 +7918,12 @@ select_alias: | TEXT_STRING_sys { $$=$1; } ; +opt_default_time_precision: + /* empty */ { $$= NOT_FIXED_DEC; } + | '(' ')' { $$= NOT_FIXED_DEC; } + | '(' real_ulong_num ')' { $$= $2; }; + ; + opt_time_precision: /* empty */ { $$= 0; } | '(' ')' { $$= 0; } @@ -8250,7 +8442,7 @@ dyncall_create_element: alloc_root(thd->mem_root, sizeof(DYNCALL_CREATE_DEF)); if ($$ == NULL) MYSQL_YYABORT; - $$->num= $1; + $$->key= $1; $$->value= $3; $$->type= (DYNAMIC_COLUMN_TYPE)$4; $$->cs= lex->charset; @@ -8458,6 +8650,14 @@ function_call_keyword: Lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_SYSTEM_FUNCTION); Lex->safe_to_cache_query= 0; } + | CURRENT_ROLE optional_braces + { + $$= new (thd->mem_root) Item_func_current_role(Lex->current_context()); + if ($$ == NULL) + MYSQL_YYABORT; + Lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_SYSTEM_FUNCTION); + Lex->safe_to_cache_query= 0; + } | DATE_SYM '(' expr ')' { $$= new (thd->mem_root) Item_date_typecast($3); @@ -8801,16 +9001,9 @@ function_call_nonkeyword: MYSQL_YYABORT; } | - COLUMN_EXISTS_SYM '(' expr ',' expr ')' - { - $$= new (thd->mem_root) Item_func_dyncol_exists($3, $5); - if ($$ == NULL) - MYSQL_YYABORT; - } - | - COLUMN_LIST_SYM '(' expr ')' + COLUMN_CHECK_SYM '(' expr ')' { - $$= new (thd->mem_root) Item_func_dyncol_list($3); + $$= new (thd->mem_root) Item_func_dyncol_check($3); if ($$ == NULL) MYSQL_YYABORT; } @@ -9613,9 +9806,7 @@ join_table: left-associative joins. */ table_ref normal_join table_ref %prec TABLE_REF_PRIORITY - { MYSQL_YYABORT_UNLESS($1 && ($$=$3)); } - | table_ref STRAIGHT_JOIN table_factor - { MYSQL_YYABORT_UNLESS($1 && ($$=$3)); $3->straight=1; } + { MYSQL_YYABORT_UNLESS($1 && ($$=$3)); $3->straight=$2; } | table_ref normal_join table_ref ON { @@ -9627,22 +9818,7 @@ join_table: } expr { - add_join_on($3,$6); - Lex->pop_context(); - Select->parsing_place= NO_MATTER; - } - | table_ref STRAIGHT_JOIN table_factor - ON - { - MYSQL_YYABORT_UNLESS($1 && $3); - /* Change the current name resolution context to a local context. */ - if (push_new_name_resolution_context(thd, $1, $3)) - MYSQL_YYABORT; - Select->parsing_place= IN_ON; - } - expr - { - $3->straight=1; + $3->straight=$2; add_join_on($3,$6); Lex->pop_context(); Select->parsing_place= NO_MATTER; @@ -9653,10 +9829,15 @@ join_table: MYSQL_YYABORT_UNLESS($1 && $3); } '(' using_list ')' - { add_join_natural($1,$3,$7,Select); $$=$3; } - | table_ref NATURAL JOIN_SYM table_factor + { + $3->straight=$2; + add_join_natural($1,$3,$7,Select); + $$=$3; + } + | table_ref NATURAL inner_join table_factor { MYSQL_YYABORT_UNLESS($1 && ($$=$4)); + $4->straight=$3; add_join_natural($1,$4,NULL,Select); } @@ -9736,10 +9917,16 @@ join_table: } ; + +inner_join: /* $$ set if using STRAIGHT_JOIN, false otherwise */ + JOIN_SYM { $$ = 0; } + | INNER_SYM JOIN_SYM { $$ = 0; } + | STRAIGHT_JOIN { $$ = 1; } + ; + normal_join: - JOIN_SYM {} - | INNER_SYM JOIN_SYM {} - | CROSS JOIN_SYM {} + inner_join { $$ = $1; } + | CROSS JOIN_SYM { $$ = 0; } ; /* @@ -10732,41 +10919,41 @@ do: */ drop: - DROP opt_temporary table_or_tables if_exists + DROP opt_temporary table_or_tables opt_if_exists { LEX *lex=Lex; lex->sql_command = SQLCOM_DROP_TABLE; lex->drop_temporary= $2; - lex->drop_if_exists= $4; + lex->check_exists= $4; YYPS->m_lock_type= TL_UNLOCK; YYPS->m_mdl_type= MDL_EXCLUSIVE; } table_list opt_restrict {} - | DROP INDEX_SYM ident ON table_ident {} + | DROP INDEX_SYM opt_if_exists ident ON table_ident {} { LEX *lex=Lex; - Alter_drop *ad= new Alter_drop(Alter_drop::KEY, $3.str); + Alter_drop *ad= new Alter_drop(Alter_drop::KEY, $4.str, $3); if (ad == NULL) MYSQL_YYABORT; lex->sql_command= SQLCOM_DROP_INDEX; lex->alter_info.reset(); lex->alter_info.flags= ALTER_DROP_INDEX; lex->alter_info.drop_list.push_back(ad); - if (!lex->current_select->add_table_to_list(lex->thd, $5, NULL, + if (!lex->current_select->add_table_to_list(lex->thd, $6, NULL, TL_OPTION_UPDATING, TL_READ_NO_INSERT, MDL_SHARED_NO_WRITE)) MYSQL_YYABORT; } - | DROP DATABASE if_exists ident + | DROP DATABASE opt_if_exists ident { LEX *lex=Lex; lex->sql_command= SQLCOM_DROP_DB; - lex->drop_if_exists=$3; + lex->check_exists=$3; lex->name= $4; } - | DROP FUNCTION_SYM if_exists ident '.' ident + | DROP FUNCTION_SYM opt_if_exists ident '.' ident { LEX *lex= thd->lex; sp_name *spname; @@ -10781,14 +10968,14 @@ drop: MYSQL_YYABORT; } lex->sql_command = SQLCOM_DROP_FUNCTION; - lex->drop_if_exists= $3; + lex->check_exists= $3; spname= new sp_name($4, $6, true); if (spname == NULL) MYSQL_YYABORT; spname->init_qname(thd); lex->spname= spname; } - | DROP FUNCTION_SYM if_exists ident + | DROP FUNCTION_SYM opt_if_exists ident { LEX *lex= thd->lex; LEX_STRING db= {0, 0}; @@ -10801,14 +10988,14 @@ drop: if (thd->db && lex->copy_db_to(&db.str, &db.length)) MYSQL_YYABORT; lex->sql_command = SQLCOM_DROP_FUNCTION; - lex->drop_if_exists= $3; + lex->check_exists= $3; spname= new sp_name(db, $4, false); if (spname == NULL) MYSQL_YYABORT; spname->init_qname(thd); lex->spname= spname; } - | DROP PROCEDURE_SYM if_exists sp_name + | DROP PROCEDURE_SYM opt_if_exists sp_name { LEX *lex=Lex; if (lex->sphead) @@ -10817,34 +11004,38 @@ drop: MYSQL_YYABORT; } lex->sql_command = SQLCOM_DROP_PROCEDURE; - lex->drop_if_exists= $3; + lex->check_exists= $3; lex->spname= $4; } | DROP USER clear_privileges user_list { Lex->sql_command = SQLCOM_DROP_USER; } - | DROP VIEW_SYM if_exists + | DROP ROLE_SYM clear_privileges role_list + { + Lex->sql_command = SQLCOM_DROP_ROLE; + } + | DROP VIEW_SYM opt_if_exists { LEX *lex= Lex; lex->sql_command= SQLCOM_DROP_VIEW; - lex->drop_if_exists= $3; + lex->check_exists= $3; YYPS->m_lock_type= TL_UNLOCK; YYPS->m_mdl_type= MDL_EXCLUSIVE; } table_list opt_restrict {} - | DROP EVENT_SYM if_exists sp_name + | DROP EVENT_SYM opt_if_exists sp_name { - Lex->drop_if_exists= $3; + Lex->check_exists= $3; Lex->spname= $4; Lex->sql_command = SQLCOM_DROP_EVENT; } - | DROP TRIGGER_SYM if_exists sp_name + | DROP TRIGGER_SYM opt_if_exists sp_name { LEX *lex= Lex; lex->sql_command= SQLCOM_DROP_TRIGGER; - lex->drop_if_exists= $3; + lex->check_exists= $3; lex->spname= $4; } | DROP TABLESPACE tablespace_name opt_ts_engine opt_ts_wait @@ -10857,10 +11048,10 @@ drop: LEX *lex= Lex; lex->alter_tablespace_info->ts_cmd_type= DROP_LOGFILE_GROUP; } - | DROP SERVER_SYM if_exists ident_or_text + | DROP SERVER_SYM opt_if_exists ident_or_text { Lex->sql_command = SQLCOM_DROP_SERVER; - Lex->drop_if_exists= $3; + Lex->check_exists= $3; Lex->server_options.server_name= $4.str; Lex->server_options.server_name_length= $4.length; } @@ -10898,9 +11089,17 @@ table_alias_ref: } ; -if_exists: - /* empty */ { $$= 0; } - | IF EXISTS { $$= 1; } +opt_if_exists: + /* empty */ + { + Lex->check_exists= FALSE; + $$= 0; + } + | IF EXISTS + { + Lex->check_exists= TRUE; + $$= 1; + } ; opt_temporary: @@ -11197,6 +11396,7 @@ single_multi: } where_clause opt_order_clause delete_limit_clause {} + opt_select_expressions {} | table_wild_list { mysql_init_multi_delete(Lex); @@ -11221,6 +11421,11 @@ single_multi: } ; +opt_select_expressions: + /* empty */ + | RETURNING_SYM select_item_list + ; + table_wild_list: table_wild_one | table_wild_list ',' table_wild_one @@ -11363,6 +11568,7 @@ show: { LEX *lex=Lex; lex->wild=0; + lex->ident=null_lex_str; mysql_init_select(lex); lex->current_select->parsing_place= SELECT_LIST; bzero((char*) &lex->create_info,sizeof(lex->create_info)); @@ -11428,6 +11634,19 @@ show_param: if (prepare_schema_table(thd, lex, 0, SCH_PLUGINS)) MYSQL_YYABORT; } + | PLUGINS_SYM SONAME_SYM TEXT_STRING_sys + { + Lex->ident= $3; + Lex->sql_command= SQLCOM_SHOW_PLUGINS; + if (prepare_schema_table(thd, Lex, 0, SCH_ALL_PLUGINS)) + MYSQL_YYABORT; + } + | PLUGINS_SYM SONAME_SYM wild_and_where + { + Lex->sql_command= SQLCOM_SHOW_PLUGINS; + if (prepare_schema_table(thd, Lex, 0, SCH_ALL_PLUGINS)) + MYSQL_YYABORT; + } | ENGINE_SYM known_storage_engines show_engine_param { Lex->create_info.db_type= $2; } | ENGINE_SYM ALL show_engine_param @@ -11454,7 +11673,7 @@ show_param: LEX *lex= Lex; lex->sql_command= SQLCOM_SHOW_BINLOG_EVENTS; } opt_limit_clause_init - | RELAYLOG_SYM EVENTS_SYM binlog_in binlog_from + | RELAYLOG_SYM optional_connection_name EVENTS_SYM binlog_in binlog_from { LEX *lex= Lex; lex->sql_command= SQLCOM_SHOW_RELAYLOG_EVENTS; @@ -11549,20 +11768,16 @@ show_param: } | GRANTS { - LEX *lex=Lex; - lex->sql_command= SQLCOM_SHOW_GRANTS; - LEX_USER *curr_user; - if (!(curr_user= (LEX_USER*) lex->thd->alloc(sizeof(st_lex_user)))) + Lex->sql_command= SQLCOM_SHOW_GRANTS; + if (!(Lex->grant_user= (LEX_USER*)thd->alloc(sizeof(LEX_USER)))) MYSQL_YYABORT; - bzero(curr_user, sizeof(st_lex_user)); - lex->grant_user= curr_user; + Lex->grant_user->user= current_user_and_current_role; } - | GRANTS FOR_SYM user + | GRANTS FOR_SYM user_or_role { LEX *lex=Lex; lex->sql_command= SQLCOM_SHOW_GRANTS; lex->grant_user=$3; - lex->grant_user->password=null_lex_str; } | CREATE DATABASE opt_if_not_exists ident { @@ -11591,9 +11806,22 @@ show_param: { Lex->sql_command = SQLCOM_SHOW_MASTER_STAT; } + | ALL SLAVES STATUS_SYM + { + Lex->sql_command = SQLCOM_SHOW_SLAVE_STAT; + Lex->verbose= 1; + } | SLAVE STATUS_SYM { + LEX *lex= thd->lex; + lex->mi.connection_name= thd->variables.default_master_connection; + lex->sql_command = SQLCOM_SHOW_SLAVE_STAT; + lex->verbose= 0; + } + | SLAVE connection_name STATUS_SYM + { Lex->sql_command = SQLCOM_SHOW_SLAVE_STAT; + Lex->verbose= 0; } | CLIENT_STATS_SYM { @@ -11672,6 +11900,13 @@ show_param: Lex->spname= $3; Lex->sql_command = SQLCOM_SHOW_CREATE_EVENT; } + | describe_command FOR_SYM expr + { + Lex->sql_command= SQLCOM_SHOW_EXPLAIN; + if (prepare_schema_table(thd, Lex, 0, SCH_EXPLAIN)) + MYSQL_YYABORT; + add_value_to_list(thd, $3); + } ; show_engine_param: @@ -11754,13 +11989,21 @@ describe: } | describe_command opt_extended_describe { Lex->describe|= DESCRIBE_NORMAL; } - select + explainable_command { LEX *lex=Lex; lex->select_lex.options|= SELECT_DESCRIBE; } ; +explainable_command: + select + | insert + | replace + | update + | delete + ; + describe_command: DESC | DESCRIBE @@ -11848,8 +12091,17 @@ flush_option: { Lex->type|= REFRESH_SLOW_LOG; } | BINARY LOGS_SYM { Lex->type|= REFRESH_BINARY_LOG; } - | RELAY LOGS_SYM - { Lex->type|= REFRESH_RELAY_LOG; } + | RELAY LOGS_SYM optional_connection_name + { + LEX *lex= Lex; + if (lex->type & REFRESH_RELAY_LOG) + { + my_error(ER_WRONG_USAGE, MYF(0), "FLUSH", "RELAY LOGS"); + MYSQL_YYABORT; + } + lex->type|= REFRESH_RELAY_LOG; + lex->relay_log_connection_name= lex->mi.connection_name; + } | QUERY_SYM CACHE_SYM { Lex->type|= REFRESH_QUERY_CACHE_FREE; } | HOSTS_SYM @@ -11857,13 +12109,23 @@ flush_option: | PRIVILEGES { Lex->type|= REFRESH_GRANT; } | LOGS_SYM - { Lex->type|= REFRESH_LOG; } + { + Lex->type|= REFRESH_LOG; + Lex->relay_log_connection_name.str= (char*) ""; + Lex->relay_log_connection_name.length= 0; + } | STATUS_SYM { Lex->type|= REFRESH_STATUS; } - | SLAVE + | SLAVE optional_connection_name { - Lex->type|= REFRESH_SLAVE; - Lex->reset_slave_info.all= false; + LEX *lex= Lex; + if (lex->type & REFRESH_SLAVE) + { + my_error(ER_WRONG_USAGE, MYF(0), "FLUSH","SLAVE"); + MYSQL_YYABORT; + } + lex->type|= REFRESH_SLAVE; + lex->reset_slave_info.all= false; } | CLIENT_STATS_SYM { Lex->type|= REFRESH_CLIENT_STATS; } @@ -11907,6 +12169,7 @@ reset_options: reset_option: SLAVE { Lex->type|= REFRESH_SLAVE; } + optional_connection_name slave_reset_options { } | MASTER_SYM { Lex->type|= REFRESH_MASTER; } | QUERY_SYM CACHE_SYM { Lex->type|= REFRESH_QUERY_CACHE;} @@ -11955,6 +12218,7 @@ kill: lex->value_list.empty(); lex->users_list.empty(); lex->sql_command= SQLCOM_KILL; + lex->kill_type= KILL_TYPE_ID; } kill_type kill_option kill_expr { @@ -11971,13 +12235,17 @@ kill_option: /* empty */ { $$= (int) KILL_CONNECTION; } | CONNECTION_SYM { $$= (int) KILL_CONNECTION; } | QUERY_SYM { $$= (int) KILL_QUERY; } + | QUERY_SYM ID_SYM + { + $$= (int) KILL_QUERY; + Lex->kill_type= KILL_TYPE_QUERY; + } ; kill_expr: expr { Lex->value_list.push_front($$); - Lex->kill_type= KILL_TYPE_ID; } | USER user { @@ -11986,6 +12254,11 @@ kill_expr: } ; + +shutdown: + SHUTDOWN { Lex->sql_command= SQLCOM_SHUTDOWN; } + ; + /* change database */ use: @@ -12902,8 +13175,7 @@ user: if (!($$=(LEX_USER*) thd->alloc(sizeof(st_lex_user)))) MYSQL_YYABORT; $$->user = $1; - $$->host.str= (char *) "%"; - $$->host.length= 1; + $$->host= null_lex_str; // User or Role, see get_current_user() $$->password= null_lex_str; $$->plugin= empty_lex_str; $$->auth= empty_lex_str; @@ -12927,26 +13199,36 @@ user: system_charset_info, 0) || check_host_name(&$$->host)) MYSQL_YYABORT; - /* - Convert hostname part of username to lowercase. - It's OK to use in-place lowercase as long as - the character set is utf8. - */ - my_casedn_str(system_charset_info, $$->host.str); + if ($$->host.str[0]) + { + /* + Convert hostname part of username to lowercase. + It's OK to use in-place lowercase as long as + the character set is utf8. + */ + my_casedn_str(system_charset_info, $$->host.str); + } + else + { + /* + fix historical undocumented convention that empty host is the + same as '%' + */ + $$->host= host_not_specified; + } } | CURRENT_USER optional_braces { - if (!($$=(LEX_USER*) thd->alloc(sizeof(st_lex_user)))) + if (!($$=(LEX_USER*)thd->calloc(sizeof(LEX_USER)))) MYSQL_YYABORT; - /* - empty LEX_USER means current_user and - will be handled in the get_current_user() function - later - */ - bzero($$, sizeof(LEX_USER)); + $$->user= current_user; + $$->plugin= empty_lex_str; + $$->auth= empty_lex_str; } ; +user_or_role: user | current_role; + /* Keyword that we allow for identifiers (except SP labels) */ keyword: keyword_sp {} @@ -12960,11 +13242,10 @@ keyword: | CHECKPOINT_SYM {} | CLOSE_SYM {} | COLUMN_ADD_SYM {} + | COLUMN_CHECK_SYM {} | COLUMN_CREATE_SYM {} | COLUMN_DELETE_SYM {} - | COLUMN_EXISTS_SYM {} | COLUMN_GET_SYM {} - | COLUMN_LIST_SYM {} | COMMENT_SYM {} | COMMIT_SYM {} | CONTAINS_SYM {} @@ -12999,6 +13280,7 @@ keyword: | SIGNED_SYM {} | SOCKET_SYM {} | SLAVE {} + | SLAVES {} | SONAME_SYM {} | START_SYM {} | STOP_SYM {} @@ -13019,6 +13301,7 @@ keyword: keyword_sp: ACTION {} | ADDDATE_SYM {} + | ADMIN_SYM {} | AFTER_SYM {} | AGAINST {} | AGGREGATE_SYM {} @@ -13029,6 +13312,7 @@ keyword_sp: | AUTHORS_SYM {} | AUTO_INC {} | AUTOEXTEND_SIZE_SYM {} + | AUTO_SYM {} | AVG_ROW_LENGTH {} | AVG_SYM {} | BINLOG_SYM {} @@ -13062,6 +13346,7 @@ keyword_sp: | CONSTRAINT_NAME_SYM {} | CONTEXT_SYM {} | CONTRIBUTORS_SYM {} + | CURRENT_POS_SYM {} | CPU_SYM {} | CUBE_SYM {} | CURSOR_NAME_SYM {} @@ -13112,6 +13397,7 @@ keyword_sp: | HARD_SYM {} | HOSTS_SYM {} | HOUR_SYM {} + | ID_SYM {} | IDENTIFIED_SYM {} | IGNORE_SERVER_IDS_SYM {} | INDEX_STATS_SYM {} @@ -13139,11 +13425,13 @@ keyword_sp: | MAX_ROWS {} | MASTER_SYM {} | MASTER_HEARTBEAT_PERIOD_SYM {} + | MASTER_GTID_POS_SYM {} | MASTER_HOST_SYM {} | MASTER_PORT_SYM {} | MASTER_LOG_FILE_SYM {} | MASTER_LOG_POS_SYM {} | MASTER_USER_SYM {} + | MASTER_USE_GTID_SYM {} | MASTER_PASSWORD_SYM {} | MASTER_SERVER_ID_SYM {} | MASTER_CONNECT_RETRY_SYM {} @@ -13231,6 +13519,7 @@ keyword_sp: | RESOURCES {} | RESUME_SYM {} | RETURNS_SYM {} + | ROLE_SYM {} | ROLLUP_SYM {} | ROUTINE_SYM {} | ROWS_SYM {} @@ -13246,6 +13535,7 @@ keyword_sp: | SIMPLE_SYM {} | SHARE_SYM {} | SHUTDOWN {} + | SLAVE_POS_SYM {} | SLOW {} | SNAPSHOT_SYM {} | SOFT_SYM {} @@ -13579,6 +13869,12 @@ option_value: MYSQL_YYABORT; lex->var_list.push_back(var); } + | ROLE_SYM ident_or_text + { + LEX *lex = Lex; + set_var_role *var= new set_var_role($2); + lex->var_list.push_back(var); + } | PASSWORD equal text_or_password { LEX *lex= thd->lex; @@ -13593,10 +13889,9 @@ option_value: my_error(ER_SP_BAD_VAR_SHADOW, MYF(0), pw.str); MYSQL_YYABORT; } - if (!(user=(LEX_USER*) thd->alloc(sizeof(LEX_USER)))) + if (!(user=(LEX_USER*) thd->calloc(sizeof(LEX_USER)))) MYSQL_YYABORT; - user->host=null_lex_str; - user->user.str=thd->security_ctx->user; + user->user= current_user; set_var_password *var= new set_var_password(user, $3); if (var == NULL) MYSQL_YYABORT; @@ -13928,13 +14223,13 @@ revoke: ; revoke_command: - grant_privileges ON opt_table grant_ident FROM user_list + grant_privileges ON opt_table grant_ident FROM user_and_role_list { LEX *lex= Lex; lex->sql_command= SQLCOM_REVOKE; lex->type= 0; } - | grant_privileges ON FUNCTION_SYM grant_ident FROM user_list + | grant_privileges ON FUNCTION_SYM grant_ident FROM user_and_role_list { LEX *lex= Lex; if (lex->columns.elements) @@ -13945,7 +14240,7 @@ revoke_command: lex->sql_command= SQLCOM_REVOKE; lex->type= TYPE_ENUM_FUNCTION; } - | grant_privileges ON PROCEDURE_SYM grant_ident FROM user_list + | grant_privileges ON PROCEDURE_SYM grant_ident FROM user_and_role_list { LEX *lex= Lex; if (lex->columns.elements) @@ -13956,7 +14251,7 @@ revoke_command: lex->sql_command= SQLCOM_REVOKE; lex->type= TYPE_ENUM_PROCEDURE; } - | ALL opt_privileges ',' GRANT OPTION FROM user_list + | ALL opt_privileges ',' GRANT OPTION FROM user_and_role_list { Lex->sql_command = SQLCOM_REVOKE_ALL; } @@ -13966,9 +14261,22 @@ revoke_command: lex->users_list.push_front ($3); lex->sql_command= SQLCOM_REVOKE; lex->type= TYPE_ENUM_PROXY; - } + } + | admin_option_for_role FROM user_and_role_list + { + Lex->sql_command= SQLCOM_REVOKE_ROLE; + if (Lex->users_list.push_front($1)) + MYSQL_YYABORT; + } ; +admin_option_for_role: + ADMIN_SYM OPTION FOR_SYM grant_role + { Lex->with_admin_option= true; $$= $4; } + | grant_role + { Lex->with_admin_option= false; $$= $1; } + ; + grant: GRANT clear_privileges grant_command {} @@ -14012,7 +14320,67 @@ grant_command: lex->users_list.push_front ($3); lex->sql_command= SQLCOM_GRANT; lex->type= TYPE_ENUM_PROXY; - } + } + | grant_role TO_SYM grant_list opt_with_admin_option + { + LEX *lex= Lex; + lex->sql_command= SQLCOM_GRANT_ROLE; + /* The first role is the one that is granted */ + if (Lex->users_list.push_front($1)) + MYSQL_YYABORT; + } + + ; + +opt_with_admin: + /* nothing */ { Lex->definer = 0; } + | WITH ADMIN_SYM user_or_role { Lex->definer = $3; } + +opt_with_admin_option: + /* nothing */ { Lex->with_admin_option= false; } + | WITH ADMIN_SYM OPTION { Lex->with_admin_option= true; } + +role_list: + grant_role + { + if (Lex->users_list.push_back($1)) + MYSQL_YYABORT; + } + | role_list ',' grant_role + { + if (Lex->users_list.push_back($3)) + MYSQL_YYABORT; + } + ; + +current_role: + CURRENT_ROLE optional_braces + { + if (!($$=(LEX_USER*) thd->calloc(sizeof(LEX_USER)))) + MYSQL_YYABORT; + $$->user= current_role; + $$->plugin= empty_lex_str; + $$->auth= empty_lex_str; + } + ; + +grant_role: + ident_or_text + { + if (!($$=(LEX_USER*) thd->alloc(sizeof(st_lex_user)))) + MYSQL_YYABORT; + $$->user = $1; + $$->host= empty_lex_str; + $$->password= null_lex_str; + $$->plugin= empty_lex_str; + $$->auth= empty_lex_str; + + if (check_string_char_length(&$$->user, ER(ER_USERNAME), + username_char_length, + system_charset_info, 0)) + MYSQL_YYABORT; + } + | current_role ; opt_table: @@ -14202,6 +14570,19 @@ grant_list: } ; +user_and_role_list: + user_or_role + { + if (Lex->users_list.push_back($1)) + MYSQL_YYABORT; + } + | user_and_role_list ',' user_or_role + { + if (Lex->users_list.push_back($3)) + MYSQL_YYABORT; + } + ; + via_or_with: VIA_SYM | WITH ; using_or_as: USING | AS ; @@ -14250,7 +14631,7 @@ grant_user: $1->plugin= $4; $1->auth= $6; } - | user + | user_or_role { $$= $1; $1->password= null_lex_str; } ; @@ -14682,9 +15063,9 @@ no_definer: ; definer: - DEFINER_SYM EQ user + DEFINER_SYM EQ user_or_role { - thd->lex->definer= get_current_user(thd, $3); + thd->lex->definer= $3; } ; diff --git a/sql/strfunc.cc b/sql/strfunc.cc index 06bd92e0bc7..452937024f7 100644 --- a/sql/strfunc.cc +++ b/sql/strfunc.cc @@ -15,6 +15,7 @@ /* Some useful string utility functions used by the MySQL server */ +#include <my_global.h> #include "sql_priv.h" #include "unireg.h" #include "strfunc.h" diff --git a/sql/structs.h b/sql/structs.h index ae71819ae09..f4b85433e3a 100644 --- a/sql/structs.h +++ b/sql/structs.h @@ -29,6 +29,7 @@ struct TABLE; class Field; +class Index_statistics; class THD; @@ -96,6 +97,11 @@ typedef struct st_key { uint block_size; uint name_length; enum ha_key_alg algorithm; + /* + The flag is on if statistical data for the index prefixes + has to be taken from the system statistical tables. + */ + bool is_statistics_from_stat_tables; /* Note that parser is used when the table is opened for use, and parser_name is used when the table is being created. @@ -115,6 +121,18 @@ typedef struct st_key { For temporary heap tables this member is NULL. */ ulong *rec_per_key; + + /* + This structure is used for statistical data on the index + that has been read from the statistical table index_stat + */ + Index_statistics *read_stats; + /* + This structure is used for statistical data on the index that + is collected by the function collect_statistics_for_table + */ + Index_statistics *collected_stats; + union { int bdb_return_if_eq; } handler; @@ -123,6 +141,9 @@ typedef struct st_key { /** reference to the list of options or NULL */ engine_option_value *option_list; ha_index_option_struct *option_struct; /* structure with parsed options */ + + double actual_rec_per_key(uint i); + } KEY; @@ -169,6 +190,14 @@ typedef int *(*update_var)(THD *, struct st_mysql_show_var *); typedef struct st_lex_user { LEX_STRING user, host, password, plugin, auth; + bool is_role() { return user.str[0] && !host.str[0]; } + void set_lex_string(LEX_STRING *l, char *buf) + { + if (is_role()) + *l= user; + else + l->length= strxmov(l->str= buf, user.str, "@", host.str, NullS) - buf; + } } LEX_USER; /* diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index d82c8f30ed2..77ea9e7a994 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -55,6 +55,9 @@ #include "../storage/perfschema/pfs_server.h" #endif /* WITH_PERFSCHEMA_STORAGE_ENGINE */ #include "threadpool.h" +#include "sql_repl.h" +#include "opt_range.h" +#include "rpl_parallel.h" /* The rule for this file: everything should be 'static'. When a sys_var @@ -801,6 +804,30 @@ static Sys_var_lexstring Sys_init_connect( DEFAULT(""), &PLock_sys_init_connect, NOT_IN_BINLOG, ON_CHECK(check_init_string)); +#ifdef HAVE_REPLICATION +static bool check_master_connection(sys_var *self, THD *thd, set_var *var) +{ + LEX_STRING tmp; + tmp.str= var->save_result.string_value.str; + tmp.length= var->save_result.string_value.length; + if (!tmp.str || check_master_connection_name(&tmp)) + { + my_error(ER_WRONG_ARGUMENTS, MYF(ME_JUST_WARNING), + var->var->name.str); + return true; + } + return false; +} + +static Sys_var_session_lexstring Sys_default_master_connection( + "default_master_connection", + "Master connection to use for all slave variables and slave commands", + SESSION_ONLY(default_master_connection), + NO_CMD_LINE, IN_SYSTEM_CHARSET, + DEFAULT(""), MAX_CONNECTION_NAME, ON_CHECK(check_master_connection), + ON_UPDATE(0)); +#endif + static Sys_var_charptr Sys_init_file( "init_file", "Read SQL commands from this file at startup", READ_ONLY GLOBAL_VAR(opt_init_file), @@ -1086,16 +1113,12 @@ static Sys_var_ulonglong Sys_max_binlog_stmt_cache_size( static bool fix_max_binlog_size(sys_var *self, THD *thd, enum_var_type type) { mysql_bin_log.set_max_size(max_binlog_size); -#ifdef HAVE_REPLICATION - if (!max_relay_log_size) - active_mi->rli.relay_log.set_max_size(max_binlog_size); -#endif return false; } static Sys_var_ulong Sys_max_binlog_size( "max_binlog_size", "Binary log will be rotated automatically when the size exceeds this " - "value. Will also apply to relay logs if max_relay_log_size is 0", + "value.", GLOBAL_VAR(max_binlog_size), CMD_LINE(REQUIRED_ARG), VALID_RANGE(IO_SIZE, 1024*1024L*1024L), DEFAULT(1024*1024L*1024L), BLOCK_SIZE(IO_SIZE), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), @@ -1185,6 +1208,438 @@ static Sys_var_ulong Sys_pseudo_thread_id( BLOCK_SIZE(1), NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(check_has_super)); +static bool +check_gtid_domain_id(sys_var *self, THD *thd, set_var *var) +{ + if (check_has_super(self, thd, var)) + return true; + if (var->type != OPT_GLOBAL && + error_if_in_trans_or_substatement(thd, + ER_STORED_FUNCTION_PREVENTS_SWITCH_GTID_DOMAIN_ID_SEQ_NO, + ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_GTID_DOMAIN_ID_SEQ_NO)) + return true; + + return false; +} + + +static Sys_var_uint Sys_gtid_domain_id( + "gtid_domain_id", + "Used with global transaction ID to identify logically independent " + "replication streams. When events can propagate through multiple " + "parallel paths (for example multiple masters), each independent " + "source server must use a distinct domain_id. For simple tree-shaped " + "replication topologies, it can be left at its default, 0.", + SESSION_VAR(gtid_domain_id), + CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, UINT_MAX32), DEFAULT(0), + BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(check_gtid_domain_id)); + + +static bool check_gtid_seq_no(sys_var *self, THD *thd, set_var *var) +{ + uint32 domain_id, server_id; + uint64 seq_no; + + if (check_has_super(self, thd, var)) + return true; + if (error_if_in_trans_or_substatement(thd, + ER_STORED_FUNCTION_PREVENTS_SWITCH_GTID_DOMAIN_ID_SEQ_NO, + ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_GTID_DOMAIN_ID_SEQ_NO)) + return true; + + domain_id= thd->variables.gtid_domain_id; + server_id= thd->variables.server_id; + seq_no= (uint64)var->value->val_uint(); + DBUG_EXECUTE_IF("ignore_set_gtid_seq_no_check", return 0;); + if (opt_gtid_strict_mode && opt_bin_log && + mysql_bin_log.check_strict_gtid_sequence(domain_id, server_id, seq_no)) + return true; + + return false; +} + + +static Sys_var_ulonglong Sys_gtid_seq_no( + "gtid_seq_no", + "Internal server usage, for replication with global transaction id. " + "When set, next event group logged to the binary log will use this " + "sequence number, not generate a new one, thus allowing to preserve " + "master's GTID in slave's binlog.", + SESSION_ONLY(gtid_seq_no), + NO_CMD_LINE, VALID_RANGE(0, ULONGLONG_MAX), DEFAULT(0), + BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(check_gtid_seq_no)); + + +#ifdef HAVE_REPLICATION +static unsigned char opt_gtid_binlog_pos_dummy; +static Sys_var_gtid_binlog_pos Sys_gtid_binlog_pos( + "gtid_binlog_pos", "Last GTID logged to the binary log, per replication" + "domain", + READ_ONLY GLOBAL_VAR(opt_gtid_binlog_pos_dummy), NO_CMD_LINE); + + +uchar * +Sys_var_gtid_binlog_pos::global_value_ptr(THD *thd, LEX_STRING *base) +{ + char buf[128]; + String str(buf, sizeof(buf), system_charset_info); + char *p; + + str.length(0); + if ((opt_bin_log && mysql_bin_log.append_state_pos(&str)) || + !(p= thd->strmake(str.ptr(), str.length()))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return NULL; + } + + return (uchar *)p; +} + + +static unsigned char opt_gtid_current_pos_dummy; +static Sys_var_gtid_current_pos Sys_gtid_current_pos( + "gtid_current_pos", "Current GTID position of the server. Per " + "replication domain, this is either the last GTID replicated by a " + "slave thread, or the GTID logged to the binary log, whichever is " + "most recent.", + READ_ONLY GLOBAL_VAR(opt_gtid_current_pos_dummy), NO_CMD_LINE); + + +uchar * +Sys_var_gtid_current_pos::global_value_ptr(THD *thd, LEX_STRING *base) +{ + String str; + char *p; + + str.length(0); + if (rpl_append_gtid_state(&str, true) || + !(p= thd->strmake(str.ptr(), str.length()))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return NULL; + } + + return (uchar *)p; +} + + +bool +Sys_var_gtid_slave_pos::do_check(THD *thd, set_var *var) +{ + String str, *res; + bool running; + + DBUG_ASSERT(var->type == OPT_GLOBAL); + + if (rpl_load_gtid_slave_state(thd)) + { + my_error(ER_CANNOT_LOAD_SLAVE_GTID_STATE, MYF(0), "mysql", + rpl_gtid_slave_state_table_name.str); + return true; + } + + mysql_mutex_lock(&LOCK_active_mi); + running= master_info_index->give_error_if_slave_running(); + mysql_mutex_unlock(&LOCK_active_mi); + if (running) + return true; + if (!(res= var->value->val_str(&str))) + return true; + if (thd->in_active_multi_stmt_transaction()) + { + my_error(ER_CANT_DO_THIS_DURING_AN_TRANSACTION, MYF(0)); + return true; + } + if (rpl_gtid_pos_check(thd, &((*res)[0]), res->length())) + return true; + + if (!(var->save_result.string_value.str= + thd->strmake(res->ptr(), res->length()))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return true; + } + var->save_result.string_value.length= res->length(); + return false; +} + + +bool +Sys_var_gtid_slave_pos::global_update(THD *thd, set_var *var) +{ + bool err; + + DBUG_ASSERT(var->type == OPT_GLOBAL); + + if (!var->value) + { + my_error(ER_NO_DEFAULT, MYF(0), var->var->name.str); + return true; + } + + mysql_mutex_unlock(&LOCK_global_system_variables); + mysql_mutex_lock(&LOCK_active_mi); + if (master_info_index->give_error_if_slave_running()) + err= true; + else + err= rpl_gtid_pos_update(thd, var->save_result.string_value.str, + var->save_result.string_value.length); + mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_global_system_variables); + return err; +} + + +uchar * +Sys_var_gtid_slave_pos::global_value_ptr(THD *thd, LEX_STRING *base) +{ + String str; + char *p; + + str.length(0); + /* + If the mysql.rpl_slave_pos table could not be loaded, then we cannot + easily automatically try to reload it here - we may be inside a statement + that already has tables locked and so opening more tables is problematic. + + But if the table is not loaded (eg. missing mysql_upgrade_db or some such), + then the slave state must be empty anyway. + */ + if ((rpl_global_gtid_slave_state.loaded && + rpl_append_gtid_state(&str, false)) || + !(p= thd->strmake(str.ptr(), str.length()))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return NULL; + } + + return (uchar *)p; +} + + +static unsigned char opt_gtid_slave_pos_dummy; +static Sys_var_gtid_slave_pos Sys_gtid_slave_pos( + "gtid_slave_pos", + "The list of global transaction IDs that were last replicated on the " + "server, one for each replication domain.", + GLOBAL_VAR(opt_gtid_slave_pos_dummy), NO_CMD_LINE); + + +static Sys_var_mybool Sys_gtid_strict_mode( + "gtid_strict_mode", + "Enforce strict seq_no ordering of events in the binary log. Slave " + "stops with an error if it encounters an event that would cause it to " + "generate an out-of-order binlog if executed.", + GLOBAL_VAR(opt_gtid_strict_mode), + CMD_LINE(OPT_ARG), DEFAULT(FALSE)); + + +struct gtid_binlog_state_data { rpl_gtid *list; uint32 list_len; }; + +bool +Sys_var_gtid_binlog_state::do_check(THD *thd, set_var *var) +{ + String str, *res; + struct gtid_binlog_state_data *data; + rpl_gtid *list; + uint32 list_len; + + DBUG_ASSERT(var->type == OPT_GLOBAL); + + if (!(res= var->value->val_str(&str))) + return true; + if (thd->in_active_multi_stmt_transaction()) + { + my_error(ER_CANT_DO_THIS_DURING_AN_TRANSACTION, MYF(0)); + return true; + } + if (!mysql_bin_log.is_open()) + { + my_error(ER_FLUSH_MASTER_BINLOG_CLOSED, MYF(0)); + return true; + } + if (!mysql_bin_log.is_empty_state()) + { + my_error(ER_BINLOG_MUST_BE_EMPTY, MYF(0)); + return true; + } + if (res->length() == 0) + list= NULL; + else if (!(list= gtid_parse_string_to_list(res->ptr(), res->length(), + &list_len))) + { + my_error(ER_INCORRECT_GTID_STATE, MYF(0)); + return true; + } + if (!(data= (gtid_binlog_state_data *)my_malloc(sizeof(*data), MYF(0)))) + { + my_free(list); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return true; + } + data->list= list; + data->list_len= list_len; + var->save_result.ptr= data; + return false; +} + + +bool +Sys_var_gtid_binlog_state::global_update(THD *thd, set_var *var) +{ + bool res; + + DBUG_ASSERT(var->type == OPT_GLOBAL); + + if (!var->value) + { + my_error(ER_NO_DEFAULT, MYF(0), var->var->name.str); + return true; + } + + struct gtid_binlog_state_data *data= + (struct gtid_binlog_state_data *)var->save_result.ptr; + mysql_mutex_unlock(&LOCK_global_system_variables); + res= (0 != reset_master(thd, data->list, data->list_len)); + mysql_mutex_lock(&LOCK_global_system_variables); + my_free(data->list); + my_free(data); + return res; +} + + +uchar * +Sys_var_gtid_binlog_state::global_value_ptr(THD *thd, LEX_STRING *base) +{ + char buf[512]; + String str(buf, sizeof(buf), system_charset_info); + char *p; + + str.length(0); + if ((opt_bin_log && mysql_bin_log.append_state(&str)) || + !(p= thd->strmake(str.ptr(), str.length()))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return NULL; + } + + return (uchar *)p; +} + + +static unsigned char opt_gtid_binlog_state_dummy; +static Sys_var_gtid_binlog_state Sys_gtid_binlog_state( + "gtid_binlog_state", + "The internal GTID state of the binlog, used to keep track of all " + "GTIDs ever logged to the binlog.", + GLOBAL_VAR(opt_gtid_binlog_state_dummy), NO_CMD_LINE); + + +static Sys_var_last_gtid Sys_last_gtid( + "last_gtid", "The GTID of the last commit (if binlogging was enabled), " + "or the empty string if none.", + READ_ONLY sys_var::ONLY_SESSION, NO_CMD_LINE); + + +uchar * +Sys_var_last_gtid::session_value_ptr(THD *thd, LEX_STRING *base) +{ + char buf[10+1+10+1+20+1]; + String str(buf, sizeof(buf), system_charset_info); + char *p; + bool first= true; + + str.length(0); + if ((thd->last_commit_gtid.seq_no > 0 && + rpl_slave_state_tostring_helper(&str, &thd->last_commit_gtid, &first)) || + !(p= thd->strmake(str.ptr(), str.length()))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return NULL; + } + + return (uchar *)p; +} + + +static bool +check_slave_parallel_threads(sys_var *self, THD *thd, set_var *var) +{ + bool running; + + mysql_mutex_lock(&LOCK_active_mi); + running= master_info_index->give_error_if_slave_running(); + mysql_mutex_unlock(&LOCK_active_mi); + if (running) + return true; + + return false; +} + +static bool +fix_slave_parallel_threads(sys_var *self, THD *thd, enum_var_type type) +{ + bool running; + bool err= false; + + mysql_mutex_unlock(&LOCK_global_system_variables); + mysql_mutex_lock(&LOCK_active_mi); + running= master_info_index->give_error_if_slave_running(); + mysql_mutex_unlock(&LOCK_active_mi); + if (running || rpl_parallel_change_thread_count(&global_rpl_thread_pool, + opt_slave_parallel_threads)) + err= true; + mysql_mutex_lock(&LOCK_global_system_variables); + + return err; +} + + +static Sys_var_ulong Sys_slave_parallel_threads( + "slave_parallel_threads", + "If non-zero, number of threads to spawn to apply in parallel events " + "on the slave that were group-committed on the master or were logged " + "with GTID in different replication domains.", + GLOBAL_VAR(opt_slave_parallel_threads), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0,16383), DEFAULT(0), BLOCK_SIZE(1), NO_MUTEX_GUARD, + NOT_IN_BINLOG, ON_CHECK(check_slave_parallel_threads), + ON_UPDATE(fix_slave_parallel_threads)); + + +static Sys_var_ulong Sys_slave_parallel_max_queued( + "slave_parallel_max_queued", + "Limit on how much memory SQL threads should use per parallel " + "replication thread when reading ahead in the relay log looking for " + "opportunities for parallel replication. Only used when " + "--slave-parallel-threads > 0.", + GLOBAL_VAR(opt_slave_parallel_max_queued), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0,2147483647), DEFAULT(131072), BLOCK_SIZE(1)); +#endif + + +static Sys_var_ulong Sys_binlog_commit_wait_count( + "binlog_commit_wait_count", + "If non-zero, binlog write will wait at most binlog_commit_wait_usec " + "microseconds for at least this many commits to queue up for group " + "commit to the binlog. This can reduce I/O on the binlog and provide " + "increased opportunity for parallel apply on the slave, but too high " + "a value will decrease commit throughput.", + GLOBAL_VAR(opt_binlog_commit_wait_count), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0, ULONG_MAX), DEFAULT(0), BLOCK_SIZE(1)); + + +static Sys_var_ulong Sys_binlog_commit_wait_usec( + "binlog_commit_wait_usec", + "Maximum time, in microseconds, to wait for more commits to queue up " + "for binlog group commit. Only takes effect if the value of " + "binlog_commit_wait_count is non-zero.", + GLOBAL_VAR(opt_binlog_commit_wait_usec), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0, ULONG_MAX), DEFAULT(100000), BLOCK_SIZE(1)); + + static bool fix_max_join_size(sys_var *self, THD *thd, enum_var_type type) { SV *sv= type == OPT_GLOBAL ? &global_system_variables : &thd->variables; @@ -1240,24 +1695,6 @@ static Sys_var_ulong Sys_max_prepared_stmt_count( VALID_RANGE(0, 1024*1024), DEFAULT(16382), BLOCK_SIZE(1), &PLock_prepared_stmt_count); -static bool fix_max_relay_log_size(sys_var *self, THD *thd, enum_var_type type) -{ -#ifdef HAVE_REPLICATION - active_mi->rli.relay_log.set_max_size(max_relay_log_size ? - max_relay_log_size: max_binlog_size); -#endif - return false; -} -static Sys_var_ulong Sys_max_relay_log_size( - "max_relay_log_size", - "If non-zero: relay log will be rotated automatically when the " - "size exceeds this value; if zero: when the size " - "exceeds max_binlog_size", - GLOBAL_VAR(max_relay_log_size), CMD_LINE(REQUIRED_ARG), - VALID_RANGE(0, 1024L*1024*1024), DEFAULT(0), BLOCK_SIZE(IO_SIZE), - NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), - ON_UPDATE(fix_max_relay_log_size)); - static Sys_var_ulong Sys_max_sort_length( "max_sort_length", "The number of bytes to use when sorting BLOB or TEXT values (only " @@ -1426,13 +1863,42 @@ static Sys_var_ulong Sys_optimizer_prune_level( SESSION_VAR(optimizer_prune_level), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, 1), DEFAULT(1), BLOCK_SIZE(1)); +static Sys_var_ulong Sys_optimizer_selectivity_sampling_limit( + "optimizer_selectivity_sampling_limit", + "Controls number of record samples to check condition selectivity", + SESSION_VAR(optimizer_selectivity_sampling_limit), + CMD_LINE(REQUIRED_ARG), + VALID_RANGE(SELECTIVITY_SAMPLING_THRESHOLD, UINT_MAX), + DEFAULT(SELECTIVITY_SAMPLING_LIMIT), BLOCK_SIZE(1)); + +static Sys_var_ulong Sys_optimizer_use_condition_selectivity( + "optimizer_use_condition_selectivity", + "Controls selectivity of which conditions the optimizer takes into " + "account to calculate cardinality of a partial join when it searches " + "for the best execution plan " + "Meaning: " + "1 - use selectivity of index backed range conditions to calculate " + "the cardinality of a partial join if the last joined table is " + "accessed by full table scan or an index scan, " + "2 - use selectivity of index backed range conditions to calculate " + "the cardinality of a partial join in any case, " + "3 - additionally always use selectivity of range conditions that are " + "not backed by any index to calculate the cardinality of a partial join, " + "4 - use histograms to calculate selectivity of range conditions that " + "are not backed by any index to calculate the cardinality of " + "a partial join." + "5 - additionally use selectivity of certain non-range predicates " + "calculated on record samples", + SESSION_VAR(optimizer_use_condition_selectivity), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(1, 5), DEFAULT(1), BLOCK_SIZE(1)); + /** Warns about deprecated value 63 */ static bool fix_optimizer_search_depth(sys_var *self, THD *thd, enum_var_type type) { SV *sv= type == OPT_GLOBAL ? &global_system_variables : &thd->variables; if (sv->optimizer_search_depth == MAX_TABLES+2) - WARN_DEPRECATED(thd, 6, 0, "optimizer-search-depth=63", + WARN_DEPRECATED(thd, 10, 1, "optimizer-search-depth=63", "a search depth less than 63"); return false; } @@ -1475,6 +1941,7 @@ export const char *optimizer_switch_names[]= "optimize_join_buffer_size", "table_elimination", "extended_keys", + "exists_to_in", "default", NullS }; /** propagates changes to @@engine_condition_pushdown */ @@ -1516,7 +1983,8 @@ static Sys_var_flagset Sys_optimizer_switch( "semijoin_with_cache, " "subquery_cache, " "table_elimination, " - "extended_keys " + "extended_keys, " + "exists_to_in " "} and val is one of {on, off, default}", SESSION_VAR(optimizer_switch), CMD_LINE(REQUIRED_ARG), optimizer_switch_names, DEFAULT(OPTIMIZER_SWITCH_DEFAULT), @@ -1970,17 +2438,27 @@ static Sys_var_charptr Sys_secure_file_priv( static bool fix_server_id(sys_var *self, THD *thd, enum_var_type type) { - server_id_supplied = 1; - thd->server_id= server_id; + if (type == OPT_GLOBAL) + { + server_id_supplied = 1; + thd->variables.server_id= global_system_variables.server_id; + /* + Historically, server_id was a global variable that is exported to + plugins. Now it is a session variable, and lives in the + global_system_variables struct, but we still need to export the + value for reading to plugins for backwards compatibility reasons. + */ + ::server_id= global_system_variables.server_id; + } return false; } static Sys_var_ulong Sys_server_id( "server_id", "Uniquely identifies the server instance in the community of " "replication partners", - GLOBAL_VAR(server_id), CMD_LINE(REQUIRED_ARG, OPT_SERVER_ID), + SESSION_VAR(server_id), CMD_LINE(REQUIRED_ARG, OPT_SERVER_ID), VALID_RANGE(0, UINT_MAX32), DEFAULT(0), BLOCK_SIZE(1), NO_MUTEX_GUARD, - NOT_IN_BINLOG, ON_CHECK(0), ON_UPDATE(fix_server_id)); + NOT_IN_BINLOG, ON_CHECK(check_has_super), ON_UPDATE(fix_server_id)); static Sys_var_mybool Sys_slave_compressed_protocol( "slave_compressed_protocol", @@ -2032,50 +2510,22 @@ static Sys_var_mybool Sys_master_verify_checksum( static const char *replicate_events_marked_for_skip_names[]= { "replicate", "filter_on_slave", "filter_on_master", 0 }; -static bool -replicate_events_marked_for_skip_check(sys_var *self, THD *thd, - set_var *var) -{ - int thread_mask; - DBUG_ENTER("sys_var_replicate_events_marked_for_skip_check"); - - /* Slave threads must be stopped to change the variable. */ - mysql_mutex_lock(&LOCK_active_mi); - lock_slave_threads(active_mi); - init_thread_mask(&thread_mask, active_mi, 0 /*not inverse*/); - unlock_slave_threads(active_mi); - mysql_mutex_unlock(&LOCK_active_mi); - if (thread_mask) // We refuse if any slave thread is running - { - my_error(ER_SLAVE_MUST_STOP, MYF(0)); - DBUG_RETURN(true); - } - DBUG_RETURN(false); -} bool Sys_var_replicate_events_marked_for_skip::global_update(THD *thd, set_var *var) { - bool result; - int thread_mask; + bool result= true; // Assume error DBUG_ENTER("Sys_var_replicate_events_marked_for_skip::global_update"); - /* Slave threads must be stopped to change the variable. */ + mysql_mutex_unlock(&LOCK_global_system_variables); mysql_mutex_lock(&LOCK_active_mi); - lock_slave_threads(active_mi); - init_thread_mask(&thread_mask, active_mi, 0 /*not inverse*/); - if (thread_mask) // We refuse if any slave thread is running - { - my_error(ER_SLAVE_MUST_STOP, MYF(0)); - result= true; - } - else + if (!master_info_index->give_error_if_slave_running()) result= Sys_var_enum::global_update(thd, var); - - unlock_slave_threads(active_mi); mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_global_system_variables); DBUG_RETURN(result); } + static Sys_var_replicate_events_marked_for_skip Replicate_events_marked_for_skip ("replicate_events_marked_for_skip", "Whether the slave should replicate events that were created with " @@ -2086,8 +2536,7 @@ static Sys_var_replicate_events_marked_for_skip Replicate_events_marked_for_skip "the slave).", GLOBAL_VAR(opt_replicate_events_marked_for_skip), CMD_LINE(REQUIRED_ARG), replicate_events_marked_for_skip_names, DEFAULT(RPL_SKIP_REPLICATE), - NO_MUTEX_GUARD, NOT_IN_BINLOG, - ON_CHECK(replicate_events_marked_for_skip_check)); + NO_MUTEX_GUARD, NOT_IN_BINLOG); #endif @@ -3145,7 +3594,7 @@ static Sys_var_have Sys_have_geometry( static Sys_var_have Sys_have_openssl( "have_openssl", "have_openssl", - READ_ONLY GLOBAL_VAR(have_ssl), NO_CMD_LINE); + READ_ONLY GLOBAL_VAR(have_openssl), NO_CMD_LINE); static Sys_var_have Sys_have_profiling( "have_profiling", "have_profiling", @@ -3298,76 +3747,52 @@ static Sys_var_mybool Sys_relay_log_recovery( "processed", GLOBAL_VAR(relay_log_recovery), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); -bool Sys_var_rpl_filter::do_check(THD *thd, set_var *var) -{ - bool status; - - /* - We must not be holding LOCK_global_system_variables here, otherwise we can - deadlock with THD::init() which is invoked from within the slave threads - with opposite locking order. - */ - mysql_mutex_assert_not_owner(&LOCK_global_system_variables); - - mysql_mutex_lock(&LOCK_active_mi); - mysql_mutex_lock(&active_mi->rli.run_lock); - - status= active_mi->rli.slave_running; - - mysql_mutex_unlock(&active_mi->rli.run_lock); - mysql_mutex_unlock(&LOCK_active_mi); - - if (status) - my_error(ER_SLAVE_MUST_STOP, MYF(0)); - else - status= Sys_var_charptr::do_string_check(thd, var, charset(thd)); - - return status; -} -void Sys_var_rpl_filter::lock(void) +bool Sys_var_rpl_filter::global_update(THD *thd, set_var *var) { - /* - Starting a slave thread causes the new thread to attempt to - acquire LOCK_global_system_variables (in THD::init) while - LOCK_active_mi is being held by the thread that initiated - the process. In order to not violate the lock order, unlock - LOCK_global_system_variables before grabbing LOCK_active_mi. - */ - mysql_mutex_unlock(&LOCK_global_system_variables); + bool result= true; // Assume error + Master_info *mi; + mysql_mutex_unlock(&LOCK_global_system_variables); mysql_mutex_lock(&LOCK_active_mi); - mysql_mutex_lock(&active_mi->rli.run_lock); -} + + if (!var->base.length) // no base name + { + mi= master_info_index-> + get_master_info(&thd->variables.default_master_connection, + MYSQL_ERROR::WARN_LEVEL_ERROR); + } + else // has base name + { + mi= master_info_index-> + get_master_info(&var->base, + MYSQL_ERROR::WARN_LEVEL_WARN); + } -void Sys_var_rpl_filter::unlock(void) -{ - mysql_mutex_unlock(&active_mi->rli.run_lock); - mysql_mutex_unlock(&LOCK_active_mi); + if (mi) + { + if (mi->rli.slave_running) + { + my_error(ER_SLAVE_MUST_STOP, MYF(0), + mi->connection_name.length, + mi->connection_name.str); + result= true; + } + else + { + result= set_filter_value(var->save_result.string_value.str, mi); + } + } + mysql_mutex_unlock(&LOCK_active_mi); mysql_mutex_lock(&LOCK_global_system_variables); + return result; } -bool Sys_var_rpl_filter::global_update(THD *thd, set_var *var) -{ - bool slave_running, status= false; - - lock(); - - if (! (slave_running= active_mi->rli.slave_running)) - status= set_filter_value(var->save_result.string_value.str); - - if (slave_running) - my_error(ER_SLAVE_MUST_STOP, MYF(0)); - - unlock(); - - return slave_running || status; -} - -bool Sys_var_rpl_filter::set_filter_value(const char *value) +bool Sys_var_rpl_filter::set_filter_value(const char *value, Master_info *mi) { bool status= true; + Rpl_filter* rpl_filter= mi ? mi->rpl_filter : global_rpl_filter; switch (opt_id) { case OPT_REPLICATE_DO_DB: @@ -3397,10 +3822,33 @@ uchar *Sys_var_rpl_filter::global_value_ptr(THD *thd, LEX_STRING *base) { char buf[256]; String tmp(buf, sizeof(buf), &my_charset_bin); + uchar *ret; + Master_info *mi; + Rpl_filter *rpl_filter; - tmp.length(0); + mysql_mutex_unlock(&LOCK_global_system_variables); + mysql_mutex_lock(&LOCK_active_mi); + if (!base->length) // no base name + { + mi= master_info_index-> + get_master_info(&thd->variables.default_master_connection, + MYSQL_ERROR::WARN_LEVEL_ERROR); + } + else // has base name + { + mi= master_info_index-> + get_master_info(base, + MYSQL_ERROR::WARN_LEVEL_WARN); + } + mysql_mutex_lock(&LOCK_global_system_variables); - lock(); + if (!mi) + { + mysql_mutex_unlock(&LOCK_active_mi); + return 0; + } + rpl_filter= mi->rpl_filter; + tmp.length(0); switch (opt_id) { case OPT_REPLICATE_DO_DB: @@ -3423,9 +3871,10 @@ uchar *Sys_var_rpl_filter::global_value_ptr(THD *thd, LEX_STRING *base) break; } - unlock(); + ret= (uchar *) thd->strmake(tmp.ptr(), tmp.length()); + mysql_mutex_unlock(&LOCK_active_mi); - return (uchar *) thd->strmake(tmp.ptr(), tmp.length()); + return ret; } static Sys_var_rpl_filter Sys_replicate_do_db( @@ -3474,72 +3923,116 @@ static Sys_var_charptr Sys_slave_load_tmpdir( READ_ONLY GLOBAL_VAR(slave_load_tmpdir), CMD_LINE(REQUIRED_ARG), IN_FS_CHARSET, DEFAULT(0)); -static bool fix_slave_net_timeout(sys_var *self, THD *thd, enum_var_type type) -{ - DEBUG_SYNC(thd, "fix_slave_net_timeout"); - - mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - DBUG_PRINT("info", ("slave_net_timeout=%u mi->heartbeat_period=%.3f", - slave_net_timeout, - (active_mi? active_mi->heartbeat_period : 0.0))); - if (active_mi && slave_net_timeout < active_mi->heartbeat_period) - push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, - ER_SLAVE_HEARTBEAT_VALUE_OUT_OF_RANGE_MAX, - ER(ER_SLAVE_HEARTBEAT_VALUE_OUT_OF_RANGE_MAX)); - mysql_mutex_unlock(&LOCK_active_mi); - mysql_mutex_lock(&LOCK_global_system_variables); - return false; -} static Sys_var_uint Sys_slave_net_timeout( "slave_net_timeout", "Number of seconds to wait for more data " - "from a master/slave connection before aborting the read", + "from any master/slave connection before aborting the read", GLOBAL_VAR(slave_net_timeout), CMD_LINE(REQUIRED_ARG), VALID_RANGE(1, LONG_TIMEOUT), DEFAULT(SLAVE_NET_TIMEOUT), BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), - ON_UPDATE(fix_slave_net_timeout)); + ON_UPDATE(0)); + + +/* + Access a multi_source variable + Return 0 + warning if it doesn't exist +*/ -static bool check_slave_skip_counter(sys_var *self, THD *thd, set_var *var) +uint Sys_var_multi_source_ulong:: +get_master_info_uint_value(THD *thd, ptrdiff_t offset) { - bool result= false; + Master_info *mi; + uint res= 0; // Default value + mysql_mutex_unlock(&LOCK_global_system_variables); mysql_mutex_lock(&LOCK_active_mi); - mysql_mutex_lock(&active_mi->rli.run_lock); - if (active_mi->rli.slave_running) + mi= master_info_index-> + get_master_info(&thd->variables.default_master_connection, + MYSQL_ERROR::WARN_LEVEL_WARN); + if (mi) { - my_message(ER_SLAVE_MUST_STOP, ER(ER_SLAVE_MUST_STOP), MYF(0)); - result= true; + mysql_mutex_lock(&mi->rli.data_lock); + res= *((uint*) (((uchar*) mi) + master_info_offset)); + mysql_mutex_unlock(&mi->rli.data_lock); } - mysql_mutex_unlock(&active_mi->rli.run_lock); - mysql_mutex_unlock(&LOCK_active_mi); - return result; + mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_global_system_variables); + return res; } -static bool fix_slave_skip_counter(sys_var *self, THD *thd, enum_var_type type) + + +bool update_multi_source_variable(sys_var *self_var, THD *thd, + enum_var_type type) { - mysql_mutex_unlock(&LOCK_global_system_variables); + Sys_var_multi_source_ulong *self= (Sys_var_multi_source_ulong*) self_var; + bool result= true; + Master_info *mi; + + if (type == OPT_GLOBAL) + mysql_mutex_unlock(&LOCK_global_system_variables); mysql_mutex_lock(&LOCK_active_mi); - mysql_mutex_lock(&active_mi->rli.run_lock); - /* - The following test should normally never be true as we test this - in the check function; To be safe against multiple - SQL_SLAVE_SKIP_COUNTER request, we do the check anyway - */ - if (!active_mi->rli.slave_running) + mi= master_info_index-> + get_master_info(&thd->variables.default_master_connection, + MYSQL_ERROR::WARN_LEVEL_ERROR); + if (mi) { - mysql_mutex_lock(&active_mi->rli.data_lock); - active_mi->rli.slave_skip_counter= sql_slave_skip_counter; - mysql_mutex_unlock(&active_mi->rli.data_lock); + mysql_mutex_lock(&mi->rli.run_lock); + mysql_mutex_lock(&mi->rli.data_lock); + result= self->update_variable(thd, mi); + mysql_mutex_unlock(&mi->rli.data_lock); + mysql_mutex_unlock(&mi->rli.run_lock); } - mysql_mutex_unlock(&active_mi->rli.run_lock); mysql_mutex_unlock(&LOCK_active_mi); - mysql_mutex_lock(&LOCK_global_system_variables); - return 0; + if (type == OPT_GLOBAL) + mysql_mutex_lock(&LOCK_global_system_variables); + return result; +} + +static bool update_slave_skip_counter(sys_var *self, THD *thd, Master_info *mi) +{ + if (mi->using_gtid != Master_info::USE_GTID_NO) + { + my_error(ER_SLAVE_SKIP_NOT_IN_GTID, MYF(0)); + return true; + } + if (mi->rli.slave_running) + { + my_error(ER_SLAVE_MUST_STOP, MYF(0), mi->connection_name.length, + mi->connection_name.str); + return true; + } + /* The value was stored temporarily in thd */ + mi->rli.slave_skip_counter= thd->variables.slave_skip_counter; + return false; } -static Sys_var_uint Sys_slave_skip_counter( - "sql_slave_skip_counter", "sql_slave_skip_counter", - GLOBAL_VAR(sql_slave_skip_counter), NO_CMD_LINE, - VALID_RANGE(0, UINT_MAX), DEFAULT(0), BLOCK_SIZE(1), - NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_slave_skip_counter), - ON_UPDATE(fix_slave_skip_counter)); + + +static Sys_var_multi_source_ulong +Sys_slave_skip_counter("sql_slave_skip_counter", + "Skip the next N events from the master log", + SESSION_VAR(slave_skip_counter), + NO_CMD_LINE, + my_offsetof(Master_info, rli.slave_skip_counter), + VALID_RANGE(0, UINT_MAX), DEFAULT(0), BLOCK_SIZE(1), + ON_UPDATE(update_slave_skip_counter)); + + +static bool update_max_relay_log_size(sys_var *self, THD *thd, Master_info *mi) +{ + mi->rli.max_relay_log_size= thd->variables.max_relay_log_size; + mi->rli.relay_log.set_max_size(mi->rli.max_relay_log_size); + return false; +} + +static Sys_var_multi_source_ulong +Sys_max_relay_log_size( "max_relay_log_size", + "relay log will be rotated automatically when the " + "size exceeds this value. If 0 are startup, it's " + "set to max_binlog_size", + SESSION_VAR(max_relay_log_size), + CMD_LINE(REQUIRED_ARG), + my_offsetof(Master_info, rli.max_relay_log_size), + VALID_RANGE(0, 1024L*1024*1024), DEFAULT(0), + BLOCK_SIZE(IO_SIZE), + ON_UPDATE(update_max_relay_log_size)); static Sys_var_charptr Sys_slave_skip_errors( "slave_skip_errors", "Tells the slave thread to continue " @@ -3769,11 +4262,12 @@ static Sys_var_ulong Sys_log_slow_rate_limit( SESSION_VAR(log_slow_rate_limit), CMD_LINE(REQUIRED_ARG), VALID_RANGE(1, UINT_MAX), DEFAULT(1), BLOCK_SIZE(1)); -static const char *log_slow_verbosity_names[]= { "innodb", "query_plan", 0 }; +static const char *log_slow_verbosity_names[]= { "innodb", "query_plan", + "explain", 0 }; static Sys_var_set Sys_log_slow_verbosity( "log_slow_verbosity", "log-slow-verbosity=[value[,value ...]] where value is one of " - "'innodb', 'query_plan'", + "'innodb', 'query_plan', 'explain' ", SESSION_VAR(log_slow_verbosity), CMD_LINE(REQUIRED_ARG), log_slow_verbosity_names, DEFAULT(LOG_SLOW_VERBOSITY_INIT)); @@ -3835,6 +4329,32 @@ static Sys_var_ulong Sys_progress_report_time( SESSION_VAR(progress_report_time), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, UINT_MAX), DEFAULT(5), BLOCK_SIZE(1)); +const char *use_stat_tables_modes[] = + {"NEVER", "COMPLEMENTARY", "PREFERABLY", 0}; +static Sys_var_enum Sys_optimizer_use_stat_tables( + "use_stat_tables", + "Specifies how to use system statistics tables. Possible values are " + "NEVER, COMPLEMENTARY, PREVERABLY", + SESSION_VAR(use_stat_tables), CMD_LINE(REQUIRED_ARG), + use_stat_tables_modes, DEFAULT(0)); + +static Sys_var_ulong Sys_histogram_size( + "histogram_size", + "Number of bytes used for a histogram. " + "If set to 0, no histograms are created by ANALYZE.", + SESSION_VAR(histogram_size), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0, 255), DEFAULT(0), BLOCK_SIZE(1)); + +extern const char *histogram_types[]; +static Sys_var_enum Sys_histogram_type( + "histogram_type", + "Specifies type of the histograms created by ANALYZE. " + "Possible values are: " + "SINGLE_PREC_HB - single precision height-balanced, " + "DOUBLE_PREC_HB - double precision height-balanced.", + SESSION_VAR(histogram_type), CMD_LINE(REQUIRED_ARG), + histogram_types, DEFAULT(0)); + static Sys_var_mybool Sys_no_thread_alarm( "debug_no_thread_alarm", "Disable system thread alarm calls. Disabling it may be useful " @@ -3892,6 +4412,8 @@ static bool check_pseudo_slave_mode(sys_var *self, THD *thd, set_var *var) #ifndef EMBEDDED_LIBRARY delete thd->rli_fake; thd->rli_fake= NULL; + delete thd->rgi_fake; + thd->rgi_fake= NULL; #endif } else if (previous_val && val) diff --git a/sql/sys_vars.h b/sql/sys_vars.h index 3cbd24f1c89..6a84fc5fbc2 100644 --- a/sql/sys_vars.h +++ b/sql/sys_vars.h @@ -29,6 +29,7 @@ #include "keycaches.h" #include "strfunc.h" #include "tztime.h" // my_tz_find, my_tz_SYSTEM, struct Time_zone +#include "rpl_mi.h" // For Multi-Source Replication /* a set of mostly trivial (as in f(X)=X) defines below to make system variable @@ -556,6 +557,7 @@ protected: } }; +class Master_info; class Sys_var_rpl_filter: public sys_var { private: @@ -567,14 +569,16 @@ public: NO_ARG, SHOW_CHAR, 0, NULL, VARIABLE_NOT_IN_BINLOG, NULL, NULL, NULL), opt_id(getopt_id) { - option.var_type= GET_STR; + option.var_type= GET_STR | GET_ASK_ADDR; } + bool do_check(THD *thd, set_var *var) + { + return Sys_var_charptr::do_string_check(thd, var, charset(thd)); + } bool check_update_type(Item_result type) { return type != STRING_RESULT; } - bool do_check(THD *thd, set_var *var); - void session_save_default(THD *thd, set_var *var) { DBUG_ASSERT(FALSE); } @@ -591,9 +595,7 @@ public: protected: uchar *global_value_ptr(THD *thd, LEX_STRING *base); - bool set_filter_value(const char *value); - void lock(void); - void unlock(void); + bool set_filter_value(const char *value, Master_info *mi); }; /** @@ -637,6 +639,93 @@ public: } }; + +/* + A LEX_STRING stored only in thd->variables + Only to be used for small buffers +*/ + +class Sys_var_session_lexstring: public sys_var +{ + size_t max_length; +public: + Sys_var_session_lexstring(const char *name_arg, + const char *comment, int flag_args, + ptrdiff_t off, size_t size, CMD_LINE getopt, + enum charset_enum is_os_charset_arg, + const char *def_val, size_t max_length_arg, + on_check_function on_check_func=0, + on_update_function on_update_func=0) + : sys_var(&all_sys_vars, name_arg, comment, flag_args, off, getopt.id, + getopt.arg_type, SHOW_CHAR, (intptr)def_val, + 0, VARIABLE_NOT_IN_BINLOG, on_check_func, on_update_func, + 0),max_length(max_length_arg) + { + option.var_type= GET_NO_ARG; + SYSVAR_ASSERT(scope() == ONLY_SESSION) + *const_cast<SHOW_TYPE*>(&show_val_type)= SHOW_LEX_STRING; + } + bool do_check(THD *thd, set_var *var) + { + char buff[STRING_BUFFER_USUAL_SIZE]; + String str(buff, sizeof(buff), system_charset_info), *res; + + if (!(res=var->value->val_str(&str))) + { + var->save_result.string_value.str= 0; /* NULL */ + var->save_result.string_value.length= 0; + } + else + { + if (res->length() > max_length) + { + my_error(ER_WRONG_STRING_LENGTH, MYF(0), + res->ptr(), name.str, (int) max_length); + return true; + } + var->save_result.string_value.str= thd->strmake(res->ptr(), + res->length()); + var->save_result.string_value.length= res->length(); + } + return false; + } + bool session_update(THD *thd, set_var *var) + { + LEX_STRING *tmp= &session_var(thd, LEX_STRING); + tmp->length= var->save_result.string_value.length; + /* Store as \0 terminated string (just to be safe) */ + strmake(tmp->str, var->save_result.string_value.str, tmp->length); + return false; + } + bool global_update(THD *thd, set_var *var) + { + DBUG_ASSERT(FALSE); + return false; + } + void session_save_default(THD *thd, set_var *var) + { + char *ptr= (char*)(intptr)option.def_value; + var->save_result.string_value.str= ptr; + var->save_result.string_value.length= strlen(ptr); + } + void global_save_default(THD *thd, set_var *var) + { + DBUG_ASSERT(FALSE); + } + uchar *session_value_ptr(THD *thd, LEX_STRING *base) + { + return (uchar*) &session_var(thd, LEX_STRING); + } + uchar *global_value_ptr(THD *thd, LEX_STRING *base) + { + DBUG_ASSERT(FALSE); + return NULL; + } + bool check_update_type(Item_result type) + { return type != STRING_RESULT; } +}; + + #ifndef DBUG_OFF /** @@session.dbug and @@global.dbug variables. @@ -1394,6 +1483,7 @@ public: }; #endif /* defined(ENABLED_DEBUG_SYNC) */ + /** The class for bit variables - a variant of boolean that stores the value in a bit. @@ -1859,25 +1949,315 @@ public: const char *comment, int flag_args, ptrdiff_t off, size_t size, CMD_LINE getopt, const char *values[], uint def_val, PolyLock *lock, - enum binlog_status_enum binlog_status_arg, - on_check_function on_check_func) + enum binlog_status_enum binlog_status_arg) :Sys_var_enum(name_arg, comment, flag_args, off, size, getopt, - values, def_val, lock, binlog_status_arg, on_check_func) + values, def_val, lock, binlog_status_arg) {} bool global_update(THD *thd, set_var *var); }; -/**************************************************************************** - Used templates -****************************************************************************/ - -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class List<set_var_base>; -template class List_iterator_fast<set_var_base>; -template class Sys_var_integer<int, GET_INT, SHOW_SINT>; -template class Sys_var_integer<uint, GET_UINT, SHOW_INT>; -template class Sys_var_integer<ulong, GET_ULONG, SHOW_LONG>; -template class Sys_var_integer<ha_rows, GET_HA_ROWS, SHOW_HA_ROWS>; -template class Sys_var_integer<ulonglong, GET_ULL, SHOW_LONGLONG>; -#endif +/* + Class for handing multi-source replication variables + Variable values are store in Master_info, but to make it possible to + access variable without locks we also store it thd->variables. + These can be used as GLOBAL or SESSION, but both points to the same + variable. This is to make things compatible with MySQL 5.5 where variables + like sql_slave_skip_counter are GLOBAL. +*/ + +class Sys_var_multi_source_ulong; +class Master_info; + +typedef bool (*on_multi_source_update_function)(sys_var *self, THD *thd, + Master_info *mi); +bool update_multi_source_variable(sys_var *self, + THD *thd, enum_var_type type); + +class Sys_var_multi_source_ulong :public Sys_var_ulong +{ + ptrdiff_t master_info_offset; + on_multi_source_update_function update_multi_source_variable_func; +public: + Sys_var_multi_source_ulong(const char *name_arg, + const char *comment, int flag_args, + ptrdiff_t off, size_t size, + CMD_LINE getopt, + ptrdiff_t master_info_offset_arg, + uint min_val, uint max_val, uint def_val, + uint block_size, + on_multi_source_update_function on_update_func) + :Sys_var_ulong(name_arg, comment, flag_args, off, size, + getopt, min_val, max_val, def_val, block_size, + 0, VARIABLE_NOT_IN_BINLOG, 0, update_multi_source_variable), + master_info_offset(master_info_offset_arg), + update_multi_source_variable_func(on_update_func) + { + } + bool session_update(THD *thd, set_var *var) + { + session_var(thd, uint)= (uint) (var->save_result.ulonglong_value); + /* Value should be moved to multi_master in on_update_func */ + return false; + } + bool global_update(THD *thd, set_var *var) + { + return session_update(thd, var); + } + void session_save_default(THD *thd, set_var *var) + { + /* Use value given in variable declaration */ + global_save_default(thd, var); + } + uchar *session_value_ptr(THD *thd,LEX_STRING *base) + { + uint *tmp, res; + tmp= (uint*) (((uchar*)&(thd->variables)) + offset); + res= get_master_info_uint_value(thd, master_info_offset); + *tmp= res; + return (uchar*) tmp; + } + uchar *global_value_ptr(THD *thd, LEX_STRING *base) + { + return session_value_ptr(thd, base); + } + uint get_master_info_uint_value(THD *thd, ptrdiff_t offset); + bool update_variable(THD *thd, Master_info *mi) + { + return update_multi_source_variable_func(this, thd, mi); + } +}; + + +/** + Class for @@global.gtid_current_pos. +*/ +class Sys_var_gtid_current_pos: public sys_var +{ +public: + Sys_var_gtid_current_pos(const char *name_arg, + const char *comment, int flag_args, ptrdiff_t off, size_t size, + CMD_LINE getopt) + : sys_var(&all_sys_vars, name_arg, comment, flag_args, off, getopt.id, + getopt.arg_type, SHOW_CHAR, 0, NULL, VARIABLE_NOT_IN_BINLOG, + NULL, NULL, NULL) + { + option.var_type= GET_STR; + } + bool do_check(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool session_update(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool global_update(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool check_update_type(Item_result type) { + DBUG_ASSERT(false); + return false; + } + void session_save_default(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + } + void global_save_default(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + } + uchar *session_value_ptr(THD *thd, LEX_STRING *base) + { + DBUG_ASSERT(false); + return NULL; + } + uchar *global_value_ptr(THD *thd, LEX_STRING *base); +}; + + +/** + Class for @@global.gtid_binlog_pos. +*/ +class Sys_var_gtid_binlog_pos: public sys_var +{ +public: + Sys_var_gtid_binlog_pos(const char *name_arg, + const char *comment, int flag_args, ptrdiff_t off, size_t size, + CMD_LINE getopt) + : sys_var(&all_sys_vars, name_arg, comment, flag_args, off, getopt.id, + getopt.arg_type, SHOW_CHAR, 0, NULL, VARIABLE_NOT_IN_BINLOG, + NULL, NULL, NULL) + { + option.var_type= GET_STR; + } + bool do_check(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool session_update(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool global_update(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool check_update_type(Item_result type) { + DBUG_ASSERT(false); + return false; + } + void session_save_default(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + } + void global_save_default(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + } + uchar *session_value_ptr(THD *thd, LEX_STRING *base) + { + DBUG_ASSERT(false); + return NULL; + } + uchar *global_value_ptr(THD *thd, LEX_STRING *base); +}; + + +/** + Class for @@global.gtid_slave_pos. +*/ +class Sys_var_gtid_slave_pos: public sys_var +{ +public: + Sys_var_gtid_slave_pos(const char *name_arg, + const char *comment, int flag_args, ptrdiff_t off, size_t size, + CMD_LINE getopt) + : sys_var(&all_sys_vars, name_arg, comment, flag_args, off, getopt.id, + getopt.arg_type, SHOW_CHAR, 0, NULL, VARIABLE_NOT_IN_BINLOG, + NULL, NULL, NULL) + { + option.var_type= GET_STR; + } + bool do_check(THD *thd, set_var *var); + bool session_update(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool global_update(THD *thd, set_var *var); + bool check_update_type(Item_result type) { return type != STRING_RESULT; } + void session_save_default(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + } + void global_save_default(THD *thd, set_var *var) + { + /* Record the attempt to use default so we can error. */ + var->value= 0; + } + uchar *session_value_ptr(THD *thd, LEX_STRING *base) + { + DBUG_ASSERT(false); + return NULL; + } + uchar *global_value_ptr(THD *thd, LEX_STRING *base); +}; + + +/** + Class for @@global.gtid_binlog_state. +*/ +class Sys_var_gtid_binlog_state: public sys_var +{ +public: + Sys_var_gtid_binlog_state(const char *name_arg, + const char *comment, int flag_args, ptrdiff_t off, size_t size, + CMD_LINE getopt) + : sys_var(&all_sys_vars, name_arg, comment, flag_args, off, getopt.id, + getopt.arg_type, SHOW_CHAR, 0, NULL, VARIABLE_NOT_IN_BINLOG, + NULL, NULL, NULL) + { + option.var_type= GET_STR; + } + bool do_check(THD *thd, set_var *var); + bool session_update(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool global_update(THD *thd, set_var *var); + bool check_update_type(Item_result type) { return type != STRING_RESULT; } + void session_save_default(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + } + void global_save_default(THD *thd, set_var *var) + { + /* Record the attempt to use default so we can error. */ + var->value= 0; + } + uchar *session_value_ptr(THD *thd, LEX_STRING *base) + { + DBUG_ASSERT(false); + return NULL; + } + uchar *global_value_ptr(THD *thd, LEX_STRING *base); +}; + + +/** + Class for @@session.last_gtid. +*/ +class Sys_var_last_gtid: public sys_var +{ +public: + Sys_var_last_gtid(const char *name_arg, + const char *comment, int flag_args, CMD_LINE getopt) + : sys_var(&all_sys_vars, name_arg, comment, flag_args, 0, getopt.id, + getopt.arg_type, SHOW_CHAR, 0, NULL, VARIABLE_NOT_IN_BINLOG, + NULL, NULL, NULL) + { + option.var_type= GET_STR; + } + bool do_check(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool session_update(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool global_update(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + return true; + } + bool check_update_type(Item_result type) { + DBUG_ASSERT(false); + return false; + } + void session_save_default(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + } + void global_save_default(THD *thd, set_var *var) + { + DBUG_ASSERT(false); + } + uchar *session_value_ptr(THD *thd, LEX_STRING *base); + uchar *global_value_ptr(THD *thd, LEX_STRING *base) + { + DBUG_ASSERT(false); + return NULL; + } +}; diff --git a/sql/table.cc b/sql/table.cc index cccda4e76fc..957e911e462 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -21,7 +21,6 @@ #include "sql_priv.h" #include "unireg.h" // REQUIRED: for other includes #include "table.h" -#include "frm_crypt.h" // get_crypt_for_frm #include "key.h" // find_ref_key #include "sql_table.h" // build_table_filename, // primary_key_name @@ -38,6 +37,8 @@ #include "my_bit.h" #include "sql_select.h" #include "sql_derived.h" +#include "sql_statistics.h" +#include "discover.h" #include "mdl.h" // MDL_wait_for_graph_visitor /* INFORMATION_SCHEMA name */ @@ -63,18 +64,12 @@ LEX_STRING parse_vcol_keyword= { C_STRING_WITH_LEN("PARSE_VCOL_EXPR ") }; /* Functions defined in this file */ -void open_table_error(TABLE_SHARE *share, int error, int db_errno, - myf errortype, int errarg); -static int open_binary_frm(THD *thd, TABLE_SHARE *share, - uchar *head, File file); static void fix_type_pointers(const char ***array, TYPELIB *point_to_type, uint types, char **names); static uint find_field(Field **fields, uchar *record, uint start, uint length); inline bool is_system_table_name(const char *name, uint length); -static ulong get_form_pos(File file, uchar *head); - /************************************************************************** Object_creation_ctx implementation. **************************************************************************/ @@ -290,8 +285,8 @@ TABLE_CATEGORY get_table_category(const LEX_STRING *db, const LEX_STRING *name) # Share */ -TABLE_SHARE *alloc_table_share(TABLE_LIST *table_list, char *key, - uint key_length) +TABLE_SHARE *alloc_table_share(const char *db, const char *table_name, + char *key, uint key_length) { MEM_ROOT mem_root; TABLE_SHARE *share; @@ -299,13 +294,11 @@ TABLE_SHARE *alloc_table_share(TABLE_LIST *table_list, char *key, char path[FN_REFLEN]; uint path_length; DBUG_ENTER("alloc_table_share"); - DBUG_PRINT("enter", ("table: '%s'.'%s'", - table_list->db, table_list->table_name)); + DBUG_PRINT("enter", ("table: '%s'.'%s'", db, table_name)); path_length= build_table_filename(path, sizeof(path) - 1, - table_list->db, - table_list->table_name, "", 0); - init_sql_alloc(&mem_root, TABLE_ALLOC_BLOCK_SIZE, 0); + db, table_name, "", 0); + init_sql_alloc(&mem_root, TABLE_ALLOC_BLOCK_SIZE, 0, MYF(0)); if (multi_alloc_root(&mem_root, &share, sizeof(*share), &key_buff, key_length, @@ -321,8 +314,9 @@ TABLE_SHARE *alloc_table_share(TABLE_LIST *table_list, char *key, strmov(share->path.str, path); share->normalized_path.str= share->path.str; share->normalized_path.length= path_length; - + share->table_category= get_table_category(& share->db, & share->table_name); share->set_refresh_version(); + share->open_errno= ENOENT; /* Since alloc_table_share() can be called without any locking (for @@ -338,6 +332,8 @@ TABLE_SHARE *alloc_table_share(TABLE_LIST *table_list, char *key, share->free_tables.empty(); share->m_flush_tickets.empty(); + init_sql_alloc(&share->stats_cb.mem_root, TABLE_ALLOC_BLOCK_SIZE, 0, MYF(0)); + memcpy((char*) &share->mem_root, (char*) &mem_root, sizeof(mem_root)); mysql_mutex_init(key_TABLE_SHARE_LOCK_ha_data, &share->LOCK_ha_data, MY_MUTEX_INIT_FAST); @@ -377,7 +373,12 @@ void init_tmp_table_share(THD *thd, TABLE_SHARE *share, const char *key, DBUG_PRINT("enter", ("table: '%s'.'%s'", key, table_name)); bzero((char*) share, sizeof(*share)); - init_sql_alloc(&share->mem_root, TABLE_ALLOC_BLOCK_SIZE, 0); + /* + This can't be MY_THREAD_SPECIFIC for slaves as they are freed + during cleanup() from Relay_log_info::close_temporary_tables() + */ + init_sql_alloc(&share->mem_root, TABLE_ALLOC_BLOCK_SIZE, 0, + MYF(thd->slave_thread ? 0 : MY_THREAD_SPECIFIC)); share->table_category= TABLE_CATEGORY_TEMPORARY; share->tmp_table= INTERNAL_TMP_TABLE; share->db.str= (char*) key; @@ -418,6 +419,16 @@ void TABLE_SHARE::destroy() uint idx; KEY *info_it; + if (tmp_table == NO_TMP_TABLE) + mysql_mutex_lock(&LOCK_ha_data); + free_root(&stats_cb.mem_root, MYF(0)); + stats_cb.stats_can_be_read= FALSE; + stats_cb.stats_is_read= FALSE; + stats_cb.histograms_can_be_read= FALSE; + stats_cb.histograms_are_read= FALSE; + if (tmp_table == NO_TMP_TABLE) + mysql_mutex_unlock(&LOCK_ha_data); + /* The mutex is initialized only for shares that are part of the TDC */ if (tmp_table == NO_TMP_TABLE) mysql_mutex_destroy(&LOCK_ha_data); @@ -442,6 +453,7 @@ void TABLE_SHARE::destroy() ha_data_destroy= NULL; } #ifdef WITH_PARTITION_STORAGE_ENGINE + plugin_unlock(NULL, default_part_plugin); if (ha_part_data_destroy) { ha_part_data_destroy(ha_part_data); @@ -543,6 +555,13 @@ inline bool is_system_table_name(const char *name, uint length) my_tolower(ci, name[2]) == 'm' && my_tolower(ci, name[3]) == 'e') || + /* one of mysql.*_stat tables */ + (my_tolower(ci, name[length-5]) == 's' && + my_tolower(ci, name[length-4]) == 't' && + my_tolower(ci, name[length-3]) == 'a' && + my_tolower(ci, name[length-2]) == 't' && + my_tolower(ci, name[length-1]) == 's') || + /* mysql.event table */ (my_tolower(ci, name[0]) == 'e' && my_tolower(ci, name[1]) == 'v' && @@ -555,27 +574,6 @@ inline bool is_system_table_name(const char *name, uint length) } -/** - Check if a string contains path elements -*/ - -static bool has_disabled_path_chars(const char *str) -{ - for (; *str; str++) - { - switch (*str) { - case FN_EXTCHAR: - case '/': - case '\\': - case '~': - case '@': - return TRUE; - } - } - return FALSE; -} - - /* Read table definition from a binary / text based .frm file @@ -590,152 +588,111 @@ static bool has_disabled_path_chars(const char *str) table_def_cache The data is returned in 'share', which is alloced by alloc_table_share().. The code assumes that share is initialized. - - RETURN VALUES - 0 ok - 1 Error (see open_table_error) - 2 Error (see open_table_error) - 3 Wrong data in .frm file - 4 Error (see open_table_error) - 5 Error (see open_table_error: charset unavailable) - 6 Unknown .frm version */ -int open_table_def(THD *thd, TABLE_SHARE *share, uint db_flags) +enum open_frm_error open_table_def(THD *thd, TABLE_SHARE *share, uint flags) { - int error, table_type; - bool error_given; + bool error_given= false; File file; - uchar head[64]; + uchar *buf; + uchar head[FRM_HEADER_SIZE]; char path[FN_REFLEN]; - MEM_ROOT **root_ptr, *old_root; + size_t frmlen, read_length; DBUG_ENTER("open_table_def"); DBUG_PRINT("enter", ("table: '%s'.'%s' path: '%s'", share->db.str, share->table_name.str, share->normalized_path.str)); - error= 1; - error_given= 0; + share->error= OPEN_FRM_OPEN_ERROR; strxmov(path, share->normalized_path.str, reg_ext, NullS); - if ((file= mysql_file_open(key_file_frm, - path, O_RDONLY | O_SHARE, MYF(0))) < 0) + if (flags & GTS_FORCE_DISCOVERY) { - /* - We don't try to open 5.0 unencoded name, if - - non-encoded name contains '@' signs, - because '@' can be misinterpreted. - It is not clear if '@' is escape character in 5.1, - or a normal character in 5.0. - - - non-encoded db or table name contain "#mysql50#" prefix. - This kind of tables must have been opened only by the - mysql_file_open() above. - */ - if (has_disabled_path_chars(share->table_name.str) || - has_disabled_path_chars(share->db.str) || - !strncmp(share->db.str, MYSQL50_TABLE_NAME_PREFIX, - MYSQL50_TABLE_NAME_PREFIX_LENGTH) || - !strncmp(share->table_name.str, MYSQL50_TABLE_NAME_PREFIX, - MYSQL50_TABLE_NAME_PREFIX_LENGTH)) - goto err_not_open; - - /* Try unencoded 5.0 name */ - uint length; - strxnmov(path, sizeof(path)-1, - mysql_data_home, "/", share->db.str, "/", - share->table_name.str, reg_ext, NullS); - length= unpack_filename(path, path) - reg_ext_length; - /* - The following is a safety test and should never fail - as the old file name should never be longer than the new one. - */ - DBUG_ASSERT(length <= share->normalized_path.length); - /* - If the old and the new names have the same length, - then table name does not have tricky characters, - so no need to check the old file name. - */ - if (length == share->normalized_path.length || - ((file= mysql_file_open(key_file_frm, - path, O_RDONLY | O_SHARE, MYF(0))) < 0)) - goto err_not_open; + DBUG_ASSERT(flags & GTS_TABLE); + DBUG_ASSERT(flags & GTS_USE_DISCOVERY); + mysql_file_delete_with_symlink(key_file_frm, path, MYF(0)); + file= -1; + } + else + file= mysql_file_open(key_file_frm, path, O_RDONLY | O_SHARE, MYF(0)); - /* Unencoded 5.0 table name found */ - path[length]= '\0'; // Remove .frm extension - strmov(share->normalized_path.str, path); - share->normalized_path.length= length; + if (file < 0) + { + if ((flags & GTS_TABLE) && (flags & GTS_USE_DISCOVERY)) + { + ha_discover_table(thd, share); + error_given= true; + } + goto err_not_open; } - error= 4; - if (mysql_file_read(file, head, 64, MYF(MY_NABP))) + if (mysql_file_read(file, head, sizeof(head), MYF(MY_NABP))) + { + share->error = my_errno == HA_ERR_FILE_TOO_SHORT + ? OPEN_FRM_CORRUPTED : OPEN_FRM_READ_ERROR; goto err; + } - if (head[0] == (uchar) 254 && head[1] == 1) + if (memcmp(head, STRING_WITH_LEN("TYPE=VIEW\n")) == 0) { - if (head[2] == FRM_VER || head[2] == FRM_VER+1 || - (head[2] >= FRM_VER+3 && head[2] <= FRM_VER+4)) - { - /* Open view only */ - if (db_flags & OPEN_VIEW_ONLY) - { - error_given= 1; - goto err; - } - table_type= 1; - } - else - { - error= 6; // Unkown .frm version - goto err; - } + share->is_view= 1; + share->error= flags & GTS_VIEW ? OPEN_FRM_OK : OPEN_FRM_NOT_A_TABLE; + goto err; } - else if (memcmp(head, STRING_WITH_LEN("TYPE=")) == 0) + if (!is_binary_frm_header(head)) { - error= 5; - if (memcmp(head+5,"VIEW",4) == 0) - { - share->is_view= 1; - if (db_flags & OPEN_VIEW) - error= 0; - } + /* No handling of text based files yet */ + share->error = OPEN_FRM_CORRUPTED; goto err; } - else + if (!(flags & GTS_TABLE)) + { + share->error = OPEN_FRM_NOT_A_VIEW; + goto err; + } + + frmlen= uint4korr(head+10); + set_if_smaller(frmlen, FRM_MAX_SIZE); // safety + + if (!(buf= (uchar*)my_malloc(frmlen, MYF(MY_THREAD_SPECIFIC|MY_WME)))) goto err; - /* No handling of text based files yet */ - if (table_type == 1) + memcpy(buf, head, sizeof(head)); + + read_length= mysql_file_read(file, buf + sizeof(head), + frmlen - sizeof(head), MYF(MY_WME)); + if (read_length == 0 || read_length == (size_t)-1) { - root_ptr= my_pthread_getspecific_ptr(MEM_ROOT**, THR_MALLOC); - old_root= *root_ptr; - *root_ptr= &share->mem_root; - error= open_binary_frm(thd, share, head, file); - *root_ptr= old_root; - error_given= 1; + share->error = OPEN_FRM_READ_ERROR; + my_free(buf); + goto err; } + mysql_file_close(file, MYF(MY_WME)); + + frmlen= read_length + sizeof(head); - share->table_category= get_table_category(& share->db, & share->table_name); + share->init_from_binary_frm_image(thd, false, buf, frmlen); + error_given= true; // init_from_binary_frm_image has already called my_error() + my_free(buf); - if (!error) - thd->status_var.opened_shares++; + goto err_not_open; err: mysql_file_close(file, MYF(MY_WME)); err_not_open: - if (error && !error_given) + if (share->error && !error_given) { - share->error= error; - open_table_error(share, error, (share->open_errno= my_errno), 0); + share->open_errno= my_errno; + open_table_error(share, share->error, share->open_errno); } - DBUG_RETURN(error); + DBUG_RETURN(share->error); } - -static bool create_key_infos(uchar *strpos, uint keys, KEY *keyinfo, - uint new_frm_ver, - uint &ext_key_parts, TABLE_SHARE *share, uint len, +static bool create_key_infos(const uchar *strpos, const uchar *frm_image_end, + uint keys, KEY *keyinfo, + uint new_frm_ver, uint &ext_key_parts, + TABLE_SHARE *share, uint len, KEY *first_keyinfo, char* &keynames) { uint i, j, n_length; @@ -770,6 +727,8 @@ static bool create_key_infos(uchar *strpos, uint keys, KEY *keyinfo, { if (new_frm_ver >= 3) { + if (strpos + 8 >= frm_image_end) + return 1; keyinfo->flags= (uint) uint2korr(strpos) ^ HA_NOSAME; keyinfo->key_length= (uint) uint2korr(strpos+2); keyinfo->key_parts= (uint) strpos[4]; @@ -779,6 +738,8 @@ static bool create_key_infos(uchar *strpos, uint keys, KEY *keyinfo, } else { + if (strpos + 4 >= frm_image_end) + return 1; keyinfo->flags= ((uint) strpos[0]) ^ HA_NOSAME; keyinfo->key_length= (uint) uint2korr(strpos+1); keyinfo->key_parts= (uint) strpos[3]; @@ -814,6 +775,8 @@ static bool create_key_infos(uchar *strpos, uint keys, KEY *keyinfo, keyinfo->rec_per_key= rec_per_key; for (j=keyinfo->key_parts ; j-- ; key_part++) { + if (strpos + (new_frm_ver >= 1 ? 9 : 7) >= frm_image_end) + return 1; *rec_per_key++=0; key_part->fieldnr= (uint16) (uint2korr(strpos) & FIELD_NR_MASK); key_part->offset= (uint) uint2korr(strpos+2)-1; @@ -869,17 +832,25 @@ static bool create_key_infos(uchar *strpos, uint keys, KEY *keyinfo, share->ext_key_parts+= keyinfo->ext_key_parts; } keynames=(char*) key_part; - strpos+= (strmov(keynames, (char *) strpos) - keynames)+1; + strpos+= strnmov(keynames, (char *) strpos, frm_image_end - strpos) - keynames; + if (*strpos++) // key names are \0-terminated + return 1; //reading index comments for (keyinfo= share->key_info, i=0; i < keys; i++, keyinfo++) { if (keyinfo->flags & HA_USES_COMMENT) { + if (strpos + 2 >= frm_image_end) + return 1; keyinfo->comment.length= uint2korr(strpos); - keyinfo->comment.str= strmake_root(&share->mem_root, (char*) strpos+2, + strpos+= 2; + + if (strpos + keyinfo->comment.length >= frm_image_end) + return 1; + keyinfo->comment.str= strmake_root(&share->mem_root, (char*) strpos, keyinfo->comment.length); - strpos+= 2 + keyinfo->comment.length; + strpos+= keyinfo->comment.length; } DBUG_ASSERT(test(keyinfo->flags & HA_USES_COMMENT) == (keyinfo->comment.length > 0)); @@ -890,31 +861,45 @@ static bool create_key_infos(uchar *strpos, uint keys, KEY *keyinfo, return 0; } -/* - Read data from a binary .frm file from MySQL 3.23 - 5.0 into TABLE_SHARE + +/** + Read data from a binary .frm file image into a TABLE_SHARE + + @note + frm bytes at the following offsets are unused in MariaDB 10.0: + + 8..9 (used to be the number of "form names") + 28..29 (used to be key_info_length) + + They're still set, for compatibility reasons, but never read. + + 42..46 are unused since 5.0 (were for RAID support) + Also, there're few unused bytes in forminfo. + */ -static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, - File file) +int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, + const uchar *frm_image, + size_t frm_length) { - int error, errarg= 0; + TABLE_SHARE *share= this; uint new_frm_ver, field_pack_length, new_field_pack_flag; uint interval_count, interval_parts, read_length, int_length; uint db_create_options, keys, key_parts, n_length; - uint key_info_length, com_length, null_bit_pos; + uint com_length, null_bit_pos; uint extra_rec_buf_length; uint i; bool use_hash; char *keynames, *names, *comment_pos; - uchar forminfo[288]; - uchar *record; - uchar *disk_buff, *strpos, *null_flags, *null_pos; - ulong pos, record_offset; + const uchar *forminfo, *extra2; + const uchar *frm_image_end = frm_image + frm_length; + uchar *record, *null_flags, *null_pos; + const uchar *disk_buff, *strpos; + ulong pos, record_offset; ulong rec_buff_length; handler *handler_file= 0; KEY *keyinfo; KEY_PART_INFO *key_part= NULL; - SQL_CRYPT *crypted=0; Field **field_ptr, *reg_field; const char **interval_array; enum legacy_db_type legacy_db_type; @@ -922,77 +907,149 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, bool null_bits_are_used; uint vcol_screen_length, UNINIT_VAR(options_len); char *vcol_screen_pos; - uchar *UNINIT_VAR(options); - uchar *extra_segment_buff= 0; + const uchar *options= 0; KEY first_keyinfo; uint len; uint ext_key_parts= 0; + plugin_ref se_plugin= 0; keyinfo= &first_keyinfo; share->ext_key_parts= 0; - DBUG_ENTER("open_binary_frm"); + MEM_ROOT **root_ptr, *old_root; + DBUG_ENTER("TABLE_SHARE::init_from_binary_frm_image"); + + root_ptr= my_pthread_getspecific_ptr(MEM_ROOT**, THR_MALLOC); + old_root= *root_ptr; + *root_ptr= &share->mem_root; + + if (write && write_frm_image(frm_image, frm_length)) + goto err; + + if (frm_length < FRM_HEADER_SIZE + FRM_FORMINFO_SIZE) + goto err; - new_field_pack_flag= head[27]; - new_frm_ver= (head[2] - FRM_VER); + new_field_pack_flag= frm_image[27]; + new_frm_ver= (frm_image[2] - FRM_VER); field_pack_length= new_frm_ver < 2 ? 11 : 17; - disk_buff= 0; - error= 3; - /* Position of the form in the form file. */ - if (!(pos= get_form_pos(file, head))) - goto err; /* purecov: inspected */ + /* Length of the MariaDB extra2 segment in the form file. */ + len = uint2korr(frm_image+4); + extra2= frm_image + 64; + + if (*extra2 != '/') // old frm had '/' there + { + const uchar *e2end= extra2 + len; + while (extra2 + 3 < e2end) + { + uchar type= *extra2++; + size_t length= *extra2++; + if (!length) + { + if (extra2 + 258 >= e2end) + goto err; + length= uint2korr(extra2); + extra2+=2; + if (length < 256) + goto err; + } + if (extra2 + length > e2end) + goto err; + switch (type) { + case EXTRA2_TABLEDEF_VERSION: + if (tabledef_version.str) // see init_from_sql_statement_string() + { + if (length != tabledef_version.length || + memcmp(extra2, tabledef_version.str, length)) + goto err; + } + else + { + tabledef_version.length= length; + tabledef_version.str= (uchar*)memdup_root(&mem_root, extra2, length); + if (!tabledef_version.str) + goto err; + } + break; + case EXTRA2_ENGINE_TABLEOPTS: + if (options) + goto err; + /* remember but delay parsing until we have read fields and keys */ + options= extra2; + options_len= length; + break; + case EXTRA2_DEFAULT_PART_ENGINE: +#ifdef WITH_PARTITION_STORAGE_ENGINE + { + LEX_STRING name= { (char*)extra2, length }; + share->default_part_plugin= ha_resolve_by_name(NULL, &name); + if (!share->default_part_plugin) + goto err; + } +#endif + break; + default: + /* abort frm parsing if it's an unknown but important extra2 value */ + if (type >= EXTRA2_ENGINE_IMPORTANT) + goto err; + } + extra2+= length; + } + if (extra2 != e2end) + goto err; + } - mysql_file_seek(file,pos,MY_SEEK_SET,MYF(0)); - if (mysql_file_read(file, forminfo,288,MYF(MY_NABP))) + if (frm_length < FRM_HEADER_SIZE + len || + !(pos= uint4korr(frm_image + FRM_HEADER_SIZE + len))) goto err; - share->frm_version= head[2]; + + forminfo= frm_image + pos; + if (forminfo + FRM_FORMINFO_SIZE >= frm_image_end) + goto err; + + share->frm_version= frm_image[2]; /* Check if .frm file created by MySQL 5.0. In this case we want to display CHAR fields as CHAR and not as VARCHAR. We do it this way as we want to keep the old frm version to enable MySQL 4.1 to read these files. */ - if (share->frm_version == FRM_VER_TRUE_VARCHAR -1 && head[33] == 5) + if (share->frm_version == FRM_VER_TRUE_VARCHAR -1 && frm_image[33] == 5) share->frm_version= FRM_VER_TRUE_VARCHAR; #ifdef WITH_PARTITION_STORAGE_ENGINE - /* - Yuck! Double-bad. Doesn't work with dynamic engine codes. - And doesn't lock the plugin. Fixed in 10.0.4 - */ - compile_time_assert(MYSQL_VERSION_ID < 100000); - if (*(head+61) && - !(share->default_part_db_type= - ha_checktype(thd, (enum legacy_db_type) (uint) *(head+61), 1, 0))) - goto err; - DBUG_PRINT("info", ("default_part_db_type = %u", head[61])); + if (frm_image[61] && !share->default_part_plugin) + { + enum legacy_db_type db_type= (enum legacy_db_type) (uint) frm_image[61]; + share->default_part_plugin= + ha_lock_engine(NULL, ha_checktype(thd, db_type, 1, 0)); + if (!share->default_part_plugin) + goto err; + } #endif - legacy_db_type= (enum legacy_db_type) (uint) *(head+3); - DBUG_ASSERT(share->db_plugin == NULL); + legacy_db_type= (enum legacy_db_type) (uint) frm_image[3]; /* if the storage engine is dynamic, no point in resolving it by its dynamically allocated legacy_db_type. We will resolve it later by name. */ if (legacy_db_type > DB_TYPE_UNKNOWN && legacy_db_type < DB_TYPE_FIRST_DYNAMIC) - share->db_plugin= ha_lock_engine(NULL, - ha_checktype(thd, legacy_db_type, 0, 0)); - share->db_create_options= db_create_options= uint2korr(head+30); + se_plugin= ha_lock_engine(NULL, ha_checktype(thd, legacy_db_type, 0, 0)); + share->db_create_options= db_create_options= uint2korr(frm_image+30); share->db_options_in_use= share->db_create_options; - share->mysql_version= uint4korr(head+51); + share->mysql_version= uint4korr(frm_image+51); share->null_field_first= 0; - if (!head[32]) // New frm file in 3.23 - { - share->avg_row_length= uint4korr(head+34); - share->transactional= (ha_choice) (head[39] & 3); - share->page_checksum= (ha_choice) ((head[39] >> 2) & 3); - share->row_type= (row_type) head[40]; - share->table_charset= get_charset((((uint) head[41]) << 8) + - (uint) head[38],MYF(0)); + if (!frm_image[32]) // New frm file in 3.23 + { + share->avg_row_length= uint4korr(frm_image+34); + share->transactional= (ha_choice) (frm_image[39] & 3); + share->page_checksum= (ha_choice) ((frm_image[39] >> 2) & 3); + share->row_type= (enum row_type) frm_image[40]; + share->table_charset= get_charset((((uint) frm_image[41]) << 8) + + (uint) frm_image[38],MYF(0)); share->null_field_first= 1; } if (!share->table_charset) { - /* unknown charset in head[38] or pre-3.23 frm */ + /* unknown charset in frm_image[38] or pre-3.23 frm */ if (use_mb(default_charset_info)) { /* Warn that we may be changing the size of character columns */ @@ -1006,15 +1063,15 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, share->db_record_offset= 1; if (db_create_options & HA_OPTION_LONG_BLOB_PTR) share->blob_ptr_size= portable_sizeof_char_ptr; - error=4; - share->max_rows= uint4korr(head+18); - share->min_rows= uint4korr(head+22); + share->max_rows= uint4korr(frm_image+18); + share->min_rows= uint4korr(frm_image+22); /* Read keyinformation */ - key_info_length= (uint) uint2korr(head+28); - mysql_file_seek(file, (ulong) uint2korr(head+6), MY_SEEK_SET, MYF(0)); - if (read_string(file,(uchar**) &disk_buff,key_info_length)) - goto err; /* purecov: inspected */ + disk_buff= frm_image + uint2korr(frm_image+6); + + if (disk_buff + 6 >= frm_image_end) + goto err; + if (disk_buff[0] & 0x80) { keys= (disk_buff[1] << 7) | (disk_buff[0] & 0x7f); @@ -1031,36 +1088,29 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, len= (uint) uint2korr(disk_buff+4); - share->reclength = uint2korr((head+16)); + share->reclength = uint2korr(frm_image+16); share->stored_rec_length= share->reclength; - if (*(head+26) == 1) + if (frm_image[26] == 1) share->system= 1; /* one-record-database */ -#ifdef HAVE_CRYPTED_FRM - else if (*(head+26) == 2) - { - crypted= get_crypt_for_frm(); - share->crypted= 1; - } -#endif - record_offset= (ulong) (uint2korr(head+6)+ - ((uint2korr(head+14) == 0xffff ? - uint4korr(head+47) : uint2korr(head+14)))); + record_offset= (ulong) (uint2korr(frm_image+6)+ + ((uint2korr(frm_image+14) == 0xffff ? + uint4korr(frm_image+47) : uint2korr(frm_image+14)))); + + if (record_offset + share->reclength >= frm_length) + goto err; - if ((n_length= uint4korr(head+55))) + if ((n_length= uint4korr(frm_image+55))) { /* Read extra data segment */ - uchar *next_chunk, *buff_end; + const uchar *next_chunk, *buff_end; DBUG_PRINT("info", ("extra segment size is %u bytes", n_length)); - if (!(extra_segment_buff= (uchar*) my_malloc(n_length + 1, MYF(MY_WME)))) - goto err; - next_chunk= extra_segment_buff; - if (mysql_file_pread(file, extra_segment_buff, - n_length, record_offset + share->reclength, - MYF(MY_NABP))) - { + next_chunk= frm_image + record_offset + share->reclength; + buff_end= next_chunk + n_length; + + if (buff_end >= frm_image_end) goto err; - } + share->connect_string.length= uint2korr(next_chunk); if (!(share->connect_string.str= strmake_root(&share->mem_root, (char*) next_chunk + 2, @@ -1070,7 +1120,6 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, goto err; } next_chunk+= share->connect_string.length + 2; - buff_end= extra_segment_buff + n_length; if (next_chunk + 2 < buff_end) { uint str_db_type_length= uint2korr(next_chunk); @@ -1079,12 +1128,9 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, name.length= str_db_type_length; plugin_ref tmp_plugin= ha_resolve_by_name(thd, &name); - if (tmp_plugin != NULL && !plugin_equals(tmp_plugin, share->db_plugin)) + if (tmp_plugin != NULL && !plugin_equals(tmp_plugin, se_plugin)) { - if (legacy_db_type > DB_TYPE_UNKNOWN && - legacy_db_type < DB_TYPE_FIRST_DYNAMIC && - legacy_db_type != ha_legacy_type( - plugin_data(tmp_plugin, handlerton *))) + if (se_plugin) { /* bad file, legacy_db_type did not match the name */ sql_print_warning("%s.frm is inconsistent: engine typecode %d, engine name %s (%d)", @@ -1094,14 +1140,11 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, } /* tmp_plugin is locked with a local lock. - we unlock the old value of share->db_plugin before + we unlock the old value of se_plugin before replacing it with a globally locked version of tmp_plugin */ - plugin_unlock(NULL, share->db_plugin); - share->db_plugin= my_plugin_lock(NULL, tmp_plugin); - DBUG_PRINT("info", ("setting dbtype to '%.*s' (%d)", - str_db_type_length, next_chunk + 2, - ha_legacy_type(share->db_type()))); + plugin_unlock(NULL, se_plugin); + se_plugin= plugin_lock(NULL, tmp_plugin); } #ifdef WITH_PARTITION_STORAGE_ENGINE else if (str_db_type_length == 9 && @@ -1110,28 +1153,23 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, /* Use partition handler tmp_plugin is locked with a local lock. - we unlock the old value of share->db_plugin before + we unlock the old value of se_plugin before replacing it with a globally locked version of tmp_plugin */ /* Check if the partitioning engine is ready */ if (!plugin_is_ready(&name, MYSQL_STORAGE_ENGINE_PLUGIN)) { - error= 8; my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), "--skip-partition"); goto err; } - plugin_unlock(NULL, share->db_plugin); - share->db_plugin= ha_lock_engine(NULL, partition_hton); - DBUG_PRINT("info", ("setting dbtype to '%.*s' (%d)", - str_db_type_length, next_chunk + 2, - ha_legacy_type(share->db_type()))); + plugin_unlock(NULL, se_plugin); + se_plugin= ha_lock_engine(NULL, partition_hton); } #endif else if (!tmp_plugin) { /* purecov: begin inspected */ - error= 8; name.str[name.length]=0; my_error(ER_UNKNOWN_STORAGE_ENGINE, MYF(0), name.str); goto err; @@ -1140,10 +1178,10 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, next_chunk+= str_db_type_length + 2; } - share->set_use_ext_keys_flag(share->db_type()->flags & HTON_EXTENDED_KEYS); + share->set_use_ext_keys_flag(plugin_hton(se_plugin)->flags & HTON_EXTENDED_KEYS); - if (create_key_infos(disk_buff + 6, keys, keyinfo, new_frm_ver, - ext_key_parts, + if (create_key_infos(disk_buff + 6, frm_image_end, keys, keyinfo, + new_frm_ver, ext_key_parts, share, len, &first_keyinfo, keynames)) goto err; @@ -1223,12 +1261,10 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, DBUG_ASSERT(next_chunk <= buff_end); - if (share->db_create_options & HA_OPTION_TEXT_CREATE_OPTIONS) + if (share->db_create_options & HA_OPTION_TEXT_CREATE_OPTIONS_legacy) { - /* - store options position, but skip till the time we will - know number of fields - */ + if (options) + goto err; options_len= uint4korr(next_chunk); options= next_chunk + 4; next_chunk+= options_len + 4; @@ -1237,35 +1273,27 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, } else { - if (create_key_infos(disk_buff + 6, keys, keyinfo, new_frm_ver, - ext_key_parts, + if (create_key_infos(disk_buff + 6, frm_image_end, keys, keyinfo, + new_frm_ver, ext_key_parts, share, len, &first_keyinfo, keynames)) goto err; } - share->key_block_size= uint2korr(head+62); + share->key_block_size= uint2korr(frm_image+62); + + if (share->db_plugin && !plugin_equals(share->db_plugin, se_plugin)) + goto err; // wrong engine (someone changed the frm under our feet?) - error=4; - extra_rec_buf_length= uint2korr(head+59); + extra_rec_buf_length= uint2korr(frm_image+59); rec_buff_length= ALIGN_SIZE(share->reclength + 1 + extra_rec_buf_length); share->rec_buff_length= rec_buff_length; if (!(record= (uchar *) alloc_root(&share->mem_root, rec_buff_length))) goto err; /* purecov: inspected */ share->default_values= record; - if (mysql_file_pread(file, record, (size_t) share->reclength, - record_offset, MYF(MY_NABP))) - goto err; /* purecov: inspected */ + memcpy(record, frm_image + record_offset, share->reclength); - mysql_file_seek(file, pos+288, MY_SEEK_SET, MYF(0)); -#ifdef HAVE_CRYPTED_FRM - if (crypted) - { - crypted->decode((char*) forminfo+256,288-256); - if (sint2korr(forminfo+284) != 0) // Should be 0 - goto err; // Wrong password - } -#endif + disk_buff= frm_image + pos + FRM_FORMINFO_SIZE; share->fields= uint2korr(forminfo+258); pos= uint2korr(forminfo+260); /* Length of all screens */ @@ -1277,6 +1305,7 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, com_length= uint2korr(forminfo+284); vcol_screen_length= uint2korr(forminfo+286); share->vfields= 0; + share->default_fields= 0; share->stored_fields= share->fields; if (forminfo[46] != (uchar)255) { @@ -1302,16 +1331,6 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, read_length=(uint) (share->fields * field_pack_length + pos+ (uint) (n_length+int_length+com_length+ vcol_screen_length)); - if (read_string(file,(uchar**) &disk_buff,read_length)) - goto err; /* purecov: inspected */ -#ifdef HAVE_CRYPTED_FRM - if (crypted) - { - crypted->decode((char*) disk_buff,read_length); - delete crypted; - crypted=0; - } -#endif strpos= disk_buff+pos; share->intervals= (TYPELIB*) (field_ptr+share->fields+1); @@ -1359,14 +1378,14 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, /* Allocate handler */ if (!(handler_file= get_new_handler(share, thd->mem_root, - share->db_type()))) + plugin_hton(se_plugin)))) goto err; record= share->default_values-1; /* Fieldstart = 1 */ null_bits_are_used= share->null_fields != 0; if (share->null_field_first) { - null_flags= null_pos= (uchar*) record+1; + null_flags= null_pos= record+1; null_bit_pos= (db_create_options & HA_OPTION_PACK_RECORD) ? 0 : 1; /* null_bytes below is only correct under the condition that @@ -1379,8 +1398,7 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, else { share->null_bytes= (share->null_fields+7)/8; - null_flags= null_pos= (uchar*) (record + 1 +share->reclength - - share->null_bytes); + null_flags= null_pos= record + 1 + share->reclength - share->null_bytes; null_bit_pos= 0; } #endif @@ -1422,7 +1440,6 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, geom_type= (Field::geometry_type) strpos[14]; charset= &my_charset_bin; #else - error= 4; // unsupported field type goto err; #endif } @@ -1433,8 +1450,16 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, charset= &my_charset_bin; else if (!(charset= get_charset(csid, MYF(0)))) { - error= 5; // Unknown or unavailable charset - errarg= (int) csid; + const char *csname= get_charset_name((uint) csid); + char tmp[10]; + if (!csname || csname[0] =='?') + { + my_snprintf(tmp, sizeof(tmp), "#%d", csid); + csname= tmp; + } + my_printf_error(ER_UNKNOWN_COLLATION, + "Unknown collation '%s' in table '%-.64s' definition", + MYF(0), csname, share->table_name.str); goto err; } } @@ -1478,10 +1503,8 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, if (opt_interval_id) interval_nr= (uint)vcol_screen_pos[3]; else if ((uint)vcol_screen_pos[0] != 1) - { - error= 4; goto err; - } + fld_stored_in_db= (bool) (uint) vcol_screen_pos[2]; vcol_expr_length= vcol_info_length - (uint)(FRM_VCOL_HEADER_SIZE(opt_interval_id)); @@ -1580,10 +1603,8 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, (TYPELIB*) 0), share->fieldnames.type_names[i]); if (!reg_field) // Not supported field type - { - error= 4; - goto err; /* purecov: inspected */ - } + goto err; + reg_field->field_index= i; reg_field->comment=comment; @@ -1608,29 +1629,18 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, if (reg_field->unireg_check == Field::NEXT_NUMBER) share->found_next_number_field= field_ptr; - if (share->timestamp_field == reg_field) - share->timestamp_field_offset= i; - if (use_hash) - { - if (my_hash_insert(&share->name_hash, - (uchar*) field_ptr)) - { - /* - Set return code 8 here to indicate that an error has - occurred but that the error message already has been - sent (OOM). - */ - error= 8; - goto err; - } - } + if (use_hash && my_hash_insert(&share->name_hash, (uchar*) field_ptr)) + goto err; if (!reg_field->stored_in_db) { share->stored_fields--; if (share->stored_rec_length>=recpos) share->stored_rec_length= recpos-1; } + if (reg_field->has_insert_default_function() || + reg_field->has_update_default_function()) + ++share->default_fields; } *field_ptr=0; // End marker /* Sanity checks: */ @@ -1773,10 +1783,8 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, (uint) key_part->offset, (uint) key_part->length); if (!key_part->fieldnr) - { - error= 4; // Wrong file goto err; - } + field= key_part->field= share->field[key_part->fieldnr-1]; key_part->type= field->key_type(); if (field->null_ptr) @@ -1924,8 +1932,6 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, } else share->primary_key= MAX_KEY; - my_free(disk_buff); - disk_buff=0; if (new_field_pack_flag <= 1) { /* Old file format with default as not null */ @@ -1934,7 +1940,7 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, null_length, 255); } - if (share->db_create_options & HA_OPTION_TEXT_CREATE_OPTIONS) + if (options) { DBUG_ASSERT(options_len); if (engine_table_options_frm_read(options, options_len, share)) @@ -1951,13 +1957,8 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, share->default_values, reg_field, &share->next_number_key_offset, &share->next_number_keypart)) < 0) - { - /* Wrong field definition */ - error= 4; - goto err; - } - else - reg_field->flags |= AUTO_INCREMENT_FLAG; + goto err; // Wrong field definition + reg_field->flags |= AUTO_INCREMENT_FLAG; } if (share->blob_fields) @@ -2001,17 +2002,18 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, if (use_hash) (void) my_hash_check(&share->name_hash); #endif - my_free(extra_segment_buff); - DBUG_RETURN (0); + + share->db_plugin= se_plugin; + share->error= OPEN_FRM_OK; + thd->status_var.opened_shares++; + *root_ptr= old_root; + DBUG_RETURN(0); err: - share->error= error; + share->error= OPEN_FRM_CORRUPTED; share->open_errno= my_errno; - share->errarg= errarg; - my_free(disk_buff); - my_free(extra_segment_buff); - delete crypted; delete handler_file; + plugin_unlock(0, se_plugin); my_hash_free(&share->name_hash); if (share->ha_data_destroy) { @@ -2026,9 +2028,168 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, } #endif /* WITH_PARTITION_STORAGE_ENGINE */ - open_table_error(share, error, share->open_errno, errarg); - DBUG_RETURN(error); -} /* open_binary_frm */ + if (!thd->is_error()) + open_table_error(share, OPEN_FRM_CORRUPTED, share->open_errno); + + *root_ptr= old_root; + DBUG_RETURN(HA_ERR_NOT_A_TABLE); +} + + +static bool sql_unusable_for_discovery(THD *thd, const char *sql) +{ + LEX *lex= thd->lex; + HA_CREATE_INFO *create_info= &lex->create_info; + + // ... not CREATE TABLE + if (lex->sql_command != SQLCOM_CREATE_TABLE) + return 1; + // ... create like + if (create_info->options & HA_LEX_CREATE_TABLE_LIKE) + return 1; + // ... create select + if (lex->select_lex.item_list.elements) + return 1; + // ... temporary + if (create_info->options & HA_LEX_CREATE_TMP_TABLE) + return 1; + // ... if exists + if (create_info->options & HA_LEX_CREATE_IF_NOT_EXISTS) + return 1; + + // XXX error out or rather ignore the following: + // ... partitioning + if (lex->part_info) + return 1; + // ... union + if (create_info->used_fields & HA_CREATE_USED_UNION) + return 1; + // ... index/data directory + if (create_info->data_file_name || create_info->index_file_name) + return 1; + // ... engine + if (create_info->used_fields & HA_CREATE_USED_ENGINE) + return 1; + + return 0; +} + +int TABLE_SHARE::init_from_sql_statement_string(THD *thd, bool write, + const char *sql, size_t sql_length) +{ + ulonglong saved_mode= thd->variables.sql_mode; + CHARSET_INFO *old_cs= thd->variables.character_set_client; + Parser_state parser_state; + bool error; + char *sql_copy; + handler *file; + LEX *old_lex; + Query_arena *arena, backup; + LEX tmp_lex; + LEX_CUSTRING frm= {0,0}; + + DBUG_ENTER("TABLE_SHARE::init_from_sql_statement_string"); + + /* + Ouch. Parser may *change* the string it's working on. + Currently (2013-02-26) it is used to permanently disable + conditional comments. + Anyway, let's copy the caller's string... + */ + if (!(sql_copy= thd->strmake(sql, sql_length))) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + + if (parser_state.init(thd, sql_copy, sql_length)) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + + thd->variables.sql_mode= MODE_NO_ENGINE_SUBSTITUTION | MODE_NO_DIR_IN_CREATE; + thd->variables.character_set_client= system_charset_info; + tmp_disable_binlog(thd); + old_lex= thd->lex; + thd->lex= &tmp_lex; + + arena= thd->stmt_arena; + if (arena->is_conventional()) + arena= 0; + else + thd->set_n_backup_active_arena(arena, &backup); + + lex_start(thd); + + if ((error= parse_sql(thd, & parser_state, NULL) || + sql_unusable_for_discovery(thd, sql_copy))) + goto ret; + + thd->lex->create_info.db_type= plugin_hton(db_plugin); + + if (tabledef_version.str) + thd->lex->create_info.tabledef_version= tabledef_version; + + file= mysql_create_frm_image(thd, db.str, table_name.str, + &thd->lex->create_info, &thd->lex->alter_info, + C_ORDINARY_CREATE, &frm); + error|= file == 0; + delete file; + + if (frm.str) + { + option_list= 0; // cleanup existing options ... + option_struct= 0; // ... if it's an assisted discovery + error= init_from_binary_frm_image(thd, write, frm.str, frm.length); + } + +ret: + my_free(const_cast<uchar*>(frm.str)); + lex_end(thd->lex); + thd->lex= old_lex; + if (arena) + thd->restore_active_arena(arena, &backup); + reenable_binlog(thd); + thd->variables.sql_mode= saved_mode; + thd->variables.character_set_client= old_cs; + if (thd->is_error() || error) + { + thd->clear_error(); + my_error(ER_SQL_DISCOVER_ERROR, MYF(0), + plugin_name(db_plugin)->str, db.str, table_name.str, + sql_copy); + DBUG_RETURN(HA_ERR_GENERIC); + } + DBUG_RETURN(0); +} + +bool TABLE_SHARE::write_frm_image(const uchar *frm, size_t len) +{ + return writefrm(normalized_path.str, db.str, table_name.str, false, frm, len); +} + + +bool TABLE_SHARE::read_frm_image(const uchar **frm, size_t *len) +{ + if (IF_PARTITIONING(partition_info_str, 0)) // cannot discover a partition + { + DBUG_ASSERT(db_type()->discover_table == 0); + return 1; + } + + if (frm_image) + { + *frm= frm_image->str; + *len= frm_image->length; + frm_image->str= 0; // pass the ownership to the caller + frm_image= 0; + return 0; + } + return readfrm(normalized_path.str, frm, len); +} + + +void TABLE_SHARE::free_frm_image(const uchar *frm) +{ + if (frm) + my_free(const_cast<uchar*>(frm)); +} + /* @brief @@ -2341,15 +2502,16 @@ end: 7 Table definition has changed in engine */ -int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, - uint db_stat, uint prgflag, uint ha_open_flags, - TABLE *outparam, bool is_create_table) +enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, + const char *alias, uint db_stat, uint prgflag, + uint ha_open_flags, TABLE *outparam, + bool is_create_table) { - int error; + enum open_frm_error error; uint records, i, bitmap_size; bool error_reported= FALSE; uchar *record, *bitmaps; - Field **field_ptr, **vfield_ptr; + Field **field_ptr, **UNINIT_VAR(vfield_ptr), **UNINIT_VAR(dfield_ptr); uint8 save_context_analysis_only= thd->lex->context_analysis_only; DBUG_ENTER("open_table_from_share"); DBUG_PRINT("enter",("name: '%s.%s' form: 0x%lx", share->db.str, @@ -2357,14 +2519,14 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, thd->lex->context_analysis_only&= ~CONTEXT_ANALYSIS_ONLY_VIEW; // not a view - error= 1; + error= OPEN_FRM_ERROR_ALREADY_ISSUED; // for OOM errors below bzero((char*) outparam, sizeof(*outparam)); outparam->in_use= thd; outparam->s= share; outparam->db_stat= db_stat; outparam->write_row_record= NULL; - init_sql_alloc(&outparam->mem_root, TABLE_ALLOC_BLOCK_SIZE, 0); + init_sql_alloc(&outparam->mem_root, TABLE_ALLOC_BLOCK_SIZE, 0, MYF(0)); if (outparam->alias.copy(alias, strlen(alias), table_alias_charset)) goto err; @@ -2386,7 +2548,6 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, DBUG_ASSERT(!db_stat); } - error= 4; outparam->reginfo.lock_type= TL_UNLOCK; outparam->current_lock= F_UNLCK; records=0; @@ -2453,9 +2614,6 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, if (share->found_next_number_field) outparam->found_next_number_field= outparam->field[(uint) (share->found_next_number_field - share->field)]; - if (share->timestamp_field) - outparam->timestamp_field= (Field_timestamp*) outparam->field[share->timestamp_field_offset]; - /* Fix key->name and key_part->field */ if (share->key_parts) @@ -2506,11 +2664,9 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, } /* - Process virtual columns, if any. + Process virtual and default columns, if any. */ - if (!share->vfields) - outparam->vfield= NULL; - else + if (share->vfields) { if (!(vfield_ptr = (Field **) alloc_root(&outparam->mem_root, (uint) ((share->vfields+1)* @@ -2518,10 +2674,24 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, goto err; outparam->vfield= vfield_ptr; + } + + if (share->default_fields) + { + if (!(dfield_ptr = (Field **) alloc_root(&outparam->mem_root, + (uint) ((share->default_fields+1)* + sizeof(Field*))))) + goto err; + + outparam->default_field= dfield_ptr; + } + if (share->vfields || share->default_fields) + { + /* Reuse the same loop both for virtual and default fields. */ for (field_ptr= outparam->field; *field_ptr; field_ptr++) { - if ((*field_ptr)->vcol_info) + if (share->vfields && (*field_ptr)->vcol_info) { if (unpack_vcol_info_from_frm(thd, &outparam->mem_root, @@ -2530,13 +2700,20 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, &(*field_ptr)->vcol_info->expr_str, &error_reported)) { - error= 4; // in case no error is reported + error= OPEN_FRM_CORRUPTED; goto err; } *(vfield_ptr++)= *field_ptr; } + if (share->default_fields && + ((*field_ptr)->has_insert_default_function() || + (*field_ptr)->has_update_default_function())) + *(dfield_ptr++)= *field_ptr; } - *vfield_ptr= 0; // End marker + if (share->vfields) + *vfield_ptr= 0; // End marker + if (share->default_fields) + *dfield_ptr= 0; // End marker } #ifdef WITH_PARTITION_STORAGE_ENGINE @@ -2565,7 +2742,7 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, tmp= mysql_unpack_partition(thd, share->partition_info_str, share->partition_info_str_len, outparam, is_create_table, - share->default_part_db_type, + plugin_hton(share->default_part_plugin), &work_part_info_used); if (tmp) { @@ -2619,7 +2796,7 @@ partititon_err: /* Allocate bitmaps */ bitmap_size= share->column_bitmap_size; - if (!(bitmaps= (uchar*) alloc_root(&outparam->mem_root, bitmap_size*5))) + if (!(bitmaps= (uchar*) alloc_root(&outparam->mem_root, bitmap_size*6))) goto err; bitmap_init(&outparam->def_read_set, (my_bitmap_map*) bitmaps, share->fields, FALSE); @@ -2631,56 +2808,53 @@ partititon_err: (my_bitmap_map*) (bitmaps+bitmap_size*3), share->fields, FALSE); bitmap_init(&outparam->eq_join_set, (my_bitmap_map*) (bitmaps+bitmap_size*4), share->fields, FALSE); + bitmap_init(&outparam->cond_set, + (my_bitmap_map*) (bitmaps+bitmap_size*5), share->fields, FALSE); outparam->default_column_bitmaps(); + outparam->cond_selectivity= 1.0; + /* The table struct is now initialized; Open the table */ - error= 2; if (db_stat) { - int ha_err; - if ((ha_err= (outparam->file-> - ha_open(outparam, share->normalized_path.str, - (db_stat & HA_READ_ONLY ? O_RDONLY : O_RDWR), - (db_stat & HA_OPEN_TEMPORARY ? HA_OPEN_TMP_TABLE : - ((db_stat & HA_WAIT_IF_LOCKED) || - (specialflag & SPECIAL_WAIT_IF_LOCKED)) ? - HA_OPEN_WAIT_IF_LOCKED : - (db_stat & (HA_ABORT_IF_LOCKED | HA_GET_INFO)) ? - HA_OPEN_ABORT_IF_LOCKED : - HA_OPEN_IGNORE_IF_LOCKED) | ha_open_flags)))) + if (db_stat & HA_OPEN_TEMPORARY) + ha_open_flags|= HA_OPEN_TMP_TABLE; + else if ((db_stat & HA_WAIT_IF_LOCKED) || + (specialflag & SPECIAL_WAIT_IF_LOCKED)) + ha_open_flags|= HA_OPEN_WAIT_IF_LOCKED; + else if (db_stat & (HA_ABORT_IF_LOCKED | HA_GET_INFO)) + ha_open_flags|= HA_OPEN_ABORT_IF_LOCKED; + else + ha_open_flags|= HA_OPEN_IGNORE_IF_LOCKED; + + int ha_err= outparam->file->ha_open(outparam, share->normalized_path.str, + (db_stat & HA_READ_ONLY ? O_RDONLY : O_RDWR), + ha_open_flags); + if (ha_err) { + share->open_errno= ha_err; /* Set a flag if the table is crashed and it can be auto. repaired */ share->crashed= (outparam->file->auto_repair(ha_err) && !(ha_open_flags & HA_OPEN_FOR_REPAIR)); + outparam->file->print_error(ha_err, MYF(0)); + error_reported= TRUE; - switch (ha_err) - { - case HA_ERR_NO_SUCH_TABLE: - /* - The table did not exists in storage engine, use same error message - as if the .frm file didn't exist - */ - error= 1; - my_errno= ENOENT; - break; - case EMFILE: - /* - Too many files opened, use same error message as if the .frm - file can't open - */ - DBUG_PRINT("error", ("open file: %s failed, too many files opened (errno: %d)", - share->normalized_path.str, ha_err)); - error= 1; - my_errno= EMFILE; - break; - default: - outparam->file->print_error(ha_err, MYF(0)); - error_reported= TRUE; - if (ha_err == HA_ERR_TABLE_DEF_CHANGED) - error= 7; - break; - } - goto err; /* purecov: inspected */ + if (ha_err == HA_ERR_TABLE_DEF_CHANGED) + error= OPEN_FRM_DISCOVER; + + /* + We're here, because .frm file was successfully opened. + + But if the table doesn't exist in the engine and the engine + supports discovery, we force rediscover to discover + the fact that table doesn't in fact exist and remove + the stray .frm file. + */ + if (share->db_type()->discover_table && + (ha_err == ENOENT || ha_err == HA_ERR_NO_SUCH_TABLE)) + error= OPEN_FRM_DISCOVER; + + goto err; } } @@ -2694,11 +2868,11 @@ partititon_err: thd->status_var.opened_tables++; thd->lex->context_analysis_only= save_context_analysis_only; - DBUG_RETURN (0); + DBUG_RETURN (OPEN_FRM_OK); err: if (! error_reported) - open_table_error(share, error, my_errno, 0); + open_table_error(share, error, my_errno); delete outparam->file; #ifdef WITH_PARTITION_STORAGE_ENGINE if (outparam->part_info) @@ -2809,158 +2983,17 @@ void free_field_buffers_larger_than(TABLE *table, uint32 size) } } -/** - Find where a form starts. - - @param head The start of the form file. - - @remark If formname is NULL then only formnames is read. +/* error message when opening a form file */ - @retval The form position. -*/ - -static ulong get_form_pos(File file, uchar *head) +void open_table_error(TABLE_SHARE *share, enum open_frm_error error, + int db_errno) { - uchar *pos, *buf; - uint names, length; - ulong ret_value=0; - DBUG_ENTER("get_form_pos"); - - names= uint2korr(head+8); - - if (!(names= uint2korr(head+8))) - DBUG_RETURN(0); - - length= uint2korr(head+4); - - mysql_file_seek(file, 64L, MY_SEEK_SET, MYF(0)); - - if (!(buf= (uchar*) my_malloc(length+names*4, MYF(MY_WME)))) - DBUG_RETURN(0); - - if (mysql_file_read(file, buf, length+names*4, MYF(MY_NABP))) - { - my_free(buf); - DBUG_RETURN(0); - } - - pos= buf+length; - ret_value= uint4korr(pos); - - my_free(buf); - - DBUG_RETURN(ret_value); -} - - -/* - Read string from a file with malloc - - NOTES: - We add an \0 at end of the read string to make reading of C strings easier -*/ - -int read_string(File file, uchar**to, size_t length) -{ - DBUG_ENTER("read_string"); - - my_free(*to); - if (!(*to= (uchar*) my_malloc(length+1,MYF(MY_WME))) || - mysql_file_read(file, *to, length, MYF(MY_NABP))) - { - my_free(*to); /* purecov: inspected */ - *to= 0; /* purecov: inspected */ - DBUG_RETURN(1); /* purecov: inspected */ - } - *((char*) *to+length)= '\0'; - DBUG_RETURN (0); -} /* read_string */ - - - /* Add a new form to a form file */ - -ulong make_new_entry(File file, uchar *fileinfo, TYPELIB *formnames, - const char *newname) -{ - uint i,bufflength,maxlength,n_length,length,names; - ulong endpos,newpos; - uchar buff[IO_SIZE]; - uchar *pos; - DBUG_ENTER("make_new_entry"); - - length=(uint) strlen(newname)+1; - n_length=uint2korr(fileinfo+4); - maxlength=uint2korr(fileinfo+6); - names=uint2korr(fileinfo+8); - newpos=uint4korr(fileinfo+10); - - if (64+length+n_length+(names+1)*4 > maxlength) - { /* Expand file */ - newpos+=IO_SIZE; - int4store(fileinfo+10,newpos); - /* Copy from file-end */ - endpos= (ulong) mysql_file_seek(file, 0L, MY_SEEK_END, MYF(0)); - bufflength= (uint) (endpos & (IO_SIZE-1)); /* IO_SIZE is a power of 2 */ - - while (endpos > maxlength) - { - mysql_file_seek(file, (ulong) (endpos-bufflength), MY_SEEK_SET, MYF(0)); - if (mysql_file_read(file, buff, bufflength, MYF(MY_NABP+MY_WME))) - DBUG_RETURN(0L); - mysql_file_seek(file, (ulong) (endpos-bufflength+IO_SIZE), MY_SEEK_SET, - MYF(0)); - if ((mysql_file_write(file, buff, bufflength, MYF(MY_NABP+MY_WME)))) - DBUG_RETURN(0); - endpos-=bufflength; bufflength=IO_SIZE; - } - bzero(buff,IO_SIZE); /* Null new block */ - mysql_file_seek(file, (ulong) maxlength, MY_SEEK_SET, MYF(0)); - if (mysql_file_write(file, buff, bufflength, MYF(MY_NABP+MY_WME))) - DBUG_RETURN(0L); - maxlength+=IO_SIZE; /* Fix old ref */ - int2store(fileinfo+6,maxlength); - for (i=names, pos= (uchar*) *formnames->type_names+n_length-1; i-- ; - pos+=4) - { - endpos=uint4korr(pos)+IO_SIZE; - int4store(pos,endpos); - } - } - - if (n_length == 1 ) - { /* First name */ - length++; - (void) strxmov((char*) buff,"/",newname,"/",NullS); - } - else - (void) strxmov((char*) buff,newname,"/",NullS); /* purecov: inspected */ - mysql_file_seek(file, 63L+(ulong) n_length, MY_SEEK_SET, MYF(0)); - if (mysql_file_write(file, buff, (size_t) length+1, MYF(MY_NABP+MY_WME)) || - (names && mysql_file_write(file, - (uchar*) (*formnames->type_names+n_length-1), - names*4, MYF(MY_NABP+MY_WME))) || - mysql_file_write(file, fileinfo+10, 4, MYF(MY_NABP+MY_WME))) - DBUG_RETURN(0L); /* purecov: inspected */ - - int2store(fileinfo+8,names+1); - int2store(fileinfo+4,n_length+length); - (void) mysql_file_chsize(file, newpos, 0, MYF(MY_WME));/* Append file with '\0' */ - DBUG_RETURN(newpos); -} /* make_new_entry */ - - - /* error message when opening a form file */ - -void open_table_error(TABLE_SHARE *share, int error, int db_errno, int errarg) -{ - int err_no; char buff[FN_REFLEN]; - myf errortype= ME_ERROR+ME_WAITTANG; // Write fatals error to log + const myf errortype= ME_ERROR+ME_WAITTANG; // Write fatals error to log DBUG_ENTER("open_table_error"); switch (error) { - case 7: - case 1: + case OPEN_FRM_OPEN_ERROR: /* Test if file didn't exists. We have to also test for EINVAL as this may happen on windows when opening a file with a not legal file name @@ -2974,55 +3007,30 @@ void open_table_error(TABLE_SHARE *share, int error, int db_errno, int errarg) errortype, buff, db_errno); } break; - case 2: - { - handler *file= 0; - const char *datext= ""; - - if (share->db_type() != NULL) - { - if ((file= get_new_handler(share, current_thd->mem_root, - share->db_type()))) - { - if (!(datext= *file->bas_ext())) - datext= ""; - } - } - err_no= (db_errno == ENOENT) ? ER_FILE_NOT_FOUND : (db_errno == EAGAIN) ? - ER_FILE_USED : ER_CANT_OPEN_FILE; - strxmov(buff, share->normalized_path.str, datext, NullS); - my_error(err_no,errortype, buff, db_errno); - delete file; + case OPEN_FRM_OK: + DBUG_ASSERT(0); // open_table_error() is never called for this one break; - } - case 5: - { - const char *csname= get_charset_name((uint) errarg); - char tmp[10]; - if (!csname || csname[0] =='?') - { - my_snprintf(tmp, sizeof(tmp), "#%d", errarg); - csname= tmp; - } - my_printf_error(ER_UNKNOWN_COLLATION, - "Unknown collation '%s' in table '%-.64s' definition", - MYF(0), csname, share->table_name.str); + case OPEN_FRM_ERROR_ALREADY_ISSUED: break; - } - case 6: - strxmov(buff, share->normalized_path.str, reg_ext, NullS); - my_printf_error(ER_NOT_FORM_FILE, - "Table '%-.64s' was created with a different version " - "of MySQL and cannot be read", - MYF(0), buff); + case OPEN_FRM_NOT_A_VIEW: + my_error(ER_WRONG_OBJECT, MYF(0), share->db.str, + share->table_name.str, "VIEW"); + break; + case OPEN_FRM_NOT_A_TABLE: + my_error(ER_WRONG_OBJECT, MYF(0), share->db.str, + share->table_name.str, "TABLE"); break; - case 8: + case OPEN_FRM_DISCOVER: + DBUG_ASSERT(0); // open_table_error() is never called for this one break; - default: /* Better wrong error than none */ - case 4: + case OPEN_FRM_CORRUPTED: strxmov(buff, share->normalized_path.str, reg_ext, NullS); my_error(ER_NOT_FORM_FILE, errortype, buff); break; + case OPEN_FRM_READ_ERROR: + strxmov(buff, share->normalized_path.str, reg_ext, NullS); + my_error(ER_ERROR_ON_READ, errortype, buff, db_errno); + break; } DBUG_VOID_RETURN; } /* open_table_error */ @@ -3126,28 +3134,6 @@ static uint find_field(Field **fields, uchar *record, uint start, uint length) } - /* Check that the integer is in the internal */ - -int set_zone(register int nr, int min_zone, int max_zone) -{ - if (nr<=min_zone) - return (min_zone); - if (nr>=max_zone) - return (max_zone); - return (nr); -} /* set_zone */ - - /* Adjust number to next larger disk buffer */ - -ulong next_io_size(register ulong pos) -{ - reg2 ulong offset; - if ((offset= pos & (IO_SIZE-1))) - return pos-offset+IO_SIZE; - return pos; -} /* next_io_size */ - - /* Store an SQL quoted string. @@ -3210,22 +3196,12 @@ void append_unescaped(String *res, const char *pos, uint length) } - /* Create a .frm file */ - -File create_frm(THD *thd, const char *name, const char *db, - const char *table, uint reclength, uchar *fileinfo, - HA_CREATE_INFO *create_info, uint keys, KEY *key_info) +void prepare_frm_header(THD *thd, uint reclength, uchar *fileinfo, + HA_CREATE_INFO *create_info, uint keys, KEY *key_info) { - register File file; - ulong length; - uchar fill[IO_SIZE]; - int create_flags= O_RDWR | O_TRUNC; ulong key_comment_total_bytes= 0; uint i; - DBUG_ENTER("create_frm"); - - if (create_info->options & HA_LEX_CREATE_TMP_TABLE) - create_flags|= O_EXCL | O_NOFOLLOW; + DBUG_ENTER("prepare_frm_header"); /* Fix this when we have new .frm files; Current limit is 4G rows (TODO) */ if (create_info->max_rows > UINT_MAX32) @@ -3233,101 +3209,77 @@ File create_frm(THD *thd, const char *name, const char *db, if (create_info->min_rows > UINT_MAX32) create_info->min_rows= UINT_MAX32; - if ((file= mysql_file_create(key_file_frm, - name, CREATE_MODE, create_flags, MYF(0))) >= 0) - { - uint key_length, tmp_key_length, tmp, csid; - bzero((char*) fileinfo,64); - /* header */ - fileinfo[0]=(uchar) 254; - fileinfo[1]= 1; - fileinfo[2]= FRM_VER+3+ test(create_info->varchar); + uint key_length, tmp_key_length, tmp, csid; + bzero((char*) fileinfo, FRM_HEADER_SIZE); + /* header */ + fileinfo[0]=(uchar) 254; + fileinfo[1]= 1; + fileinfo[2]= FRM_VER+3+ test(create_info->varchar); - fileinfo[3]= (uchar) ha_legacy_type( - ha_checktype(thd,ha_legacy_type(create_info->db_type),0,0)); - fileinfo[4]=1; - int2store(fileinfo+6,IO_SIZE); /* Next block starts here */ - /* - Keep in sync with pack_keys() in unireg.cc - For each key: - 8 bytes for the key header - 9 bytes for each key-part (MAX_REF_PARTS) - NAME_LEN bytes for the name - 1 byte for the NAMES_SEP_CHAR (before the name) - For all keys: - 6 bytes for the header - 1 byte for the NAMES_SEP_CHAR (after the last name) - 9 extra bytes (padding for safety? alignment?) - */ - for (i= 0; i < keys; i++) - { - DBUG_ASSERT(test(key_info[i].flags & HA_USES_COMMENT) == - (key_info[i].comment.length > 0)); - if (key_info[i].flags & HA_USES_COMMENT) - key_comment_total_bytes += 2 + key_info[i].comment.length; - } + fileinfo[3]= (uchar) ha_legacy_type( + ha_checktype(thd,ha_legacy_type(create_info->db_type),0,0)); - key_length= keys * (8 + MAX_REF_PARTS * 9 + NAME_LEN + 1) + 16 - + key_comment_total_bytes; - - length= next_io_size((ulong) (IO_SIZE+key_length+reclength+ - create_info->extra_size)); - int4store(fileinfo+10,length); - tmp_key_length= (key_length < 0xffff) ? key_length : 0xffff; - int2store(fileinfo+14,tmp_key_length); - int2store(fileinfo+16,reclength); - int4store(fileinfo+18,create_info->max_rows); - int4store(fileinfo+22,create_info->min_rows); - /* fileinfo[26] is set in mysql_create_frm() */ - fileinfo[27]=2; // Use long pack-fields - /* fileinfo[28 & 29] is set to key_info_length in mysql_create_frm() */ - create_info->table_options|=HA_OPTION_LONG_BLOB_PTR; // Use portable blob pointers - int2store(fileinfo+30,create_info->table_options); - fileinfo[32]=0; // No filename anymore - fileinfo[33]=5; // Mark for 5.0 frm file - int4store(fileinfo+34,create_info->avg_row_length); - csid= (create_info->default_table_charset ? - create_info->default_table_charset->number : 0); - fileinfo[38]= (uchar) csid; - fileinfo[39]= (uchar) ((uint) create_info->transactional | - ((uint) create_info->page_checksum << 2)); - fileinfo[40]= (uchar) create_info->row_type; - /* Next few bytes where for RAID support */ - fileinfo[41]= (uchar) (csid >> 8); - fileinfo[42]= 0; - fileinfo[43]= 0; - fileinfo[44]= 0; - fileinfo[45]= 0; - fileinfo[46]= 0; - int4store(fileinfo+47, key_length); - tmp= MYSQL_VERSION_ID; // Store to avoid warning from int4store - int4store(fileinfo+51, tmp); - int4store(fileinfo+55, create_info->extra_size); - /* - 59-60 is reserved for extra_rec_buf_length, - 61 for default_part_db_type - */ - int2store(fileinfo+62, create_info->key_block_size); - bzero(fill,IO_SIZE); - for (; length > IO_SIZE ; length-= IO_SIZE) - { - if (mysql_file_write(file, fill, IO_SIZE, MYF(MY_WME | MY_NABP))) - { - (void) mysql_file_close(file, MYF(0)); - (void) mysql_file_delete(key_file_frm, name, MYF(0)); - return(-1); - } - } - } - else - { - if (my_errno == ENOENT) - my_error(ER_BAD_DB_ERROR,MYF(0),db); - else - my_error(ER_CANT_CREATE_TABLE,MYF(0),table,my_errno); - } - DBUG_RETURN(file); -} /* create_frm */ + /* + Keep in sync with pack_keys() in unireg.cc + For each key: + 8 bytes for the key header + 9 bytes for each key-part (MAX_REF_PARTS) + NAME_LEN bytes for the name + 1 byte for the NAMES_SEP_CHAR (before the name) + For all keys: + 6 bytes for the header + 1 byte for the NAMES_SEP_CHAR (after the last name) + 9 extra bytes (padding for safety? alignment?) + */ + for (i= 0; i < keys; i++) + { + DBUG_ASSERT(test(key_info[i].flags & HA_USES_COMMENT) == + (key_info[i].comment.length > 0)); + if (key_info[i].flags & HA_USES_COMMENT) + key_comment_total_bytes += 2 + key_info[i].comment.length; + } + + key_length= keys * (8 + MAX_REF_PARTS * 9 + NAME_LEN + 1) + 16 + + key_comment_total_bytes; + + int2store(fileinfo+8,1); + tmp_key_length= (key_length < 0xffff) ? key_length : 0xffff; + int2store(fileinfo+14,tmp_key_length); + int2store(fileinfo+16,reclength); + int4store(fileinfo+18,create_info->max_rows); + int4store(fileinfo+22,create_info->min_rows); + /* fileinfo[26] is set in mysql_create_frm() */ + fileinfo[27]=2; // Use long pack-fields + /* fileinfo[28 & 29] is set to key_info_length in mysql_create_frm() */ + create_info->table_options|=HA_OPTION_LONG_BLOB_PTR; // Use portable blob pointers + int2store(fileinfo+30,create_info->table_options); + fileinfo[32]=0; // No filename anymore + fileinfo[33]=5; // Mark for 5.0 frm file + int4store(fileinfo+34,create_info->avg_row_length); + csid= (create_info->default_table_charset ? + create_info->default_table_charset->number : 0); + fileinfo[38]= (uchar) csid; + fileinfo[39]= (uchar) ((uint) create_info->transactional | + ((uint) create_info->page_checksum << 2)); + fileinfo[40]= (uchar) create_info->row_type; + /* Next few bytes where for RAID support */ + fileinfo[41]= (uchar) (csid >> 8); + fileinfo[42]= 0; + fileinfo[43]= 0; + fileinfo[44]= 0; + fileinfo[45]= 0; + fileinfo[46]= 0; + int4store(fileinfo+47, key_length); + tmp= MYSQL_VERSION_ID; // Store to avoid warning from int4store + int4store(fileinfo+51, tmp); + int4store(fileinfo+55, create_info->extra_size); + /* + 59-60 is reserved for extra_rec_buf_length, + 61 for default_part_db_type + */ + int2store(fileinfo+62, create_info->key_block_size); + DBUG_VOID_RETURN; +} /* prepare_fileinfo */ void update_create_info_from_table(HA_CREATE_INFO *create_info, TABLE *table) @@ -3356,7 +3308,7 @@ rename_file_ext(const char * from,const char * to,const char * ext) char from_b[FN_REFLEN],to_b[FN_REFLEN]; (void) strxmov(from_b,from,ext,NullS); (void) strxmov(to_b,to,ext,NullS); - return (mysql_file_rename(key_file_frm, from_b, to_b, MYF(MY_WME))); + return mysql_file_rename(key_file_frm, from_b, to_b, MYF(0)); } @@ -3727,6 +3679,46 @@ Table_check_intact::check(TABLE *table, const TABLE_FIELD_DEF *table_def) } } + if (table_def->primary_key_parts) + { + if (table->s->primary_key == MAX_KEY) + { + report_error(0, "Incorrect definition of table %s.%s: " + "missing primary key.", table->s->db.str, + table->alias.c_ptr()); + error= TRUE; + } + else + { + KEY *pk= &table->s->key_info[table->s->primary_key]; + if (pk->key_parts != table_def->primary_key_parts) + { + report_error(0, "Incorrect definition of table %s.%s: " + "Expected primary key to have %u columns, but instead " + "found %u columns.", table->s->db.str, + table->alias.c_ptr(), table_def->primary_key_parts, + pk->key_parts); + error= TRUE; + } + else + { + for (i= 0; i < pk->key_parts; ++i) + { + if (table_def->primary_key_columns[i] + 1 != pk->key_part[i].fieldnr) + { + report_error(0, "Incorrect definition of table %s.%s: Expected " + "primary key part %u to refer to column %u, but " + "instead found column %u.", table->s->db.str, + table->alias.c_ptr(), i + 1, + table_def->primary_key_columns[i] + 1, + pk->key_part[i].fieldnr); + error= TRUE; + } + } + } + } + } + if (! error) table->s->table_field_def_cache= table_def; @@ -3957,17 +3949,21 @@ void TABLE::init(THD *thd, TABLE_LIST *tl) file->ha_start_of_new_statement(); reginfo.impossible_range= 0; created= TRUE; + cond_selectivity= 1.0; + cond_selectivity_sampling_explain= NULL; /* Catch wrong handling of the auto_increment_field_not_null. */ DBUG_ASSERT(!auto_increment_field_not_null); auto_increment_field_not_null= FALSE; - if (timestamp_field) - timestamp_field_type= timestamp_field->get_auto_set_type(); - pos_in_table_list= tl; clear_column_bitmaps(); + for (Field **f_ptr= field ; *f_ptr ; f_ptr++) + { + (*f_ptr)->next_equal_field= NULL; + (*f_ptr)->cond_selectivity= 1.0; + } DBUG_ASSERT(key_read == 0); @@ -4811,7 +4807,7 @@ void TABLE_LIST::register_want_access(ulong want_access) Load security context information for this view SYNOPSIS - TABLE_LIST::prepare_view_securety_context() + TABLE_LIST::prepare_view_security_context() thd [in] thread handler RETURN @@ -4820,9 +4816,9 @@ void TABLE_LIST::register_want_access(ulong want_access) */ #ifndef NO_EMBEDDED_ACCESS_CHECKS -bool TABLE_LIST::prepare_view_securety_context(THD *thd) +bool TABLE_LIST::prepare_view_security_context(THD *thd) { - DBUG_ENTER("TABLE_LIST::prepare_view_securety_context"); + DBUG_ENTER("TABLE_LIST::prepare_view_security_context"); DBUG_PRINT("enter", ("table: %s", alias)); DBUG_ASSERT(!prelocking_placeholder && view); @@ -4929,7 +4925,7 @@ bool TABLE_LIST::prepare_security(THD *thd) Security_context *save_security_ctx= thd->security_ctx; DBUG_ASSERT(!prelocking_placeholder); - if (prepare_view_securety_context(thd)) + if (prepare_view_security_context(thd)) DBUG_RETURN(TRUE); thd->security_ctx= find_view_security_context(thd); while ((tbl= tb++)) @@ -5904,6 +5900,51 @@ void TABLE::mark_virtual_columns_for_write(bool insert_fl) /** + Check if a table has a default function either for INSERT or UPDATE-like + operation + @retval true there is a default function + @retval false there is no default function +*/ + +bool TABLE::has_default_function(bool is_update) +{ + Field **dfield_ptr, *dfield; + bool res= false; + for (dfield_ptr= default_field; *dfield_ptr; dfield_ptr++) + { + dfield= (*dfield_ptr); + if (is_update) + res= dfield->has_update_default_function(); + else + res= dfield->has_insert_default_function(); + if (res) + return res; + } + return res; +} + + +/** + Add all fields that have a default function to the table write set. +*/ + +void TABLE::mark_default_fields_for_write() +{ + Field **dfield_ptr, *dfield; + enum_sql_command cmd= in_use->lex->sql_command; + for (dfield_ptr= default_field; *dfield_ptr; dfield_ptr++) + { + dfield= (*dfield_ptr); + if (((sql_command_flags[cmd] & CF_INSERTS_DATA) && + dfield->has_insert_default_function()) || + ((sql_command_flags[cmd] & CF_UPDATES_DATA) && + dfield->has_update_default_function())) + bitmap_set_bit(write_set, dfield->field_index); + } +} + + +/** @brief Allocate space for keys @@ -6020,6 +6061,7 @@ bool TABLE::add_tmp_key(uint key, uint key_parts, keyinfo->algorithm= HA_KEY_ALG_UNDEF; keyinfo->flags= HA_GENERATED_KEY; keyinfo->ext_key_flags= keyinfo->flags; + keyinfo->is_statistics_from_stat_tables= FALSE; if (unique) keyinfo->flags|= HA_NOSAME; sprintf(buf, "key%i", key); @@ -6030,6 +6072,8 @@ bool TABLE::add_tmp_key(uint key, uint key_parts, if (!keyinfo->rec_per_key) return TRUE; bzero(keyinfo->rec_per_key, sizeof(ulong)*key_parts); + keyinfo->read_stats= NULL; + keyinfo->collected_stats= NULL; for (i= 0; i < key_parts; i++) { @@ -6512,6 +6556,56 @@ int update_virtual_fields(THD *thd, TABLE *table, DBUG_RETURN(0); } + +/** + Update all DEFAULT and/or ON INSERT fields. + + @details + Compute and set the default value of all fields with a default function. + There are two kinds of default functions - one is used for INSERT-like + operations, the other for UPDATE-like operations. Depending on the field + definition and the current operation one or the other kind of update + function is evaluated. + + @retval + 0 Success + @retval + >0 Error occurred when storing a virtual field value +*/ + +int TABLE::update_default_fields() +{ + DBUG_ENTER("update_default_fields"); + Field **dfield_ptr, *dfield; + int res= 0; + enum_sql_command cmd= in_use->lex->sql_command; + + DBUG_ASSERT(default_field); + + /* Iterate over virtual fields in the table */ + for (dfield_ptr= default_field; *dfield_ptr; dfield_ptr++) + { + dfield= (*dfield_ptr); + /* + If an explicit default value for a filed overrides the default, + do not update the field with its automatic default value. + */ + if (!(dfield->flags & HAS_EXPLICIT_VALUE)) + { + if (sql_command_flags[cmd] & CF_INSERTS_DATA) + res= dfield->evaluate_insert_default_function(); + if (sql_command_flags[cmd] & CF_UPDATES_DATA) + res= dfield->evaluate_update_default_function(); + if (res) + DBUG_RETURN(res); + } + /* Unset the explicit default flag for the next record. */ + dfield->flags&= ~HAS_EXPLICIT_VALUE; + } + DBUG_RETURN(res); +} + + /* @brief Reset const_table flag @@ -6723,6 +6817,7 @@ int TABLE_LIST::fetch_number_of_rows() { table->file->stats.records= ((select_union*)derived->result)->records; set_if_bigger(table->file->stats.records, 2); + table->used_stat_records= table->file->stats.records; } else error= table->file->info(HA_STATUS_VARIABLE | HA_STATUS_NO_LOCK); @@ -6819,11 +6914,11 @@ uint TABLE_SHARE::actual_n_key_parts(THD *thd) } -/***************************************************************************** -** Instansiate templates -*****************************************************************************/ +double KEY::actual_rec_per_key(uint i) +{ + if (rec_per_key == 0) + return 0; + return (is_statistics_from_stat_tables ? + read_stats->get_avg_frequency(i) : (double) rec_per_key[i]); +} -#ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION -template class List<String>; -template class List_iterator<String>; -#endif diff --git a/sql/table.h b/sql/table.h index 83b2a7a99a9..b4aa0739f74 100644 --- a/sql/table.h +++ b/sql/table.h @@ -29,6 +29,7 @@ #include "handler.h" /* row_type, ha_choice, handler */ #include "mysql_com.h" /* enum_field_types */ #include "thr_lock.h" /* thr_lock_type */ +#include "filesort_utils.h" /* Structs that defines the TABLE */ @@ -45,6 +46,7 @@ struct TABLE_LIST; class ACL_internal_schema_access; class ACL_internal_table_access; class Field; +class Table_statistics; /* Used to identify NESTED_JOIN structures within a join (applicable only to @@ -249,7 +251,8 @@ typedef struct st_grant_info @details The version of this copy is found in GRANT_INFO::version. */ - GRANT_TABLE *grant_table; + GRANT_TABLE *grant_table_user; + GRANT_TABLE *grant_table_role; /** @brief Used for cache invalidation when caching privilege information. @@ -306,11 +309,13 @@ enum enum_vcol_update_mode VCOL_UPDATE_ALL }; -typedef struct st_filesort_info +class Filesort_info { + /// Buffer for sorting keys. + Filesort_buffer filesort_buffer; + +public: IO_CACHE *io_cache; /* If sorted through filesort */ - uchar **sort_keys; /* Buffer for sorting keys */ - uint keys; /* Number of key pointers in buffer */ uchar *buffpek; /* Buffer for buffpek structures */ uint buffpek_len; /* Max number of buffpeks in the buffer */ uchar *addon_buf; /* Pointer to a buffer if sorted with fields */ @@ -319,28 +324,40 @@ typedef struct st_filesort_info void (*unpack)(struct st_sort_addon_field *, uchar *, uchar *); /* To unpack back */ uchar *record_pointers; /* If sorted in memory */ ha_rows found_records; /* How many records in sort */ -} FILESORT_INFO; + /** Sort filesort_buffer */ + void sort_buffer(Sort_param *param, uint count) + { filesort_buffer.sort_buffer(param, count); } -/* - Values in this enum are used to indicate how a tables TIMESTAMP field - should be treated. It can be set to the current timestamp on insert or - update or both. - WARNING: The values are used for bit operations. If you change the - enum, you must keep the bitwise relation of the values. For example: - (int) TIMESTAMP_AUTO_SET_ON_BOTH must be equal to - (int) TIMESTAMP_AUTO_SET_ON_INSERT | (int) TIMESTAMP_AUTO_SET_ON_UPDATE. - We use an enum here so that the debugger can display the value names. -*/ -enum timestamp_auto_set_type -{ - TIMESTAMP_NO_AUTO_SET= 0, TIMESTAMP_AUTO_SET_ON_INSERT= 1, - TIMESTAMP_AUTO_SET_ON_UPDATE= 2, TIMESTAMP_AUTO_SET_ON_BOTH= 3 + /** + Accessors for Filesort_buffer (which @c). + */ + uchar *get_record_buffer(uint idx) + { return filesort_buffer.get_record_buffer(idx); } + + uchar **get_sort_keys() + { return filesort_buffer.get_sort_keys(); } + + uchar **alloc_sort_buffer(uint num_records, uint record_length) + { return filesort_buffer.alloc_sort_buffer(num_records, record_length); } + + bool check_sort_buffer_properties(uint num_records, uint record_length) + { + return filesort_buffer.check_sort_buffer_properties(num_records, + record_length); + } + + void free_sort_buffer() + { filesort_buffer.free_sort_buffer(); } + + void init_record_pointers() + { filesort_buffer.init_record_pointers(); } + + size_t sort_buffer_size() const + { return filesort_buffer.sort_buffer_size(); } }; -#define clear_timestamp_auto_bits(_target_, _bits_) \ - (_target_)= (enum timestamp_auto_set_type)((int)(_target_) & ~(int)(_bits_)) -class Field_timestamp; + class Field_blob; class Table_triggers_list; @@ -477,6 +494,8 @@ typedef struct st_table_field_def { uint count; const TABLE_FIELD_TYPE *field; + uint primary_key_parts; + const uint *primary_key_columns; } TABLE_FIELD_DEF; @@ -546,6 +565,34 @@ typedef I_P_List <Wait_for_flush, Wait_for_flush_list; +enum open_frm_error { + OPEN_FRM_OK = 0, + OPEN_FRM_OPEN_ERROR, + OPEN_FRM_READ_ERROR, + OPEN_FRM_CORRUPTED, + OPEN_FRM_DISCOVER, + OPEN_FRM_ERROR_ALREADY_ISSUED, + OPEN_FRM_NOT_A_VIEW, + OPEN_FRM_NOT_A_TABLE +}; + +/** + Control block to access table statistics loaded + from persistent statistical tables +*/ + +struct TABLE_STATISTICS_CB +{ + MEM_ROOT mem_root; /* MEM_ROOT to allocate statistical data for the table */ + Table_statistics *table_stats; /* Structure to access the statistical data */ + bool stats_can_be_read; /* Memory for statistical data is allocated */ + bool stats_is_read; /* Statistical data for table has been read + from statistical tables */ + bool histograms_can_be_read; + bool histograms_are_read; +}; + + /** This structure is shared between different table objects. There is one instance of table share per one table in the database. @@ -574,16 +621,19 @@ struct TABLE_SHARE I_P_List <TABLE, TABLE_share> used_tables; I_P_List <TABLE, TABLE_share> free_tables; + LEX_CUSTRING tabledef_version; + engine_option_value *option_list; /* text options for table */ ha_table_option_struct *option_struct; /* structure with parsed options */ /* The following is copied to each TABLE on OPEN */ Field **field; Field **found_next_number_field; - Field *timestamp_field; /* Used only during open */ KEY *key_info; /* data of keys in database */ uint *blob_field; /* Index to blobs in Field arrray*/ + TABLE_STATISTICS_CB stats_cb; + uchar *default_values; /* row with default values */ LEX_STRING comment; /* Comment about table */ CHARSET_INFO *table_charset; /* Default charset of string fields */ @@ -622,8 +672,8 @@ struct TABLE_SHARE plugin_ref db_plugin; /* storage engine plugin */ inline handlerton *db_type() const /* table_type for handler */ { - // DBUG_ASSERT(db_plugin); - return db_plugin ? plugin_data(db_plugin, handlerton*) : NULL; + return is_view ? view_pseudo_hton : + db_plugin ? plugin_hton(db_plugin) : NULL; } enum row_type row_type; /* How rows are stored */ enum tmp_table_type tmp_table; @@ -652,7 +702,6 @@ struct TABLE_SHARE uint uniques; /* Number of UNIQUE index */ uint null_fields; /* number of null fields */ uint blob_fields; /* number of blob fields */ - uint timestamp_field_offset; /* Field number for timestamp field */ uint varchar_fields; /* number of varchar fields */ uint db_create_options; /* Create options from database */ uint db_options_in_use; /* Options in use */ @@ -663,10 +712,12 @@ struct TABLE_SHARE uint next_number_index; /* autoincrement key number */ uint next_number_key_offset; /* autoinc keypart offset in a key */ uint next_number_keypart; /* autoinc keypart number in a key */ - uint error, open_errno, errarg; /* error from open_table_def() */ + enum open_frm_error error; /* error from open_table_def() */ + uint open_errno; /* error from open_table_def() */ uint column_bitmap_size; uchar frm_version; uint vfields; /* Number of computed (virtual) fields */ + uint default_fields; /* Number of default fields */ bool use_ext_keys; /* Extended keys can be used */ bool null_field_first; bool system; /* Set if system table (one record) */ @@ -691,7 +742,7 @@ struct TABLE_SHARE char *partition_info_str; uint partition_info_str_len; uint partition_info_buffer_size; - handlerton *default_part_db_type; + plugin_ref default_part_plugin; #endif /** @@ -942,6 +993,40 @@ struct TABLE_SHARE } uint actual_n_key_parts(THD *thd); + + LEX_CUSTRING *frm_image; ///< only during CREATE TABLE (@sa ha_create_table) + + /* + populates TABLE_SHARE from the table description in the binary frm image. + if 'write' is true, this frm image is also written into a corresponding + frm file, that serves as a persistent metadata cache to avoid + discovering the table over and over again + */ + int init_from_binary_frm_image(THD *thd, bool write, + const uchar *frm_image, size_t frm_length); + + /* + populates TABLE_SHARE from the table description, specified as the + complete CREATE TABLE sql statement. + if 'write' is true, this frm image is also written into a corresponding + frm file, that serves as a persistent metadata cache to avoid + discovering the table over and over again + */ + int init_from_sql_statement_string(THD *thd, bool write, + const char *sql, size_t sql_length); + /* + writes the frm image to an frm file, corresponding to this table + */ + bool write_frm_image(const uchar *frm_image, size_t frm_length); + + /* + returns an frm image for this table. + the memory is allocated and must be freed later + */ + bool read_frm_image(const uchar **frm_image, size_t *frm_length); + + /* frees the memory allocated in read_frm_image */ + void free_frm_image(const uchar *frm); }; @@ -953,6 +1038,7 @@ enum index_hint_type INDEX_HINT_FORCE }; +struct st_cond_statistic; #define CHECK_ROW_FOR_NULLS_TO_REJECT (1 << 0) #define REJECT_ROW_DUE_TO_NULL_FIELDS (1 << 1) @@ -1013,8 +1099,9 @@ public: Field *next_number_field; /* Set if next_number is activated */ Field *found_next_number_field; /* Set on open */ - Field_timestamp *timestamp_field; Field **vfield; /* Pointer to virtual fields*/ + /* Fields that are updated automatically on INSERT or UPDATE. */ + Field **default_field; /* Table's triggers, 0 if there are no of them */ Table_triggers_list *triggers; @@ -1027,6 +1114,7 @@ public: my_bitmap_map *bitmap_init_value; MY_BITMAP def_read_set, def_write_set, def_vcol_set, tmp_set; MY_BITMAP eq_join_set; /* used to mark equi-joined fields */ + MY_BITMAP cond_set; /* used to mark fields from sargable conditions*/ MY_BITMAP *read_set, *write_set, *vcol_set; /* Active column sets */ /* The ID of the query that opened and is using this table. Has different @@ -1048,6 +1136,15 @@ public: */ query_id_t query_id; + /* + This structure is used for statistical data on the table that + is collected by the function collect_statistics_for_table + */ + Table_statistics *collected_stats; + + /* The estimate of the number of records in the table used by optimizer */ + ha_rows used_stat_records; + /* For each key that has quick_keys.is_set(key) == TRUE: estimate of #records and max #key parts that range access would use. @@ -1070,19 +1167,9 @@ public: */ ha_rows quick_condition_rows; - /* - If this table has TIMESTAMP field with auto-set property (pointed by - timestamp_field member) then this variable indicates during which - operations (insert only/on update/in both cases) we should set this - field to current timestamp. If there are no such field in this table - or we should not automatically set its value during execution of current - statement then the variable contains TIMESTAMP_NO_AUTO_SET (i.e. 0). - - Value of this variable is set for each statement in open_table() and - if needed cleared later in statement processing code (see mysql_update() - as example). - */ - timestamp_auto_set_type timestamp_field_type; + double cond_selectivity; + List<st_cond_statistic> *cond_selectivity_sampling_explain; + table_map map; /* ID bit of table (1,2,4,8,16...) */ uint lock_position; /* Position in MYSQL_LOCK.table */ @@ -1152,7 +1239,12 @@ public: See TABLE_LIST::process_index_hints(). */ bool force_index_group; - bool distinct,const_table,no_rows, used_for_duplicate_elimination; + /* + TRUE<=> this table was created with create_tmp_table(... distinct=TRUE..) + call + */ + bool distinct; + bool const_table,no_rows, used_for_duplicate_elimination; /** Forces DYNAMIC Aria row format for internal temporary tables. */ @@ -1186,7 +1278,7 @@ public: REGINFO reginfo; /* field connections */ MEM_ROOT mem_root; GRANT_INFO grant; - FILESORT_INFO sort; + Filesort_info sort; /* The arena which the items for expressions from the table definition are associated with. @@ -1200,6 +1292,8 @@ public: bool no_partitions_used; /* If true, all partitions have been pruned away */ #endif uint max_keys; /* Size of allocated key_info array. */ + bool stats_is_read; /* Persistent statistics is read for the table */ + bool histograms_are_read; MDL_ticket *mdl_ticket; void init(THD *thd, TABLE_LIST *tl); @@ -1217,6 +1311,8 @@ public: void mark_columns_needed_for_insert(void); bool mark_virtual_col(Field *field); void mark_virtual_columns_for_write(bool insert_fl); + void mark_default_fields_for_write(); + bool has_default_function(bool is_update); inline void column_bitmaps_set(MY_BITMAP *read_set_arg, MY_BITMAP *write_set_arg) { @@ -1303,6 +1399,8 @@ public: bool update_const_key_parts(COND *conds); uint actual_n_key_parts(KEY *keyinfo); ulong actual_key_flags(KEY *keyinfo); + int update_default_fields(); + inline ha_rows stat_records() { return used_stat_records; } }; @@ -1580,7 +1678,7 @@ struct TABLE_LIST /** Prepare TABLE_LIST that consists of one table instance to use in - simple_open_and_lock_tables + open_and_lock_tables */ inline void init_one_table(const char *db_name_arg, size_t db_length_arg, @@ -1893,7 +1991,6 @@ struct TABLE_LIST /* For transactional locking. */ int lock_timeout; /* NOWAIT or WAIT [X] */ bool lock_transactional; /* If transactional lock requested. */ - bool internal_tmp_table; /** TRUE if an alias for this table was specified in the SQL. */ bool is_alias; /** TRUE if the table is referred to in the statement using a fully @@ -2004,7 +2101,7 @@ struct TABLE_LIST bool prepare_security(THD *thd); #ifndef NO_EMBEDDED_ACCESS_CHECKS Security_context *find_view_security_context(THD *thd); - bool prepare_view_securety_context(THD *thd); + bool prepare_view_security_context(THD *thd); #endif /* Cleanup for re-execution in a prepared statement or a stored @@ -2157,9 +2254,9 @@ private: #else inline void set_check_merged() {} #endif - /** See comments for set_metadata_id() */ + /** See comments for set_table_ref_id() */ enum enum_table_ref_type m_table_ref_type; - /** See comments for set_metadata_id() */ + /** See comments for set_table_ref_id() */ ulong m_table_ref_version; }; @@ -2413,26 +2510,36 @@ static inline void dbug_tmp_restore_column_maps(MY_BITMAP *read_set, #endif } +enum get_table_share_flags { + GTS_TABLE = 1, + GTS_VIEW = 2, + GTS_NOLOCK = 4, + GTS_USE_DISCOVERY = 8, + GTS_FORCE_DISCOVERY = 16 +}; size_t max_row_length(TABLE *table, const uchar *data); - void init_mdl_requests(TABLE_LIST *table_list); -int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, - uint db_stat, uint prgflag, uint ha_open_flags, - TABLE *outparam, bool is_create_table); +enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, + const char *alias, uint db_stat, uint prgflag, + uint ha_open_flags, TABLE *outparam, + bool is_create_table); bool unpack_vcol_info_from_frm(THD *thd, MEM_ROOT *mem_root, TABLE *table, Field *field, LEX_STRING *vcol_expr, bool *error_reported); -TABLE_SHARE *alloc_table_share(TABLE_LIST *table_list, char *key, - uint key_length); +TABLE_SHARE *alloc_table_share(const char *db, const char *table_name, + char *key, uint key_length); void init_tmp_table_share(THD *thd, TABLE_SHARE *share, const char *key, uint key_length, const char *table_name, const char *path); void free_table_share(TABLE_SHARE *share); -int open_table_def(THD *thd, TABLE_SHARE *share, uint db_flags); -void open_table_error(TABLE_SHARE *share, int error, int db_errno, int errarg); +enum open_frm_error open_table_def(THD *thd, TABLE_SHARE *share, + uint flags = GTS_TABLE); + +void open_table_error(TABLE_SHARE *share, enum open_frm_error error, + int db_errno); void update_create_info_from_table(HA_CREATE_INFO *info, TABLE *form); bool check_and_convert_db_name(LEX_STRING *db, bool preserve_lettercase); bool check_db_name(LEX_STRING *db); @@ -2443,20 +2550,24 @@ char *get_field(MEM_ROOT *mem, Field *field); bool get_field(MEM_ROOT *mem, Field *field, class String *res); int closefrm(TABLE *table, bool free_share); -int read_string(File file, uchar* *to, size_t length); void free_blobs(TABLE *table); void free_field_buffers_larger_than(TABLE *table, uint32 size); -int set_zone(int nr,int min_zone,int max_zone); ulong get_form_pos(File file, uchar *head, TYPELIB *save_names); -ulong make_new_entry(File file,uchar *fileinfo,TYPELIB *formnames, - const char *newname); -ulong next_io_size(ulong pos); void append_unescaped(String *res, const char *pos, uint length); -File create_frm(THD *thd, const char *name, const char *db, - const char *table, uint reclength, uchar *fileinfo, - HA_CREATE_INFO *create_info, uint keys, KEY *key_info); +void prepare_frm_header(THD *thd, uint reclength, uchar *fileinfo, + HA_CREATE_INFO *create_info, uint keys, KEY *key_info); char *fn_rext(char *name); +/* Check that the integer is in the internal */ +static inline int set_zone(int nr,int min_zone,int max_zone) +{ + if (nr <= min_zone) + return min_zone; + if (nr >= max_zone) + return max_zone; + return nr; +} + /* performance schema */ extern LEX_STRING PERFORMANCE_SCHEMA_DB_NAME; diff --git a/sql/thr_malloc.cc b/sql/thr_malloc.cc index cedcbefc26f..8c7db0673ac 100644 --- a/sql/thr_malloc.cc +++ b/sql/thr_malloc.cc @@ -61,9 +61,10 @@ extern "C" { } } -void init_sql_alloc(MEM_ROOT *mem_root, uint block_size, uint pre_alloc) +void init_sql_alloc(MEM_ROOT *mem_root, uint block_size, uint pre_alloc, + myf my_flags) { - init_alloc_root(mem_root, block_size, pre_alloc); + init_alloc_root(mem_root, block_size, pre_alloc, my_flags); mem_root->error_handler=sql_alloc_error_handler; } diff --git a/sql/thr_malloc.h b/sql/thr_malloc.h index 81b7d3cc238..0b17c5cdaf1 100644 --- a/sql/thr_malloc.h +++ b/sql/thr_malloc.h @@ -20,7 +20,8 @@ typedef struct st_mem_root MEM_ROOT; -void init_sql_alloc(MEM_ROOT *root, uint block_size, uint pre_alloc_size); +void init_sql_alloc(MEM_ROOT *root, uint block_size, uint pre_alloc_size, + myf my_flags); void *sql_alloc(size_t); void *sql_calloc(size_t); char *sql_strdup(const char *str); diff --git a/sql/transaction.cc b/sql/transaction.cc index ae38e920a1d..b3bc3bf8838 100644 --- a/sql/transaction.cc +++ b/sql/transaction.cc @@ -139,6 +139,11 @@ bool trans_begin(THD *thd, uint flags) } thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_KEEP_LOG); + + /* + The following set should not be needed as the flag should always be 0 + when we come here. We should at some point change this to an assert. + */ thd->transaction.all.modified_non_trans_table= FALSE; if (res) diff --git a/sql/tztime.cc b/sql/tztime.cc index da23d6fc8c2..3e79f1bc39e 100644 --- a/sql/tztime.cc +++ b/sql/tztime.cc @@ -1641,7 +1641,7 @@ my_tz_init(THD *org_thd, const char *default_tzname, my_bool bootstrap) my_hash_free(&tz_names); goto end; } - init_sql_alloc(&tz_storage, 32 * 1024, 0); + init_sql_alloc(&tz_storage, 32 * 1024, 0, MYF(0)); mysql_mutex_init(key_tz_LOCK, &tz_LOCK, MY_MUTEX_INIT_FAST); tz_inited= 1; @@ -1804,7 +1804,7 @@ end: else { /* Remember that we don't have a THD */ - my_pthread_setspecific_ptr(THR_THD, 0); + set_current_thd(0); my_pthread_setspecific_ptr(THR_MALLOC, 0); } @@ -2514,12 +2514,13 @@ scan_tz_dir(char * name_end, uint symlink_recursion_level, uint verbose) char *name_end_tmp; uint i; - if (!(cur_dir= my_dir(fullname, MYF(MY_WANT_STAT)))) + /* Sort directory data, to pass mtr tests on different platforms. */ + if (!(cur_dir= my_dir(fullname, MYF(MY_WANT_STAT|MY_WANT_SORT)))) return 1; name_end= strmake(name_end, "/", FN_REFLEN - (name_end - fullname)); - for (i= 0; i < cur_dir->number_off_files; i++) + for (i= 0; i < cur_dir->number_of_files; i++) { if (cur_dir->dir_entry[i].name[0] != '.') { @@ -2570,7 +2571,7 @@ scan_tz_dir(char * name_end, uint symlink_recursion_level, uint verbose) } else if (MY_S_ISREG(cur_dir->dir_entry[i].mystat->st_mode)) { - init_alloc_root(&tz_storage, 32768, 0); + init_alloc_root(&tz_storage, 32768, 0, MYF(MY_THREAD_SPECIFIC)); if (!tz_load(fullname, &tz_info, &tz_storage)) print_tz_as_sql(root_name_end + 1, &tz_info); else @@ -2737,7 +2738,7 @@ main(int argc, char **argv) First argument is timezonefile. The second is timezonename if opt_leap is not given */ - init_alloc_root(&tz_storage, 32768, 0); + init_alloc_root(&tz_storage, 32768, 0, MYF(0)); if (tz_load(argv[0], &tz_info, &tz_storage)) { @@ -2808,7 +2809,7 @@ main(int argc, char **argv) MY_INIT(argv[0]); - init_alloc_root(&tz_storage, 32768, 0); + init_alloc_root(&tz_storage, 32768, MYF(0)); /* let us set some well known timezone */ setenv("TZ", "MET", 1); diff --git a/sql/uniques.cc b/sql/uniques.cc index 72411be5cd6..0c1c34d495b 100644 --- a/sql/uniques.cc +++ b/sql/uniques.cc @@ -86,11 +86,13 @@ Unique::Unique(qsort_cmp2 comp_func, void * comp_func_fixed_arg, full_size= size; if (min_dupl_count_arg) full_size+= sizeof(element_count); + with_counters= test(min_dupl_count_arg); my_b_clear(&file); - init_tree(&tree, (ulong) (max_in_memory_size / 16), 0, size, comp_func, 0, - NULL, comp_func_fixed_arg); + init_tree(&tree, (ulong) (max_in_memory_size / 16), 0, size, comp_func, + NULL, comp_func_fixed_arg, MYF(MY_THREAD_SPECIFIC)); /* If the following fail's the next add will also fail */ - my_init_dynamic_array(&file_ptrs, sizeof(BUFFPEK), 16, 16); + my_init_dynamic_array(&file_ptrs, sizeof(BUFFPEK), 16, 16, + MYF(MY_THREAD_SPECIFIC)); /* If you change the following, change it in get_max_elements function, too. */ @@ -427,6 +429,22 @@ static int buffpek_compare(void *arg, uchar *key_ptr1, uchar *key_ptr2) C_MODE_END +inline +element_count get_counter_from_merged_element(void *ptr, uint ofs) +{ + element_count cnt; + memcpy((uchar *) &cnt, (uchar *) ptr + ofs, sizeof(element_count)); + return cnt; +} + + +inline +void put_counter_into_merged_element(void *ptr, uint ofs, element_count cnt) +{ + memcpy((uchar *) ptr + ofs, (uchar *) &cnt, sizeof(element_count)); +} + + /* DESCRIPTION @@ -456,6 +474,8 @@ C_MODE_END file file with all trees dumped. Trees in the file must contain sorted unique values. Cache must be initialized in read mode. + with counters take into account counters for equal merged + elements RETURN VALUE 0 ok <> 0 error @@ -465,7 +485,7 @@ static bool merge_walk(uchar *merge_buffer, ulong merge_buffer_size, uint key_length, BUFFPEK *begin, BUFFPEK *end, tree_walk_action walk_action, void *walk_action_arg, qsort_cmp2 compare, void *compare_arg, - IO_CACHE *file) + IO_CACHE *file, bool with_counters) { BUFFPEK_COMPARE_CONTEXT compare_context = { compare, compare_arg }; QUEUE queue; @@ -484,6 +504,8 @@ static bool merge_walk(uchar *merge_buffer, ulong merge_buffer_size, uint bytes_read; /* to hold return value of read_to_buffer */ BUFFPEK *top; int res= 1; + uint cnt_ofs= key_length - (with_counters ? sizeof(element_count) : 0); + element_count cnt; /* Invariant: queue must contain top element from each tree, until a tree is not completely walked through. @@ -542,9 +564,17 @@ static bool merge_walk(uchar *merge_buffer, ulong merge_buffer_size, /* new top has been obtained; if old top is unique, apply the action */ if (compare(compare_arg, old_key, top->key)) { - if (walk_action(old_key, 1, walk_action_arg)) + cnt= with_counters ? + get_counter_from_merged_element(old_key, cnt_ofs) : 1; + if (walk_action(old_key, cnt, walk_action_arg)) goto end; } + else if (with_counters) + { + cnt= get_counter_from_merged_element(top->key, cnt_ofs); + cnt+= get_counter_from_merged_element(old_key, cnt_ofs); + put_counter_into_merged_element(top->key, cnt_ofs, cnt); + } } /* Applying walk_action to the tail of the last tree: this is safe because @@ -555,7 +585,10 @@ static bool merge_walk(uchar *merge_buffer, ulong merge_buffer_size, { do { - if (walk_action(top->key, 1, walk_action_arg)) + + cnt= with_counters ? + get_counter_from_merged_element(top->key, cnt_ofs) : 1; + if (walk_action(top->key, cnt, walk_action_arg)) goto end; top->key+= key_length; } @@ -608,9 +641,9 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg) if (flush_io_cache(&file) || reinit_io_cache(&file, READ_CACHE, 0L, 0, 0)) return 1; ulong buff_sz= (max_in_memory_size / full_size + 1) * full_size; - if (!(merge_buffer= (uchar *) my_malloc((ulong) buff_sz, MYF(0)))) + if (!(merge_buffer= (uchar *) my_malloc(buff_sz, MYF(MY_THREAD_SPECIFIC)))) return 1; - if (buff_sz < (ulong) (full_size * (file_ptrs.elements + 1))) + if (buff_sz < full_size * (file_ptrs.elements + 1UL)) res= merge(table, merge_buffer, buff_sz >= full_size * MERGEBUFF2) ; if (!res) @@ -619,7 +652,7 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg) (BUFFPEK *) file_ptrs.buffer, (BUFFPEK *) file_ptrs.buffer + file_ptrs.elements, action, walk_action_arg, - tree.compare, tree.custom_arg, &file); + tree.compare, tree.custom_arg, &file, with_counters); } my_free(merge_buffer); return res; @@ -644,7 +677,6 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg) bool Unique::merge(TABLE *table, uchar *buff, bool without_last_merge) { - SORTPARAM sort_param; IO_CACHE *outfile= table->sort.io_cache; BUFFPEK *file_ptr= (BUFFPEK*) file_ptrs.buffer; uint maxbuffer= file_ptrs.elements - 1; @@ -654,7 +686,7 @@ bool Unique::merge(TABLE *table, uchar *buff, bool without_last_merge) /* Open cached file if it isn't open */ if (!outfile) outfile= table->sort.io_cache= (IO_CACHE*) my_malloc(sizeof(IO_CACHE), - MYF(MY_ZEROFILL)); + MYF(MY_THREAD_SPECIFIC|MY_ZEROFILL)); if (!outfile || (! my_b_inited(outfile) && open_cached_file(outfile,mysql_tmpdir,TEMP_PREFIX,READ_RECORD_BUFFER, @@ -662,6 +694,7 @@ bool Unique::merge(TABLE *table, uchar *buff, bool without_last_merge) return 1; reinit_io_cache(outfile,WRITE_CACHE,0L,0,0); + Sort_param sort_param; bzero((char*) &sort_param,sizeof(sort_param)); sort_param.max_rows= elements; sort_param.sort_form= table; @@ -669,10 +702,12 @@ bool Unique::merge(TABLE *table, uchar *buff, bool without_last_merge) full_size; sort_param.min_dupl_count= min_dupl_count; sort_param.res_length= 0; - sort_param.keys= (uint) (max_in_memory_size / sort_param.sort_length); + sort_param.max_keys_per_buffer= + (uint) (max_in_memory_size / sort_param.sort_length); sort_param.not_killable= 1; - sort_param.unique_buff= buff + (sort_param.keys * sort_param.sort_length); + sort_param.unique_buff= buff +(sort_param.max_keys_per_buffer * + sort_param.sort_length); sort_param.compare= (qsort2_cmp) buffpek_compare; sort_param.cmp_context.key_compare= tree.compare; @@ -722,7 +757,7 @@ bool Unique::get(TABLE *table) { /* Whole tree is in memory; Don't use disk if you don't need to */ if ((record_pointers=table->sort.record_pointers= (uchar*) - my_malloc(size * tree.elements_in_tree, MYF(0)))) + my_malloc(size * tree.elements_in_tree, MYF(MY_THREAD_SPECIFIC)))) { tree_walk_action action= min_dupl_count ? (tree_walk_action) unique_intersect_write_to_ptrs : @@ -739,7 +774,7 @@ bool Unique::get(TABLE *table) return 1; ulong buff_sz= (max_in_memory_size / full_size + 1) * full_size; - if (!(sort_buffer= (uchar*) my_malloc(buff_sz, MYF(0)))) + if (!(sort_buffer= (uchar*) my_malloc(buff_sz, MYF(MY_THREAD_SPECIFIC)))) return 1; if (merge(table, sort_buffer, FALSE)) diff --git a/sql/unireg.cc b/sql/unireg.cc index edcfe9eb934..d90db0ebab8 100644 --- a/sql/unireg.cc +++ b/sql/unireg.cc @@ -1,5 +1,6 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates. + Copyright (c) 2009, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -27,9 +28,9 @@ #include "sql_priv.h" #include "unireg.h" #include "sql_partition.h" // struct partition_info -#include "sql_table.h" // check_duplicate_warning #include "sql_class.h" // THD, Internal_error_handler #include "create_options.h" +#include "discover.h" #include <m_ctype.h> #include <assert.h> @@ -38,137 +39,88 @@ /* threshold for safe_alloca */ #define ALLOCA_THRESHOLD 2048 -static uchar * pack_screens(List<Create_field> &create_fields, - uint *info_length, uint *screens, bool small_file); -static uint pack_keys(uchar *keybuff,uint key_count, KEY *key_info, - ulong data_offset); -static bool pack_header(uchar *forminfo,enum legacy_db_type table_type, - List<Create_field> &create_fields, - uint info_length, uint screens, uint table_options, - ulong data_offset, handler *file); +static uint pack_keys(uchar *,uint, KEY *, ulong); +static bool pack_header(uchar *, List<Create_field> &, uint, ulong, handler *); static uint get_interval_id(uint *,List<Create_field> &, Create_field *); -static bool pack_fields(File file, List<Create_field> &create_fields, - ulong data_offset); -static bool make_empty_rec(THD *thd, int file, enum legacy_db_type table_type, - uint table_options, - List<Create_field> &create_fields, - uint reclength, ulong data_offset, - handler *handler); - -/** - An interceptor to hijack ER_TOO_MANY_FIELDS error from - pack_screens and retry again without UNIREG screens. +static bool pack_fields(uchar *, List<Create_field> &, ulong); +static size_t packed_fields_length(List<Create_field> &); +static bool make_empty_rec(THD *, uchar *, uint, List<Create_field> &, uint, ulong); - XXX: what is a UNIREG screen? -*/ +static uchar *extra2_write_len(uchar *pos, size_t len) +{ + if (len < 255) + *pos++= len; + else + { + /* + At the moment we support options_len up to 64K. + We can easily extend it in the future, if the need arises. + */ + DBUG_ASSERT(len <= 65535); + int2store(pos + 1, len); + pos+= 3; + } + return pos; +} -struct Pack_header_error_handler: public Internal_error_handler +static uchar *extra2_write(uchar *pos, enum extra2_frm_value_type type, + LEX_STRING *str) { - virtual bool handle_condition(THD *thd, - uint sql_errno, - const char* sqlstate, - MYSQL_ERROR::enum_warning_level level, - const char* msg, - MYSQL_ERROR ** cond_hdl); - bool is_handled; - Pack_header_error_handler() :is_handled(FALSE) {} -}; - - -bool -Pack_header_error_handler:: -handle_condition(THD *, - uint sql_errno, - const char*, - MYSQL_ERROR::enum_warning_level, - const char*, - MYSQL_ERROR ** cond_hdl) + *pos++ = type; + pos= extra2_write_len(pos, str->length); + memcpy(pos, str->str, str->length); + return pos + str->length; +} + +static uchar *extra2_write(uchar *pos, enum extra2_frm_value_type type, + LEX_CUSTRING *str) { - *cond_hdl= NULL; - is_handled= (sql_errno == ER_TOO_MANY_FIELDS); - return is_handled; + return extra2_write(pos, type, reinterpret_cast<LEX_STRING *>(str)); } -/* +/** Create a frm (table definition) file - SYNOPSIS - mysql_create_frm() - thd Thread handler - file_name Path for file (including database and .frm) - db Name of database - table Name of table - create_info create info parameters - create_fields Fields to create - keys number of keys to create - key_info Keys to create - db_file Handler to use. May be zero, in which case we use - create_info->db_type - RETURN - false ok - true error + @param thd Thread handler + @param table Name of table + @param create_info create info parameters + @param create_fields Fields to create + @param keys number of keys to create + @param key_info Keys to create + @param db_file Handler to use. + + @return the generated frm image as a LEX_CUSTRING, + or null LEX_CUSTRING (str==0) in case of an error. */ -bool mysql_create_frm(THD *thd, const char *file_name, - const char *db, const char *table, - HA_CREATE_INFO *create_info, - List<Create_field> &create_fields, - uint keys, KEY *key_info, - handler *db_file) +LEX_CUSTRING build_frm_image(THD *thd, const char *table, + HA_CREATE_INFO *create_info, + List<Create_field> &create_fields, + uint keys, KEY *key_info, handler *db_file) { LEX_STRING str_db_type; - uint reclength, info_length, screens, key_info_length, maxlength, tmp_len, i; + uint reclength, key_info_length, tmp_len, i; ulong key_buff_length; - File file; ulong filepos, data_offset; uint options_len; - uchar fileinfo[64],forminfo[288],*keybuff; - uchar *screen_buff; - char buff[128]; -#ifdef WITH_PARTITION_STORAGE_ENGINE - partition_info *part_info= thd->work_part_info; -#endif - Pack_header_error_handler pack_header_error_handler; + uchar fileinfo[FRM_HEADER_SIZE],forminfo[FRM_FORMINFO_SIZE]; + const partition_info *part_info= IF_PARTITIONING(thd->work_part_info, 0); int error; - DBUG_ENTER("mysql_create_frm"); - - DBUG_ASSERT(*fn_rext((char*)file_name)); // Check .frm extension - - if (!(screen_buff=pack_screens(create_fields,&info_length,&screens,0))) - DBUG_RETURN(1); - DBUG_ASSERT(db_file != NULL); + uchar *frm_ptr, *pos; + LEX_CUSTRING frm= {0,0}; + DBUG_ENTER("build_frm_image"); /* If fixed row records, we need one bit to check for deleted rows */ if (!(create_info->table_options & HA_OPTION_PACK_RECORD)) create_info->null_bits++; data_offset= (create_info->null_bits + 7) / 8; - thd->push_internal_handler(&pack_header_error_handler); - - error= pack_header(forminfo, ha_legacy_type(create_info->db_type), - create_fields,info_length, - screens, create_info->table_options, + error= pack_header(forminfo, create_fields, create_info->table_options, data_offset, db_file); - thd->pop_internal_handler(); - if (error) - { - my_free(screen_buff); - if (! pack_header_error_handler.is_handled) - DBUG_RETURN(1); - - // Try again without UNIREG screens (to get more columns) - if (!(screen_buff=pack_screens(create_fields,&info_length,&screens,1))) - DBUG_RETURN(1); - if (pack_header(forminfo, ha_legacy_type(create_info->db_type), - create_fields,info_length, - screens, create_info->table_options, data_offset, db_file)) - { - my_free(screen_buff); - DBUG_RETURN(1); - } - } + DBUG_RETURN(frm); + reclength=uint2korr(forminfo+266); /* Calculate extra data segment length */ @@ -184,12 +136,8 @@ bool mysql_create_frm(THD *thd, const char *file_name, => Total 6 byte */ create_info->extra_size+= 6; -#ifdef WITH_PARTITION_STORAGE_ENGINE if (part_info) - { create_info->extra_size+= part_info->part_info_len; - } -#endif for (i= 0; i < keys; i++) { @@ -201,13 +149,6 @@ bool mysql_create_frm(THD *thd, const char *file_name, create_fields, keys, key_info); DBUG_PRINT("info", ("Options length: %u", options_len)); - if (options_len) - { - create_info->table_options|= HA_OPTION_TEXT_CREATE_OPTIONS; - create_info->extra_size+= (options_len + 4); - } - else - create_info->table_options&= ~HA_OPTION_TEXT_CREATE_OPTIONS; /* This gives us the byte-position of the character at @@ -245,190 +186,164 @@ bool mysql_create_frm(THD *thd, const char *file_name, (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES))) { my_error(ER_TOO_LONG_TABLE_COMMENT, MYF(0), - real_table_name, static_cast<ulong>(TABLE_COMMENT_MAXLEN)); - my_free(screen_buff); - DBUG_RETURN(1); + real_table_name, TABLE_COMMENT_MAXLEN); + DBUG_RETURN(frm); } char warn_buff[MYSQL_ERRMSG_SIZE]; my_snprintf(warn_buff, sizeof(warn_buff), ER(ER_TOO_LONG_TABLE_COMMENT), - real_table_name, static_cast<ulong>(TABLE_COMMENT_MAXLEN)); - /* do not push duplicate warnings */ - if (!check_duplicate_warning(current_thd, warn_buff, strlen(warn_buff))) - push_warning(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN, - ER_TOO_LONG_TABLE_COMMENT, warn_buff); + real_table_name, TABLE_COMMENT_MAXLEN); + push_warning(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN, + ER_TOO_LONG_TABLE_COMMENT, warn_buff); create_info->comment.length= tmp_len; } /* If table comment is longer than TABLE_COMMENT_INLINE_MAXLEN bytes, store the comment in an extra segment (up to TABLE_COMMENT_MAXLEN bytes). - Pre 6.0, the limit was 60 characters, with no extra segment-handling. + Pre 5.5, the limit was 60 characters, with no extra segment-handling. */ if (create_info->comment.length > TABLE_COMMENT_INLINE_MAXLEN) { forminfo[46]=255; create_info->extra_size+= 2 + create_info->comment.length; } - else{ + else + { strmake((char*) forminfo+47, create_info->comment.str ? create_info->comment.str : "", create_info->comment.length); forminfo[46]=(uchar) create_info->comment.length; } - if ((file=create_frm(thd, file_name, db, table, reclength, fileinfo, - create_info, keys, key_info)) < 0) + if (!create_info->tabledef_version.str) { - my_free(screen_buff); - DBUG_RETURN(1); + uchar *to= (uchar*) thd->alloc(MY_UUID_SIZE); + if (unlikely(!to)) + DBUG_RETURN(frm); + my_uuid(to); + create_info->tabledef_version.str= to; + create_info->tabledef_version.length= MY_UUID_SIZE; } + DBUG_ASSERT(create_info->tabledef_version.length > 0); + DBUG_ASSERT(create_info->tabledef_version.length <= 255); + + prepare_frm_header(thd, reclength, fileinfo, create_info, keys, key_info); + + /* one byte for a type, one or three for a length */ + uint extra2_size= 1 + 1 + create_info->tabledef_version.length; + if (options_len) + extra2_size+= 1 + (options_len > 255 ? 3 : 1) + options_len; + + if (part_info) + extra2_size+= 1 + 1 + hton_name(part_info->default_engine_type)->length; key_buff_length= uint4korr(fileinfo+47); - keybuff=(uchar*) my_malloc(key_buff_length, MYF(0)); - key_info_length= pack_keys(keybuff, keys, key_info, data_offset); - /* - Ensure that there are no forms in this newly created form file. - Even if the form file exists, create_frm must truncate it to - ensure one form per form file. - */ - DBUG_ASSERT(uint2korr(fileinfo+8) == 0); + frm.length= FRM_HEADER_SIZE; // fileinfo; + frm.length+= extra2_size + 4; // mariadb extra2 frm segment + + int2store(fileinfo+4, extra2_size); + int2store(fileinfo+6, frm.length); // Position to key information + frm.length+= key_buff_length; + frm.length+= reclength; // row with default values + frm.length+= create_info->extra_size; + + filepos= frm.length; + frm.length+= FRM_FORMINFO_SIZE; // forminfo + frm.length+= packed_fields_length(create_fields); + + frm_ptr= (uchar*) my_malloc(frm.length, MYF(MY_WME | MY_ZEROFILL | + MY_THREAD_SPECIFIC)); + if (!frm_ptr) + DBUG_RETURN(frm); + + /* write the extra2 segment */ + pos = frm_ptr + 64; + compile_time_assert(EXTRA2_TABLEDEF_VERSION != '/'); + pos= extra2_write(pos, EXTRA2_TABLEDEF_VERSION, + &create_info->tabledef_version); - if (!(filepos= make_new_entry(file, fileinfo, NULL, ""))) - goto err; - maxlength=(uint) next_io_size((ulong) (uint2korr(forminfo)+1000)); - int2store(forminfo+2,maxlength); - int4store(fileinfo+10,(ulong) (filepos+maxlength)); + if (part_info) + pos= extra2_write(pos, EXTRA2_DEFAULT_PART_ENGINE, + hton_name(part_info->default_engine_type)); + + if (options_len) + { + *pos++= EXTRA2_ENGINE_TABLEOPTS; + pos= extra2_write_len(pos, options_len); + pos= engine_table_options_frm_image(pos, create_info->option_list, + create_fields, keys, key_info); + } + + int4store(pos, filepos); // end of the extra2 segment + pos+= 4; + + DBUG_ASSERT(pos == frm_ptr + uint2korr(fileinfo+6)); + key_info_length= pack_keys(pos, keys, key_info, data_offset); + + int2store(forminfo+2, frm.length - filepos); + int4store(fileinfo+10, frm.length); fileinfo[26]= (uchar) test((create_info->max_rows == 1) && (create_info->min_rows == 1) && (keys == 0)); int2store(fileinfo+28,key_info_length); -#ifdef WITH_PARTITION_STORAGE_ENGINE if (part_info) { fileinfo[61]= (uchar) ha_legacy_type(part_info->default_engine_type); DBUG_PRINT("info", ("part_db_type = %d", fileinfo[61])); } -#endif - int2store(fileinfo+59,db_file->extra_rec_buf_length()); - if (mysql_file_pwrite(file, fileinfo, 64, 0L, MYF_RW) || - mysql_file_pwrite(file, keybuff, key_info_length, - (ulong) uint2korr(fileinfo+6), MYF_RW)) - goto err; - mysql_file_seek(file, - (ulong) uint2korr(fileinfo+6) + (ulong) key_buff_length, - MY_SEEK_SET, MYF(0)); - if (make_empty_rec(thd,file,ha_legacy_type(create_info->db_type), - create_info->table_options, - create_fields,reclength, data_offset, db_file)) - goto err; + int2store(fileinfo+59,db_file->extra_rec_buf_length()); - int2store(buff, create_info->connect_string.length); - if (mysql_file_write(file, (const uchar*)buff, 2, MYF(MY_NABP)) || - mysql_file_write(file, (const uchar*)create_info->connect_string.str, - create_info->connect_string.length, MYF(MY_NABP))) - goto err; + memcpy(frm_ptr, fileinfo, FRM_HEADER_SIZE); - int2store(buff, str_db_type.length); - if (mysql_file_write(file, (const uchar*)buff, 2, MYF(MY_NABP)) || - mysql_file_write(file, (const uchar*)str_db_type.str, - str_db_type.length, MYF(MY_NABP))) + pos+= key_buff_length; + if (make_empty_rec(thd, pos, create_info->table_options, create_fields, + reclength, data_offset)) goto err; -#ifdef WITH_PARTITION_STORAGE_ENGINE + pos+= reclength; + int2store(pos, create_info->connect_string.length); + pos+= 2; + memcpy(pos, create_info->connect_string.str, create_info->connect_string.length); + pos+= create_info->connect_string.length; + int2store(pos, str_db_type.length); + pos+= 2; + memcpy(pos, str_db_type.str, str_db_type.length); + pos+= str_db_type.length; + if (part_info) { char auto_partitioned= part_info->is_auto_partitioned ? 1 : 0; - int4store(buff, part_info->part_info_len); - if (mysql_file_write(file, (const uchar*)buff, 4, MYF_RW) || - mysql_file_write(file, (const uchar*)part_info->part_info_string, - part_info->part_info_len + 1, MYF_RW) || - mysql_file_write(file, (const uchar*)&auto_partitioned, 1, MYF_RW)) - goto err; + int4store(pos, part_info->part_info_len); + pos+= 4; + memcpy(pos, part_info->part_info_string, part_info->part_info_len + 1); + pos+= part_info->part_info_len + 1; + *pos++= auto_partitioned; } else -#endif { - bzero((uchar*) buff, 6); - if (mysql_file_write(file, (uchar*) buff, 6, MYF_RW)) - goto err; + pos+= 6; } for (i= 0; i < keys; i++) { if (key_info[i].parser_name) { - if (mysql_file_write(file, (const uchar*)key_info[i].parser_name->str, - key_info[i].parser_name->length + 1, MYF(MY_NABP))) - goto err; + memcpy(pos, key_info[i].parser_name->str, key_info[i].parser_name->length + 1); + pos+= key_info[i].parser_name->length + 1; } } - if (forminfo[46] == (uchar)255) + if (forminfo[46] == (uchar)255) // New style MySQL 5.5 table comment { - uchar comment_length_buff[2]; - int2store(comment_length_buff,create_info->comment.length); - if (mysql_file_write(file, comment_length_buff, 2, MYF(MY_NABP)) || - mysql_file_write(file, (uchar*) create_info->comment.str, - create_info->comment.length, MYF(MY_NABP))) - goto err; + int2store(pos, create_info->comment.length); + pos+=2; + memcpy(pos, create_info->comment.str, create_info->comment.length); + pos+= create_info->comment.length; } - if (options_len) - { - uchar *optbuff= (uchar *)my_safe_alloca(options_len + 4, ALLOCA_THRESHOLD); - my_bool error; - DBUG_PRINT("info", ("Create options length: %u", options_len)); - if (!optbuff) - goto err; - int4store(optbuff, options_len); - engine_table_options_frm_image(optbuff + 4, - create_info->option_list, - create_fields, - keys, key_info); - error= my_write(file, optbuff, options_len + 4, MYF_RW); - my_safe_afree(optbuff, options_len + 4, ALLOCA_THRESHOLD); - if (error) - goto err; - } - - mysql_file_seek(file, filepos, MY_SEEK_SET, MYF(0)); - if (mysql_file_write(file, forminfo, 288, MYF_RW) || - mysql_file_write(file, screen_buff, info_length, MYF_RW) || - pack_fields(file, create_fields, data_offset)) + memcpy(frm_ptr + filepos, forminfo, 288); + if (pack_fields(frm_ptr + filepos + 288, create_fields, data_offset)) goto err; -#ifdef HAVE_CRYPTED_FRM - if (create_info->password) - { - char tmp=2,*disk_buff=0; - SQL_CRYPT *crypted=new SQL_CRYPT(create_info->password); - if (!crypted || mysql_file_pwrite(file, &tmp, 1, 26, MYF_RW))// Mark crypted - goto err; - uint read_length=uint2korr(forminfo)-256; - mysql_file_seek(file, filepos+256, MY_SEEK_SET, MYF(0)); - if (read_string(file,(uchar**) &disk_buff,read_length)) - goto err; - crypted->encode(disk_buff,read_length); - delete crypted; - if (mysql_file_pwrite(file, disk_buff, read_length, filepos+256, MYF_RW)) - { - my_free(disk_buff); - goto err; - } - my_free(disk_buff); - } -#endif - - my_free(screen_buff); - my_free(keybuff); - - if (opt_sync_frm && !(create_info->options & HA_LEX_CREATE_TMP_TABLE) && - (mysql_file_sync(file, MYF(MY_WME)) || - my_sync_dir_by_file(file_name, MYF(MY_WME)))) - goto err2; - - if (mysql_file_close(file, MYF(MY_WME))) - goto err3; - { /* Restore all UCS2 intervals. @@ -445,17 +360,14 @@ bool mysql_create_frm(THD *thd, const char *file_name, } } } - DBUG_RETURN(0); + + frm.str= frm_ptr; + DBUG_RETURN(frm); err: - my_free(screen_buff); - my_free(keybuff); -err2: - (void) mysql_file_close(file, MYF(MY_WME)); -err3: - mysql_file_delete(key_file_frm, file_name, MYF(0)); - DBUG_RETURN(1); -} /* mysql_create_frm */ + my_free(frm_ptr); + DBUG_RETURN(frm); +} /* @@ -464,130 +376,58 @@ err3: SYNOPSIS rea_create_table() thd Thread handler + frm binary frm image of the table to create path Name of file (including database, without .frm) db Data base name table_name Table name create_info create info parameters - create_fields Fields to create - keys number of keys to create - key_info Keys to create - file Handler to use + file Handler to use or NULL if only frm needs to be created RETURN 0 ok 1 error */ -int rea_create_table(THD *thd, const char *path, - const char *db, const char *table_name, - HA_CREATE_INFO *create_info, - List<Create_field> &create_fields, - uint keys, KEY *key_info, handler *file) +int rea_create_table(THD *thd, LEX_CUSTRING *frm, + const char *path, const char *db, const char *table_name, + HA_CREATE_INFO *create_info, handler *file) { DBUG_ENTER("rea_create_table"); - char frm_name[FN_REFLEN]; - strxmov(frm_name, path, reg_ext, NullS); - if (mysql_create_frm(thd, frm_name, db, table_name, create_info, - create_fields, keys, key_info, file)) + if (file) + { + // TODO don't write frm for temp tables + if (create_info->tmp_table() && + writefrm(path, db, table_name, true, frm->str, frm->length)) + goto err_handler; - DBUG_RETURN(1); + if (thd->variables.keep_files_on_create) + create_info->options|= HA_CREATE_KEEP_FILES; + + if (file->ha_create_partitioning_metadata(path, NULL, CHF_CREATE_FLAG) || + ha_create_table(thd, path, db, table_name, create_info, frm)) + { + file->ha_create_partitioning_metadata(path, NULL, CHF_DELETE_FLAG); + goto err_handler; + } + } + else + { + if (writefrm(path, db, table_name, false, frm->str, frm->length)) + goto err_handler; + } - // Make sure mysql_create_frm din't remove extension - DBUG_ASSERT(*fn_rext(frm_name)); - if (thd->variables.keep_files_on_create) - create_info->options|= HA_CREATE_KEEP_FILES; - if (!create_info->frm_only && - (file->ha_create_handler_files(path, NULL, CHF_CREATE_FLAG, - create_info) || - ha_create_table(thd, path, db, table_name, create_info, 0))) - goto err_handler; DBUG_RETURN(0); err_handler: - (void) file->ha_create_handler_files(path, NULL, CHF_DELETE_FLAG, create_info); + char frm_name[FN_REFLEN]; + strxmov(frm_name, path, reg_ext, NullS); mysql_file_delete(key_file_frm, frm_name, MYF(0)); DBUG_RETURN(1); } /* rea_create_table */ - /* Pack screens to a screen for save in a form-file */ - -static uchar *pack_screens(List<Create_field> &create_fields, - uint *info_length, uint *screens, - bool small_file) -{ - reg1 uint i; - uint row,start_row,end_row,fields_on_screen; - uint length,cols; - uchar *info,*pos,*start_screen; - uint fields=create_fields.elements; - List_iterator<Create_field> it(create_fields); - DBUG_ENTER("pack_screens"); - - start_row=4; end_row=22; cols=80; fields_on_screen=end_row+1-start_row; - - *screens=(fields-1)/fields_on_screen+1; - length= (*screens) * (SC_INFO_LENGTH+ (cols>> 1)+4); - - Create_field *field; - while ((field=it++)) - length+=(uint) strlen(field->field_name)+1+TE_INFO_LENGTH+cols/2; - - if (!(info=(uchar*) my_malloc(length,MYF(MY_WME)))) - DBUG_RETURN(0); - - start_screen=0; - row=end_row; - pos=info; - it.rewind(); - for (i=0 ; i < fields ; i++) - { - Create_field *cfield=it++; - if (row++ == end_row) - { - if (i) - { - length=(uint) (pos-start_screen); - int2store(start_screen,length); - start_screen[2]=(uchar) (fields_on_screen+1); - start_screen[3]=(uchar) (fields_on_screen); - } - row=start_row; - start_screen=pos; - pos+=4; - pos[0]= (uchar) start_row-2; /* Header string */ - pos[1]= (uchar) (cols >> 2); - pos[2]= (uchar) (cols >> 1) +1; - strfill((char *) pos+3,(uint) (cols >> 1),' '); - pos+=(cols >> 1)+4; - } - length=(uint) strlen(cfield->field_name); - if (length > cols-3) - length=cols-3; - - if (!small_file) - { - pos[0]=(uchar) row; - pos[1]=0; - pos[2]=(uchar) (length+1); - pos=(uchar*) strmake((char*) pos+3,cfield->field_name,length)+1; - } - cfield->row=(uint8) row; - cfield->col=(uint8) (length+1); - cfield->sc_length=(uint8) min(cfield->length,cols-(length+2)); - } - length=(uint) (pos-start_screen); - int2store(start_screen,length); - start_screen[2]=(uchar) (row-start_row+2); - start_screen[3]=(uchar) (row-start_row+1); - - *info_length=(uint) (pos-info); - DBUG_RETURN(info); -} /* pack_screens */ - - - /* Pack keyinfo and keynames to keybuff for save in form-file. */ +/* Pack keyinfo and keynames to keybuff for save in form-file. */ static uint pack_keys(uchar *keybuff, uint key_count, KEY *keyinfo, ulong data_offset) @@ -670,12 +510,10 @@ static uint pack_keys(uchar *keybuff, uint key_count, KEY *keyinfo, } /* pack_keys */ - /* Make formheader */ +/* Make formheader */ -static bool pack_header(uchar *forminfo, enum legacy_db_type table_type, - List<Create_field> &create_fields, - uint info_length, uint screens, uint table_options, - ulong data_offset, handler *file) +static bool pack_header(uchar *forminfo, List<Create_field> &create_fields, + uint table_options, ulong data_offset, handler *file) { uint length,int_count,int_length,no_empty, int_parts; uint time_stamp_pos,null_fields; @@ -694,8 +532,7 @@ static bool pack_header(uchar *forminfo, enum legacy_db_type table_type, com_length=vcol_info_length=0; n_length=2L; - /* Check fields */ - + /* Check fields */ List_iterator<Create_field> it(create_fields); Create_field *field; while ((field=it++)) @@ -707,21 +544,18 @@ static bool pack_header(uchar *forminfo, enum legacy_db_type table_type, COLUMN_COMMENT_MAXLEN); if (tmp_len < field->comment.length) { - if ((current_thd->variables.sql_mode & - (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES))) - { - my_error(ER_TOO_LONG_FIELD_COMMENT, MYF(0), field->field_name, - static_cast<ulong>(COLUMN_COMMENT_MAXLEN)); + myf myf_warning= ME_JUST_WARNING; + ulonglong sql_mode= current_thd->variables.sql_mode; + + if (sql_mode & (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES)) + myf_warning= 0; + + my_error(ER_TOO_LONG_FIELD_COMMENT, myf_warning, field->field_name, + COLUMN_COMMENT_MAXLEN); + + if (!myf_warning) DBUG_RETURN(1); - } - char warn_buff[MYSQL_ERRMSG_SIZE]; - my_snprintf(warn_buff, sizeof(warn_buff), ER(ER_TOO_LONG_FIELD_COMMENT), - field->field_name, - static_cast<ulong>(COLUMN_COMMENT_MAXLEN)); - /* do not push duplicate warnings */ - if (!check_duplicate_warning(current_thd, warn_buff, strlen(warn_buff))) - push_warning(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN, - ER_TOO_LONG_FIELD_COMMENT, warn_buff); + field->comment.length= tmp_len; } if (field->vcol_info) @@ -747,7 +581,7 @@ static bool pack_header(uchar *forminfo, enum legacy_db_type table_type, expressions saved in the frm file for virtual columns. */ vcol_info_length+= field->vcol_info->expr_str.length+ - FRM_VCOL_HEADER_SIZE(field->interval!=NULL); + FRM_VCOL_HEADER_SIZE(field->interval); } totlength+= field->length; @@ -824,8 +658,7 @@ static bool pack_header(uchar *forminfo, enum legacy_db_type table_type, } int_length+=int_count*2; // 255 prefix + 0 suffix - /* Save values in forminfo */ - + /* Save values in forminfo */ if (reclength > (ulong) file->max_record_length()) { my_error(ER_TOO_BIG_ROWSIZE, MYF(0), static_cast<long>(file->max_record_length())); @@ -833,7 +666,7 @@ static bool pack_header(uchar *forminfo, enum legacy_db_type table_type, } /* Hack to avoid bugs with small static rows in MySQL */ reclength=max(file->min_record_length(table_options),reclength); - if (info_length+(ulong) create_fields.elements*FCOMP+288+ + if ((ulong) create_fields.elements*FCOMP+FRM_FORMINFO_SIZE+ n_length+int_length+com_length+vcol_info_length > 65535L || int_count > 255) { @@ -841,13 +674,13 @@ static bool pack_header(uchar *forminfo, enum legacy_db_type table_type, DBUG_RETURN(1); } - bzero((char*)forminfo,288); - length=(info_length+create_fields.elements*FCOMP+288+n_length+int_length+ + bzero((char*)forminfo,FRM_FORMINFO_SIZE); + length=(create_fields.elements*FCOMP+FRM_FORMINFO_SIZE+n_length+int_length+ com_length+vcol_info_length); int2store(forminfo,length); - forminfo[256] = (uint8) screens; + forminfo[256] = 0; int2store(forminfo+258,create_fields.elements); - int2store(forminfo+260,info_length); + int2store(forminfo+260,0); int2store(forminfo+262,totlength); int2store(forminfo+264,no_empty); int2store(forminfo+266,reclength); @@ -861,13 +694,11 @@ static bool pack_header(uchar *forminfo, enum legacy_db_type table_type, int2store(forminfo+282,null_fields); int2store(forminfo+284,com_length); int2store(forminfo+286,vcol_info_length); - /* forminfo+288 is free to use for additional information */ DBUG_RETURN(0); } /* pack_header */ - /* get each unique interval each own id */ - +/* get each unique interval each own id */ static uint get_interval_id(uint *int_count,List<Create_field> &create_fields, Create_field *last_field) { @@ -894,29 +725,57 @@ static uint get_interval_id(uint *int_count,List<Create_field> &create_fields, } - /* Save fields, fieldnames and intervals */ +static size_t packed_fields_length(List<Create_field> &create_fields) +{ + Create_field *field; + size_t length= 0; + DBUG_ENTER("packed_fields_length"); + + List_iterator<Create_field> it(create_fields); + uint int_count=0; + while ((field=it++)) + { + if (field->interval_id > int_count) + { + int_count= field->interval_id; + length++; + for (int i=0; field->interval->type_names[i]; i++) + { + length+= field->interval->type_lengths[i]; + length++; + } + length++; + } + if (field->vcol_info) + { + length+= field->vcol_info->expr_str.length + + FRM_VCOL_HEADER_SIZE(field->interval); + } + length+= FCOMP; + length+= strlen(field->field_name)+1; + length+= field->comment.length; + } + length++; + length++; + DBUG_RETURN(length); +} + +/* Save fields, fieldnames and intervals */ -static bool pack_fields(File file, List<Create_field> &create_fields, +static bool pack_fields(uchar *buff, List<Create_field> &create_fields, ulong data_offset) { - reg2 uint i; uint int_count, comment_length= 0, vcol_info_length=0; - uchar buff[MAX_FIELD_WIDTH]; Create_field *field; DBUG_ENTER("pack_fields"); - /* Write field info */ - + /* Write field info */ List_iterator<Create_field> it(create_fields); - int_count=0; while ((field=it++)) { uint recpos; uint cur_vcol_expr_len= 0; - buff[0]= (uchar) field->row; - buff[1]= (uchar) field->col; - buff[2]= (uchar) field->sc_length; int2store(buff+3, field->length); /* The +1 is here becasue the col offset in .frm file have offset 1 */ recpos= field->offset+1 + (uint) data_offset; @@ -950,40 +809,29 @@ static bool pack_fields(File file, List<Create_field> &create_fields, the additional data saved for the virtual field */ buff[12]= cur_vcol_expr_len= field->vcol_info->expr_str.length + - FRM_VCOL_HEADER_SIZE(field->interval!=NULL); - vcol_info_length+= cur_vcol_expr_len + - FRM_VCOL_HEADER_SIZE(field->interval!=NULL); + FRM_VCOL_HEADER_SIZE(field->interval); + vcol_info_length+= cur_vcol_expr_len; buff[13]= (uchar) MYSQL_TYPE_VIRTUAL; } int2store(buff+15, field->comment.length); comment_length+= field->comment.length; set_if_bigger(int_count,field->interval_id); - if (mysql_file_write(file, buff, FCOMP, MYF_RW)) - DBUG_RETURN(1); + buff+= FCOMP; } - /* Write fieldnames */ - buff[0]=(uchar) NAMES_SEP_CHAR; - if (mysql_file_write(file, buff, 1, MYF_RW)) - DBUG_RETURN(1); - i=0; + /* Write fieldnames */ + *buff++= NAMES_SEP_CHAR; it.rewind(); while ((field=it++)) { - char *pos= strmov((char*) buff,field->field_name); - *pos++=NAMES_SEP_CHAR; - if (i == create_fields.elements-1) - *pos++=0; - if (mysql_file_write(file, buff, (size_t) (pos-(char*) buff), MYF_RW)) - DBUG_RETURN(1); - i++; + buff= (uchar*)strmov((char*) buff, field->field_name); + *buff++=NAMES_SEP_CHAR; } + *buff++= 0; - /* Write intervals */ + /* Write intervals */ if (int_count) { - String tmp((char*) buff,sizeof(buff), &my_charset_bin); - tmp.length(0); it.rewind(); int_count=0; while ((field=it++)) @@ -1025,34 +873,30 @@ static bool pack_fields(File file, List<Create_field> &create_fields, } int_count= field->interval_id; - tmp.append(sep); - for (const char **pos=field->interval->type_names ; *pos ; pos++) + *buff++= sep; + for (int i=0; field->interval->type_names[i]; i++) { - tmp.append(*pos); - tmp.append(sep); + memcpy(buff, field->interval->type_names[i], field->interval->type_lengths[i]); + buff+= field->interval->type_lengths[i]; + *buff++= sep; } - tmp.append('\0'); // End of intervall + *buff++= 0; + } } - if (mysql_file_write(file, (uchar*) tmp.ptr(), tmp.length(), MYF_RW)) - DBUG_RETURN(1); } if (comment_length) { it.rewind(); - int_count=0; while ((field=it++)) { - if (field->comment.length) - if (mysql_file_write(file, (uchar*) field->comment.str, - field->comment.length, MYF_RW)) - DBUG_RETURN(1); + memcpy(buff, field->comment.str, field->comment.length); + buff+= field->comment.length; } } if (vcol_info_length) { it.rewind(); - int_count=0; while ((field=it++)) { /* @@ -1065,18 +909,13 @@ static bool pack_fields(File file, List<Create_field> &create_fields, */ if (field->vcol_info && field->vcol_info->expr_str.length) { - buff[0]= (uchar)(1 + test(field->interval_id)); - buff[1]= (uchar) field->sql_type; - buff[2]= (uchar) field->stored_in_db; - if (field->interval_id) - buff[3]= (uchar) field->interval_id; - if (my_write(file, buff, 3 + test(field->interval_id), MYF_RW)) - DBUG_RETURN(1); - if (my_write(file, - (uchar*) field->vcol_info->expr_str.str, - field->vcol_info->expr_str.length, - MYF_RW)) - DBUG_RETURN(1); + *buff++= (uchar)(1 + test(field->interval)); + *buff++= (uchar) field->sql_type; + *buff++= (uchar) field->stored_in_db; + if (field->interval) + *buff++= (uchar) field->interval_id; + memcpy(buff, field->vcol_info->expr_str.str, field->vcol_info->expr_str.length); + buff+= field->vcol_info->expr_str.length; } } } @@ -1084,19 +923,16 @@ static bool pack_fields(File file, List<Create_field> &create_fields, } - /* save an empty record on start of formfile */ +/* save an empty record on start of formfile */ -static bool make_empty_rec(THD *thd, File file,enum legacy_db_type table_type, - uint table_options, +static bool make_empty_rec(THD *thd, uchar *buff, uint table_options, List<Create_field> &create_fields, - uint reclength, - ulong data_offset, - handler *handler) + uint reclength, ulong data_offset) { int error= 0; Field::utype type; uint null_count; - uchar *buff,*null_pos; + uchar *null_pos; TABLE table; TABLE_SHARE share; Create_field *field; @@ -1108,11 +944,6 @@ static bool make_empty_rec(THD *thd, File file,enum legacy_db_type table_type, bzero((char*) &share, sizeof(share)); table.s= &share; - if (!(buff=(uchar*) my_malloc((size_t) reclength,MYF(MY_WME | MY_ZEROFILL)))) - { - DBUG_RETURN(1); - } - table.in_use= thd; table.s->blob_ptr_size= portable_sizeof_char_ptr; @@ -1198,10 +1029,7 @@ static bool make_empty_rec(THD *thd, File file,enum legacy_db_type table_type, if (null_count & 7) *(null_pos + null_count / 8)|= ~(((uchar) 1 << (null_count & 7)) - 1); - error= mysql_file_write(file, buff, (size_t) reclength, MYF_RW) != 0; - err: - my_free(buff); thd->count_cuted_fields= old_count_cuted_fields; DBUG_RETURN(error); } /* make_empty_rec */ diff --git a/sql/unireg.h b/sql/unireg.h index a35c25f92ee..c867f50197d 100644 --- a/sql/unireg.h +++ b/sql/unireg.h @@ -23,8 +23,6 @@ /* Extra functions used by unireg library */ -typedef struct st_ha_create_information HA_CREATE_INFO; - #ifndef NO_ALARM_LOOP #define NO_ALARM_LOOP /* lib5 and popen can't use alarm */ #endif @@ -88,7 +86,7 @@ typedef struct st_ha_create_information HA_CREATE_INFO; #define READ_ALL 1 /* openfrm: Read all parameters */ #define CHANGE_FRM 2 /* openfrm: open .frm as O_RDWR */ #define READ_KEYINFO 4 /* L{s nyckeldata fr}n filen */ -#define EXTRA_RECORD 8 /* Reservera plats f|r extra record */ +#define EXTRA_RECORD 8 /* Reserve space for an extra record */ #define DONT_OPEN_TABLES 8 /* Don't open database-files (frd) */ #define DONT_OPEN_MASTER_REG 16 /* Don't open first reg-file (prt) */ #define EXTRA_LONG_RECORD 16 /* Plats f|r dubbel s|k-record */ @@ -169,15 +167,45 @@ typedef struct st_ha_create_information HA_CREATE_INFO; #include "sql_list.h" /* List<> */ #include "field.h" /* Create_field */ -bool mysql_create_frm(THD *thd, const char *file_name, - const char *db, const char *table, - HA_CREATE_INFO *create_info, - List<Create_field> &create_field, - uint key_count,KEY *key_info,handler *db_type); -int rea_create_table(THD *thd, const char *path, - const char *db, const char *table_name, - HA_CREATE_INFO *create_info, - List<Create_field> &create_field, - uint key_count,KEY *key_info, - handler *file); +/* + Types of values in the MariaDB extra2 frm segment. + Each value is written as + type: 1 byte + length: 1 byte (1..255) or \0 and 2 bytes. + binary value of the 'length' bytes. + + Older MariaDB servers can ignore values of unknown types if + the type code is less than 128 (EXTRA2_ENGINE_IMPORTANT). + Otherwise older (but newer than 10.0.1) servers are required + to report an error. +*/ +enum extra2_frm_value_type { + EXTRA2_TABLEDEF_VERSION=0, + EXTRA2_DEFAULT_PART_ENGINE=1, + +#define EXTRA2_ENGINE_IMPORTANT 128 + + EXTRA2_ENGINE_TABLEOPTS=128, +}; + +int rea_create_table(THD *thd, LEX_CUSTRING *frm, + const char *path, const char *db, const char *table_name, + HA_CREATE_INFO *create_info, handler *file); +LEX_CUSTRING build_frm_image(THD *thd, const char *table, + HA_CREATE_INFO *create_info, + List<Create_field> &create_fields, + uint keys, KEY *key_info, handler *db_file); + +#define FRM_HEADER_SIZE 64 +#define FRM_FORMINFO_SIZE 288 +#define FRM_MAX_SIZE (256*1024) + +static inline bool is_binary_frm_header(uchar *head) +{ + return head[0] == 254 + && head[1] == 1 + && head[2] >= FRM_VER + && head[2] <= FRM_VER+4; +} + #endif |