From c0e10f375ad619d825ef7c21232cf5946bdf5be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 10 Nov 2017 09:07:45 +0200 Subject: Fix a -Wimplicit-fallthrough warning --- storage/maria/ma_loghandler.c | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/maria/ma_loghandler.c b/storage/maria/ma_loghandler.c index 556e7a93ef1..ef99307b441 100644 --- a/storage/maria/ma_loghandler.c +++ b/storage/maria/ma_loghandler.c @@ -6367,7 +6367,6 @@ my_bool translog_write_record(LSN *lsn, short_trid, &parts, trn, hook_arg); break; case LOGRECTYPE_NOT_ALLOWED: - DBUG_ASSERT(0); default: DBUG_ASSERT(0); rc= 1; -- cgit v1.2.1 From 2913f615f050f356f7be178e5d91650b86b33e4e Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Mon, 13 Nov 2017 16:30:02 +0100 Subject: MDEV-8949: COLUMN_CREATE unicode name breakage Use utf-mb4 if it is possible. --- include/ma_dyncol.h | 6 ++++++ mysql-test/r/ctype_utf8mb4.result | 27 +++++++++++++++++++++++++++ mysql-test/t/ctype_utf8mb4.test | 20 ++++++++++++++++++++ mysys/ma_dyncol.c | 3 +-- sql/item_cmpfunc.cc | 6 +++--- sql/item_strfunc.cc | 17 +++++++++-------- sql/item_strfunc.h | 7 ++++--- 7 files changed, 70 insertions(+), 16 deletions(-) diff --git a/include/ma_dyncol.h b/include/ma_dyncol.h index cb03ccb5382..300474e061e 100644 --- a/include/ma_dyncol.h +++ b/include/ma_dyncol.h @@ -63,6 +63,12 @@ typedef struct st_mysql_lex_string LEX_STRING; /* NO and OK is the same used just to show semantics */ #define ER_DYNCOL_NO ER_DYNCOL_OK +#ifdef HAVE_CHARSET_utf8mb4 +#define DYNCOL_UTF (&my_charset_utf8mb4_general_ci) +#else +#define DYNCOL_UTF (&my_charset_utf8_general_ci) +#endif + enum enum_dyncol_func_result { ER_DYNCOL_OK= 0, diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result index 50382b5d5ca..ff9c155e445 100644 --- a/mysql-test/r/ctype_utf8mb4.result +++ b/mysql-test/r/ctype_utf8mb4.result @@ -3379,5 +3379,32 @@ F09F988E78 78F09F988E78 DROP TABLE t1; # +# MDEV-8949: COLUMN_CREATE unicode name breakage +# +SET NAMES utf8mb4; +SELECT COLUMN_JSON(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1)); +COLUMN_JSON(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1)) +{"😎":1} +SELECT COLUMN_LIST(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1)); +COLUMN_LIST(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1)) +`😎` +SELECT COLUMN_GET(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1), _utf8mb4 0xF09F988E +as int); +COLUMN_GET(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1), _utf8mb4 0xF09F988E +as int) +1 +CREATE TABLE t1 AS SELECT +COLUMN_LIST(COLUMN_CREATE('a',1)), +COLUMN_JSON(COLUMN_CREATE('b',1)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `COLUMN_LIST(COLUMN_CREATE('a',1))` longtext CHARACTER SET utf8mb4, + `COLUMN_JSON(COLUMN_CREATE('b',1))` longtext CHARACTER SET utf8mb4 +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +SET NAMES default; +# End of 10.0 tests +# # End of tests # diff --git a/mysql-test/t/ctype_utf8mb4.test b/mysql-test/t/ctype_utf8mb4.test index cf1c103137e..10786183ad0 100644 --- a/mysql-test/t/ctype_utf8mb4.test +++ b/mysql-test/t/ctype_utf8mb4.test @@ -1876,6 +1876,26 @@ LOAD DATA INFILE '../../std_data/loaddata/mdev-11343.txt' INTO TABLE t1 CHARACTE SELECT HEX(a) FROM t1; DROP TABLE t1; +--echo # +--echo # MDEV-8949: COLUMN_CREATE unicode name breakage +--echo # + +SET NAMES utf8mb4; +SELECT COLUMN_JSON(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1)); +SELECT COLUMN_LIST(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1)); +SELECT COLUMN_GET(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1), _utf8mb4 0xF09F988E +as int); + +CREATE TABLE t1 AS SELECT + COLUMN_LIST(COLUMN_CREATE('a',1)), + COLUMN_JSON(COLUMN_CREATE('b',1)); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +SET NAMES default; + +--echo # End of 10.0 tests + --echo # --echo # End of tests --echo # diff --git a/mysys/ma_dyncol.c b/mysys/ma_dyncol.c index d0d6254d11c..9f6df107316 100644 --- a/mysys/ma_dyncol.c +++ b/mysys/ma_dyncol.c @@ -4184,8 +4184,7 @@ mariadb_dyncol_json_internal(DYNAMIC_COLUMN *str, DYNAMIC_STRING *json, } else { - if ((rc= mariadb_dyncol_val_str(json, &val, - &my_charset_utf8_general_ci, '"')) < 0) + if ((rc= mariadb_dyncol_val_str(json, &val, DYNCOL_UTF, '"')) < 0) goto err; } } diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index b086b084f97..807ce828131 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -6642,7 +6642,7 @@ longlong Item_func_dyncol_exists::val_int() null_value= 1; return 1; } - if (my_charset_same(nm->charset(), &my_charset_utf8_general_ci)) + if (my_charset_same(nm->charset(), DYNCOL_UTF)) { buf.str= (char *) nm->ptr(); buf.length= nm->length(); @@ -6652,11 +6652,11 @@ longlong Item_func_dyncol_exists::val_int() uint strlen; uint dummy_errors; buf.str= (char *)sql_alloc((strlen= nm->length() * - my_charset_utf8_general_ci.mbmaxlen + 1)); + DYNCOL_UTF->mbmaxlen + 1)); if (buf.str) { buf.length= - copy_and_convert(buf.str, strlen, &my_charset_utf8_general_ci, + copy_and_convert(buf.str, strlen, DYNCOL_UTF, nm->ptr(), nm->length(), nm->charset(), &dummy_errors); } diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index c3043adac47..c1bca3e12eb 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -4444,7 +4444,7 @@ bool Item_func_dyncol_create::prepare_arguments(bool force_names_arg) if (res) { // guaranty UTF-8 string for names - if (my_charset_same(res->charset(), &my_charset_utf8_general_ci)) + if (my_charset_same(res->charset(), DYNCOL_UTF)) { keys_str[i].length= res->length(); keys_str[i].str= sql_strmake(res->ptr(), res->length()); @@ -4455,11 +4455,11 @@ bool Item_func_dyncol_create::prepare_arguments(bool force_names_arg) uint dummy_errors; char *str= (char *)sql_alloc((strlen= res->length() * - my_charset_utf8_general_ci.mbmaxlen + 1)); + DYNCOL_UTF->mbmaxlen + 1)); if (str) { keys_str[i].length= - copy_and_convert(str, strlen, &my_charset_utf8_general_ci, + copy_and_convert(str, strlen, DYNCOL_UTF, res->ptr(), res->length(), res->charset(), &dummy_errors); keys_str[i].str= str; @@ -4680,9 +4680,10 @@ String *Item_func_dyncol_json::val_str(String *str) size_t length, alloc_length; dynstr_reassociate(&json, &ptr, &length, &alloc_length); str->reassociate(ptr, (uint32) length, (uint32) alloc_length, - &my_charset_utf8_general_ci); + DYNCOL_UTF); null_value= FALSE; } + str->set_charset(DYNCOL_UTF); return str; null: @@ -4781,7 +4782,7 @@ bool Item_dyncol_get::get_dyn_value(DYNAMIC_COLUMN_VALUE *val, String *tmp) return 1; } - if (my_charset_same(nm->charset(), &my_charset_utf8_general_ci)) + if (my_charset_same(nm->charset(), DYNCOL_UTF)) { buf.str= (char *) nm->ptr(); buf.length= nm->length(); @@ -4791,11 +4792,11 @@ bool Item_dyncol_get::get_dyn_value(DYNAMIC_COLUMN_VALUE *val, String *tmp) uint strlen; uint dummy_errors; buf.str= (char *)sql_alloc((strlen= nm->length() * - my_charset_utf8_general_ci.mbmaxlen + 1)); + DYNCOL_UTF->mbmaxlen + 1)); if (buf.str) { buf.length= - copy_and_convert(buf.str, strlen, &my_charset_utf8_general_ci, + copy_and_convert(buf.str, strlen, DYNCOL_UTF, nm->ptr(), nm->length(), nm->charset(), &dummy_errors); } @@ -5229,7 +5230,6 @@ String *Item_func_dyncol_list::val_str(String *str) goto null; str->length(0); - str->set_charset(&my_charset_utf8_general_ci); for (i= 0; i < count; i++) { append_identifier(current_thd, str, names[i].str, names[i].length); @@ -5239,6 +5239,7 @@ String *Item_func_dyncol_list::val_str(String *str) null_value= FALSE; if (names) my_free(names); + str->set_charset(DYNCOL_UTF); return str; null: diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 1b133d1d885..6520591145b 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -1177,14 +1177,14 @@ public: class Item_func_dyncol_json: public Item_str_func { public: - Item_func_dyncol_json(Item *str) :Item_str_func(str) {} + Item_func_dyncol_json(Item *str) :Item_str_func(str) + {collation.set(DYNCOL_UTF);} const char *func_name() const{ return "column_json"; } String *val_str(String *); void fix_length_and_dec() { max_length= MAX_BLOB_WIDTH; maybe_null= 1; - collation.set(&my_charset_bin); decimals= 0; } }; @@ -1218,7 +1218,8 @@ public: class Item_func_dyncol_list: public Item_str_func { public: - Item_func_dyncol_list(Item *str) :Item_str_func(str) {}; + Item_func_dyncol_list(Item *str) :Item_str_func(str) + {collation.set(DYNCOL_UTF);}; void fix_length_and_dec() { maybe_null= 1; max_length= MAX_BLOB_WIDTH; }; const char *func_name() const{ return "column_list"; } String *val_str(String *); -- cgit v1.2.1 From ea1739f90d07c9f252b63a19cd9425b05abaf33a Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Tue, 14 Nov 2017 11:29:52 +0300 Subject: removed garbase struct member --- storage/innobase/include/que0que.h | 3 --- storage/xtradb/include/que0que.h | 3 --- 2 files changed, 6 deletions(-) diff --git a/storage/innobase/include/que0que.h b/storage/innobase/include/que0que.h index ba8828623af..f37581cbf8f 100644 --- a/storage/innobase/include/que0que.h +++ b/storage/innobase/include/que0que.h @@ -383,9 +383,6 @@ struct que_thr_t{ UT_LIST_NODE_T(que_thr_t) thrs; /*!< list of thread nodes of the fork node */ - UT_LIST_NODE_T(que_thr_t) - trx_thrs; /*!< lists of threads in wait list of - the trx */ UT_LIST_NODE_T(que_thr_t) queue; /*!< list of runnable thread nodes in the server task queue */ diff --git a/storage/xtradb/include/que0que.h b/storage/xtradb/include/que0que.h index e5b2a1ba3fc..005f28d2af1 100644 --- a/storage/xtradb/include/que0que.h +++ b/storage/xtradb/include/que0que.h @@ -384,9 +384,6 @@ struct que_thr_t{ UT_LIST_NODE_T(que_thr_t) thrs; /*!< list of thread nodes of the fork node */ - UT_LIST_NODE_T(que_thr_t) - trx_thrs; /*!< lists of threads in wait list of - the trx */ UT_LIST_NODE_T(que_thr_t) queue; /*!< list of runnable thread nodes in the server task queue */ -- cgit v1.2.1 From 02e35ef5f237b03c3e735fa7afe21dd92086cbc5 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 15 Nov 2017 15:52:03 +0400 Subject: MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1) --- mysql-test/r/ctype_utf8.result | 15 +++++++++++++++ mysql-test/r/func_str.result | 21 +++++++++++++++++++++ mysql-test/t/ctype_utf8.test | 12 ++++++++++++ mysql-test/t/func_str.test | 12 ++++++++++++ sql/item_strfunc.cc | 14 ++++++++++++++ sql/item_strfunc.h | 1 + 6 files changed, 75 insertions(+) diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result index e292b64cfa6..7a3ff3732c3 100644 --- a/mysql-test/r/ctype_utf8.result +++ b/mysql-test/r/ctype_utf8.result @@ -10113,5 +10113,20 @@ DROP FUNCTION iswellformed; DROP TABLE allbytes; # End of ctype_backslash.inc # +# MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1) +# +SET NAMES utf8; +SELECT CHAR(0xDF USING latin1); +CHAR(0xDF USING latin1) +ß +CREATE OR REPLACE VIEW v1 AS SELECT CHAR(0xDF USING latin1) AS c; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select char(0xdf using latin1) AS `c` utf8 utf8_general_ci +SELECT * FROM v1; +c +ß +DROP VIEW v1; +# # End of 10.0 tests # diff --git a/mysql-test/r/func_str.result b/mysql-test/r/func_str.result index 8f08d7acd25..9a9b9f0d73d 100644 --- a/mysql-test/r/func_str.result +++ b/mysql-test/r/func_str.result @@ -4530,3 +4530,24 @@ latin2_general_ci # # End of 5.6 tests # +# +# Start of 10.0 tests +# +# +# MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1) +# +EXPLAIN EXTENDED SELECT CHAR(0xDF USING latin1); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1003 select char(0xdf using latin1) AS `CHAR(0xDF USING latin1)` +EXPLAIN EXTENDED SELECT CHAR(0xDF USING `binary`); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1003 select char(0xdf) AS `CHAR(0xDF USING ``binary``)` +EXPLAIN EXTENDED SELECT CHAR(0xDF); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1003 select char(0xdf) AS `CHAR(0xDF)` diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test index 4b181182c46..2524daffa9e 100644 --- a/mysql-test/t/ctype_utf8.test +++ b/mysql-test/t/ctype_utf8.test @@ -1868,6 +1868,18 @@ SELECT _utf8 0x7E, _utf8 X'7E', _utf8 B'01111110'; let $ctype_unescape_combinations=selected; --source include/ctype_unescape.inc +--echo # +--echo # MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1) +--echo # + +SET NAMES utf8; +SELECT CHAR(0xDF USING latin1); +CREATE OR REPLACE VIEW v1 AS SELECT CHAR(0xDF USING latin1) AS c; +SHOW CREATE VIEW v1; +SELECT * FROM v1; +DROP VIEW v1; + + --echo # --echo # End of 10.0 tests --echo # diff --git a/mysql-test/t/func_str.test b/mysql-test/t/func_str.test index 6369609bea3..bdeca8d0d86 100644 --- a/mysql-test/t/func_str.test +++ b/mysql-test/t/func_str.test @@ -1742,3 +1742,15 @@ EXECUTE stmt; --echo # --echo # End of 5.6 tests --echo # + +--echo # +--echo # Start of 10.0 tests +--echo # + +--echo # +--echo # MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1) +--echo # + +EXPLAIN EXTENDED SELECT CHAR(0xDF USING latin1); +EXPLAIN EXTENDED SELECT CHAR(0xDF USING `binary`); +EXPLAIN EXTENDED SELECT CHAR(0xDF); diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index c1bca3e12eb..3ee8a0e4970 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -2905,6 +2905,20 @@ String *Item_func_make_set::val_str(String *str) } +void Item_func_char::print(String *str, enum_query_type query_type) +{ + str->append(Item_func_char::func_name()); + str->append('('); + print_args(str, 0, query_type); + if (collation.collation != &my_charset_bin) + { + str->append(C_STRING_WITH_LEN(" using ")); + str->append(collation.collation->csname); + } + str->append(')'); +} + + String *Item_func_char::val_str(String *str) { DBUG_ASSERT(fixed == 1); diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 6520591145b..aa3486d4e73 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -702,6 +702,7 @@ public: max_length= arg_count * 4; } const char *func_name() const { return "char"; } + void print(String *str, enum_query_type query_type); }; -- cgit v1.2.1 From 923ea5dbf6644fab088e35122523b2b8ef03b7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 16 Nov 2017 13:18:22 +0200 Subject: MDEV-9663: InnoDB assertion failure: *cursor->index->name == TEMP_INDEX_PREFIX Imported missing test case from MySQL 5.7 for commit 25781c154396dbbc21023786aa3be070057d6999 Author: Annamalai Gurusami Date: Mon Feb 24 14:00:03 2014 +0530 Bug #17604730 ASSERTION: *CURSOR->INDEX->NAME == TEMP_INDEX_PREFIX --- mysql-test/suite/innodb/r/innodb-replace-debug.result | 9 +++++++++ mysql-test/suite/innodb/t/innodb-replace-debug.test | 13 +++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 mysql-test/suite/innodb/r/innodb-replace-debug.result create mode 100644 mysql-test/suite/innodb/t/innodb-replace-debug.test diff --git a/mysql-test/suite/innodb/r/innodb-replace-debug.result b/mysql-test/suite/innodb/r/innodb-replace-debug.result new file mode 100644 index 00000000000..0de351efded --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-replace-debug.result @@ -0,0 +1,9 @@ +# +# Bug#17604730 ASSERTION: *CURSOR->INDEX->NAME == TEMP_INDEX_PREFIX +# +create table t1 (f1 int primary key, f2 int, f3 int, unique key k1(f2), +key k2(f3)) engine=innodb; +insert into t1 values (14, 24, 34); +set debug = '+d,row_ins_sec_index_entry_timeout'; +replace into t1 values (14, 25, 34); +drop table t1; diff --git a/mysql-test/suite/innodb/t/innodb-replace-debug.test b/mysql-test/suite/innodb/t/innodb-replace-debug.test new file mode 100644 index 00000000000..250c90ba5b1 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-replace-debug.test @@ -0,0 +1,13 @@ +--source include/have_innodb.inc +--source include/have_debug.inc + +--echo # +--echo # Bug#17604730 ASSERTION: *CURSOR->INDEX->NAME == TEMP_INDEX_PREFIX +--echo # + +create table t1 (f1 int primary key, f2 int, f3 int, unique key k1(f2), + key k2(f3)) engine=innodb; +insert into t1 values (14, 24, 34); +set debug = '+d,row_ins_sec_index_entry_timeout'; +replace into t1 values (14, 25, 34); +drop table t1; -- cgit v1.2.1 From 93326ef051350787a3b289f68137365224a5e77a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 16 Nov 2017 13:21:07 +0200 Subject: MDEV-9663: InnoDB assertion failure: *cursor->index->name == TEMP_INDEX_PREFIX MariaDB adjustments to test case innodb-replace-debug. MariaDB 10.0 does not seem to be affected. --- mysql-test/suite/innodb/r/innodb-replace-debug.result | 6 +++++- mysql-test/suite/innodb/t/innodb-replace-debug.test | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-replace-debug.result b/mysql-test/suite/innodb/r/innodb-replace-debug.result index 0de351efded..84bc9dc9769 100644 --- a/mysql-test/suite/innodb/r/innodb-replace-debug.result +++ b/mysql-test/suite/innodb/r/innodb-replace-debug.result @@ -4,6 +4,10 @@ create table t1 (f1 int primary key, f2 int, f3 int, unique key k1(f2), key k2(f3)) engine=innodb; insert into t1 values (14, 24, 34); -set debug = '+d,row_ins_sec_index_entry_timeout'; +set @@debug_dbug = '+d,row_ins_sec_index_entry_timeout'; replace into t1 values (14, 25, 34); +select * from t1; +f1 f2 f3 +14 25 34 drop table t1; +set @@debug_dbug = '-d,row_ins_sec_index_entry_timeout'; diff --git a/mysql-test/suite/innodb/t/innodb-replace-debug.test b/mysql-test/suite/innodb/t/innodb-replace-debug.test index 250c90ba5b1..5cec9e1febf 100644 --- a/mysql-test/suite/innodb/t/innodb-replace-debug.test +++ b/mysql-test/suite/innodb/t/innodb-replace-debug.test @@ -8,6 +8,8 @@ create table t1 (f1 int primary key, f2 int, f3 int, unique key k1(f2), key k2(f3)) engine=innodb; insert into t1 values (14, 24, 34); -set debug = '+d,row_ins_sec_index_entry_timeout'; +set @@debug_dbug = '+d,row_ins_sec_index_entry_timeout'; replace into t1 values (14, 25, 34); +select * from t1; drop table t1; +set @@debug_dbug = '-d,row_ins_sec_index_entry_timeout'; -- cgit v1.2.1 From d8ccc61f76d56b761e52564701814739abc190d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 16 Nov 2017 14:03:02 +0200 Subject: MDEV-9663: InnoDB assertion failure: *cursor->index->name == TEMP_INDEX_PREFIX Add missing instrumentation to row0ins.cc. --- storage/innobase/row/row0ins.cc | 4 ++++ storage/xtradb/row/row0ins.cc | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 47f09ac764b..2c1b784109d 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2918,6 +2918,10 @@ row_ins_sec_index_entry( mem_heap_t* offsets_heap; mem_heap_t* heap; + DBUG_EXECUTE_IF("row_ins_sec_index_entry_timeout", { + DBUG_SET("-d,row_ins_sec_index_entry_timeout"); + return(DB_LOCK_WAIT);}); + if (!index->table->foreign_set.empty()) { err = row_ins_check_foreign_constraints(index->table, index, entry, thr); diff --git a/storage/xtradb/row/row0ins.cc b/storage/xtradb/row/row0ins.cc index 6dcb11c8bb5..c9cdee9e6be 100644 --- a/storage/xtradb/row/row0ins.cc +++ b/storage/xtradb/row/row0ins.cc @@ -2991,6 +2991,10 @@ row_ins_sec_index_entry( mem_heap_t* offsets_heap; mem_heap_t* heap; + DBUG_EXECUTE_IF("row_ins_sec_index_entry_timeout", { + DBUG_SET("-d,row_ins_sec_index_entry_timeout"); + return(DB_LOCK_WAIT);}); + if (!index->table->foreign_set.empty()) { err = row_ins_check_foreign_constraints(index->table, index, entry, thr); -- cgit v1.2.1 From 622466644dda1e3c72f5fe276e3a8127b20c1182 Mon Sep 17 00:00:00 2001 From: Ian Gilfillan Date: Mon, 20 Nov 2017 11:00:44 +0200 Subject: mysql_uprade --help and man page fixes --- client/mysql_upgrade.c | 4 ++-- man/mysql_upgrade.1 | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/client/mysql_upgrade.c b/client/mysql_upgrade.c index e131aabd8b6..cecf212ad92 100644 --- a/client/mysql_upgrade.c +++ b/client/mysql_upgrade.c @@ -164,8 +164,8 @@ static struct my_option my_long_options[]= "server with which it was built/distributed.", &opt_version_check, &opt_version_check, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, - {"write-binlog", OPT_WRITE_BINLOG, "All commands including those, " - "issued by mysqlcheck, are written to the binary log.", + {"write-binlog", OPT_WRITE_BINLOG, "All commands including those " + "issued by mysqlcheck are written to the binary log.", &opt_write_binlog, &opt_write_binlog, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} diff --git a/man/mysql_upgrade.1 b/man/mysql_upgrade.1 index 3948a670436..200037de41a 100644 --- a/man/mysql_upgrade.1 +++ b/man/mysql_upgrade.1 @@ -691,8 +691,7 @@ it was built/distributed. Defaults to on; use \fB\-\-skip\-version\-check\fR to .sp Cause binary logging to be enabled while \fBmysql_upgrade\fR -runs\&. This is the default behavior; to disable binary logging during the upgrade, use the inverse of this option (that is, start the program with -\fB\-\-skip\-write\-binlog\fR)\&. +runs\&. .RE .SH "COPYRIGHT" .br -- cgit v1.2.1 From 1374f958c120ee0fdf3f4b017f1289aba87dae07 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 7 Dec 2017 17:22:24 +0200 Subject: Fixed failing tokudb tests This was caused by my earlier fix for rpl.rpl_row_log_innodb :( --- storage/tokudb/mysql-test/rpl/r/rpl_tokudb_row_log.result | 2 -- storage/tokudb/mysql-test/rpl/r/rpl_tokudb_stm_log.result | 2 -- 2 files changed, 4 deletions(-) diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_row_log.result b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_row_log.result index 8cbbda48c1f..f283b3adf80 100644 --- a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_row_log.result +++ b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_row_log.result @@ -215,7 +215,6 @@ master-bin.000001 # Xid # # COMMIT /* XID */ master-bin.000001 # Rotate # # master-bin.000002;pos=POS include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info -master-bin.000002 # Binlog_checkpoint # # master-bin.000002 master-bin.000002 # Gtid # # GTID #-#-# master-bin.000002 # Query # # use `test`; create table t3 (a int)ENGINE=TokuDB master-bin.000002 # Gtid # # GTID #-#-# @@ -253,7 +252,6 @@ slave-bin.000001 # Query # # use `test`; create table t3 (a int)ENGINE=TokuDB slave-bin.000001 # Rotate # # slave-bin.000002;pos=POS include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info -slave-bin.000002 # Binlog_checkpoint # # slave-bin.000002 slave-bin.000002 # Gtid # # GTID #-#-# slave-bin.000002 # Query # # use `test`; create table t2 (n int)ENGINE=TokuDB slave-bin.000002 # Gtid # # BEGIN GTID #-#-# diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_stm_log.result b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_stm_log.result index 42defbe07b5..0334000f12e 100644 --- a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_stm_log.result +++ b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_stm_log.result @@ -215,7 +215,6 @@ master-bin.000001 # Xid # # COMMIT /* XID */ master-bin.000001 # Rotate # # master-bin.000002;pos=POS include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info -master-bin.000002 # Binlog_checkpoint # # master-bin.000002 master-bin.000002 # Gtid # # GTID #-#-# master-bin.000002 # Query # # use `test`; create table t3 (a int)ENGINE=TokuDB master-bin.000002 # Gtid # # GTID #-#-# @@ -252,7 +251,6 @@ slave-bin.000001 # Query # # use `test`; create table t3 (a int)ENGINE=TokuDB slave-bin.000001 # Rotate # # slave-bin.000002;pos=POS include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info -slave-bin.000002 # Binlog_checkpoint # # slave-bin.000002 slave-bin.000002 # Gtid # # GTID #-#-# slave-bin.000002 # Query # # use `test`; create table t2 (n int)ENGINE=TokuDB slave-bin.000002 # Gtid # # BEGIN GTID #-#-# -- cgit v1.2.1 From e66bb5726716b7852c880e02b0acf0f5b9a1e8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sat, 9 Dec 2017 11:20:46 +0200 Subject: MDEV-12837: WSREP: BF lock wait long This is 10.1 version where no merge error exists. wsrep_on_check New check function. Galera can't be enabled if innodb-lock-schedule-algorithm=VATS. innobase_kill_query In Galera async kill we could own lock mutex. innobase_init If Variance-Aware-Transaction-Sheduling Algorithm (VATS) is used on Galera we refuse to start InnoDB. Changed innodb-lock-schedule-algorithm as read-only parameter as it was designed to be. lock_rec_other_has_expl_req, lock_rec_other_has_conflicting, lock_rec_lock_slow lock_table_other_has_incompatible lock_rec_insert_check_and_lock Change pointer to conflicting lock to normal pointer as this pointer contents could be changed later. --- mysql-test/suite/sys_vars/r/sysvars_innodb.result | 2 +- mysql-test/suite/sys_vars/t/wsrep_on_basic.opt | 1 + sql/sys_vars.cc | 3 +- sql/wsrep_var.cc | 16 ++++ sql/wsrep_var.h | 3 +- storage/innobase/handler/ha_innodb.cc | 17 +++- storage/innobase/lock/lock0lock.cc | 94 +++++++++++++++-------- storage/innobase/lock/lock0wait.cc | 39 +++++----- storage/xtradb/handler/ha_innodb.cc | 17 +++- storage/xtradb/lock/lock0lock.cc | 91 ++++++++++++++-------- storage/xtradb/lock/lock0wait.cc | 39 +++++----- 11 files changed, 215 insertions(+), 107 deletions(-) create mode 100644 mysql-test/suite/sys_vars/t/wsrep_on_basic.opt diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index acdbd007e6e..161f740dbfb 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1347,7 +1347,7 @@ NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST fcfs,vats -READ_ONLY NO +READ_ONLY YES COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOCK_WAIT_TIMEOUT SESSION_VALUE 50 diff --git a/mysql-test/suite/sys_vars/t/wsrep_on_basic.opt b/mysql-test/suite/sys_vars/t/wsrep_on_basic.opt new file mode 100644 index 00000000000..aa1fb6cb155 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/wsrep_on_basic.opt @@ -0,0 +1 @@ +--innodb-lock-schedule-algorithm=FCFS diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 8eb3d35a96e..303633939c3 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -4895,7 +4895,8 @@ static Sys_var_mybool Sys_wsrep_on ( "wsrep_on", "To enable wsrep replication ", SESSION_VAR(wsrep_on), CMD_LINE(OPT_ARG), DEFAULT(FALSE), - NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(wsrep_on_check), ON_UPDATE(wsrep_on_update)); static Sys_var_charptr Sys_wsrep_start_position ( diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index b21041eb0f8..ad1f4ec0eac 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -52,12 +52,28 @@ int wsrep_init_vars() return 0; } +extern ulong innodb_lock_schedule_algorithm; + bool wsrep_on_update (sys_var *self, THD* thd, enum_var_type var_type) { if (var_type == OPT_GLOBAL) { // FIXME: this variable probably should be changed only per session thd->variables.wsrep_on = global_system_variables.wsrep_on; } + + return false; +} + +bool wsrep_on_check(sys_var *self, THD* thd, set_var* var) +{ + bool new_wsrep_on= (bool)var->save_result.ulonglong_value; + + if (new_wsrep_on && innodb_lock_schedule_algorithm != 0) { + my_message(ER_WRONG_ARGUMENTS, " WSREP (galera) can't be enabled " + "if innodb_lock_schedule_algorithm=VATS. Please configure" + " innodb_lock_schedule_algorithm=FCFS and restart.", MYF(0)); + return true; + } return false; } diff --git a/sql/wsrep_var.h b/sql/wsrep_var.h index 7530fd98870..55eb2fbc501 100644 --- a/sql/wsrep_var.h +++ b/sql/wsrep_var.h @@ -41,7 +41,8 @@ int wsrep_init_vars(); #define DEFAULT_ARGS (THD* thd, enum_var_type var_type) #define INIT_ARGS (const char* opt) -extern bool wsrep_causal_reads_update UPDATE_ARGS; +extern bool wsrep_causal_reads_update UPDATE_ARGS; +extern bool wsrep_on_check CHECK_ARGS; extern bool wsrep_on_update UPDATE_ARGS; extern bool wsrep_sync_wait_update UPDATE_ARGS; extern bool wsrep_start_position_check CHECK_ARGS; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 9a7258bc924..2d319439fad 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3478,6 +3478,17 @@ innobase_init( goto error; } +#ifdef WITH_WSREP + /* Currently, Galera does not support VATS lock schedule algorithm. */ + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS + && global_system_variables.wsrep_on) { + /* Do not allow InnoDB startup with VATS and Galera */ + sql_print_error("In Galera environment Variance-Aware-Transaction-Sheduling Algorithm" + " is not supported."); + goto error; + } +#endif /* WITH_WSREP */ + #ifndef HAVE_LZ4 if (innodb_compression_algorithm == PAGE_LZ4_ALGORITHM) { sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" @@ -4882,8 +4893,8 @@ innobase_kill_query( wsrep_thd_is_BF(current_thd, FALSE), lock_get_info(trx->lock.wait_lock).c_str()); - if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE) && - trx->abort_type == TRX_SERVER_ABORT) { + if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE) + && trx->abort_type == TRX_SERVER_ABORT) { ut_ad(!lock_mutex_own()); lock_mutex_enter(); } @@ -19234,7 +19245,7 @@ static MYSQL_SYSVAR_ULONG(doublewrite_batch_size, srv_doublewrite_batch_size, #endif /* defined UNIV_DEBUG || defined UNIV_PERF_DEBUG */ static MYSQL_SYSVAR_ENUM(lock_schedule_algorithm, innodb_lock_schedule_algorithm, - PLUGIN_VAR_RQCMDARG, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "The algorithm Innodb uses for deciding which locks to grant next when" " a lock is released. Possible values are" " FCFS" diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 6b2c309e432..61385891b76 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -925,15 +925,21 @@ lock_reset_lock_and_trx_wait( ib_logf(IB_LOG_LEVEL_INFO, "Trx id " TRX_ID_FMT - " is waiting a lock in statement %s" + " is waiting a lock " " for this trx id " TRX_ID_FMT - " and statement %s wait_lock %p", + " wait_lock %p", lock->trx->id, - stmt ? stmt : "NULL", trx_id, - stmt2 ? stmt2 : "NULL", lock->trx->lock.wait_lock); + if (stmt) { + ib_logf(IB_LOG_LEVEL_INFO, " SQL1: %s\n", stmt); + } + + if (stmt2) { + ib_logf(IB_LOG_LEVEL_INFO, " SQL2: %s\n", stmt2); + } + ut_ad(lock->trx->lock.wait_lock == lock); } @@ -1151,7 +1157,7 @@ lock_rec_has_to_wait( type_mode, lock_is_on_supremum); fprintf(stderr, "conflicts states: my %d locked %d\n", - wsrep_thd_conflict_state(trx->mysql_thd, FALSE), + wsrep_thd_conflict_state(trx->mysql_thd, FALSE), wsrep_thd_conflict_state(lock2->trx->mysql_thd, FALSE) ); lock_rec_print(stderr, lock2); if (for_locking) return FALSE; @@ -1687,7 +1693,7 @@ lock_rec_discard(lock_t* in_lock); Checks if some other transaction has a lock request in the queue. @return lock or NULL */ static -const lock_t* +lock_t* lock_rec_other_has_expl_req( /*========================*/ enum lock_mode mode, /*!< in: LOCK_S or LOCK_X */ @@ -1704,7 +1710,7 @@ lock_rec_other_has_expl_req( requests by all transactions are taken into account */ { - const lock_t* lock; + lock_t* lock; ut_ad(lock_mutex_own()); ut_ad(mode == LOCK_X || mode == LOCK_S); @@ -1713,7 +1719,7 @@ lock_rec_other_has_expl_req( for (lock = lock_rec_get_first(block, heap_no); lock != NULL; - lock = lock_rec_get_next_const(heap_no, lock)) { + lock = lock_rec_get_next(heap_no, lock)) { if (lock->trx != trx && (gap @@ -1800,7 +1806,7 @@ Checks if some other transaction has a conflicting explicit lock request in the queue, so that we have to wait. @return lock or NULL */ static -const lock_t* +lock_t* lock_rec_other_has_conflicting( /*===========================*/ enum lock_mode mode, /*!< in: LOCK_S or LOCK_X, @@ -1812,7 +1818,7 @@ lock_rec_other_has_conflicting( ulint heap_no,/*!< in: heap number of the record */ const trx_t* trx) /*!< in: our transaction */ { - const lock_t* lock; + lock_t* lock; ibool is_supremum; ut_ad(lock_mutex_own()); @@ -1821,13 +1827,16 @@ lock_rec_other_has_conflicting( for (lock = lock_rec_get_first(block, heap_no); lock != NULL; - lock = lock_rec_get_next_const(heap_no, lock)) { + lock = lock_rec_get_next(heap_no, lock)) { #ifdef WITH_WSREP if (lock_rec_has_to_wait(TRUE, trx, mode, lock, is_supremum)) { if (wsrep_on_trx(trx)) { trx_mutex_enter(lock->trx); - wsrep_kill_victim(trx, lock); + /* Below function will roll back either trx + or lock->trx depending on priority of the + transaction. */ + wsrep_kill_victim(const_cast(trx), lock); trx_mutex_exit(lock->trx); } #else @@ -2023,15 +2032,17 @@ wsrep_print_wait_locks( { if (wsrep_debug && c_lock->trx->lock.wait_lock != c_lock) { fprintf(stderr, "WSREP: c_lock != wait lock\n"); - if (lock_get_type_low(c_lock) & LOCK_TABLE) + if (lock_get_type_low(c_lock) & LOCK_TABLE) { lock_table_print(stderr, c_lock); - else + } else { lock_rec_print(stderr, c_lock); + } - if (lock_get_type_low(c_lock->trx->lock.wait_lock) & LOCK_TABLE) + if (lock_get_type_low(c_lock->trx->lock.wait_lock) & LOCK_TABLE) { lock_table_print(stderr, c_lock->trx->lock.wait_lock); - else + } else { lock_rec_print(stderr, c_lock->trx->lock.wait_lock); + } } } #endif /* WITH_WSREP */ @@ -2217,8 +2228,8 @@ lock_rec_create( if (wsrep_debug) { fprintf( stderr, - "WSREP: c_lock canceled %llu\n", - (ulonglong) c_lock->trx->id); + "WSREP: c_lock canceled " TRX_ID_FMT "\n", + c_lock->trx->id); } /* have to bail out here to avoid lock_set_lock... */ @@ -2519,6 +2530,16 @@ lock_rec_enqueue_waiting( err = DB_LOCK_WAIT; } +#ifdef WITH_WSREP + if (!lock_get_wait(lock) && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + if (wsrep_debug) { + fprintf(stderr, "WSREP: BF thread got lock granted early, ID " TRX_ID_FMT + "\n", + lock->trx->id); + } + return(DB_SUCCESS); + } +#endif /* WITH_WSREP */ // Move it only when it does not cause a deadlock. if (err != DB_DEADLOCK && innodb_lock_schedule_algorithm @@ -2814,7 +2835,7 @@ lock_rec_lock_slow( /* The trx already has a strong enough lock on rec: do nothing */ #ifdef WITH_WSREP - } else if ((c_lock = (ib_lock_t*)lock_rec_other_has_conflicting( + } else if ((c_lock = lock_rec_other_has_conflicting( static_cast(mode), block, heap_no, trx))) { #else @@ -2946,6 +2967,15 @@ lock_rec_has_to_wait_in_queue( #ifdef WITH_WSREP if (wsrep_thd_is_BF(wait_lock->trx->mysql_thd, FALSE) && wsrep_thd_is_BF(lock->trx->mysql_thd, TRUE)) { + if (wsrep_debug) { + fprintf(stderr, + "BF-BF lock conflict " TRX_ID_FMT + " : " TRX_ID_FMT "\n", + wait_lock->trx->id, + lock->trx->id); + lock_rec_print(stderr, wait_lock); + lock_rec_print(stderr, lock); + } /* don't wait for another BF lock */ continue; } @@ -3104,7 +3134,7 @@ lock_grant_and_move_on_page( && !lock_rec_has_to_wait_in_queue(lock)) { lock_grant(lock, false); - + if (previous != NULL) { /* Move the lock to the head of the list. */ HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); @@ -4978,8 +5008,8 @@ lock_table_create( } if (wsrep_debug) { - fprintf(stderr, "WSREP: c_lock canceled %llu\n", - (ulonglong) c_lock->trx->id); + fprintf(stderr, "WSREP: c_lock canceled " TRX_ID_FMT "\n", + c_lock->trx->id); } } if (c_lock) { @@ -5251,7 +5281,7 @@ Checks if other transactions have an incompatible mode lock request in the lock queue. @return lock or NULL */ UNIV_INLINE -const lock_t* +lock_t* lock_table_other_has_incompatible( /*==============================*/ const trx_t* trx, /*!< in: transaction, or NULL if all @@ -5262,7 +5292,7 @@ lock_table_other_has_incompatible( const dict_table_t* table, /*!< in: table */ enum lock_mode mode) /*!< in: lock mode */ { - const lock_t* lock; + lock_t* lock; ut_ad(lock_mutex_own()); @@ -5315,7 +5345,7 @@ lock_table( #endif trx_t* trx; dberr_t err; - const lock_t* wait_for; + lock_t* wait_for; ut_ad(table != NULL); ut_ad(thr != NULL); @@ -5362,13 +5392,13 @@ lock_table( if (wait_for != NULL) { #ifdef WITH_WSREP - err = lock_table_enqueue_waiting((ib_lock_t*)wait_for, mode | flags, table, thr); + err = lock_table_enqueue_waiting(wait_for, mode | flags, table, thr); #else err = lock_table_enqueue_waiting(mode | flags, table, thr); #endif } else { #ifdef WITH_WSREP - lock_table_create(c_lock, table, mode | flags, trx); + lock_table_create(c_lock, table, mode | flags, trx); #else lock_table_create(table, mode | flags, trx); #endif @@ -7036,10 +7066,10 @@ lock_rec_insert_check_and_lock( on the successor, which produced an unnecessary deadlock. */ #ifdef WITH_WSREP - if ((c_lock = (ib_lock_t*)lock_rec_other_has_conflicting( - static_cast( - LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION), - block, next_rec_heap_no, trx))) { + if ((c_lock = lock_rec_other_has_conflicting( + static_cast( + LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION), + block, next_rec_heap_no, trx))) { #else if (lock_rec_other_has_conflicting( static_cast( @@ -7052,7 +7082,7 @@ lock_rec_insert_check_and_lock( #ifdef WITH_WSREP err = lock_rec_enqueue_waiting(c_lock, - LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION, + LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION, block, next_rec_heap_no, index, thr); #else err = lock_rec_enqueue_waiting( diff --git a/storage/innobase/lock/lock0wait.cc b/storage/innobase/lock/lock0wait.cc index ca9d05a4829..a0f557e18e5 100644 --- a/storage/innobase/lock/lock0wait.cc +++ b/storage/innobase/lock/lock0wait.cc @@ -191,22 +191,25 @@ lock_wait_table_reserve_slot( /*********************************************************************//** check if lock timeout was for priority thread, as a side effect trigger lock monitor +@param[in] trx transaction owning the lock +@param[in] locked true if trx and lock_sys_mutex is ownd @return false for regular lock timeout */ -static ibool +static +bool wsrep_is_BF_lock_timeout( -/*====================*/ - trx_t* trx) /* in: trx to check for lock priority */ + const trx_t* trx, + bool locked = true) { - if (wsrep_on_trx(trx) && - wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { - fprintf(stderr, "WSREP: BF lock wait long\n"); - srv_print_innodb_monitor = TRUE; - srv_print_innodb_lock_monitor = TRUE; - os_event_set(srv_monitor_event); - return TRUE; - } - return FALSE; - } + if (wsrep_on_trx(trx) + && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + fprintf(stderr, "WSREP: BF lock wait long for trx " TRX_ID_FMT "\n", trx->id); + srv_print_innodb_monitor = TRUE; + srv_print_innodb_lock_monitor = TRUE; + os_event_set(srv_monitor_event); + return true; + } + return false; +} #endif /* WITH_WSREP */ /***************************************************************//** @@ -402,15 +405,15 @@ lock_wait_suspend_thread( if (lock_wait_timeout < 100000000 && wait_time > (double) lock_wait_timeout) { #ifdef WITH_WSREP - if (!wsrep_on_trx(trx) || - (!wsrep_is_BF_lock_timeout(trx) && - trx->error_state != DB_DEADLOCK)) { + if (!wsrep_on_trx(trx) || + (!wsrep_is_BF_lock_timeout(trx) && + trx->error_state != DB_DEADLOCK)) { #endif /* WITH_WSREP */ - trx->error_state = DB_LOCK_WAIT_TIMEOUT; + trx->error_state = DB_LOCK_WAIT_TIMEOUT; #ifdef WITH_WSREP - } + } #endif /* WITH_WSREP */ MONITOR_INC(MONITOR_TIMEOUT); } diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 28e3b23b667..8487c5b2510 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -3934,6 +3934,17 @@ innobase_init( } } +#ifdef WITH_WSREP + /* Currently, Galera does not support VATS lock schedule algorithm. */ + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS + && global_system_variables.wsrep_on) { + /* Do not allow InnoDB startup with VATS and Galera */ + sql_print_error("In Galera environment Variance-Aware-Transaction-Sheduling Algorithm" + " is not supported."); + goto error; + } +#endif /* WITH_WSREP */ + #ifndef HAVE_LZ4 if (innodb_compression_algorithm == PAGE_LZ4_ALGORITHM) { sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" @@ -5485,8 +5496,8 @@ innobase_kill_connection( wsrep_thd_is_BF(current_thd, FALSE), lock_get_info(trx->lock.wait_lock).c_str()); - if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE) && - trx->abort_type == TRX_SERVER_ABORT) { + if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE) + && trx->abort_type == TRX_SERVER_ABORT) { ut_ad(!lock_mutex_own()); lock_mutex_enter(); } @@ -20500,7 +20511,7 @@ static MYSQL_SYSVAR_ENUM(empty_free_list_algorithm, &innodb_empty_free_list_algorithm_typelib); static MYSQL_SYSVAR_ENUM(lock_schedule_algorithm, innodb_lock_schedule_algorithm, - PLUGIN_VAR_RQCMDARG, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "The algorithm Innodb uses for deciding which locks to grant next when" " a lock is released. Possible values are" " FCFS" diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index ddaeff69f10..20e3f5adeb7 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -937,14 +937,21 @@ lock_reset_lock_and_trx_wait( ib_logf(IB_LOG_LEVEL_INFO, "Trx id " TRX_ID_FMT - " is waiting a lock in statement %s" + " is waiting a lock " " for this trx id " TRX_ID_FMT - " and statement %s wait_lock %p", + " wait_lock %p", lock->trx->id, - stmt ? stmt : "NULL", trx_id, - stmt2 ? stmt2 : "NULL", lock->trx->lock.wait_lock); + + if (stmt) { + ib_logf(IB_LOG_LEVEL_INFO, " SQL1: %s\n", stmt); + } + + if (stmt2) { + ib_logf(IB_LOG_LEVEL_INFO, " SQL2: %s\n", stmt2); + } + ut_ad(lock->trx->lock.wait_lock == lock); } @@ -1162,7 +1169,7 @@ lock_rec_has_to_wait( type_mode, lock_is_on_supremum); fprintf(stderr, "conflicts states: my %d locked %d\n", - wsrep_thd_conflict_state(trx->mysql_thd, FALSE), + wsrep_thd_conflict_state(trx->mysql_thd, FALSE), wsrep_thd_conflict_state(lock2->trx->mysql_thd, FALSE) ); lock_rec_print(stderr, lock2); if (for_locking) return FALSE; @@ -1714,7 +1721,7 @@ lock_rec_other_has_expl_req( ulint heap_no,/*!< in: heap number of the record */ trx_id_t trx_id) /*!< in: transaction */ { - const lock_t* lock; + lock_t* lock; ut_ad(lock_mutex_own()); ut_ad(mode == LOCK_X || mode == LOCK_S); @@ -1723,7 +1730,7 @@ lock_rec_other_has_expl_req( for (lock = lock_rec_get_first(block, heap_no); lock != NULL; - lock = lock_rec_get_next_const(heap_no, lock)) { + lock = lock_rec_get_next(heap_no, lock)) { if (lock->trx->id != trx_id && (gap @@ -1810,7 +1817,7 @@ Checks if some other transaction has a conflicting explicit lock request in the queue, so that we have to wait. @return lock or NULL */ static -const lock_t* +lock_t* lock_rec_other_has_conflicting( /*===========================*/ enum lock_mode mode, /*!< in: LOCK_S or LOCK_X, @@ -1822,7 +1829,7 @@ lock_rec_other_has_conflicting( ulint heap_no,/*!< in: heap number of the record */ const trx_t* trx) /*!< in: our transaction */ { - const lock_t* lock; + lock_t* lock; ibool is_supremum; ut_ad(lock_mutex_own()); @@ -1831,13 +1838,16 @@ lock_rec_other_has_conflicting( for (lock = lock_rec_get_first(block, heap_no); lock != NULL; - lock = lock_rec_get_next_const(heap_no, lock)) { + lock = lock_rec_get_next(heap_no, lock)) { #ifdef WITH_WSREP if (lock_rec_has_to_wait(TRUE, trx, mode, lock, is_supremum)) { if (wsrep_on_trx(trx)) { trx_mutex_enter(lock->trx); - wsrep_kill_victim(trx, lock); + /* Below function will roll back either trx + or lock->trx depending on priority of the + transaction. */ + wsrep_kill_victim(const_cast(trx), lock); trx_mutex_exit(lock->trx); } #else @@ -2045,15 +2055,17 @@ wsrep_print_wait_locks( { if (wsrep_debug && c_lock->trx->lock.wait_lock != c_lock) { fprintf(stderr, "WSREP: c_lock != wait lock\n"); - if (lock_get_type_low(c_lock) & LOCK_TABLE) + if (lock_get_type_low(c_lock) & LOCK_TABLE) { lock_table_print(stderr, c_lock); - else + } else { lock_rec_print(stderr, c_lock); + } - if (lock_get_type_low(c_lock->trx->lock.wait_lock) & LOCK_TABLE) + if (lock_get_type_low(c_lock->trx->lock.wait_lock) & LOCK_TABLE) { lock_table_print(stderr, c_lock->trx->lock.wait_lock); - else + } else { lock_rec_print(stderr, c_lock->trx->lock.wait_lock); + } } } #endif /* WITH_WSREP */ @@ -2358,8 +2370,8 @@ lock_rec_create( if (wsrep_debug) { fprintf( stderr, - "WSREP: c_lock canceled %llu\n", - (ulonglong) c_lock->trx->id); + "WSREP: c_lock canceled " TRX_ID_FMT "\n", + c_lock->trx->id); } /* have to bail out here to avoid lock_set_lock... */ @@ -2551,6 +2563,16 @@ lock_rec_enqueue_waiting( err = DB_LOCK_WAIT; } +#ifdef WITH_WSREP + if (!lock_get_wait(lock) && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + if (wsrep_debug) { + fprintf(stderr, "WSREP: BF thread got lock granted early, ID " TRX_ID_FMT + "\n", + lock->trx->id); + } + return(DB_SUCCESS); + } +#endif /* WITH_WSREP */ // Move it only when it does not cause a deadlock. if (err != DB_DEADLOCK && innodb_lock_schedule_algorithm @@ -2981,6 +3003,15 @@ lock_rec_has_to_wait_in_queue( #ifdef WITH_WSREP if (wsrep_thd_is_BF(wait_lock->trx->mysql_thd, FALSE) && wsrep_thd_is_BF(lock->trx->mysql_thd, TRUE)) { + if (wsrep_debug) { + fprintf(stderr, + "BF-BF lock conflict " TRX_ID_FMT + " : " TRX_ID_FMT "\n", + wait_lock->trx->id, + lock->trx->id); + lock_rec_print(stderr, wait_lock); + lock_rec_print(stderr, lock); + } /* don't wait for another BF lock */ continue; } @@ -3139,7 +3170,7 @@ lock_grant_and_move_on_page( && !lock_rec_has_to_wait_in_queue(lock)) { lock_grant(lock, false); - + if (previous != NULL) { /* Move the lock to the head of the list. */ HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); @@ -5017,8 +5048,8 @@ lock_table_create( } if (wsrep_debug) { - fprintf(stderr, "WSREP: c_lock canceled %llu\n", - (ulonglong) c_lock->trx->id); + fprintf(stderr, "WSREP: c_lock canceled " TRX_ID_FMT "\n", + c_lock->trx->id); } } if (c_lock) { @@ -5297,7 +5328,7 @@ Checks if other transactions have an incompatible mode lock request in the lock queue. @return lock or NULL */ UNIV_INLINE -const lock_t* +lock_t* lock_table_other_has_incompatible( /*==============================*/ const trx_t* trx, /*!< in: transaction, or NULL if all @@ -5308,7 +5339,7 @@ lock_table_other_has_incompatible( const dict_table_t* table, /*!< in: table */ enum lock_mode mode) /*!< in: lock mode */ { - const lock_t* lock; + lock_t* lock; ut_ad(lock_mutex_own()); @@ -5361,7 +5392,7 @@ lock_table( #endif trx_t* trx; dberr_t err; - const lock_t* wait_for; + lock_t* wait_for; ut_ad(table != NULL); ut_ad(thr != NULL); @@ -5412,13 +5443,13 @@ lock_table( if (wait_for != NULL) { #ifdef WITH_WSREP - err = lock_table_enqueue_waiting((ib_lock_t*)wait_for, mode | flags, table, thr); + err = lock_table_enqueue_waiting(wait_for, mode | flags, table, thr); #else err = lock_table_enqueue_waiting(mode | flags, table, thr); #endif } else { #ifdef WITH_WSREP - lock_table_create(c_lock, table, mode | flags, trx); + lock_table_create(c_lock, table, mode | flags, trx); #else lock_table_create(table, mode | flags, trx); #endif @@ -7101,10 +7132,10 @@ lock_rec_insert_check_and_lock( on the successor, which produced an unnecessary deadlock. */ #ifdef WITH_WSREP - if ((c_lock = (ib_lock_t*)lock_rec_other_has_conflicting( - static_cast( - LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION), - block, next_rec_heap_no, trx))) { + if ((c_lock = lock_rec_other_has_conflicting( + static_cast( + LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION), + block, next_rec_heap_no, trx))) { #else if (lock_rec_other_has_conflicting( static_cast( @@ -7117,7 +7148,7 @@ lock_rec_insert_check_and_lock( #ifdef WITH_WSREP err = lock_rec_enqueue_waiting(c_lock, - LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION, + LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION, block, next_rec_heap_no, index, thr); #else err = lock_rec_enqueue_waiting( diff --git a/storage/xtradb/lock/lock0wait.cc b/storage/xtradb/lock/lock0wait.cc index ca9d05a4829..a0f557e18e5 100644 --- a/storage/xtradb/lock/lock0wait.cc +++ b/storage/xtradb/lock/lock0wait.cc @@ -191,22 +191,25 @@ lock_wait_table_reserve_slot( /*********************************************************************//** check if lock timeout was for priority thread, as a side effect trigger lock monitor +@param[in] trx transaction owning the lock +@param[in] locked true if trx and lock_sys_mutex is ownd @return false for regular lock timeout */ -static ibool +static +bool wsrep_is_BF_lock_timeout( -/*====================*/ - trx_t* trx) /* in: trx to check for lock priority */ + const trx_t* trx, + bool locked = true) { - if (wsrep_on_trx(trx) && - wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { - fprintf(stderr, "WSREP: BF lock wait long\n"); - srv_print_innodb_monitor = TRUE; - srv_print_innodb_lock_monitor = TRUE; - os_event_set(srv_monitor_event); - return TRUE; - } - return FALSE; - } + if (wsrep_on_trx(trx) + && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + fprintf(stderr, "WSREP: BF lock wait long for trx " TRX_ID_FMT "\n", trx->id); + srv_print_innodb_monitor = TRUE; + srv_print_innodb_lock_monitor = TRUE; + os_event_set(srv_monitor_event); + return true; + } + return false; +} #endif /* WITH_WSREP */ /***************************************************************//** @@ -402,15 +405,15 @@ lock_wait_suspend_thread( if (lock_wait_timeout < 100000000 && wait_time > (double) lock_wait_timeout) { #ifdef WITH_WSREP - if (!wsrep_on_trx(trx) || - (!wsrep_is_BF_lock_timeout(trx) && - trx->error_state != DB_DEADLOCK)) { + if (!wsrep_on_trx(trx) || + (!wsrep_is_BF_lock_timeout(trx) && + trx->error_state != DB_DEADLOCK)) { #endif /* WITH_WSREP */ - trx->error_state = DB_LOCK_WAIT_TIMEOUT; + trx->error_state = DB_LOCK_WAIT_TIMEOUT; #ifdef WITH_WSREP - } + } #endif /* WITH_WSREP */ MONITOR_INC(MONITOR_TIMEOUT); } -- cgit v1.2.1 From feb8296ee61cbd349a8bc13cac4e126306039fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sat, 9 Dec 2017 11:21:23 +0200 Subject: MDEV-14401: Stored procedure that declares a handler that catches ER_LOCK_DEADLOCK error causes thd->is_error() assertion This was missing bug fix from MySQL wsrep i.e. Galera. Problem was that if stored procedure declares a handler that catches deadlock error, then the error may have been cleared in method sp_rcontext::handle_sql_condition(). Use wsrep_conflict_state correctly to determine is the error already sent to client. Add test case for both this bug and MDEV-12837: WSREP: BF lock wait long. Test requires both fixes to pass. --- .../suite/galera/r/galera_bf_lock_wait.result | 18 ++++++++ mysql-test/suite/galera/t/galera_bf_lock_wait.test | 52 ++++++++++++++++++++++ sql/sql_parse.cc | 33 +++++++++----- 3 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_bf_lock_wait.result create mode 100644 mysql-test/suite/galera/t/galera_bf_lock_wait.test diff --git a/mysql-test/suite/galera/r/galera_bf_lock_wait.result b/mysql-test/suite/galera/r/galera_bf_lock_wait.result new file mode 100644 index 00000000000..4e6019ec8ad --- /dev/null +++ b/mysql-test/suite/galera/r/galera_bf_lock_wait.result @@ -0,0 +1,18 @@ +CREATE TABLE t1 ENGINE=InnoDB select 1 as a, 1 as b union select 2, 2; +ALTER TABLE t1 add primary key(a); +CREATE PROCEDURE p1() +BEGIN +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION rollback; +WHILE 1 DO +start transaction; +update t1 set b=connection_id() where a=1; +commit; +END WHILE; +END| +call p1; +call p1; +call p1; +call p1; +checking error log for 'BF lock wait long' message for 10 times every 10 seconds ... +drop table t1; +drop procedure p1; diff --git a/mysql-test/suite/galera/t/galera_bf_lock_wait.test b/mysql-test/suite/galera/t/galera_bf_lock_wait.test new file mode 100644 index 00000000000..e3a9077a888 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_bf_lock_wait.test @@ -0,0 +1,52 @@ +--source include/galera_cluster.inc +--source include/big_test.inc + +CREATE TABLE t1 ENGINE=InnoDB select 1 as a, 1 as b union select 2, 2; +ALTER TABLE t1 add primary key(a); + +DELIMITER |; + +CREATE PROCEDURE p1() +BEGIN + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION rollback; + WHILE 1 DO + start transaction; + update t1 set b=connection_id() where a=1; + commit; + END WHILE; +END| + + +DELIMITER ;| + +--connect node_1_p1, 127.0.0.1, root, , test, $NODE_MYPORT_1 +send call p1; +--connect node_1_p2, 127.0.0.1, root, , test, $NODE_MYPORT_1 +send call p1; +--connect node_2_p1, 127.0.0.1, root, , test, $NODE_MYPORT_2 +send call p1; +--connect node_2_p2, 127.0.0.1, root, , test, $NODE_MYPORT_2 +send call p1; + +connection default; +let $counter=10; +let $sleep_period=10; + +echo checking error log for 'BF lock wait long' message for $counter times every $sleep_period seconds ...; +while($counter > 0) +{ +--disable_query_log +--disable_result_log + eval do sleep($sleep_period); +--enable_query_log +--enable_result_log + +# use error 0,1 instead if want test to continue + --error 1 + exec grep 'BF lock wait long' $MYSQLTEST_VARDIR/log/mysqld.*.err; + dec $counter; +} + +drop table t1; +drop procedure p1; + diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 7337a8aeb21..5cea264e4a8 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -5041,14 +5041,19 @@ end_with_restore_list: thd->print_aborted_warning(3, "RELEASE"); } #ifdef WITH_WSREP - if (WSREP(thd) && (thd->wsrep_conflict_state != NO_CONFLICT && - thd->wsrep_conflict_state != REPLAYING)) - { - DBUG_ASSERT(thd->is_error()); // the error is already issued + if (WSREP(thd)) { + + if (thd->wsrep_conflict_state == NO_CONFLICT || + thd->wsrep_conflict_state == REPLAYING) + { + my_ok(thd); + } + } else { +#endif /* WITH_WSREP */ + my_ok(thd); +#ifdef WITH_WSREP } - else #endif /* WITH_WSREP */ - my_ok(thd); break; } case SQLCOM_ROLLBACK: @@ -5085,13 +5090,16 @@ end_with_restore_list: if (tx_release) thd->set_killed(KILL_CONNECTION); #ifdef WITH_WSREP - if (WSREP(thd) && thd->wsrep_conflict_state != NO_CONFLICT) - { - DBUG_ASSERT(thd->is_error()); // the error is already issued + if (WSREP(thd)) { + if (thd->wsrep_conflict_state == NO_CONFLICT) { + my_ok(thd); + } + } else { +#endif /* WITH_WSREP */ + my_ok(thd); +#ifdef WITH_WSREP } - else #endif /* WITH_WSREP */ - my_ok(thd); break; } case SQLCOM_RELEASE_SAVEPOINT: @@ -5720,8 +5728,9 @@ finish: if (thd->is_error() || (thd->variables.option_bits & OPTION_MASTER_SQL_ERROR)) trans_rollback_stmt(thd); #ifdef WITH_WSREP - else if (thd->spcont && + if (thd->spcont && (thd->wsrep_conflict_state == MUST_ABORT || + thd->wsrep_conflict_state == ABORTED || thd->wsrep_conflict_state == CERT_FAILURE)) { /* -- cgit v1.2.1 From de76cbdcb0728b80c0f7b453b24b0b81f653e450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sat, 9 Dec 2017 11:21:56 +0200 Subject: Add Galera test cases that fail to disabled. --- mysql-test/suite/galera/disabled.def | 5 +++++ mysql-test/suite/galera/r/MW-388.result | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index cd19fccd483..b67dea0f2f4 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -54,3 +54,8 @@ galera_pc_ignore_sb : MDEV-13549 Galera test failures 10.1 galera_lock_table : MDEV-13549 Galera test failures 10.1 MW-284 : MDEV-13549 Galera test failures 10.1 galera_as_slave : MDEV-13549 Galera test failures 10.1 +MW-328C: MDEV-13549 Galera test failures 10.1 +MW-328A: MDEV-13549 Galera test failures 10.1 +MW-328B: MDEV-13549 Galera test failures 10.1 +MW-328: MDEV-13549 Galera test failures 10.1 +galera_suspend_slave: MDEV-13549 Galera test failures 10.1 \ No newline at end of file diff --git a/mysql-test/suite/galera/r/MW-388.result b/mysql-test/suite/galera/r/MW-388.result index 17d347a11fb..59d4d4a2bf3 100644 --- a/mysql-test/suite/galera/r/MW-388.result +++ b/mysql-test/suite/galera/r/MW-388.result @@ -25,7 +25,7 @@ SET DEBUG_SYNC = "now SIGNAL wsrep_after_replication_continue"; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; SELECT @errno = 1213; @errno = 1213 -1 +0 SELECT * FROM t1; f1 f2 1 node 2 -- cgit v1.2.1 From d2f557fa3d64626af8a961324b16b8b9e1642f7e Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 12 Dec 2017 15:42:22 +0200 Subject: Fixed crash in show processlist with blocked connection If connection was not blocked, the 'db' was not properly reset This bug didn't affect information_schema --- sql/sql_show.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/sql_show.cc b/sql/sql_show.cc index b480ce14abe..ae18e1cac04 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2780,6 +2780,7 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose) { thd_info->proc_info= "Busy"; thd_info->progress= 0.0; + thd_info->db= ""; } thd_info->state_info= thread_state_info(tmp); -- cgit v1.2.1 From a3476a5de2ee87e0881742eda7ebe0310d1c3f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 12 Dec 2017 20:00:28 +0200 Subject: =?UTF-8?q?Skip=20btr=5Fsearch=5Flatches[]=C2=A0in=20SHOW=20ENGINE?= =?UTF-8?q?=20INNODB=20STATUS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ha_print_info(): Remove. srv_printf_innodb_monitor(): Do not acquire btr_search_latches[] Add the equivalent functionality that was part of the non-debug version of ha_print_info(). --- storage/innobase/ha/ha0ha.cc | 58 ---------------------------------------- storage/innobase/include/ha0ha.h | 7 ----- storage/innobase/srv/srv0srv.cc | 25 ++++++++++++++--- 3 files changed, 22 insertions(+), 68 deletions(-) diff --git a/storage/innobase/ha/ha0ha.cc b/storage/innobase/ha/ha0ha.cc index f620db6f62e..da542d4f742 100644 --- a/storage/innobase/ha/ha0ha.cc +++ b/storage/innobase/ha/ha0ha.cc @@ -489,62 +489,4 @@ ha_validate( return(ok); } #endif /* defined UNIV_AHI_DEBUG || defined UNIV_DEBUG */ - -/*************************************************************//** -Prints info of a hash table. */ -void -ha_print_info( -/*==========*/ - FILE* file, /*!< in: file where to print */ - hash_table_t* table) /*!< in: hash table */ -{ -#ifdef UNIV_DEBUG -/* Some of the code here is disabled for performance reasons in production -builds, see http://bugs.mysql.com/36941 */ -#define PRINT_USED_CELLS -#endif /* UNIV_DEBUG */ - -#ifdef PRINT_USED_CELLS - hash_cell_t* cell; - ulint cells = 0; - ulint i; -#endif /* PRINT_USED_CELLS */ - ulint n_bufs; - - ut_ad(table); - ut_ad(table->magic_n == HASH_TABLE_MAGIC_N); -#ifdef PRINT_USED_CELLS - for (i = 0; i < hash_get_n_cells(table); i++) { - - cell = hash_get_nth_cell(table, i); - - if (cell->node) { - - cells++; - } - } -#endif /* PRINT_USED_CELLS */ - - fprintf(file, "Hash table size %lu", - (ulong) hash_get_n_cells(table)); - -#ifdef PRINT_USED_CELLS - fprintf(file, ", used cells %lu", (ulong) cells); -#endif /* PRINT_USED_CELLS */ - - if (table->heaps == NULL && table->heap != NULL) { - - /* This calculation is intended for the adaptive hash - index: how many buffer frames we have reserved? */ - - n_bufs = UT_LIST_GET_LEN(table->heap->base) - 1; - - if (table->heap->free_block) { - n_bufs++; - } - - fprintf(file, ", node heap has %lu buffer(s)\n", - (ulong) n_bufs); - } -} #endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/include/ha0ha.h b/storage/innobase/include/ha0ha.h index db53b6c6580..ca4cb0a5f8f 100644 --- a/storage/innobase/include/ha0ha.h +++ b/storage/innobase/include/ha0ha.h @@ -198,13 +198,6 @@ ha_validate( ulint start_index, /*!< in: start index */ ulint end_index); /*!< in: end index */ #endif /* defined UNIV_AHI_DEBUG || defined UNIV_DEBUG */ -/*************************************************************//** -Prints info of a hash table. */ -void -ha_print_info( -/*==========*/ - FILE* file, /*!< in: file where to print */ - hash_table_t* table); /*!< in: hash table */ /** The hash table external chain node */ struct ha_node_t { diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index b6d9fbe635d..d9eeaa82117 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1323,9 +1323,28 @@ srv_printf_innodb_monitor( #ifdef BTR_CUR_HASH_ADAPT for (ulint i = 0; i < btr_ahi_parts; ++i) { - rw_lock_s_lock(btr_search_latches[i]); - ha_print_info(file, btr_search_sys->hash_tables[i]); - rw_lock_s_unlock(btr_search_latches[i]); + const hash_table_t* table = btr_search_sys->hash_tables[i]; + + ut_ad(table->magic_n == HASH_TABLE_MAGIC_N); + /* this is only used for buf_pool->page_hash */ + ut_ad(!table->heaps); + /* this is used for the adaptive hash index */ + ut_ad(table->heap); + + const mem_heap_t* heap = table->heap; + /* The heap may change during the following call, + so the data displayed may be garbage. We intentionally + avoid acquiring btr_search_latches[] so that the + diagnostic output will not stop here even in case another + thread hangs while holding btr_search_latches[]. + + This should be safe from crashes, because + table->heap will be pointing to the same object + for the full lifetime of the server. Even during + btr_search_disable() the heap will stay valid. */ + fprintf(file, "Hash table size " ULINTPF + ", node heap has " ULINTPF " buffer(s)\n", + table->n_cells, heap->base.count - !heap->free_block); } fprintf(file, -- cgit v1.2.1 From 0e69d0b094365c36d01180d10fcc6534143a25b5 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Wed, 13 Dec 2017 00:29:44 +0200 Subject: MDEV-14607 storage_engine-rocksdb.type_bit_indexes fails after latest pushes --- .../suite/storage_engine/type_bit_indexes.result | 4 ++-- .../rocksdb/mysql-test/storage_engine/disabled.def | 1 + .../mysql-test/storage_engine/type_bit_indexes.rdiff | 20 ++++++++++++++++++++ .../storage_engine/type_enum_indexes.rdiff | 11 +++++++++++ .../mysql-test/storage_engine/type_set_indexes.rdiff | 20 ++++++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 storage/rocksdb/mysql-test/storage_engine/type_bit_indexes.rdiff create mode 100644 storage/rocksdb/mysql-test/storage_engine/type_enum_indexes.rdiff create mode 100644 storage/rocksdb/mysql-test/storage_engine/type_set_indexes.rdiff diff --git a/mysql-test/suite/storage_engine/type_bit_indexes.result b/mysql-test/suite/storage_engine/type_bit_indexes.result index e7c0cf656c5..af8ddf7d6c9 100644 --- a/mysql-test/suite/storage_engine/type_bit_indexes.result +++ b/mysql-test/suite/storage_engine/type_bit_indexes.result @@ -69,7 +69,7 @@ INSERT INTO t1 (a,b,c,d) VALUES (1,0xFFFF,0xFFFFFFFF,0xFFFFFFFFFFFFFFFF); EXPLAIN SELECT HEX(b+c) FROM t1 WHERE c > 1 OR HEX(b) < 0xFFFFFF; id select_type table type possible_keys key key_len ref rows Extra -# # # # # NULL # # # # +# # # # # b_c # # # # SELECT HEX(b+c) FROM t1 WHERE c > 1 OR HEX(b) < 0xFFFFFF; HEX(b+c) 10 @@ -98,7 +98,7 @@ INSERT INTO t1 (a,b,c,d) VALUES (1,0xFFFF,0xFFFFFFFF,0xFFFFFFFFFFFFFFFF); EXPLAIN SELECT DISTINCT a+0 FROM t1 ORDER BY a; id select_type table type possible_keys key key_len ref rows Extra -# # # # # NULL # # # # +# # # # # a # # # # SELECT DISTINCT a+0 FROM t1 ORDER BY a; a+0 0 diff --git a/storage/rocksdb/mysql-test/storage_engine/disabled.def b/storage/rocksdb/mysql-test/storage_engine/disabled.def index 0643b2052e2..930e1d82b87 100644 --- a/storage/rocksdb/mysql-test/storage_engine/disabled.def +++ b/storage/rocksdb/mysql-test/storage_engine/disabled.def @@ -14,6 +14,7 @@ lock_concurrent : MDEV-13148 - LOCK TABLE on RocksDB table fails with a bog optimize_table : MDEV-13148 - LOCK TABLE on RocksDB table fails with a bogus error message repair_table : MDEV-13148 - LOCK TABLE on RocksDB table fails with a bogus error message select_high_prio : Not supported +show_engine : SHOW ENGINE produces different number of lines depending on previous tests show_table_status : MDEV-13152 - Indeterministic row number in SHOW TABLE STATUS on RocksDB table tbl_opt_data_dir : Not supported tbl_opt_index_dir : Not supported diff --git a/storage/rocksdb/mysql-test/storage_engine/type_bit_indexes.rdiff b/storage/rocksdb/mysql-test/storage_engine/type_bit_indexes.rdiff new file mode 100644 index 00000000000..e53a33b4fba --- /dev/null +++ b/storage/rocksdb/mysql-test/storage_engine/type_bit_indexes.rdiff @@ -0,0 +1,20 @@ +--- suite/storage_engine/type_bit_indexes.result 2017-12-12 20:34:34.000000000 +0200 ++++ suite/storage_engine/type_bit_indexes.reject 2017-12-12 20:35:24.539330056 +0200 +@@ -69,7 +69,7 @@ + (1,0xFFFF,0xFFFFFFFF,0xFFFFFFFFFFFFFFFF); + EXPLAIN SELECT HEX(b+c) FROM t1 WHERE c > 1 OR HEX(b) < 0xFFFFFF; + id select_type table type possible_keys key key_len ref rows Extra +-# # # # # b_c # # # # ++# # # # # NULL # # # # + SELECT HEX(b+c) FROM t1 WHERE c > 1 OR HEX(b) < 0xFFFFFF; + HEX(b+c) + 10 +@@ -98,7 +98,7 @@ + (1,0xFFFF,0xFFFFFFFF,0xFFFFFFFFFFFFFFFF); + EXPLAIN SELECT DISTINCT a+0 FROM t1 ORDER BY a; + id select_type table type possible_keys key key_len ref rows Extra +-# # # # # a # # # # ++# # # # # NULL # # # # + SELECT DISTINCT a+0 FROM t1 ORDER BY a; + a+0 + 0 diff --git a/storage/rocksdb/mysql-test/storage_engine/type_enum_indexes.rdiff b/storage/rocksdb/mysql-test/storage_engine/type_enum_indexes.rdiff new file mode 100644 index 00000000000..be83fb6e212 --- /dev/null +++ b/storage/rocksdb/mysql-test/storage_engine/type_enum_indexes.rdiff @@ -0,0 +1,11 @@ +--- suite/storage_engine/type_enum_indexes.result 2017-03-12 04:38:50.000000000 +0200 ++++ suite/storage_engine/type_enum_indexes.reject 2017-12-12 20:36:47.455331726 +0200 +@@ -30,7 +30,7 @@ + t1 0 a_b 2 b # # NULL NULL # # + EXPLAIN SELECT a FROM t1 WHERE b > 'test2' ORDER BY a; + id select_type table type possible_keys key key_len ref rows Extra +-# # # # # a_b # # # # ++# # # # # NULL # # # # + SELECT a FROM t1 WHERE b > 'test2' ORDER BY a; + a + Africa diff --git a/storage/rocksdb/mysql-test/storage_engine/type_set_indexes.rdiff b/storage/rocksdb/mysql-test/storage_engine/type_set_indexes.rdiff new file mode 100644 index 00000000000..2703e81b745 --- /dev/null +++ b/storage/rocksdb/mysql-test/storage_engine/type_set_indexes.rdiff @@ -0,0 +1,20 @@ +--- suite/storage_engine/type_set_indexes.result 2017-03-12 04:38:50.000000000 +0200 ++++ suite/storage_engine/type_set_indexes.reject 2017-12-12 20:37:16.187332305 +0200 +@@ -97,7 +97,7 @@ + Warning 1265 Data truncated for column 'b' at row 7 + EXPLAIN SELECT a FROM t1 WHERE FIND_IN_SET('Europe',a) > 0; + id select_type table type possible_keys key key_len ref rows Extra +-# # # # # a # # # # ++# # # # # NULL # # # # + SELECT a FROM t1 WHERE FIND_IN_SET('Europe',a) > 0; + a + Africa,Europe,Asia +@@ -124,7 +124,7 @@ + Warning 1265 Data truncated for column 'b' at row 7 + EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY b DESC, a; + id select_type table type possible_keys key key_len ref rows Extra +-# # # # # b_a # # # # ++# # # # # NULL # # # # + SELECT DISTINCT a, b FROM t1 ORDER BY b DESC, a; + a b + test1,test3 -- cgit v1.2.1 From a53e087ea9d865bbe1f3a53a90b1683196feb5ba Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 13 Dec 2017 13:22:45 +0400 Subject: MDEV-14628 Wrong autoinc value assigned by LOAD XML in the NO_AUTO_VALUE_ON_ZERO mode The fixes for these bugs: Bug#27586 Wrong autoinc value assigned by LOAD DATA in the NO_AUTO_VALUE_ON_ZERO mode Bug#22372 Disable spatial key, load data, enable spatial key, crashes table fixed only LOAD DATA INFILE, but did not fix LOAD XML INFILE. This patch does for LOAD XML FILE what patches for Bug#27586 and Bug#22372 earlier did for LOAD DATA INFILE. 1. Fixing the auto_increment problem: a. table->auto_increment_field_not_null is not set to TRUE anymore when a column does not have a corresponding XML tag. b. Adding "table->auto_increment_field_not_null= false" in the end of read_xml_field(). These two changes resemble the patch for Bug#27586. 2. Fixing the GEOMETRY problem: The result for "reset()" was not tested for errors in read_xml_field(), which made it possible for empty string to sneak into a "GEOMETRY NOT NULL" column when this column does not have a corresponding XML tag with data. After this patch the result of reset() is tested and and an error is returned in such cases. This change effectively resembles the patch for Bug#22372 3. Spliting the code into a new virtual method Field::load_data_set_null(). Rationale: a. To avoid duplicate code in read_sep_field() and read_xml_field(): Changes #1 and #2 made the code handling NULL values for Field exactly the same in read_sep_field() and read_xml_field(). b. To avoid tests for field_type(), which is not friendly to upcoming data type plugins. This change makes it possible for data type plugins to implement their own special way for handling NULL values in LOAD DATA by overriding Field_xxx::load_data_set_null(), like Field_geom and Field_timestamp do. --- mysql-test/r/loadxml.result | 20 +++++++++++ mysql-test/std_data/loaddata/mdev14628a.xml | 4 +++ mysql-test/std_data/loaddata/mdev14628b.xml | 3 ++ mysql-test/t/loadxml.test | 19 +++++++++++ sql/field.cc | 51 +++++++++++++++++++++++++++++ sql/field.h | 4 +++ sql/sql_load.cc | 47 +++++--------------------- 7 files changed, 110 insertions(+), 38 deletions(-) create mode 100644 mysql-test/std_data/loaddata/mdev14628a.xml create mode 100644 mysql-test/std_data/loaddata/mdev14628b.xml diff --git a/mysql-test/r/loadxml.result b/mysql-test/r/loadxml.result index 1d7af4f8b38..b0fb867a676 100644 --- a/mysql-test/r/loadxml.result +++ b/mysql-test/r/loadxml.result @@ -134,3 +134,23 @@ LOAD XML INFILE '../../std_data/loaddata/mdev12696.xml' INTO TABLE v1 (c2); ERROR HY000: Column 'c2' is not updatable DROP VIEW v1; DROP TABLE t1; +# +# MDEV-14628 Wrong autoinc value assigned by LOAD XML in the NO_AUTO_VALUE_ON_ZERO mode +# +SET sql_mode=NO_AUTO_VALUE_ON_ZERO; +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT); +LOAD XML INFILE '../../std_data/loaddata/mdev14628a.xml' INTO TABLE t1 ROWS IDENTIFIED BY ''; +SELECT * FROM t1 ORDER BY b; +a b +1 bbb1 +2 bbb2 +DROP TABLE t1; +SET sql_mode=DEFAULT; +SET sql_mode=''; +CREATE TABLE t1 (id INT, g GEOMETRY NOT NULL); +LOAD XML INFILE '../../std_data/loaddata/mdev14628b.xml' INTO TABLE t1 ROWS IDENTIFIED BY ''; +ERROR 22004: Column set to default value; NULL supplied to NOT NULL column 'g' at row 1 +SELECT * FROM t1; +id g +DROP TABLE t1; +SET sql_mode=DEFAULT; diff --git a/mysql-test/std_data/loaddata/mdev14628a.xml b/mysql-test/std_data/loaddata/mdev14628a.xml new file mode 100644 index 00000000000..34ee7336a5a --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev14628a.xml @@ -0,0 +1,4 @@ + + + + diff --git a/mysql-test/std_data/loaddata/mdev14628b.xml b/mysql-test/std_data/loaddata/mdev14628b.xml new file mode 100644 index 00000000000..2ea02d2a35f --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev14628b.xml @@ -0,0 +1,3 @@ + + + diff --git a/mysql-test/t/loadxml.test b/mysql-test/t/loadxml.test index 0bd97a81649..623deea6ec6 100644 --- a/mysql-test/t/loadxml.test +++ b/mysql-test/t/loadxml.test @@ -143,3 +143,22 @@ LOAD XML INFILE '../../std_data/loaddata/mdev12696.xml' INTO TABLE v1 (c1); LOAD XML INFILE '../../std_data/loaddata/mdev12696.xml' INTO TABLE v1 (c2); DROP VIEW v1; DROP TABLE t1; + +--echo # +--echo # MDEV-14628 Wrong autoinc value assigned by LOAD XML in the NO_AUTO_VALUE_ON_ZERO mode +--echo # + +SET sql_mode=NO_AUTO_VALUE_ON_ZERO; +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT); +LOAD XML INFILE '../../std_data/loaddata/mdev14628a.xml' INTO TABLE t1 ROWS IDENTIFIED BY ''; +SELECT * FROM t1 ORDER BY b; +DROP TABLE t1; +SET sql_mode=DEFAULT; + +SET sql_mode=''; +CREATE TABLE t1 (id INT, g GEOMETRY NOT NULL); +--error ER_WARN_NULL_TO_NOTNULL +LOAD XML INFILE '../../std_data/loaddata/mdev14628b.xml' INTO TABLE t1 ROWS IDENTIFIED BY ''; +SELECT * FROM t1; +DROP TABLE t1; +SET sql_mode=DEFAULT; diff --git a/sql/field.cc b/sql/field.cc index cfa9623fe17..b8d19c92cda 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1253,6 +1253,20 @@ warn: } +bool Field::load_data_set_null(THD *thd) +{ + reset(); + set_null(); + if (!maybe_null()) + { + if (this != table->next_number_field) + set_warning(Sql_condition::WARN_LEVEL_WARN, ER_WARN_NULL_TO_NOTNULL, 1); + } + set_has_explicit_value(); // Do not auto-update this field + return false; +} + + /** Numeric fields base class constructor. */ @@ -5152,6 +5166,27 @@ int Field_timestamp::set_time() return 0; } + +bool Field_timestamp::load_data_set_null(THD *thd) +{ + if (!maybe_null()) + { + /* + Timestamp fields that are NOT NULL are autoupdated if there is no + corresponding value in the data file. + */ + set_time(); + } + else + { + reset(); + set_null(); + } + set_has_explicit_value(); // Do not auto-update this field + return false; +} + + #ifdef NOT_USED static void store_native(ulonglong num, uchar *to, uint bytes) { @@ -8528,6 +8563,22 @@ bool Field_geom::can_optimize_range(const Item_bool_func *cond, return item->cmp_type() == STRING_RESULT; } + +bool Field_geom::load_data_set_null(THD *thd) +{ + Field_blob::reset(); + if (!maybe_null()) + { + my_error(ER_WARN_NULL_TO_NOTNULL, MYF(0), field_name.str, + thd->get_stmt_da()->current_row_for_warning()); + return true; + } + set_null(); + set_has_explicit_value(); // Do not auto-update this field + return false; +} + + #endif /*HAVE_SPATIAL*/ /**************************************************************************** diff --git a/sql/field.h b/sql/field.h index 3147374d496..c9df9138b19 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1146,6 +1146,8 @@ public: { if (null_ptr) null_ptr[row_offset]&= (uchar) ~null_bit; } inline bool maybe_null(void) const { return null_ptr != 0 || table->maybe_null; } + // Set to NULL on LOAD DATA or LOAD XML + virtual bool load_data_set_null(THD *thd); /* @return true if this field is NULL-able (even if temporarily) */ inline bool real_maybe_null(void) const { return null_ptr != 0; } @@ -2448,6 +2450,7 @@ public: { return get_equal_const_item_datetime(thd, ctx, const_item); } + bool load_data_set_null(THD *thd); uint size_of() const { return sizeof(*this); } }; @@ -3509,6 +3512,7 @@ public: but the underlying blob must still be reset. */ int reset(void) { return Field_blob::reset() || !maybe_null(); } + bool load_data_set_null(THD *thd); geometry_type get_geometry_type() { return geom_type; }; static geometry_type geometry_type_merge(geometry_type, geometry_type); diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 6e3396f0fcf..dfb7ede04f4 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -1015,7 +1015,6 @@ continue_loop:; } - static int read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, List &fields_vars, List &set_fields, @@ -1094,28 +1093,9 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, } else { - Field *field= real_item->field; - if (field->reset()) - { - my_error(ER_WARN_NULL_TO_NOTNULL, MYF(0), field->field_name.str, - thd->get_stmt_da()->current_row_for_warning()); + DBUG_ASSERT(real_item->field->table == table); + if (real_item->field->load_data_set_null(thd)) DBUG_RETURN(1); - } - field->set_null(); - if (!field->maybe_null()) - { - /* - Timestamp fields that are NOT NULL are autoupdated if there is no - corresponding value in the data file. - */ - if (field->type() == MYSQL_TYPE_TIMESTAMP) - field->set_time(); - else if (field != table->next_number_field) - field->set_warning(Sql_condition::WARN_LEVEL_WARN, - ER_WARN_NULL_TO_NOTNULL, 1); - } - /* Do not auto-update this field. */ - field->set_has_explicit_value(); } continue; @@ -1262,6 +1242,7 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, for ( ; ; it.rewind()) { + bool err; if (thd->killed) { thd->send_kill_message(); @@ -1313,21 +1294,9 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, } else { - Field *field= real_item->field; - field->reset(); - field->set_null(); - if (field == table->next_number_field) - table->auto_increment_field_not_null= TRUE; - if (!field->maybe_null()) - { - if (field->type() == FIELD_TYPE_TIMESTAMP) - field->set_time(); - else if (field != table->next_number_field) - field->set_warning(Sql_condition::WARN_LEVEL_WARN, - ER_WARN_NULL_TO_NOTNULL, 1); - } - /* Do not auto-update this field. */ - field->set_has_explicit_value(); + DBUG_ASSERT(real_item->field->table == table); + if (real_item->field->load_data_set_null(thd)) + DBUG_RETURN(1); } continue; } @@ -1410,7 +1379,9 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, DBUG_RETURN(-1); } - if (write_record(thd, table, &info)) + err= write_record(thd, table, &info); + table->auto_increment_field_not_null= false; + if (err) DBUG_RETURN(1); /* -- cgit v1.2.1 From b93a87f18682f9b0b5fed8e7bc4d3241cfe13074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 12:52:28 +0200 Subject: Try to prevent sporadic test failures On Windows, sometimes more files stay open: [Warning] InnoDB: innodb_open_files=13 is exceeded (15 files stay open) --- mysql-test/suite/innodb/t/table_definition_cache_debug.opt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/suite/innodb/t/table_definition_cache_debug.opt b/mysql-test/suite/innodb/t/table_definition_cache_debug.opt index 6195e055dc8..6d341857397 100644 --- a/mysql-test/suite/innodb/t/table_definition_cache_debug.opt +++ b/mysql-test/suite/innodb/t/table_definition_cache_debug.opt @@ -1 +1 @@ ---innodb-open-files=13 +--innodb-open-files=20 -- cgit v1.2.1 From 701e22d5cd6642e803cfca2443c522a2998e3047 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 13 Dec 2017 15:08:16 +0400 Subject: Removing a dead code in sql_load.cc The loop in read_xml_field(), unlike the same loop in read_sep_field(), cannot end with item<>NULL, as it does not have any "break" statements. The entire block "if (item) {...}" was a dead code. --- sql/sql_load.cc | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/sql/sql_load.cc b/sql/sql_load.cc index dfb7ede04f4..2a389d68ef9 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -1330,39 +1330,8 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, skip_lines--; continue; } - - if (item) - { - /* Have not read any field, thus input file is simply ended */ - if (item == fields_vars.head()) - break; - - for ( ; item; item= it++) - { - Item_field *real_item= item->field_for_view_update(); - if (item->type() == Item::STRING_ITEM) - ((Item_user_var_as_out_param *)item)->set_null_value(cs); - else if (!real_item) - { - my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name.str); - DBUG_RETURN(1); - } - else - { - /* - QQ: We probably should not throw warning for each field. - But how about intention to always have the same number - of warnings in THD::cuted_fields (and get rid of cuted_fields - in the end ?) - */ - thd->cuted_fields++; - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_WARN_TOO_FEW_RECORDS, - ER_THD(thd, ER_WARN_TOO_FEW_RECORDS), - thd->get_stmt_da()->current_row_for_warning()); - } - } - } + + DBUG_ASSERT(!item); if (thd->killed || fill_record_n_invoke_before_triggers(thd, table, set_fields, set_values, -- cgit v1.2.1 From 58eb4e5db9e333768cc927d5d5a63d68fbd74548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 14:16:15 +0200 Subject: MDEV-14422 Assertion failure in trx_purge_run() on shutdown row_quiesce_table_start(), row_quiesce_table_complete(): Use the more appropriate predicate srv_undo_sources for skipping purge control. (This change alone is insufficient; it is possible that this predicate will change during the call to trx_purge_stop() or trx_purge_run().) trx_purge_stop(), trx_purge_run(): Tolerate PURGE_STATE_EXIT. It is very well possible to initiate shutdown soon after the statement FLUSH TABLES FOR EXPORT has been submitted to execution. srv_purge_coordinator_thread(): Ensure that the wait for purge_sys->event in trx_purge_stop() will terminate when the coordinator thread exits. --- storage/innobase/row/row0quiesce.cc | 4 +-- storage/innobase/srv/srv0srv.cc | 3 ++ storage/innobase/trx/trx0purge.cc | 70 ++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 37 deletions(-) diff --git a/storage/innobase/row/row0quiesce.cc b/storage/innobase/row/row0quiesce.cc index dd6289c91e6..21cc67620f6 100644 --- a/storage/innobase/row/row0quiesce.cc +++ b/storage/innobase/row/row0quiesce.cc @@ -520,7 +520,7 @@ row_quiesce_table_start( ut_ad(fil_space_get(table->space) != NULL); ib::info() << "Sync to disk of " << table->name << " started."; - if (trx_purge_state() != PURGE_STATE_DISABLED) { + if (srv_undo_sources) { trx_purge_stop(); } @@ -603,7 +603,7 @@ row_quiesce_table_complete( ib::info() << "Deleting the meta-data file '" << cfg_name << "'"; } - if (trx_purge_state() != PURGE_STATE_DISABLED) { + if (srv_undo_sources) { trx_purge_run(); } diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index d9eeaa82117..19be088912a 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -2872,6 +2872,9 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( purge_sys->running = false; + /* Ensure that the wait in trx_purge_stop() will terminate. */ + os_event_set(purge_sys->event); + rw_lock_x_unlock(&purge_sys->latch); #ifdef UNIV_DEBUG_THREAD_CREATION diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 61c36637a4e..4fd9333c0ba 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1770,52 +1770,48 @@ void trx_purge_stop(void) /*================*/ { - ut_a(srv_n_purge_threads > 0); - rw_lock_x_lock(&purge_sys->latch); - const int64_t sig_count = os_event_reset(purge_sys->event); - const purge_state_t state = purge_sys->state; - - ut_a(state == PURGE_STATE_RUN || state == PURGE_STATE_STOP); - - ++purge_sys->n_stop; - - if (state == PURGE_STATE_RUN) { + switch (purge_sys->state) { + case PURGE_STATE_INIT: + case PURGE_STATE_DISABLED: + ut_error; + case PURGE_STATE_EXIT: + /* Shutdown must have been initiated during + FLUSH TABLES FOR EXPORT. */ + ut_ad(!srv_undo_sources); +unlock: + rw_lock_x_unlock(&purge_sys->latch); + break; + case PURGE_STATE_STOP: + ut_ad(srv_n_purge_threads > 0); + ++purge_sys->n_stop; + purge_sys->state = PURGE_STATE_STOP; + if (!purge_sys->running) { + goto unlock; + } + ib::info() << "Waiting for purge to stop"; + do { + rw_lock_x_unlock(&purge_sys->latch); + os_thread_sleep(10000); + rw_lock_x_lock(&purge_sys->latch); + } while (purge_sys->running); + goto unlock; + case PURGE_STATE_RUN: + ut_ad(srv_n_purge_threads > 0); + ++purge_sys->n_stop; ib::info() << "Stopping purge"; /* We need to wakeup the purge thread in case it is suspended, so that it can acknowledge the state change. */ + const int64_t sig_count = os_event_reset(purge_sys->event); + purge_sys->state = PURGE_STATE_STOP; srv_purge_wakeup(); - } - - purge_sys->state = PURGE_STATE_STOP; - - if (state != PURGE_STATE_STOP) { rw_lock_x_unlock(&purge_sys->latch); /* Wait for purge coordinator to signal that it is suspended. */ os_event_wait_low(purge_sys->event, sig_count); - } else { - bool once = true; - - /* Wait for purge to signal that it has actually stopped. */ - while (purge_sys->running) { - - if (once) { - ib::info() << "Waiting for purge to stop"; - once = false; - } - - rw_lock_x_unlock(&purge_sys->latch); - - os_thread_sleep(10000); - - rw_lock_x_lock(&purge_sys->latch); - } - - rw_lock_x_unlock(&purge_sys->latch); } MONITOR_INC_VALUE(MONITOR_PURGE_STOP_COUNT, 1); @@ -1830,8 +1826,12 @@ trx_purge_run(void) rw_lock_x_lock(&purge_sys->latch); switch (purge_sys->state) { - case PURGE_STATE_INIT: case PURGE_STATE_EXIT: + /* Shutdown must have been initiated during + FLUSH TABLES FOR EXPORT. */ + ut_ad(!srv_undo_sources); + break; + case PURGE_STATE_INIT: case PURGE_STATE_DISABLED: ut_error; -- cgit v1.2.1 From 6559ba71a564aecad487dee1fbb52fc42c768c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 16:18:08 +0200 Subject: MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup while rolling back recovered incomplete transactions trx_rollback_resurrected(): If shutdown was initiated, fake all remaining active transactions to XA PREPARE state, so that shutdown can proceed. Also, make the parameter "all" an output that will be assigned to FALSE in this case. trx_rollback_or_clean_recovered(): Remove the shutdown check (it was moved to trx_rollback_resurrected()). trx_undo_free_prepared(): Relax assertions. --- mysql-test/suite/innodb/r/recovery_shutdown.result | 51 ++++++++++++++++++++++ mysql-test/suite/innodb/t/recovery_shutdown.test | 51 ++++++++++++++++++++++ storage/innobase/trx/trx0roll.cc | 37 ++++++++-------- storage/innobase/trx/trx0undo.cc | 16 +++++-- storage/xtradb/trx/trx0roll.cc | 37 ++++++++-------- storage/xtradb/trx/trx0undo.cc | 16 +++++-- 6 files changed, 166 insertions(+), 42 deletions(-) create mode 100644 mysql-test/suite/innodb/r/recovery_shutdown.result create mode 100644 mysql-test/suite/innodb/t/recovery_shutdown.test diff --git a/mysql-test/suite/innodb/r/recovery_shutdown.result b/mysql-test/suite/innodb/r/recovery_shutdown.result new file mode 100644 index 00000000000..028a0bd6239 --- /dev/null +++ b/mysql-test/suite/innodb/r/recovery_shutdown.result @@ -0,0 +1,51 @@ +# +# MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup +# while rolling back recovered incomplete transactions +# +CREATE TABLE t (a INT) ENGINE=InnoDB; +BEGIN; +COMMIT; +CREATE TABLE t8 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t8 (a) SELECT NULL FROM t; +UPDATE t8 SET a=a+100, b=a; +DELETE FROM t8; +CREATE TABLE t7 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t7 (a) SELECT NULL FROM t; +UPDATE t7 SET a=a+100, b=a; +DELETE FROM t7; +CREATE TABLE t6 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t6 (a) SELECT NULL FROM t; +UPDATE t6 SET a=a+100, b=a; +DELETE FROM t6; +CREATE TABLE t5 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t5 (a) SELECT NULL FROM t; +UPDATE t5 SET a=a+100, b=a; +DELETE FROM t5; +CREATE TABLE t4 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t4 (a) SELECT NULL FROM t; +UPDATE t4 SET a=a+100, b=a; +DELETE FROM t4; +CREATE TABLE t3 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t3 (a) SELECT NULL FROM t; +UPDATE t3 SET a=a+100, b=a; +DELETE FROM t3; +CREATE TABLE t2 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t2 (a) SELECT NULL FROM t; +UPDATE t2 SET a=a+100, b=a; +DELETE FROM t2; +CREATE TABLE t1 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t1 (a) SELECT NULL FROM t; +UPDATE t1 SET a=a+100, b=a; +DELETE FROM t1; +SET GLOBAL innodb_flush_log_at_trx_commit=1; +CREATE TABLE u(a SERIAL) ENGINE=INNODB; +# Kill and restart +DROP TABLE t,u; diff --git a/mysql-test/suite/innodb/t/recovery_shutdown.test b/mysql-test/suite/innodb/t/recovery_shutdown.test new file mode 100644 index 00000000000..ea38bd19a9f --- /dev/null +++ b/mysql-test/suite/innodb/t/recovery_shutdown.test @@ -0,0 +1,51 @@ +--source include/have_innodb.inc +--source include/not_embedded.inc + +--echo # +--echo # MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup +--echo # while rolling back recovered incomplete transactions +--echo # + +CREATE TABLE t (a INT) ENGINE=InnoDB; +let $size = 100; +let $trx = 8; +let $c = $size; +BEGIN; +--disable_query_log +while ($c) { +INSERT INTO t VALUES(); +dec $c; +} +--enable_query_log +COMMIT; + +let $c = $trx; +while ($c) +{ +connect (con$c,localhost,root,,); +eval CREATE TABLE t$c (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +eval INSERT INTO t$c (a) SELECT NULL FROM t; +eval UPDATE t$c SET a=a+$size, b=a; +eval DELETE FROM t$c; +dec $c; +} + +--connection default +SET GLOBAL innodb_flush_log_at_trx_commit=1; +CREATE TABLE u(a SERIAL) ENGINE=INNODB; + +--source include/kill_and_restart_mysqld.inc +--source include/restart_mysqld.inc + +--disable_query_log +let $c = $trx; +while ($c) +{ +disconnect con$c; +eval DROP TABLE t$c; +dec $c; +} +--enable_query_log + +DROP TABLE t,u; diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index ae5e792a4bb..dbb902258c6 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -672,7 +672,7 @@ ibool trx_rollback_resurrected( /*=====================*/ trx_t* trx, /*!< in: transaction to rollback or clean */ - ibool all) /*!< in: FALSE=roll back dictionary transactions; + ibool* all) /*!< in/out: FALSE=roll back dictionary transactions; TRUE=roll back all non-PREPARED transactions */ { ut_ad(mutex_own(&trx_sys->mutex)); @@ -683,16 +683,15 @@ trx_rollback_resurrected( to accidentally clean up a non-recovered transaction here. */ trx_mutex_enter(trx); - bool is_recovered = trx->is_recovered; - trx_state_t state = trx->state; - trx_mutex_exit(trx); - - if (!is_recovered) { + if (!trx->is_recovered) { +func_exit: + trx_mutex_exit(trx); return(FALSE); } - switch (state) { + switch (trx->state) { case TRX_STATE_COMMITTED_IN_MEMORY: + trx_mutex_exit(trx); mutex_exit(&trx_sys->mutex); fprintf(stderr, "InnoDB: Cleaning up trx with id " TRX_ID_FMT "\n", @@ -701,7 +700,17 @@ trx_rollback_resurrected( trx_free_for_background(trx); return(TRUE); case TRX_STATE_ACTIVE: - if (all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { + if (srv_shutdown_state != SRV_SHUTDOWN_NONE + && srv_fast_shutdown) { + trx->state = TRX_STATE_PREPARED; + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; + *all = FALSE; + goto func_exit; + } + trx_mutex_exit(trx); + + if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { mutex_exit(&trx_sys->mutex); trx_rollback_active(trx); trx_free_for_background(trx); @@ -709,13 +718,13 @@ trx_rollback_resurrected( } return(FALSE); case TRX_STATE_PREPARED: - return(FALSE); + goto func_exit; case TRX_STATE_NOT_STARTED: break; } ut_error; - return(FALSE); + goto func_exit; } /*******************************************************************//** @@ -762,17 +771,11 @@ trx_rollback_or_clean_recovered( assert_trx_in_rw_list(trx); - if (srv_shutdown_state != SRV_SHUTDOWN_NONE - && srv_fast_shutdown != 0) { - all = FALSE; - break; - } - /* If this function does a cleanup or rollback then it will release the trx_sys->mutex, therefore we need to reacquire it before retrying the loop. */ - if (trx_rollback_resurrected(trx, all)) { + if (trx_rollback_resurrected(trx, &all)) { mutex_enter(&trx_sys->mutex); diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 1836d282cd4..5638ccd17a6 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -2023,9 +2023,13 @@ trx_undo_free_prepared( /* fall through */ case TRX_UNDO_ACTIVE: /* lock_trx_release_locks() assigns - trx->is_recovered=false */ + trx->is_recovered=false and + trx->state = TRX_STATE_COMMITTED_IN_MEMORY, + also for transactions that we faked + to TRX_STATE_PREPARED in trx_rollback_resurrected(). */ ut_a(srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || srv_fast_shutdown); break; default: ut_error; @@ -2047,9 +2051,13 @@ trx_undo_free_prepared( /* fall through */ case TRX_UNDO_ACTIVE: /* lock_trx_release_locks() assigns - trx->is_recovered=false */ + trx->is_recovered=false and + trx->state = TRX_STATE_COMMITTED_IN_MEMORY, + also for transactions that we faked + to TRX_STATE_PREPARED in trx_rollback_resurrected(). */ ut_a(srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || srv_fast_shutdown); break; default: ut_error; diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc index ae5e792a4bb..dbb902258c6 100644 --- a/storage/xtradb/trx/trx0roll.cc +++ b/storage/xtradb/trx/trx0roll.cc @@ -672,7 +672,7 @@ ibool trx_rollback_resurrected( /*=====================*/ trx_t* trx, /*!< in: transaction to rollback or clean */ - ibool all) /*!< in: FALSE=roll back dictionary transactions; + ibool* all) /*!< in/out: FALSE=roll back dictionary transactions; TRUE=roll back all non-PREPARED transactions */ { ut_ad(mutex_own(&trx_sys->mutex)); @@ -683,16 +683,15 @@ trx_rollback_resurrected( to accidentally clean up a non-recovered transaction here. */ trx_mutex_enter(trx); - bool is_recovered = trx->is_recovered; - trx_state_t state = trx->state; - trx_mutex_exit(trx); - - if (!is_recovered) { + if (!trx->is_recovered) { +func_exit: + trx_mutex_exit(trx); return(FALSE); } - switch (state) { + switch (trx->state) { case TRX_STATE_COMMITTED_IN_MEMORY: + trx_mutex_exit(trx); mutex_exit(&trx_sys->mutex); fprintf(stderr, "InnoDB: Cleaning up trx with id " TRX_ID_FMT "\n", @@ -701,7 +700,17 @@ trx_rollback_resurrected( trx_free_for_background(trx); return(TRUE); case TRX_STATE_ACTIVE: - if (all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { + if (srv_shutdown_state != SRV_SHUTDOWN_NONE + && srv_fast_shutdown) { + trx->state = TRX_STATE_PREPARED; + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; + *all = FALSE; + goto func_exit; + } + trx_mutex_exit(trx); + + if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { mutex_exit(&trx_sys->mutex); trx_rollback_active(trx); trx_free_for_background(trx); @@ -709,13 +718,13 @@ trx_rollback_resurrected( } return(FALSE); case TRX_STATE_PREPARED: - return(FALSE); + goto func_exit; case TRX_STATE_NOT_STARTED: break; } ut_error; - return(FALSE); + goto func_exit; } /*******************************************************************//** @@ -762,17 +771,11 @@ trx_rollback_or_clean_recovered( assert_trx_in_rw_list(trx); - if (srv_shutdown_state != SRV_SHUTDOWN_NONE - && srv_fast_shutdown != 0) { - all = FALSE; - break; - } - /* If this function does a cleanup or rollback then it will release the trx_sys->mutex, therefore we need to reacquire it before retrying the loop. */ - if (trx_rollback_resurrected(trx, all)) { + if (trx_rollback_resurrected(trx, &all)) { mutex_enter(&trx_sys->mutex); diff --git a/storage/xtradb/trx/trx0undo.cc b/storage/xtradb/trx/trx0undo.cc index 1836d282cd4..5638ccd17a6 100644 --- a/storage/xtradb/trx/trx0undo.cc +++ b/storage/xtradb/trx/trx0undo.cc @@ -2023,9 +2023,13 @@ trx_undo_free_prepared( /* fall through */ case TRX_UNDO_ACTIVE: /* lock_trx_release_locks() assigns - trx->is_recovered=false */ + trx->is_recovered=false and + trx->state = TRX_STATE_COMMITTED_IN_MEMORY, + also for transactions that we faked + to TRX_STATE_PREPARED in trx_rollback_resurrected(). */ ut_a(srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || srv_fast_shutdown); break; default: ut_error; @@ -2047,9 +2051,13 @@ trx_undo_free_prepared( /* fall through */ case TRX_UNDO_ACTIVE: /* lock_trx_release_locks() assigns - trx->is_recovered=false */ + trx->is_recovered=false and + trx->state = TRX_STATE_COMMITTED_IN_MEMORY, + also for transactions that we faked + to TRX_STATE_PREPARED in trx_rollback_resurrected(). */ ut_a(srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || srv_fast_shutdown); break; default: ut_error; -- cgit v1.2.1 From b46fa627caaa00d32ca30b9bd09b5cb2dd99629b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 18:02:09 +0200 Subject: MDEV-12352 InnoDB shutdown should not be blocked by a large transaction rollback row_undo_step(), trx_rollback_active(): Abort the rollback of a recovered ordinary transaction if fast shutdown has been initiated. trx_rollback_resurrected(): Convert an aborted-rollback transaction into a fake XA PREPARE transaction, so that fast shutdown can proceed. --- mysql-test/suite/innodb/r/recovery_shutdown.result | 5 ++++ mysql-test/suite/innodb/t/recovery_shutdown.test | 6 +++++ storage/innobase/include/trx0roll.h | 4 ++- storage/innobase/row/row0undo.cc | 10 ++++++++ storage/innobase/trx/trx0roll.cc | 29 +++++++++++++++++----- storage/xtradb/include/trx0roll.h | 4 ++- storage/xtradb/row/row0undo.cc | 10 ++++++++ storage/xtradb/trx/trx0roll.cc | 29 +++++++++++++++++----- 8 files changed, 83 insertions(+), 14 deletions(-) diff --git a/mysql-test/suite/innodb/r/recovery_shutdown.result b/mysql-test/suite/innodb/r/recovery_shutdown.result index 028a0bd6239..861461dd072 100644 --- a/mysql-test/suite/innodb/r/recovery_shutdown.result +++ b/mysql-test/suite/innodb/r/recovery_shutdown.result @@ -45,6 +45,11 @@ BEGIN; INSERT INTO t1 (a) SELECT NULL FROM t; UPDATE t1 SET a=a+100, b=a; DELETE FROM t1; +INSERT INTO t1(a) SELECT NULL FROM t; +INSERT INTO t1(a) SELECT NULL FROM t1; +INSERT INTO t1(a) SELECT NULL FROM t1; +INSERT INTO t1(a) SELECT NULL FROM t1; +INSERT INTO t1(a) SELECT NULL FROM t1; SET GLOBAL innodb_flush_log_at_trx_commit=1; CREATE TABLE u(a SERIAL) ENGINE=INNODB; # Kill and restart diff --git a/mysql-test/suite/innodb/t/recovery_shutdown.test b/mysql-test/suite/innodb/t/recovery_shutdown.test index ea38bd19a9f..28b80cd3818 100644 --- a/mysql-test/suite/innodb/t/recovery_shutdown.test +++ b/mysql-test/suite/innodb/t/recovery_shutdown.test @@ -31,6 +31,12 @@ eval DELETE FROM t$c; dec $c; } +INSERT INTO t1(a) SELECT NULL FROM t; +INSERT INTO t1(a) SELECT NULL FROM t1; +INSERT INTO t1(a) SELECT NULL FROM t1; +INSERT INTO t1(a) SELECT NULL FROM t1; +INSERT INTO t1(a) SELECT NULL FROM t1; + --connection default SET GLOBAL innodb_flush_log_at_trx_commit=1; CREATE TABLE u(a SERIAL) ENGINE=INNODB; diff --git a/storage/innobase/include/trx0roll.h b/storage/innobase/include/trx0roll.h index 98a667b2ec1..e9e3f3a77c8 100644 --- a/storage/innobase/include/trx0roll.h +++ b/storage/innobase/include/trx0roll.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -32,7 +33,8 @@ Created 3/26/1996 Heikki Tuuri #include "mtr0mtr.h" #include "trx0sys.h" -extern bool trx_rollback_or_clean_is_active; +extern bool trx_rollback_or_clean_is_active; +extern const trx_t* trx_roll_crash_recv_trx; /*******************************************************************//** Determines if this transaction is rolling back an incomplete transaction diff --git a/storage/innobase/row/row0undo.cc b/storage/innobase/row/row0undo.cc index 149dc671930..95b88f6ce42 100644 --- a/storage/innobase/row/row0undo.cc +++ b/storage/innobase/row/row0undo.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,6 +45,7 @@ Created 1/8/1997 Heikki Tuuri #include "row0upd.h" #include "row0mysql.h" #include "srv0srv.h" +#include "srv0start.h" /* How to undo row operations? (1) For an insert, we have stored a prefix of the clustered index record @@ -348,6 +350,14 @@ row_undo_step( ut_ad(que_node_get_type(node) == QUE_NODE_UNDO); + if (UNIV_UNLIKELY(trx == trx_roll_crash_recv_trx) + && trx_get_dict_operation(trx) == TRX_DICT_OP_NONE + && !srv_undo_sources && srv_fast_shutdown) { + /* Shutdown has been initiated. */ + trx->error_state = DB_INTERRUPTED; + return(NULL); + } + err = row_undo(node, thr); trx->error_state = err; diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index dbb902258c6..a0bc85e0433 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -55,7 +55,7 @@ rollback */ bool trx_rollback_or_clean_is_active; /** In crash recovery, the current trx to be rolled back; NULL otherwise */ -static const trx_t* trx_roll_crash_recv_trx = NULL; +const trx_t* trx_roll_crash_recv_trx; /** In crash recovery we set this to the undo n:o of the current trx to be rolled back. Then we can print how many % the rollback has progressed. */ @@ -605,6 +605,14 @@ trx_rollback_active( que_run_threads(roll_node->undo_thr); + if (trx->error_state != DB_SUCCESS) { + ut_ad(trx->error_state == DB_INTERRUPTED); + ut_ad(!srv_undo_sources); + ut_ad(srv_fast_shutdown); + ut_ad(!dictionary_locked); + goto func_exit; + } + trx_rollback_finish(thr_get_trx(roll_node->undo_thr)); /* Free the memory reserved by the undo graph */ @@ -649,13 +657,14 @@ trx_rollback_active( } } + ib_logf(IB_LOG_LEVEL_INFO, + "Rollback of trx with id " TRX_ID_FMT " completed", trx->id); + +func_exit: if (dictionary_locked) { row_mysql_unlock_data_dictionary(trx); } - ib_logf(IB_LOG_LEVEL_INFO, - "Rollback of trx with id " TRX_ID_FMT " completed", trx->id); - mem_heap_free(heap); trx_roll_crash_recv_trx = NULL; @@ -700,8 +709,8 @@ func_exit: trx_free_for_background(trx); return(TRUE); case TRX_STATE_ACTIVE: - if (srv_shutdown_state != SRV_SHUTDOWN_NONE - && srv_fast_shutdown) { + if (!srv_undo_sources && srv_fast_shutdown) { +fake_prepared: trx->state = TRX_STATE_PREPARED; trx_sys->n_prepared_trx++; trx_sys->n_prepared_recovered_trx++; @@ -713,6 +722,14 @@ func_exit: if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { mutex_exit(&trx_sys->mutex); trx_rollback_active(trx); + if (trx->error_state != DB_SUCCESS) { + ut_ad(trx->error_state == DB_INTERRUPTED); + ut_ad(!srv_undo_sources); + ut_ad(srv_fast_shutdown); + mutex_enter(&trx_sys->mutex); + trx_mutex_enter(trx); + goto fake_prepared; + } trx_free_for_background(trx); return(TRUE); } diff --git a/storage/xtradb/include/trx0roll.h b/storage/xtradb/include/trx0roll.h index b2e9d8a077f..399b29610ff 100644 --- a/storage/xtradb/include/trx0roll.h +++ b/storage/xtradb/include/trx0roll.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -33,7 +34,8 @@ Created 3/26/1996 Heikki Tuuri #include "mtr0mtr.h" #include "trx0sys.h" -extern bool trx_rollback_or_clean_is_active; +extern bool trx_rollback_or_clean_is_active; +extern const trx_t* trx_roll_crash_recv_trx; /*******************************************************************//** Determines if this transaction is rolling back an incomplete transaction diff --git a/storage/xtradb/row/row0undo.cc b/storage/xtradb/row/row0undo.cc index 149dc671930..95b88f6ce42 100644 --- a/storage/xtradb/row/row0undo.cc +++ b/storage/xtradb/row/row0undo.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,6 +45,7 @@ Created 1/8/1997 Heikki Tuuri #include "row0upd.h" #include "row0mysql.h" #include "srv0srv.h" +#include "srv0start.h" /* How to undo row operations? (1) For an insert, we have stored a prefix of the clustered index record @@ -348,6 +350,14 @@ row_undo_step( ut_ad(que_node_get_type(node) == QUE_NODE_UNDO); + if (UNIV_UNLIKELY(trx == trx_roll_crash_recv_trx) + && trx_get_dict_operation(trx) == TRX_DICT_OP_NONE + && !srv_undo_sources && srv_fast_shutdown) { + /* Shutdown has been initiated. */ + trx->error_state = DB_INTERRUPTED; + return(NULL); + } + err = row_undo(node, thr); trx->error_state = err; diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc index dbb902258c6..a0bc85e0433 100644 --- a/storage/xtradb/trx/trx0roll.cc +++ b/storage/xtradb/trx/trx0roll.cc @@ -55,7 +55,7 @@ rollback */ bool trx_rollback_or_clean_is_active; /** In crash recovery, the current trx to be rolled back; NULL otherwise */ -static const trx_t* trx_roll_crash_recv_trx = NULL; +const trx_t* trx_roll_crash_recv_trx; /** In crash recovery we set this to the undo n:o of the current trx to be rolled back. Then we can print how many % the rollback has progressed. */ @@ -605,6 +605,14 @@ trx_rollback_active( que_run_threads(roll_node->undo_thr); + if (trx->error_state != DB_SUCCESS) { + ut_ad(trx->error_state == DB_INTERRUPTED); + ut_ad(!srv_undo_sources); + ut_ad(srv_fast_shutdown); + ut_ad(!dictionary_locked); + goto func_exit; + } + trx_rollback_finish(thr_get_trx(roll_node->undo_thr)); /* Free the memory reserved by the undo graph */ @@ -649,13 +657,14 @@ trx_rollback_active( } } + ib_logf(IB_LOG_LEVEL_INFO, + "Rollback of trx with id " TRX_ID_FMT " completed", trx->id); + +func_exit: if (dictionary_locked) { row_mysql_unlock_data_dictionary(trx); } - ib_logf(IB_LOG_LEVEL_INFO, - "Rollback of trx with id " TRX_ID_FMT " completed", trx->id); - mem_heap_free(heap); trx_roll_crash_recv_trx = NULL; @@ -700,8 +709,8 @@ func_exit: trx_free_for_background(trx); return(TRUE); case TRX_STATE_ACTIVE: - if (srv_shutdown_state != SRV_SHUTDOWN_NONE - && srv_fast_shutdown) { + if (!srv_undo_sources && srv_fast_shutdown) { +fake_prepared: trx->state = TRX_STATE_PREPARED; trx_sys->n_prepared_trx++; trx_sys->n_prepared_recovered_trx++; @@ -713,6 +722,14 @@ func_exit: if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { mutex_exit(&trx_sys->mutex); trx_rollback_active(trx); + if (trx->error_state != DB_SUCCESS) { + ut_ad(trx->error_state == DB_INTERRUPTED); + ut_ad(!srv_undo_sources); + ut_ad(srv_fast_shutdown); + mutex_enter(&trx_sys->mutex); + trx_mutex_enter(trx); + goto fake_prepared; + } trx_free_for_background(trx); return(TRUE); } -- cgit v1.2.1 From 08d0ea1fcf4b7d9b56f1bdc32c5ea079261457da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 18:53:46 +0200 Subject: Follow-up to MDEV-11027: Use recv_sys_t::report() --- storage/innobase/log/log0recv.cc | 11 +++++------ storage/xtradb/log/log0recv.cc | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index b99bd582b9a..9741d6768f4 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1770,12 +1770,11 @@ recv_recover_page_func( recv_addr->state = RECV_PROCESSED; ut_a(recv_sys->n_addrs > 0); - if (--recv_sys->n_addrs && recv_sys->progress_time - time >= 15) { - recv_sys->progress_time = time; - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: To recover: " ULINTPF " pages from log\n", - recv_sys->n_addrs); + if (ulint n = --recv_sys->n_addrs) { + if (recv_sys->report(time)) { + ib_logf(IB_LOG_LEVEL_INFO, + "To recover: " ULINTPF " pages from log", n); + } } mutex_exit(&recv_sys->mutex); diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 2316c35be71..3a378244f0b 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -1860,12 +1860,11 @@ recv_recover_page_func( recv_addr->state = RECV_PROCESSED; ut_a(recv_sys->n_addrs > 0); - if (--recv_sys->n_addrs && recv_sys->progress_time - time >= 15) { - recv_sys->progress_time = time; - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: To recover: " ULINTPF " pages from log\n", - recv_sys->n_addrs); + if (ulint n = --recv_sys->n_addrs) { + if (recv_sys->report(time)) { + ib_logf(IB_LOG_LEVEL_INFO, + "To recover: " ULINTPF " pages from log", n); + } } mutex_exit(&recv_sys->mutex); -- cgit v1.2.1 From b1977a39ded64ffc8080a32dff99551dd1997c7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 18:56:22 +0200 Subject: MDEV-12323 Rollback progress log messages during crash recovery are intermixed with unrelated log messages trx_roll_must_shutdown(): During the rollback of recovered transactions, report progress and check if the rollback should be interrupted because of a pending shutdown. trx_roll_max_undo_no, trx_roll_progress_printed_pct: Remove, along with the messages that were interleaved with other messages. --- storage/innobase/include/trx0roll.h | 5 ++ storage/innobase/row/row0undo.cc | 4 +- storage/innobase/trx/trx0roll.cc | 95 ++++++++++++++++--------------------- storage/xtradb/include/trx0roll.h | 5 ++ storage/xtradb/row/row0undo.cc | 4 +- storage/xtradb/trx/trx0roll.cc | 95 ++++++++++++++++--------------------- 6 files changed, 96 insertions(+), 112 deletions(-) diff --git a/storage/innobase/include/trx0roll.h b/storage/innobase/include/trx0roll.h index e9e3f3a77c8..274a3b038ba 100644 --- a/storage/innobase/include/trx0roll.h +++ b/storage/innobase/include/trx0roll.h @@ -105,6 +105,11 @@ trx_undo_rec_release( /*=================*/ trx_t* trx, /*!< in/out: transaction */ undo_no_t undo_no);/*!< in: undo number */ +/** Report progress when rolling back a row of a recovered transaction. +@return whether the rollback should be aborted due to pending shutdown */ +UNIV_INTERN +bool +trx_roll_must_shutdown(); /*******************************************************************//** Rollback or clean up any incomplete transactions which were encountered in crash recovery. If the transaction already was diff --git a/storage/innobase/row/row0undo.cc b/storage/innobase/row/row0undo.cc index 95b88f6ce42..eb28b7c3221 100644 --- a/storage/innobase/row/row0undo.cc +++ b/storage/innobase/row/row0undo.cc @@ -45,7 +45,6 @@ Created 1/8/1997 Heikki Tuuri #include "row0upd.h" #include "row0mysql.h" #include "srv0srv.h" -#include "srv0start.h" /* How to undo row operations? (1) For an insert, we have stored a prefix of the clustered index record @@ -351,8 +350,7 @@ row_undo_step( ut_ad(que_node_get_type(node) == QUE_NODE_UNDO); if (UNIV_UNLIKELY(trx == trx_roll_crash_recv_trx) - && trx_get_dict_operation(trx) == TRX_DICT_OP_NONE - && !srv_undo_sources && srv_fast_shutdown) { + && trx_roll_must_shutdown()) { /* Shutdown has been initiated. */ trx->error_state = DB_INTERRUPTED; return(NULL); diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index a0bc85e0433..de4f48a1506 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -57,13 +57,6 @@ bool trx_rollback_or_clean_is_active; /** In crash recovery, the current trx to be rolled back; NULL otherwise */ const trx_t* trx_roll_crash_recv_trx; -/** In crash recovery we set this to the undo n:o of the current trx to be -rolled back. Then we can print how many % the rollback has progressed. */ -static undo_no_t trx_roll_max_undo_no; - -/** Auxiliary variable which tells the previous progress % we printed */ -static ulint trx_roll_progress_printed_pct; - /****************************************************************//** Finishes a transaction rollback. */ static @@ -551,8 +544,6 @@ trx_rollback_active( que_thr_t* thr; roll_node_t* roll_node; dict_table_t* table; - ib_int64_t rows_to_undo; - const char* unit = ""; ibool dictionary_locked = FALSE; heap = mem_heap_create(512); @@ -571,30 +562,8 @@ trx_rollback_active( ut_a(thr == que_fork_start_command(fork)); - mutex_enter(&trx_sys->mutex); - trx_roll_crash_recv_trx = trx; - trx_roll_max_undo_no = trx->undo_no; - - trx_roll_progress_printed_pct = 0; - - rows_to_undo = trx_roll_max_undo_no; - - mutex_exit(&trx_sys->mutex); - - if (rows_to_undo > 1000000000) { - rows_to_undo = rows_to_undo / 1000000; - unit = "M"; - } - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Rolling back trx with id " TRX_ID_FMT ", %lu%s" - " rows to undo\n", - trx->id, - (ulong) rows_to_undo, unit); - if (trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { row_mysql_lock_data_dictionary(trx); dictionary_locked = TRUE; @@ -744,6 +713,48 @@ fake_prepared: goto func_exit; } +/** Report progress when rolling back a row of a recovered transaction. +@return whether the rollback should be aborted due to pending shutdown */ +UNIV_INTERN +bool +trx_roll_must_shutdown() +{ + const trx_t* trx = trx_roll_crash_recv_trx; + ut_ad(trx); + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE)); + + if (trx_get_dict_operation(trx) == TRX_DICT_OP_NONE + && !srv_undo_sources && srv_fast_shutdown) { + return true; + } + + ib_time_t time = ut_time(); + mutex_enter(&trx_sys->mutex); + mutex_enter(&recv_sys->mutex); + + if (recv_sys->report(time)) { + ulint n_trx = 0, n_rows = 0; + for (const trx_t* t = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); + t != NULL; + t = UT_LIST_GET_NEXT(trx_list, t)) { + + assert_trx_in_rw_list(t); + if (t->is_recovered + && trx_state_eq(t, TRX_STATE_ACTIVE)) { + n_trx++; + n_rows += t->undo_no; + } + } + ib_logf(IB_LOG_LEVEL_INFO, + "To roll back: " ULINTPF " transactions, " + ULINTPF " rows", n_trx, n_rows); + } + + mutex_exit(&recv_sys->mutex); + mutex_exit(&trx_sys->mutex); + return false; +} + /*******************************************************************//** Rollback or clean up any incomplete transactions which were encountered in crash recovery. If the transaction already was @@ -1119,7 +1130,6 @@ trx_roll_pop_top_rec_of_trx( undo_no_t undo_no; ibool is_insert; trx_rseg_t* rseg; - ulint progress_pct; mtr_t mtr; rseg = trx->rseg; @@ -1177,27 +1187,6 @@ try_again: ut_ad(undo_no + 1 == trx->undo_no); - /* We print rollback progress info if we are in a crash recovery - and the transaction has at least 1000 row operations to undo. */ - - if (trx == trx_roll_crash_recv_trx && trx_roll_max_undo_no > 1000) { - - progress_pct = 100 - (ulint) - ((undo_no * 100) / trx_roll_max_undo_no); - if (progress_pct != trx_roll_progress_printed_pct) { - if (trx_roll_progress_printed_pct == 0) { - fprintf(stderr, - "\nInnoDB: Progress in percents:" - " %lu", (ulong) progress_pct); - } else { - fprintf(stderr, - " %lu", (ulong) progress_pct); - } - fflush(stderr); - trx_roll_progress_printed_pct = progress_pct; - } - } - trx->undo_no = undo_no; if (!trx_undo_arr_store_info(trx, undo_no)) { diff --git a/storage/xtradb/include/trx0roll.h b/storage/xtradb/include/trx0roll.h index 399b29610ff..565079b17b4 100644 --- a/storage/xtradb/include/trx0roll.h +++ b/storage/xtradb/include/trx0roll.h @@ -106,6 +106,11 @@ trx_undo_rec_release( /*=================*/ trx_t* trx, /*!< in/out: transaction */ undo_no_t undo_no);/*!< in: undo number */ +/** Report progress when rolling back a row of a recovered transaction. +@return whether the rollback should be aborted due to pending shutdown */ +UNIV_INTERN +bool +trx_roll_must_shutdown(); /*******************************************************************//** Rollback or clean up any incomplete transactions which were encountered in crash recovery. If the transaction already was diff --git a/storage/xtradb/row/row0undo.cc b/storage/xtradb/row/row0undo.cc index 95b88f6ce42..eb28b7c3221 100644 --- a/storage/xtradb/row/row0undo.cc +++ b/storage/xtradb/row/row0undo.cc @@ -45,7 +45,6 @@ Created 1/8/1997 Heikki Tuuri #include "row0upd.h" #include "row0mysql.h" #include "srv0srv.h" -#include "srv0start.h" /* How to undo row operations? (1) For an insert, we have stored a prefix of the clustered index record @@ -351,8 +350,7 @@ row_undo_step( ut_ad(que_node_get_type(node) == QUE_NODE_UNDO); if (UNIV_UNLIKELY(trx == trx_roll_crash_recv_trx) - && trx_get_dict_operation(trx) == TRX_DICT_OP_NONE - && !srv_undo_sources && srv_fast_shutdown) { + && trx_roll_must_shutdown()) { /* Shutdown has been initiated. */ trx->error_state = DB_INTERRUPTED; return(NULL); diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc index a0bc85e0433..de4f48a1506 100644 --- a/storage/xtradb/trx/trx0roll.cc +++ b/storage/xtradb/trx/trx0roll.cc @@ -57,13 +57,6 @@ bool trx_rollback_or_clean_is_active; /** In crash recovery, the current trx to be rolled back; NULL otherwise */ const trx_t* trx_roll_crash_recv_trx; -/** In crash recovery we set this to the undo n:o of the current trx to be -rolled back. Then we can print how many % the rollback has progressed. */ -static undo_no_t trx_roll_max_undo_no; - -/** Auxiliary variable which tells the previous progress % we printed */ -static ulint trx_roll_progress_printed_pct; - /****************************************************************//** Finishes a transaction rollback. */ static @@ -551,8 +544,6 @@ trx_rollback_active( que_thr_t* thr; roll_node_t* roll_node; dict_table_t* table; - ib_int64_t rows_to_undo; - const char* unit = ""; ibool dictionary_locked = FALSE; heap = mem_heap_create(512); @@ -571,30 +562,8 @@ trx_rollback_active( ut_a(thr == que_fork_start_command(fork)); - mutex_enter(&trx_sys->mutex); - trx_roll_crash_recv_trx = trx; - trx_roll_max_undo_no = trx->undo_no; - - trx_roll_progress_printed_pct = 0; - - rows_to_undo = trx_roll_max_undo_no; - - mutex_exit(&trx_sys->mutex); - - if (rows_to_undo > 1000000000) { - rows_to_undo = rows_to_undo / 1000000; - unit = "M"; - } - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Rolling back trx with id " TRX_ID_FMT ", %lu%s" - " rows to undo\n", - trx->id, - (ulong) rows_to_undo, unit); - if (trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { row_mysql_lock_data_dictionary(trx); dictionary_locked = TRUE; @@ -744,6 +713,48 @@ fake_prepared: goto func_exit; } +/** Report progress when rolling back a row of a recovered transaction. +@return whether the rollback should be aborted due to pending shutdown */ +UNIV_INTERN +bool +trx_roll_must_shutdown() +{ + const trx_t* trx = trx_roll_crash_recv_trx; + ut_ad(trx); + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE)); + + if (trx_get_dict_operation(trx) == TRX_DICT_OP_NONE + && !srv_undo_sources && srv_fast_shutdown) { + return true; + } + + ib_time_t time = ut_time(); + mutex_enter(&trx_sys->mutex); + mutex_enter(&recv_sys->mutex); + + if (recv_sys->report(time)) { + ulint n_trx = 0, n_rows = 0; + for (const trx_t* t = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); + t != NULL; + t = UT_LIST_GET_NEXT(trx_list, t)) { + + assert_trx_in_rw_list(t); + if (t->is_recovered + && trx_state_eq(t, TRX_STATE_ACTIVE)) { + n_trx++; + n_rows += t->undo_no; + } + } + ib_logf(IB_LOG_LEVEL_INFO, + "To roll back: " ULINTPF " transactions, " + ULINTPF " rows", n_trx, n_rows); + } + + mutex_exit(&recv_sys->mutex); + mutex_exit(&trx_sys->mutex); + return false; +} + /*******************************************************************//** Rollback or clean up any incomplete transactions which were encountered in crash recovery. If the transaction already was @@ -1119,7 +1130,6 @@ trx_roll_pop_top_rec_of_trx( undo_no_t undo_no; ibool is_insert; trx_rseg_t* rseg; - ulint progress_pct; mtr_t mtr; rseg = trx->rseg; @@ -1177,27 +1187,6 @@ try_again: ut_ad(undo_no + 1 == trx->undo_no); - /* We print rollback progress info if we are in a crash recovery - and the transaction has at least 1000 row operations to undo. */ - - if (trx == trx_roll_crash_recv_trx && trx_roll_max_undo_no > 1000) { - - progress_pct = 100 - (ulint) - ((undo_no * 100) / trx_roll_max_undo_no); - if (progress_pct != trx_roll_progress_printed_pct) { - if (trx_roll_progress_printed_pct == 0) { - fprintf(stderr, - "\nInnoDB: Progress in percents:" - " %lu", (ulong) progress_pct); - } else { - fprintf(stderr, - " %lu", (ulong) progress_pct); - } - fflush(stderr); - trx_roll_progress_printed_pct = progress_pct; - } - } - trx->undo_no = undo_no; if (!trx_undo_arr_store_info(trx, undo_no)) { -- cgit v1.2.1 From 2fe990df9f25ead34d9d7a39e19f939fbbfea127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 20:41:32 +0200 Subject: Fix the grammar of an error message --- storage/innobase/handler/ha_innodb.cc | 2 +- storage/xtradb/handler/ha_innodb.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 2d319439fad..6e98bab66af 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3483,7 +3483,7 @@ innobase_init( if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS && global_system_variables.wsrep_on) { /* Do not allow InnoDB startup with VATS and Galera */ - sql_print_error("In Galera environment Variance-Aware-Transaction-Sheduling Algorithm" + sql_print_error("In Galera, innodb_lock_schedule_algorithm=vats" " is not supported."); goto error; } diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 8487c5b2510..dd7c5b8324a 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -3939,7 +3939,7 @@ innobase_init( if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS && global_system_variables.wsrep_on) { /* Do not allow InnoDB startup with VATS and Galera */ - sql_print_error("In Galera environment Variance-Aware-Transaction-Sheduling Algorithm" + sql_print_error("In Galera, innodb_lock_schedule_algorithm=vats" " is not supported."); goto error; } -- cgit v1.2.1 From e9cc486c97d030e27e9c0ad8770b3d8a25afc27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 20:42:53 +0200 Subject: Fix a typo: schedule, scheduling --- mysys/thr_alarm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index 9d917d3dd59..61ef3657161 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -273,7 +273,7 @@ void thr_end_alarm(thr_alarm_t *alarmed) /* Come here when some alarm in queue is due. Mark all alarms with are finnished in list. - Shedule alarms to be sent again after 1-10 sec (many alarms at once) + Schedule alarms to be sent again after 1-10 sec (many alarms at once) If alarm_aborted is set then all alarms are given and resent every second. */ @@ -425,7 +425,7 @@ void end_thr_alarm(my_bool free_structures) if (alarm_aborted != 1) /* If memory not freed */ { mysql_mutex_lock(&LOCK_alarm); - DBUG_PRINT("info",("Resheduling %d waiting alarms",alarm_queue.elements)); + DBUG_PRINT("info",("Rescheduling %d waiting alarms",alarm_queue.elements)); alarm_aborted= -1; /* mark aborted */ if (alarm_queue.elements || (alarm_thread_running && free_structures)) { -- cgit v1.2.1 From 9d76b2749835a4fdbd009433c5a762da2769c4dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 22:30:13 +0200 Subject: Follow-up fix for MDEV-12352: Plug a memory leak trx_rollback_active(): When aborting the rollback, free the query graph. --- storage/innobase/trx/trx0roll.cc | 2 ++ storage/xtradb/trx/trx0roll.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index de4f48a1506..ab488b06121 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -579,6 +579,8 @@ trx_rollback_active( ut_ad(!srv_undo_sources); ut_ad(srv_fast_shutdown); ut_ad(!dictionary_locked); + que_graph_free(static_cast( + roll_node->undo_thr->common.parent)); goto func_exit; } diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc index de4f48a1506..ab488b06121 100644 --- a/storage/xtradb/trx/trx0roll.cc +++ b/storage/xtradb/trx/trx0roll.cc @@ -579,6 +579,8 @@ trx_rollback_active( ut_ad(!srv_undo_sources); ut_ad(srv_fast_shutdown); ut_ad(!dictionary_locked); + que_graph_free(static_cast( + roll_node->undo_thr->common.parent)); goto func_exit; } -- cgit v1.2.1 From 806380494353bb1389507def0256b3308e5cedbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 23:14:54 +0200 Subject: Re-remove the file kill_and_restart_mysqld.inc --- mysql-test/include/kill_and_restart_mysqld.inc | 15 -------- mysql-test/suite/innodb/r/update_time.result | 1 - mysql-test/suite/innodb/t/update_time.test | 3 +- mysql-test/suite/innodb_gis/r/kill_server.result | 1 - .../suite/innodb_gis/r/rtree_compress2.result | 1 - mysql-test/suite/innodb_gis/r/rtree_crash.result | 41 ---------------------- .../suite/innodb_gis/r/rtree_recovery.result | 2 -- mysql-test/suite/innodb_gis/r/types.result | 1 - mysql-test/suite/innodb_gis/t/kill_server.test | 3 +- mysql-test/suite/innodb_gis/t/rtree_compress2.test | 3 +- mysql-test/suite/innodb_gis/t/rtree_recovery.test | 5 +-- mysql-test/suite/innodb_gis/t/types.test | 3 +- 12 files changed, 11 insertions(+), 68 deletions(-) delete mode 100644 mysql-test/include/kill_and_restart_mysqld.inc delete mode 100644 mysql-test/suite/innodb_gis/r/rtree_crash.result diff --git a/mysql-test/include/kill_and_restart_mysqld.inc b/mysql-test/include/kill_and_restart_mysqld.inc deleted file mode 100644 index b67fb7350b4..00000000000 --- a/mysql-test/include/kill_and_restart_mysqld.inc +++ /dev/null @@ -1,15 +0,0 @@ -if (!$restart_parameters) -{ - let $restart_parameters = restart; -} - ---let $_server_id= `SELECT @@server_id` ---let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect - ---echo # Kill and $restart_parameters ---exec echo "$restart_parameters" > $_expect_file_name ---shutdown_server 0 ---source include/wait_until_disconnected.inc ---enable_reconnect ---source include/wait_until_connected_again.inc ---disable_reconnect diff --git a/mysql-test/suite/innodb/r/update_time.result b/mysql-test/suite/innodb/r/update_time.result index ab3eeb08dc7..d8b9069b1ae 100644 --- a/mysql-test/suite/innodb/r/update_time.result +++ b/mysql-test/suite/innodb/r/update_time.result @@ -43,7 +43,6 @@ XA PREPARE 'xatrx'; CONNECT con1,localhost,root,,; call mtr.add_suppression("Found 1 prepared XA transactions"); FLUSH TABLES; -# Kill and restart SELECT update_time FROM information_schema.tables WHERE table_name = 't'; update_time NULL diff --git a/mysql-test/suite/innodb/t/update_time.test b/mysql-test/suite/innodb/t/update_time.test index e2213c5d9b7..1a00e3b9f73 100644 --- a/mysql-test/suite/innodb/t/update_time.test +++ b/mysql-test/suite/innodb/t/update_time.test @@ -67,7 +67,8 @@ CONNECT (con1,localhost,root,,); call mtr.add_suppression("Found 1 prepared XA transactions"); FLUSH TABLES; ---source include/kill_and_restart_mysqld.inc +--let $shutdown_timeout=0 +--source include/restart_mysqld.inc SELECT update_time FROM information_schema.tables WHERE table_name = 't'; diff --git a/mysql-test/suite/innodb_gis/r/kill_server.result b/mysql-test/suite/innodb_gis/r/kill_server.result index 579317872b4..eac4feb442f 100644 --- a/mysql-test/suite/innodb_gis/r/kill_server.result +++ b/mysql-test/suite/innodb_gis/r/kill_server.result @@ -36,6 +36,5 @@ call mtr.add_suppression("InnoDB: A copy of page \[page id: space=[0-9]+, page n START TRANSACTION; CALL insert_t1(5000); COMMIT; -# Kill and restart drop procedure insert_t1; drop table t1; diff --git a/mysql-test/suite/innodb_gis/r/rtree_compress2.result b/mysql-test/suite/innodb_gis/r/rtree_compress2.result index 3522a1097aa..04940f2b3b1 100644 --- a/mysql-test/suite/innodb_gis/r/rtree_compress2.result +++ b/mysql-test/suite/innodb_gis/r/rtree_compress2.result @@ -20,6 +20,5 @@ call mtr.add_suppression("InnoDB: A copy of page \[page id: space=[0-9]+, page n START TRANSACTION; CALL insert_t1(5000); COMMIT; -# Kill and restart drop procedure insert_t1; drop table t1; diff --git a/mysql-test/suite/innodb_gis/r/rtree_crash.result b/mysql-test/suite/innodb_gis/r/rtree_crash.result deleted file mode 100644 index 12ca15a420d..00000000000 --- a/mysql-test/suite/innodb_gis/r/rtree_crash.result +++ /dev/null @@ -1,41 +0,0 @@ -create table t1 (c1 int, c2 geometry not null, spatial index (c2))engine=innodb; -create procedure insert_t1(IN total int) -begin -declare i int default 1; -while (i <= total) DO -insert into t1 values (i, Point(i, i)); -set i = i + 1; -end while; -end| -CALL insert_t1(5000); -select count(*) from t1; -count(*) -5000 -check table t1; -Table Op Msg_type Msg_text -test.t1 check status OK -truncate table t1; -CALL insert_t1(10000); -select count(*) from t1; -count(*) -10000 -drop index c2 on t1; -create spatial index idx on t1(c2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `c1` int(11) DEFAULT NULL, - `c2` geometry NOT NULL, - SPATIAL KEY `idx` (`c2`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -check table t1; -Table Op Msg_type Msg_text -test.t1 check status OK -truncate table t1; -call mtr.add_suppression("InnoDB: page [0-9]+ in the doublewrite buffer is not within space bounds.*"); -START TRANSACTION; -CALL insert_t1(5000); -COMMIT; -# Kill and restart -drop procedure insert_t1; -drop table t1; diff --git a/mysql-test/suite/innodb_gis/r/rtree_recovery.result b/mysql-test/suite/innodb_gis/r/rtree_recovery.result index d2f4409f38c..3572e3e0bed 100644 --- a/mysql-test/suite/innodb_gis/r/rtree_recovery.result +++ b/mysql-test/suite/innodb_gis/r/rtree_recovery.result @@ -18,7 +18,6 @@ end while; end| CALL insert_t1(367); COMMIT; -# Kill and restart check table t1; Table Op Msg_type Msg_text test.t1 check status OK @@ -29,7 +28,6 @@ CALL update_t1(367); SET @poly1 = ST_GeomFromText('POLYGON((10000 10000, 10000 10350, 10350 10350, 10350 10000, 10000 10000))'); delete from t1 where ST_Contains(@poly1, c2); COMMIT; -# Kill and restart check table t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb_gis/r/types.result b/mysql-test/suite/innodb_gis/r/types.result index ac1802085d6..7d823927df1 100644 --- a/mysql-test/suite/innodb_gis/r/types.result +++ b/mysql-test/suite/innodb_gis/r/types.result @@ -53,7 +53,6 @@ COMMIT; INSERT INTO t_wl6455 VALUES(11, POINT(11,11)); BEGIN; INSERT INTO t_wl6455 VALUES(1, POINT(1,1)); -# Kill and restart CHECK TABLE t_wl6455; Table Op Msg_type Msg_text test.t_wl6455 check status OK diff --git a/mysql-test/suite/innodb_gis/t/kill_server.test b/mysql-test/suite/innodb_gis/t/kill_server.test index 9b6469606ca..028bbbdd40d 100644 --- a/mysql-test/suite/innodb_gis/t/kill_server.test +++ b/mysql-test/suite/innodb_gis/t/kill_server.test @@ -58,7 +58,8 @@ CALL insert_t1(5000); COMMIT; ---source include/kill_and_restart_mysqld.inc +--let $shutdown_timeout=0 +--source include/restart_mysqld.inc # Clean up. drop procedure insert_t1; diff --git a/mysql-test/suite/innodb_gis/t/rtree_compress2.test b/mysql-test/suite/innodb_gis/t/rtree_compress2.test index 5956fc9a774..86ef1171833 100644 --- a/mysql-test/suite/innodb_gis/t/rtree_compress2.test +++ b/mysql-test/suite/innodb_gis/t/rtree_compress2.test @@ -51,7 +51,8 @@ CALL insert_t1(5000); COMMIT; ---source include/kill_and_restart_mysqld.inc +--let $shutdown_timeout=0 +--source include/restart_mysqld.inc # Clean up. drop procedure insert_t1; diff --git a/mysql-test/suite/innodb_gis/t/rtree_recovery.test b/mysql-test/suite/innodb_gis/t/rtree_recovery.test index 9e332c089f2..d995048ceac 100644 --- a/mysql-test/suite/innodb_gis/t/rtree_recovery.test +++ b/mysql-test/suite/innodb_gis/t/rtree_recovery.test @@ -44,7 +44,8 @@ delimiter ;| CALL insert_t1(367); COMMIT; ---source include/kill_and_restart_mysqld.inc +--let $shutdown_timeout=0 +--source include/restart_mysqld.inc # Check table. check table t1; @@ -63,7 +64,7 @@ SET @poly1 = ST_GeomFromText('POLYGON((10000 10000, 10000 10350, 10350 10350, 10 delete from t1 where ST_Contains(@poly1, c2); COMMIT; ---source include/kill_and_restart_mysqld.inc +--source include/restart_mysqld.inc # Check table. check table t1; diff --git a/mysql-test/suite/innodb_gis/t/types.test b/mysql-test/suite/innodb_gis/t/types.test index bd03e7b1dbe..65f65e5ae1f 100644 --- a/mysql-test/suite/innodb_gis/t/types.test +++ b/mysql-test/suite/innodb_gis/t/types.test @@ -73,7 +73,8 @@ INSERT INTO t_wl6455 VALUES(11, POINT(11,11)); BEGIN; INSERT INTO t_wl6455 VALUES(1, POINT(1,1)); ---source include/kill_and_restart_mysqld.inc +--let $shutdown_timeout=0 +--source include/restart_mysqld.inc CHECK TABLE t_wl6455; SELECT ST_AsText(g) FROM t_wl6455; -- cgit v1.2.1 From 84726906c947e3021a9fd7628493bfe775e2f26f Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Sat, 2 Dec 2017 07:00:04 +0530 Subject: MDEV-10177 Invisible Columns and Invisible Index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature Definition:- This feature adds invisible column functionality to server. There is 4 level of "invisibility": 1. Not invisible (NOT_INVISIBLE) — Normal columns created by the user 2. A little bit invisible (USER_DEFINED_INVISIBLE) — columns that the user has marked invisible. They aren't shown in SELECT * and they don't require values in INSERT table VALUE (...). Otherwise they behave as normal columns. 3. More invisible (SYSTEM_INVISIBLE) — Can be queried explicitly, otherwise invisible from everything. Think ROWID sytem column. Because they're invisible from ALTER TABLE and from CREATE TABLE they cannot be created or dropped, they're created by the system. User cant not create a column name which is same as of SYSTEM_INVISIBLE. 4. Very invisible (COMPLETELY_INVISIBLE) — as above, but cannot be queried either. They can only show up in EXPLAIN EXTENDED (might be possible for a very invisible indexed virtual column) but otherwise they don't exist for the user.If user creates a columns which has same name as of COMPLETELY_INVISIBLE then COMPLETELY_INVISIBLE column is renamed again. So it is completely invisible from user. Invisible Index(HA_INVISIBLE_KEY):- Creation of invisible columns require a new type of index which will be only visible to system. User cant see/alter/create/delete this index. If user creates a index which is same name as of invisible index then it will be renamed. Syntax Details:- Only USER_DEFINED_INVISIBLE column can be created by user. This can be created by adding INVISIBLE suffix after column definition. Create table t1( a int invisible, b int); Rules:- There are some rules/restrictions related to use of invisible columns 1. All the columns in table cant be invisible. Create table t1(a int invisible); \\error Create table t1(a int invisible, b int invisble); \\error 2. If you want invisible column to be NOT NULL then you have to supply Default value for the column. Create table t1(a int, b int not null); \\error 3. If you create a view/create table with select * then this wont copy invisible fields. So newly created view/table wont have any invisible columns. Create table t2 as select * from t1;//t2 wont have t1 invisible column Create view v1 as select * from t1;//v1 wont have t1 invisible column 4. Invisibility wont be forwarded to next table in any case of create table/view as select */(a,b,c) from table. Create table t2 as select a,b,c from t1; // t2 will have t1 invisible // column(b), but this wont be invisible in t2 Create view v1 as select a,b,c from t1; // v1 will have t1 invisible // column(b), but this wont be invisible in v1 Implementation Details:- Parsing:- INVISIBLE_SYM is added into vcol_attribute(so its like unique suffix), It is also added into keyword_sp_not_data_type so that table can have column with name invisible. Implementation detail is given by each modified function/created function. (Some function are left as they were self explanatory) (m= Modified, n= Newly Created) mysql_prepare_create_table(m):- Extra checks for invisible columns are added. Also some DEBUG_EXECUTE_IF are also added for test cases. mysql_prepare_alter_table(m):- Now this will drop all the COMPLETELY_INVISIBLE column and HA_INVISIBLE_KEY index. Further Modifications are made to stop drop/change/delete of SYSTEM_INVISIBLE column. build_frm_image(m):- Now this allows incorporating field_visibility status into frm image. To remain compatible with old frms field_visibility info will be only written when any of the field is not NOT_INVISIBLE. extra2_write_additional_field_properties(n):- This will write field visibility info into buffer. We first write EXTRA2_FIELD_FLAGS into buffer/frm , then each next char will have field_visibility for each field. init_from_binary_frm_image(m):- Now if we get EXTRA2_FIELD_FLAGS, then we will read the next n(n= number of fields) chars and set the field_visibility. We also increment thd->status_var.feature_invisible_columns. One important thing to note if we find out that key contains a field whose visibility is > USER_DEFINED_INVISIBLE then , we declare this key as invisible key. sql_show.cc is changed accordingly to make show table, show keys correct. mysql_insert(m):- If we get to know that we are doing insert in this way insert into t1 values(1,1); without explicitly specifying columns, then we check for if we have invisible fields if yes then we reset the whole record, Why ? Because first we want hidden columns to get default/null value. Second thing auto_increment has property no default and no null which voilates invisible key rule 2, And because of this it was giving error. Reseting table->record[0] eliminates this issue. More info put breakpoint on handler::write_row and see auto_increment value. fill_record(m):- we continue loop if we find invisible column because this is already reseted/will get its value if it is default. Test cases:- Since we can not directly add > USER_DEFINED_INVISIBLE column then I have debug_dbug to create it in mysql_prepare_create_table. Patch Credit:- Serg Golubchik --- include/my_base.h | 3 +- mysql-test/r/features.result | 1 + mysql-test/r/invisible_binlog.result | 65 +++++ mysql-test/r/invisible_field.result | 526 +++++++++++++++++++++++++++++++++++ mysql-test/t/invisible_binlog.test | 32 +++ mysql-test/t/invisible_field.test | 216 ++++++++++++++ sql/field.cc | 6 +- sql/field.h | 6 +- sql/lex.h | 1 + sql/mysqld.cc | 1 + sql/share/errmsg-utf8.txt | 2 + sql/sql_base.cc | 19 +- sql/sql_class.h | 10 +- sql/sql_insert.cc | 5 +- sql/sql_show.cc | 36 ++- sql/sql_table.cc | 165 ++++++++++- sql/sql_yacc.yy | 8 +- sql/sql_yacc_ora.yy | 6 + sql/table.cc | 25 +- sql/table.h | 11 + sql/unireg.cc | 35 ++- sql/unireg.h | 1 + 22 files changed, 1144 insertions(+), 36 deletions(-) create mode 100644 mysql-test/r/invisible_binlog.result create mode 100644 mysql-test/r/invisible_field.result create mode 100644 mysql-test/t/invisible_binlog.test create mode 100644 mysql-test/t/invisible_field.test diff --git a/include/my_base.h b/include/my_base.h index 1e7cacd3426..71dca2e831a 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -282,7 +282,8 @@ enum ha_base_keytype { This flag can be calculated -- it's based on key lengths comparison. */ #define HA_KEY_HAS_PART_KEY_SEG 65536 - +/* Internal Flag Can be calcaluted */ +#define HA_INVISIBLE_KEY 2<<18 /* Automatic bits in key-flag */ #define HA_SPACE_PACK_USED 4 /* Test for if SPACE_PACK used */ diff --git a/mysql-test/r/features.result b/mysql-test/r/features.result index c6d1a6b0bac..fc276e2f885 100644 --- a/mysql-test/r/features.result +++ b/mysql-test/r/features.result @@ -8,6 +8,7 @@ Feature_delay_key_write 0 Feature_dynamic_columns 0 Feature_fulltext 0 Feature_gis 0 +Feature_invisible_columns 0 Feature_locale 0 Feature_subquery 0 Feature_timezone 0 diff --git a/mysql-test/r/invisible_binlog.result b/mysql-test/r/invisible_binlog.result new file mode 100644 index 00000000000..088bc858e28 --- /dev/null +++ b/mysql-test/r/invisible_binlog.result @@ -0,0 +1,65 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1(a int , b int invisible); +insert into t1 values(1); +insert into t1(a,b) values(2,2); +select a,b from t1; +a b +1 NULL +2 2 +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES NULL INVISIBLE +create table t2(a int , b int invisible default 5); +insert into t2 values(1); +insert into t2(a,b) values(2,2); +select a,b from t2; +a b +1 5 +2 2 +desc t2; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES 5 INVISIBLE +connection slave; +select * from t1; +a +1 +2 +select a,b from t1; +a b +1 NULL +2 2 +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES NULL INVISIBLE +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) INVISIBLE DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +select * from t2; +a +1 +2 +select a,b from t2; +a b +1 5 +2 2 +desc t2; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES 5 INVISIBLE +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) INVISIBLE DEFAULT 5 +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +connection master; +drop table t1,t2; +include/rpl_end.inc diff --git a/mysql-test/r/invisible_field.result b/mysql-test/r/invisible_field.result new file mode 100644 index 00000000000..22bc1063649 --- /dev/null +++ b/mysql-test/r/invisible_field.result @@ -0,0 +1,526 @@ +FLUSH STATUS; +create table t1(abc int primary key, xyz int invisible); +SHOW STATUS LIKE 'Feature_invisible_columns'; +Variable_name Value +Feature_invisible_columns 1 +desc t1; +Field Type Null Key Default Extra +abc int(11) NO PRI NULL +xyz int(11) YES NULL INVISIBLE +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `abc` int(11) NOT NULL, + `xyz` int(11) INVISIBLE DEFAULT NULL, + PRIMARY KEY (`abc`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +select * from INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='test' and TABLE_NAME='t1'; +TABLE_CATALOG TABLE_SCHEMA TABLE_NAME COLUMN_NAME ORDINAL_POSITION COLUMN_DEFAULT IS_NULLABLE DATA_TYPE CHARACTER_MAXIMUM_LENGTH CHARACTER_OCTET_LENGTH NUMERIC_PRECISION NUMERIC_SCALE DATETIME_PRECISION CHARACTER_SET_NAME COLLATION_NAME COLUMN_TYPE COLUMN_KEY EXTRA PRIVILEGES COLUMN_COMMENT IS_GENERATED GENERATION_EXPRESSION +def test t1 abc 1 NULL NO int NULL NULL 10 0 NULL NULL NULL int(11) PRI select,insert,update,references NEVER NULL +def test t1 xyz 2 NULL YES int NULL NULL 10 0 NULL NULL NULL int(11) INVISIBLE select,insert,update,references NEVER NULL +drop table t1; +create table t1(a1 int invisible); +ERROR 42000: A table must have at least 1 column +create table t1(a1 blob,invisible(a1)); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '(a1))' at line 1 +create table t1(a1 int primary key invisible ,a2 int unique invisible , a3 blob,a4 int not null invisible unique); +ERROR HY000: Invisible column `a1` must have a default value +create table t1(abc int not null invisible); +ERROR HY000: Invisible column `abc` must have a default value +create table t1(a int invisible, b int); +insert into t1 values(1); +insert into t1(a) values(2); +insert into t1(b) values(3); +insert into t1(a,b) values(5,5); +select * from t1; +b +1 +NULL +3 +5 +select a,b from t1; +a b +NULL 1 +2 NULL +NULL 3 +5 5 +delete from t1; +insert into t1 values(1),(2),(3),(4); +select * from t1; +b +1 +2 +3 +4 +select a from t1; +a +NULL +NULL +NULL +NULL +drop table t1; +#more complex case of invisible +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES NULL INVISIBLE +c int(11) NO PRI NULL auto_increment, INVISIBLE +d blob YES NULL +e int(11) YES UNI NULL +f int(11) YES NULL +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +select * from t1; +a d e f +1 d blob 1 1 +1 d blob 11 1 +1 d blob 2 1 +1 d blob 3 1 +1 d blob 41 1 +select a,b,c,d,e,f from t1; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +drop table t1; +#more complex case of invisible with sql_mode=NO_AUTO_VALUE_ON_ZERO +set sql_mode='NO_AUTO_VALUE_ON_ZERO'; +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES NULL INVISIBLE +c int(11) NO PRI NULL auto_increment, INVISIBLE +d blob YES NULL +e int(11) YES UNI NULL +f int(11) YES NULL +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +select * from t1; +a d e f +1 d blob 1 1 +1 d blob 11 1 +1 d blob 2 1 +1 d blob 3 1 +1 d blob 41 1 +select a,b,c,d,e,f from t1; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +drop table t1; +set sql_mode=''; +create table sdsdsd(a int , b int, invisible(a,b)); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '(a,b))' at line 1 +create table t1(a int,abc int as (a mod 3) virtual invisible); +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +abc int(11) YES NULL VIRTUAL GENERATED, INVISIBLE +insert into t1 values(1,default); +ERROR 21S01: Column count doesn't match value count at row 1 +insert into t1 values(1),(22),(233); +select * from t1; +a +1 +22 +233 +select a,abc from t1; +a abc +1 1 +22 1 +233 2 +drop table t1; +create table t1(abc int primary key invisible auto_increment, a int); +desc t1; +Field Type Null Key Default Extra +abc int(11) NO PRI NULL auto_increment, INVISIBLE +a int(11) YES NULL +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `abc` int(11) NOT NULL INVISIBLE AUTO_INCREMENT, + `a` int(11) DEFAULT NULL, + PRIMARY KEY (`abc`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +select * from t1; +a +1 +2 +3 +select abc,a from t1; +abc a +1 1 +2 2 +3 3 +delete from t1; +insert into t1 values(1),(2),(3),(4),(6); +select abc,a from t1; +abc a +4 1 +5 2 +6 3 +7 4 +8 6 +drop table t1; +create table t1(abc int); +alter table t1 change abc ss int invisible; +ERROR 42000: A table must have at least 1 column +alter table t1 add column xyz int; +alter table t1 modify column abc int ; +desc t1; +Field Type Null Key Default Extra +abc int(11) YES NULL +xyz int(11) YES NULL +insert into t1 values(22); +ERROR 21S01: Column count doesn't match value count at row 1 +alter table t1 modify column abc int invisible; +desc t1; +Field Type Null Key Default Extra +abc int(11) YES NULL INVISIBLE +xyz int(11) YES NULL +insert into t1 values(12); +drop table t1; +#some test on copy table structure with table data; +#table with invisible fields and unique keys; +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES NULL INVISIBLE +c int(11) NO PRI NULL auto_increment, INVISIBLE +d blob YES NULL +e int(11) YES UNI NULL +f int(11) YES NULL +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +select * from t1; +a d e f +1 d blob 1 1 +1 d blob 11 1 +1 d blob 2 1 +1 d blob 3 1 +1 d blob 41 1 +select a,b,c,d,e,f from t1; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +#this won't copy invisible fields and keys; +create table t2 as select * from t1; +desc t2; +Field Type Null Key Default Extra +a int(11) YES NULL +d blob YES NULL +e int(11) YES NULL +f int(11) YES NULL +select * from t2; +a d e f +1 d blob 1 1 +1 d blob 11 1 +1 d blob 2 1 +1 d blob 3 1 +1 d blob 41 1 +select a,b,c,d,e,f from t2; +ERROR 42S22: Unknown column 'b' in 'field list' +drop table t2; +#now this will copy invisible fields +create table t2 as select a,b,c,d,e,f from t1; +desc t2; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES NULL +c int(11) NO 0 +d blob YES NULL +e int(11) YES NULL +f int(11) YES NULL +select * from t2; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +select a,b,c,d,e,f from t2; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +drop table t2,t1; +#some test related to copy of data from one table to another; +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +select a,b,c,d,e,f from t1; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +create table t2(a int , b int invisible , c int invisible , d blob , e int unique, f int); +insert into t2 select * from t1; +select a,b,c,d,e,f from t2; +a b c d e f +1 NULL NULL d blob 1 1 +1 NULL NULL d blob 11 1 +1 NULL NULL d blob 2 1 +1 NULL NULL d blob 3 1 +1 NULL NULL d blob 41 1 +truncate t2; +insert into t2 (a,b,c,d,e,f) select a,b,c,d,e,f from t1; +select a,b,c,d,e,f from t2; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +truncate t2; +drop table t1,t2; +#some test related to creating view on table with invisible column; +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +create view v as select * from t1; +desc v; +Field Type Null Key Default Extra +a int(11) YES NULL +d blob YES NULL +e int(11) YES NULL +f int(11) YES NULL +select * from v; +a d e f +1 d blob 1 1 +1 d blob 11 1 +1 d blob 2 1 +1 d blob 3 1 +1 d blob 41 1 +#v does not have invisible column; +select a,b,c,d,e,f from v; +ERROR 42S22: Unknown column 'b' in 'field list' +insert into v values(1,21,32,4); +select * from v; +a d e f +1 d blob 1 1 +1 d blob 11 1 +1 d blob 2 1 +1 d blob 3 1 +1 d blob 41 1 +1 21 32 4 +insert into v(a,b,c,d,e,f) values(1,12,3,4,5,6); +ERROR 42S22: Unknown column 'b' in 'field list' +drop view v; +create view v as select a,b,c,d,e,f from t1; +desc v; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES NULL +c int(11) NO 0 +d blob YES NULL +e int(11) YES NULL +f int(11) YES NULL +select * from v; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +1 NULL 6 21 32 4 +#v does have invisible column but they aren't invisible anymore. +select a,b,c,d,e,f from v; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +1 NULL 6 21 32 4 +insert into v values(1,26,33,4,45,66); +select a,b,c,d,e,f from v; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +1 NULL 6 21 32 4 +1 26 33 4 45 66 +insert into v(a,b,c,d,e,f) values(1,32,31,41,5,6); +select a,b,c,d,e,f from v; +a b c d e f +1 NULL 1 d blob 1 1 +1 NULL 2 d blob 11 1 +1 NULL 3 d blob 2 1 +1 NULL 4 d blob 3 1 +1 NULL 5 d blob 41 1 +1 NULL 6 21 32 4 +1 26 33 4 45 66 +1 32 31 41 5 6 +drop view v; +drop table t1; +#now invisible column in where and some join query +create table t1 (a int unique , b int invisible unique, c int unique invisible); +insert into t1(a,b,c) values(1,1,1); +insert into t1(a,b,c) values(2,2,2); +insert into t1(a,b,c) values(3,3,3); +insert into t1(a,b,c) values(4,4,4); +insert into t1(a,b,c) values(21,21,26); +insert into t1(a,b,c) values(31,31,35); +insert into t1(a,b,c) values(41,41,45); +insert into t1(a,b,c) values(22,22,24); +insert into t1(a,b,c) values(32,32,33); +insert into t1(a,b,c) values(42,42,43); +explain select * from t1 where b=3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 const b b 5 const 1 +select * from t1 where b=3; +a +3 +explain select * from t1 where c=3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 const c c 5 const 1 +select * from t1 where c=3; +a +3 +create table t2 as select a,b,c from t1; +desc t2; +Field Type Null Key Default Extra +a int(11) YES NULL +b int(11) YES NULL +c int(11) YES NULL +explain select * from t1,t2 where t1.b = t2.c and t1.c = t2.b; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 10 +1 SIMPLE t1 ALL b,c NULL NULL NULL 10 Using where; Using join buffer (flat, BNL join) +select * from t1,t2 where t1.b = t2.c and t1.c = t2.b; +a a b c +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 4 4 4 +drop table t1,t2; +#Unhide invisible columns +create table t1 (a int primary key, b int invisible, c int invisible unique); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) NOT NULL, + `b` int(11) INVISIBLE DEFAULT NULL, + `c` int(11) INVISIBLE DEFAULT NULL, + PRIMARY KEY (`a`), + UNIQUE KEY `c` (`c`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +desc t1; +Field Type Null Key Default Extra +a int(11) NO PRI NULL +b int(11) YES NULL INVISIBLE +c int(11) YES UNI NULL INVISIBLE +alter table t1 modify column b int; +desc t1; +Field Type Null Key Default Extra +a int(11) NO PRI NULL +b int(11) YES NULL +c int(11) YES UNI NULL INVISIBLE +alter table t1 change column c d int; +desc t1; +Field Type Null Key Default Extra +a int(11) NO PRI NULL +b int(11) YES NULL +d int(11) YES UNI NULL +drop table t1; +SHOW STATUS LIKE 'Feature_invisible_columns'; +Variable_name Value +Feature_invisible_columns 50 +#invisible is non reserved +create table t1(a int unique , invisible int invisible, c int ); +desc t1; +Field Type Null Key Default Extra +a int(11) YES UNI NULL +invisible int(11) YES NULL INVISIBLE +c int(11) YES NULL +alter table t1 change column invisible hid int invisible; +desc t1; +Field Type Null Key Default Extra +a int(11) YES UNI NULL +hid int(11) YES NULL INVISIBLE +c int(11) YES NULL +drop table t1; +CREATE TABLE t1 (b int); +INSERT t1 values(1); +INSERT t1 values(2); +INSERT t1 values(3); +INSERT t1 values(4); +INSERT t1 values(5); +CREATE TABLE t2 (a int invisible) SELECT * FROM t1; +select * from t2 order by b; +b +1 +2 +3 +4 +5 +select a,b from t2 order by b; +a b +NULL 1 +NULL 2 +NULL 3 +NULL 4 +NULL 5 +CREATE TABLE t3 (b int, a int invisible) SELECT * FROM t1; +select * from t3 order by b; +b +1 +2 +3 +4 +5 +select a,b from t3 order by b; +a b +NULL 1 +NULL 2 +NULL 3 +NULL 4 +NULL 5 +CREATE TABLE t4 (b int invisible) SELECT * FROM t1; +ERROR 42000: A table must have at least 1 column +CREATE TABLE t5 (a int invisible) SELECT b as a FROM t1; +ERROR 42000: A table must have at least 1 column +drop table t1,t2,t3; +create table t1 (a int , b int invisible default 3, c int , d int invisible default 6); +CREATE PROCEDURE +insert_t1(a int, b int) +MODIFIES SQL DATA +insert into t1 values(a,b); +// +call insert_t1(1,1); +call insert_t1(2,2); +select * from t1 order by a; +a c +1 1 +2 2 +select a,b,c,d from t1 order by a; +a b c d +1 3 1 6 +2 3 2 6 +DROP PROCEDURE insert_t1; +delete from t1; +prepare insert_1 from "insert into t1 values(@a,@c)"; +prepare insert_2 from "insert into t1(a,b,c) values(@a,@b,@c)"; +set @a=1, @c=1; +execute insert_1; +set @a=2,@b=2, @c=2; +execute insert_2; +select a,b,c,d from t1 order by a; +a b c d +1 3 1 6 +2 2 2 6 +drop table t1; diff --git a/mysql-test/t/invisible_binlog.test b/mysql-test/t/invisible_binlog.test new file mode 100644 index 00000000000..654afcc39c3 --- /dev/null +++ b/mysql-test/t/invisible_binlog.test @@ -0,0 +1,32 @@ +--source include/master-slave.inc + +--connection master +create table t1(a int , b int invisible); +insert into t1 values(1); +insert into t1(a,b) values(2,2); +select a,b from t1; +desc t1; + +create table t2(a int , b int invisible default 5); +insert into t2 values(1); +insert into t2(a,b) values(2,2); +select a,b from t2; +desc t2; + + +--sync_slave_with_master +select * from t1; +select a,b from t1; +desc t1; +show create table t1; + +select * from t2; +select a,b from t2; +desc t2; +show create table t2; + + +--connection master +drop table t1,t2; + +--source include/rpl_end.inc diff --git a/mysql-test/t/invisible_field.test b/mysql-test/t/invisible_field.test new file mode 100644 index 00000000000..55f3463a310 --- /dev/null +++ b/mysql-test/t/invisible_field.test @@ -0,0 +1,216 @@ +FLUSH STATUS; +create table t1(abc int primary key, xyz int invisible); +SHOW STATUS LIKE 'Feature_invisible_columns'; +desc t1; +show create table t1; +select * from INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='test' and TABLE_NAME='t1'; +drop table t1; +--error ER_TABLE_MUST_HAVE_COLUMNS +create table t1(a1 int invisible); +--error ER_PARSE_ERROR +create table t1(a1 blob,invisible(a1)); +--error ER_INVISIBLE_NOT_NULL_WITHOUT_DEFAULT +create table t1(a1 int primary key invisible ,a2 int unique invisible , a3 blob,a4 int not null invisible unique); +--error ER_INVISIBLE_NOT_NULL_WITHOUT_DEFAULT +create table t1(abc int not null invisible); +create table t1(a int invisible, b int); +#should automatically add null +insert into t1 values(1); +insert into t1(a) values(2); +insert into t1(b) values(3); +insert into t1(a,b) values(5,5); +select * from t1; +select a,b from t1; +delete from t1; +insert into t1 values(1),(2),(3),(4); +select * from t1; +select a from t1; +drop table t1; + +--echo #more complex case of invisible +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +desc t1; +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +select * from t1; +select a,b,c,d,e,f from t1; +drop table t1; + +--echo #more complex case of invisible with sql_mode=NO_AUTO_VALUE_ON_ZERO +set sql_mode='NO_AUTO_VALUE_ON_ZERO'; +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +desc t1; +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +select * from t1; +select a,b,c,d,e,f from t1; +drop table t1; +set sql_mode=''; + +--error ER_PARSE_ERROR +create table sdsdsd(a int , b int, invisible(a,b)); +create table t1(a int,abc int as (a mod 3) virtual invisible); +desc t1; +--error ER_WRONG_VALUE_COUNT_ON_ROW +insert into t1 values(1,default); +insert into t1 values(1),(22),(233); +select * from t1; +select a,abc from t1; +drop table t1; +create table t1(abc int primary key invisible auto_increment, a int); +desc t1; +show create table t1; +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +select * from t1; +select abc,a from t1; +delete from t1; +insert into t1 values(1),(2),(3),(4),(6); +select abc,a from t1; +drop table t1; +create table t1(abc int); +--error ER_TABLE_MUST_HAVE_COLUMNS +alter table t1 change abc ss int invisible; +alter table t1 add column xyz int; +alter table t1 modify column abc int ; +desc t1; +--error ER_WRONG_VALUE_COUNT_ON_ROW +insert into t1 values(22); +alter table t1 modify column abc int invisible; +desc t1; +insert into t1 values(12); +drop table t1; +--echo #some test on copy table structure with table data; +--echo #table with invisible fields and unique keys; +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +desc t1; +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +select * from t1; +select a,b,c,d,e,f from t1; +--echo #this won't copy invisible fields and keys; +create table t2 as select * from t1; +desc t2; +select * from t2; +--error ER_BAD_FIELD_ERROR +select a,b,c,d,e,f from t2; +drop table t2; +--echo #now this will copy invisible fields +create table t2 as select a,b,c,d,e,f from t1; +desc t2; +select * from t2; +select a,b,c,d,e,f from t2; +drop table t2,t1; +--echo #some test related to copy of data from one table to another; +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +select a,b,c,d,e,f from t1; +create table t2(a int , b int invisible , c int invisible , d blob , e int unique, f int); +insert into t2 select * from t1; +select a,b,c,d,e,f from t2; +truncate t2; +insert into t2 (a,b,c,d,e,f) select a,b,c,d,e,f from t1; +select a,b,c,d,e,f from t2; +truncate t2; +drop table t1,t2; +--echo #some test related to creating view on table with invisible column; +create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int); +insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1); +create view v as select * from t1; +desc v; +select * from v; +--echo #v does not have invisible column; +--error ER_BAD_FIELD_ERROR +select a,b,c,d,e,f from v; +insert into v values(1,21,32,4); +select * from v; +--error ER_BAD_FIELD_ERROR +insert into v(a,b,c,d,e,f) values(1,12,3,4,5,6); +drop view v; +create view v as select a,b,c,d,e,f from t1; +desc v; +select * from v; +--echo #v does have invisible column but they aren't invisible anymore. +select a,b,c,d,e,f from v; +insert into v values(1,26,33,4,45,66); +select a,b,c,d,e,f from v; +insert into v(a,b,c,d,e,f) values(1,32,31,41,5,6); +select a,b,c,d,e,f from v; +drop view v; +drop table t1; +--echo #now invisible column in where and some join query +create table t1 (a int unique , b int invisible unique, c int unique invisible); +insert into t1(a,b,c) values(1,1,1); +insert into t1(a,b,c) values(2,2,2); +insert into t1(a,b,c) values(3,3,3); +insert into t1(a,b,c) values(4,4,4); +insert into t1(a,b,c) values(21,21,26); +insert into t1(a,b,c) values(31,31,35); +insert into t1(a,b,c) values(41,41,45); +insert into t1(a,b,c) values(22,22,24); +insert into t1(a,b,c) values(32,32,33); +insert into t1(a,b,c) values(42,42,43); +explain select * from t1 where b=3; +select * from t1 where b=3; +explain select * from t1 where c=3; +select * from t1 where c=3; +create table t2 as select a,b,c from t1; +desc t2; +explain select * from t1,t2 where t1.b = t2.c and t1.c = t2.b; +select * from t1,t2 where t1.b = t2.c and t1.c = t2.b; +drop table t1,t2; +--echo #Unhide invisible columns +create table t1 (a int primary key, b int invisible, c int invisible unique); +show create table t1; +desc t1; +alter table t1 modify column b int; +desc t1; +alter table t1 change column c d int; +desc t1; +drop table t1; +SHOW STATUS LIKE 'Feature_invisible_columns'; +--echo #invisible is non reserved +create table t1(a int unique , invisible int invisible, c int ); +desc t1; +alter table t1 change column invisible hid int invisible; +desc t1; +drop table t1; +##Internal temp table +CREATE TABLE t1 (b int); +INSERT t1 values(1); +INSERT t1 values(2); +INSERT t1 values(3); +INSERT t1 values(4); +INSERT t1 values(5); +CREATE TABLE t2 (a int invisible) SELECT * FROM t1; +select * from t2 order by b; +select a,b from t2 order by b; +CREATE TABLE t3 (b int, a int invisible) SELECT * FROM t1; +select * from t3 order by b; +select a,b from t3 order by b; +--error ER_TABLE_MUST_HAVE_COLUMNS +CREATE TABLE t4 (b int invisible) SELECT * FROM t1; +--error ER_TABLE_MUST_HAVE_COLUMNS +CREATE TABLE t5 (a int invisible) SELECT b as a FROM t1; +drop table t1,t2,t3; + +create table t1 (a int , b int invisible default 3, c int , d int invisible default 6); +DELIMITER //; +CREATE PROCEDURE +insert_t1(a int, b int) +MODIFIES SQL DATA +insert into t1 values(a,b); +// +DELIMITER ;// +call insert_t1(1,1); +call insert_t1(2,2); +select * from t1 order by a; +select a,b,c,d from t1 order by a; +DROP PROCEDURE insert_t1; +delete from t1; +prepare insert_1 from "insert into t1 values(@a,@c)"; +prepare insert_2 from "insert into t1(a,b,c) values(@a,@b,@c)"; +set @a=1, @c=1; +execute insert_1; +set @a=2,@b=2, @c=2; +execute insert_2; +select a,b,c,d from t1 order by a; +drop table t1; \ No newline at end of file diff --git a/sql/field.cc b/sql/field.cc index 7d8f8a9af20..5ad1981b752 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1603,7 +1603,8 @@ String *Field::val_int_as_str(String *val_buffer, bool unsigned_val) Field::Field(uchar *ptr_arg,uint32 length_arg,uchar *null_ptr_arg, uchar null_bit_arg, utype unireg_check_arg, const LEX_CSTRING *field_name_arg) - :ptr(ptr_arg), null_ptr(null_ptr_arg), table(0), orig_table(0), + :ptr(ptr_arg), field_visibility(NOT_INVISIBLE), + null_ptr(null_ptr_arg), table(0), orig_table(0), table_name(0), field_name(*field_name_arg), option_list(0), option_struct(0), key_start(0), part_of_key(0), part_of_key_not_clustered(0), part_of_sortkey(0), @@ -2195,6 +2196,7 @@ Field *Field::make_new_field(MEM_ROOT *root, TABLE *new_table, tmp->flags&= (NOT_NULL_FLAG | BLOB_FLAG | UNSIGNED_FLAG | ZEROFILL_FLAG | BINARY_FLAG | ENUM_FLAG | SET_FLAG); tmp->reset_fields(); + tmp->field_visibility= NOT_INVISIBLE; return tmp; } @@ -10645,6 +10647,7 @@ Column_definition::Column_definition(THD *thd, Field *old_field, option_list= old_field->option_list; pack_flag= 0; compression_method_ptr= 0; + field_visibility= old_field->field_visibility; if (orig_field) { @@ -10782,6 +10785,7 @@ Column_definition::redefine_stage1_common(const Column_definition *dup_field, flags= dup_field->flags; interval= dup_field->interval; vcol_info= dup_field->vcol_info; + field_visibility= dup_field->field_visibility; } diff --git a/sql/field.h b/sql/field.h index 624921b628e..7338a8ff679 100644 --- a/sql/field.h +++ b/sql/field.h @@ -674,6 +674,8 @@ public: { DBUG_ASSERT(0); } uchar *ptr; // Position to field in record + + field_visible_type field_visibility; /** Byte where the @c NULL bit is stored inside a record. If this Field is a @c NOT @c NULL field, this member is @c NULL. @@ -4046,6 +4048,7 @@ public: max number of characters. */ ulonglong length; + field_visible_type field_visibility; /* The value of `length' as set by parser: is the number of characters for most of the types, or of bytes for BLOBs or numeric types. @@ -4076,7 +4079,7 @@ public: :Type_handler_hybrid_field_type(&type_handler_null), compression_method_ptr(0), comment(null_clex_str), - on_update(NULL), length(0), decimals(0), + on_update(NULL), length(0),field_visibility(NOT_INVISIBLE), decimals(0), flags(0), pack_length(0), key_length(0), unireg_check(Field::NONE), interval(0), charset(&my_charset_bin), srid(0), geom_type(Field::GEOM_GEOMETRY), @@ -4553,5 +4556,6 @@ bool check_expression(Virtual_column_info *vcol, LEX_CSTRING *name, #define f_no_default(x) ((x) & FIELDFLAG_NO_DEFAULT) #define f_bit_as_char(x) ((x) & FIELDFLAG_TREAT_BIT_AS_CHAR) #define f_is_hex_escape(x) ((x) & FIELDFLAG_HEX_ESCAPE) +#define f_visibility(x) (static_cast ((x) & 3)) #endif /* FIELD_INCLUDED */ diff --git a/sql/lex.h b/sql/lex.h index a0068069660..da985ad26a6 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -273,6 +273,7 @@ static SYMBOL symbols[] = { { "HAVING", SYM(HAVING)}, { "HELP", SYM(HELP_SYM)}, { "HIGH_PRIORITY", SYM(HIGH_PRIORITY)}, + { "INVISIBLE", SYM(INVISIBLE_SYM)}, { "HOST", SYM(HOST_SYM)}, { "HOSTS", SYM(HOSTS_SYM)}, { "HOUR", SYM(HOUR_SYM)}, diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 1f679ee6e9e..f3cb39959a7 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -8490,6 +8490,7 @@ SHOW_VAR status_vars[]= { {"Feature_dynamic_columns", (char*) offsetof(STATUS_VAR, feature_dynamic_columns), SHOW_LONG_STATUS}, {"Feature_fulltext", (char*) offsetof(STATUS_VAR, feature_fulltext), SHOW_LONG_STATUS}, {"Feature_gis", (char*) offsetof(STATUS_VAR, feature_gis), SHOW_LONG_STATUS}, + {"Feature_invisible_columns", (char*) offsetof(STATUS_VAR, feature_invisible_columns), SHOW_LONG_STATUS}, {"Feature_locale", (char*) offsetof(STATUS_VAR, feature_locale), SHOW_LONG_STATUS}, {"Feature_subquery", (char*) offsetof(STATUS_VAR, feature_subquery), SHOW_LONG_STATUS}, {"Feature_timezone", (char*) offsetof(STATUS_VAR, feature_timezone), SHOW_LONG_STATUS}, diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 1b7696bc056..949e3bd9d17 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -7810,3 +7810,5 @@ ER_INVALID_AGGREGATE_FUNCTION eng "Aggregate specific instruction(FETCH GROUP NEXT ROW) missing from the aggregate function" ER_INVALID_VALUE_TO_LIMIT eng "Limit only accepts integer values" +ER_INVISIBLE_NOT_NULL_WITHOUT_DEFAULT + eng "Invisible column %`s must have a default value" diff --git a/sql/sql_base.cc b/sql/sql_base.cc index a08078c5cff..ef8d4afdd3c 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -5528,7 +5528,10 @@ find_field_in_table(THD *thd, TABLE *table, const char *name, uint length, if (field_ptr && *field_ptr) { - *cached_field_index_ptr= (uint)(field_ptr - table->field); + if ((*field_ptr)->field_visibility == COMPLETELY_INVISIBLE) + DBUG_RETURN((Field*)0); + + *cached_field_index_ptr= field_ptr - table->field; field= *field_ptr; } else @@ -7607,6 +7610,14 @@ insert_fields(THD *thd, Name_resolution_context *context, const char *db_name, for (; !field_iterator.end_of_fields(); field_iterator.next()) { + /* + field() is always NULL for views (see, e.g. Field_iterator_view or + Field_iterator_natural_join). + But view fields can never be invisible. + */ + if ((field= field_iterator.field()) && + field->field_visibility != NOT_INVISIBLE) + continue; Item *item; if (!(item= field_iterator.create_item(thd))) @@ -8213,7 +8224,6 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List &values, ? table->next_number_field->field_index : ~0U; DBUG_ENTER("fill_record"); - if (!*ptr) { /* No fields to update, quite strange!*/ @@ -8236,7 +8246,10 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List &values, /* Ensure that all fields are from the same table */ DBUG_ASSERT(field->table == table); - value=v++; + if (field->field_visibility != NOT_INVISIBLE) + continue; + else + value=v++; if (field->field_index == autoinc_index) table->auto_increment_field_not_null= TRUE; if (field->vcol_info) diff --git a/sql/sql_class.h b/sql/sql_class.h index 46dea27dc7a..4249bc6bb5b 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -304,22 +304,25 @@ public: LEX_CSTRING name; engine_option_value *option_list; bool generated; + bool invisible; Key(enum Keytype type_par, const LEX_CSTRING *name_arg, ha_key_alg algorithm_arg, bool generated_arg, DDL_options_st ddl_options) :DDL_options(ddl_options), type(type_par), key_create_info(default_key_create_info), - name(*name_arg), option_list(NULL), generated(generated_arg) + name(*name_arg), option_list(NULL), generated(generated_arg), + invisible(false) { key_create_info.algorithm= algorithm_arg; - } + } Key(enum Keytype type_par, const LEX_CSTRING *name_arg, KEY_CREATE_INFO *key_info_arg, bool generated_arg, List *cols, engine_option_value *create_opt, DDL_options_st ddl_options) :DDL_options(ddl_options), type(type_par), key_create_info(*key_info_arg), columns(*cols), - name(*name_arg), option_list(create_opt), generated(generated_arg) + name(*name_arg), option_list(create_opt), generated(generated_arg), + invisible(false) {} Key(const Key &rhs, MEM_ROOT *mem_root); virtual ~Key() {} @@ -791,6 +794,7 @@ typedef struct system_status_var ulong feature_dynamic_columns; /* +1 when creating a dynamic column */ ulong feature_fulltext; /* +1 when MATCH is used */ ulong feature_gis; /* +1 opening a table with GIS features */ + ulong feature_invisible_columns; /* +1 opening a table with invisible column */ ulong feature_locale; /* +1 when LOCALE is set */ ulong feature_subquery; /* +1 when subqueries are used */ ulong feature_timezone; /* +1 when XPATH is used */ diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 51c6c80fd66..0cf121fdd59 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -222,7 +222,7 @@ static int check_insert_fields(THD *thd, TABLE_LIST *table_list, table_list->view_db.str, table_list->view_name.str); DBUG_RETURN(-1); } - if (values.elements != table->s->fields) + if (values.elements != table->s->visible_fields) { my_error(ER_WRONG_VALUE_COUNT_ON_ROW, MYF(0), 1L); DBUG_RETURN(-1); @@ -980,7 +980,8 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, No field list, all fields are set explicitly: INSERT INTO t1 VALUES (values) */ - if (thd->lex->used_tables) // Column used in values() + if (thd->lex->used_tables || // Column used in values() + table->s->visible_fields != table->s->fields) restore_record(table,s->default_values); // Get empty record else { diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 8ac472bd626..d0a0293bf1b 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2162,13 +2162,18 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet, */ old_map= tmp_use_all_columns(table, table->read_set); + bool not_the_first_field= false; for (ptr=table->field ; (field= *ptr); ptr++) { + uint flags = field->flags; - if (ptr != table->field) + if (field->field_visibility > USER_DEFINED_INVISIBLE) + continue; + if (not_the_first_field) packet->append(STRING_WITH_LEN(",\n")); + not_the_first_field= true; packet->append(STRING_WITH_LEN(" ")); append_identifier(thd,packet,field->field_name.str, field->field_name.length); @@ -2221,6 +2226,10 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet, packet->append(STRING_WITH_LEN(" NULL")); } + if (field->field_visibility == USER_DEFINED_INVISIBLE) + { + packet->append(STRING_WITH_LEN(" INVISIBLE")); + } def_value.set(def_value_buf, sizeof(def_value_buf), system_charset_info); if (get_field_default_value(thd, field, &def_value, 1)) { @@ -2262,6 +2271,8 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet, for (uint i=0 ; i < share->keys ; i++,key_info++) { + if (key_info->flags & HA_INVISIBLE_KEY) + continue; KEY_PART_INFO *key_part= key_info->key_part; bool found_primary=0; packet->append(STRING_WITH_LEN(",\n ")); @@ -5711,6 +5722,8 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables, for (; (field= *ptr) ; ptr++) { + if(field->field_visibility > USER_DEFINED_INVISIBLE) + continue; uchar *pos; char tmp[MAX_FIELD_WIDTH]; String type(tmp,sizeof(tmp), system_charset_info); @@ -5768,11 +5781,11 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables, table->field[16]->store((const char*) pos, strlen((const char*) pos), cs); + StringBuffer<256> buf; if (field->unireg_check == Field::NEXT_NUMBER) - table->field[17]->store(STRING_WITH_LEN("auto_increment"), cs); + buf.set(STRING_WITH_LEN("auto_increment"),cs); if (print_on_update_clause(field, &type, true)) - table->field[17]->store(type.ptr(), type.length(), cs); - + buf.set(type.ptr(), type.length(),cs); if (field->vcol_info) { String gen_s(tmp,sizeof(tmp), system_charset_info); @@ -5783,13 +5796,20 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables, table->field[20]->store(STRING_WITH_LEN("ALWAYS"), cs); if (field->vcol_info->stored_in_db) - table->field[17]->store(STRING_WITH_LEN("STORED GENERATED"), cs); + buf.set(STRING_WITH_LEN("STORED GENERATED"), cs); else - table->field[17]->store(STRING_WITH_LEN("VIRTUAL GENERATED"), cs); + buf.set(STRING_WITH_LEN("VIRTUAL GENERATED"), cs); } else table->field[20]->store(STRING_WITH_LEN("NEVER"), cs); - + /*Invisible can coexist with auto_increment and virtual */ + if (field->field_visibility == USER_DEFINED_INVISIBLE) + { + if (buf.length()) + buf.append(STRING_WITH_LEN(", ")); + buf.append(STRING_WITH_LEN("INVISIBLE"),cs); + } + table->field[17]->store(buf.ptr(), buf.length(), cs); table->field[19]->store(field->comment.str, field->comment.length, cs); if (schema_table_store_record(thd, table)) DBUG_RETURN(1); @@ -6369,6 +6389,8 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, } for (uint i=0 ; i < show_table->s->keys ; i++,key_info++) { + if ((key_info->flags & HA_INVISIBLE_KEY)) + continue; KEY_PART_INFO *key_part= key_info->key_part; LEX_CSTRING *str; LEX_CSTRING unknown= {STRING_WITH_LEN("?unknown field?") }; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index dbf198d722a..44774d5fcff 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -62,12 +62,16 @@ const char *primary_key_name="PRIMARY"; -static bool check_if_keyname_exists(const char *name,KEY *start, KEY *end); +static int check_if_keyname_exists(const char *name,KEY *start, KEY *end); static char *make_unique_key_name(THD *thd, const char *field_name, KEY *start, KEY *end); static void make_unique_constraint_name(THD *thd, LEX_CSTRING *name, List *vcol, uint *nr); +static const +char * make_unique_invisible_field_name(THD *thd, const char *field_name, + List *fields); + static int copy_data_between_tables(THD *thd, TABLE *from,TABLE *to, List &create, bool ignore, uint order_num, ORDER *order, @@ -3263,8 +3267,70 @@ bool Column_definition::prepare_stage1_check_typelib_default() } return false; } +/* + This function adds a invisible field to field_list + SYNOPSIS + mysql_add_invisible_field() + thd Thread Object + field_list list of all table fields + field_name name/prefix of invisible field + ( Prefix in the case when it is + *COMPLETELY_INVISIBLE* + and given name is duplicate) + type_handler field data type + field_visibility + default value + RETURN VALUE + Create_field pointer +*/ +int mysql_add_invisible_field(THD *thd, List * field_list, + const char *field_name, Type_handler *type_handler, + field_visible_type field_visibility, Item* default_value) +{ + Create_field *fld= new(thd->mem_root)Create_field(); + const char *new_name= NULL; + /* Get unique field name if field_visibility == COMPLETELY_INVISIBLE */ + if (field_visibility == COMPLETELY_INVISIBLE) + { + if ((new_name= make_unique_invisible_field_name(thd, field_name, + field_list))) + { + fld->field_name.str= new_name; + fld->field_name.length= strlen(new_name); + } + else + return 1; //Should not happen + } + else + { + fld->field_name.str= thd->strmake(field_name, strlen(field_name)); + fld->field_name.length= strlen(field_name); + } + fld->set_handler(type_handler); + fld->field_visibility= field_visibility; + if (default_value) + { + Virtual_column_info *v= new (thd->mem_root) Virtual_column_info(); + v->expr= default_value; + v->utf8= 0; + fld->default_value= v; + } + field_list->push_front(fld, thd->mem_root); + return 0; +} - +Key * +mysql_add_invisible_index(THD *thd, List *key_list, + LEX_CSTRING* field_name, enum Key::Keytype type) +{ + Key *key= NULL; + key= new (thd->mem_root) Key(type, &null_clex_str, HA_KEY_ALG_UNDEF, + false, DDL_options(DDL_options::OPT_NONE)); + key->columns.push_back(new(thd->mem_root) Key_part_spec(field_name, 0), + thd->mem_root); + key_list->push_back(key, thd->mem_root); + return key; +} /* Preparation for table creation @@ -3440,7 +3506,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, while ((sql_field=it++)) { DBUG_ASSERT(sql_field->charset != 0); - if (sql_field->prepare_stage2(file, file->ha_table_flags())) DBUG_RETURN(TRUE); if (sql_field->real_field_type() == MYSQL_TYPE_VARCHAR) @@ -3460,8 +3525,18 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, */ if (sql_field->stored_in_db()) record_offset+= sql_field->pack_length; + if (sql_field->field_visibility == USER_DEFINED_INVISIBLE && + sql_field->flags & NOT_NULL_FLAG && + sql_field->flags & NO_DEFAULT_VALUE_FLAG) + { + my_error(ER_INVISIBLE_NOT_NULL_WITHOUT_DEFAULT, MYF(0), + sql_field->field_name.str); + DBUG_RETURN(TRUE); + } } - /* Update virtual fields' offset*/ + /* Update virtual fields' offset and give error if + All fields are invisible */ + bool is_all_invisible= true; it.rewind(); while ((sql_field=it++)) { @@ -3470,6 +3545,13 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, sql_field->offset= record_offset; record_offset+= sql_field->pack_length; } + if (sql_field->field_visibility == NOT_INVISIBLE) + is_all_invisible= false; + } + if (is_all_invisible) + { + my_error(ER_TABLE_MUST_HAVE_COLUMNS, MYF(0)); + DBUG_RETURN(TRUE); } if (auto_increment > 1) { @@ -3720,11 +3802,21 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, &column->field_name, &sql_field->field_name)) field++; + /* + Either field is not present or field visibility is > + USER_DEFINED_INVISIBLE + */ if (!sql_field) { my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), column->field_name.str); DBUG_RETURN(TRUE); } + if (sql_field->field_visibility > USER_DEFINED_INVISIBLE && + !key->invisible) + { + my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), column->field_name.str); + DBUG_RETURN(TRUE); + } while ((dup_column= cols2++) != column) { if (!lex_string_cmp(system_charset_info, @@ -5034,17 +5126,36 @@ err: /* ** Give the key name after the first field with an optional '_#' after + @returns + 0 if keyname does not exists + [1..) index + 1 of duplicate key name **/ -static bool +static int check_if_keyname_exists(const char *name, KEY *start, KEY *end) { - for (KEY *key=start ; key != end ; key++) + uint i= 1; + for (KEY *key=start; key != end ; key++, i++) if (!my_strcasecmp(system_charset_info, name, key->name.str)) - return 1; + return i; return 0; } +/** + Returns 1 if field name exists otherwise 0 +*/ +static bool +check_if_field_name_exists(const char *name, List * fields) +{ + Create_field *fld; + List_iteratorit(*fields); + while ((fld = it++)) + { + if (!my_strcasecmp(system_charset_info, fld->field_name.str, name)) + return 1; + } + return 0; +} static char * make_unique_key_name(THD *thd, const char *field_name,KEY *start,KEY *end) @@ -5102,6 +5213,33 @@ static void make_unique_constraint_name(THD *thd, LEX_CSTRING *name, } } +/** + COMPLETELY_INVISIBLE are internally created. They are completely invisible + to Alter command (Opposite of SYSTEM_INVISIBLE which throws an + error when same name column is added by Alter). So in the case of when + user added a same column name as of COMPLETELY_INVISIBLE , we change + COMPLETELY_INVISIBLE column name. +*/ +static const +char * make_unique_invisible_field_name(THD *thd, const char *field_name, + List *fields) +{ + if (!check_if_field_name_exists(field_name, fields)) + return field_name; + char buff[MAX_FIELD_NAME], *buff_end; + buff_end= strmake_buf(buff, field_name); + if (buff_end - buff < 5) + return NULL; // Should not happen + + for (uint i=1 ; i < 10000; i++) + { + char *real_end= int10_to_str(i, buff_end, 10); + if (check_if_field_name_exists(buff, fields)) + continue; + return (const char *)thd->strmake(buff, real_end - buff); + } + return NULL; //Should not happen +} /**************************************************************************** ** Alter a table definition @@ -7530,6 +7668,8 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, bitmap_clear_all(&table->tmp_set); for (f_ptr=table->field ; (field= *f_ptr) ; f_ptr++) { + if (field->field_visibility == COMPLETELY_INVISIBLE) + continue; Alter_drop *drop; if (field->type() == MYSQL_TYPE_VARCHAR) create_info->varchar= TRUE; @@ -7541,7 +7681,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, !my_strcasecmp(system_charset_info,field->field_name.str, drop->name)) break; } - if (drop) + if (drop && field->field_visibility < SYSTEM_INVISIBLE) { /* Reset auto_increment value if it was dropped */ if (MTYP_TYPENR(field->unireg_check) == Field::NEXT_NUMBER && @@ -7566,7 +7706,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, &def->change)) break; } - if (def) + if (def && field->field_visibility < SYSTEM_INVISIBLE) { // Field is changed def->field=field; /* @@ -7621,12 +7761,12 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, def_it.rewind(); while ((def=def_it++)) // Add new columns { + Create_field *find; if (def->change.str && ! def->field) { /* Check if there is modify for newly added field. */ - Create_field *find; find_it.rewind(); while((find=find_it++)) { @@ -7666,7 +7806,6 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, new_create_list.push_back(def, thd->mem_root); else { - Create_field *find; if (def->change.str) { find_it.rewind(); @@ -7753,9 +7892,11 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, Collect all keys which isn't in drop list. Add only those for which some fields exists. */ - + for (uint i=0 ; i < table->s->keys ; i++,key_info++) { + if (key_info->flags & HA_INVISIBLE_KEY) + continue; const char *key_name= key_info->name.str; Alter_drop *drop; drop_it.rewind(); diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index b2227bdbbbf..2e51cb6da11 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1129,6 +1129,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token HEX_NUM %token HEX_STRING %token HIGH_PRIORITY +%token INVISIBLE_SYM %token HOST_SYM %token HOSTS_SYM %token HOUR_MICROSECOND_SYM @@ -6428,6 +6429,10 @@ vcol_attribute: lex->alter_info.flags|= Alter_info::ALTER_ADD_INDEX; } | COMMENT_SYM TEXT_STRING_sys { Lex->last_field->comment= $2; } + | INVISIBLE_SYM + { + Lex->last_field->field_visibility= USER_DEFINED_INVISIBLE; + } ; parse_vcol_expr: @@ -6812,7 +6817,7 @@ attribute: } | AUTO_INC { Lex->last_field->flags|= AUTO_INCREMENT_FLAG | NOT_NULL_FLAG; } | SERIAL_SYM DEFAULT VALUE_SYM - { + { LEX *lex=Lex; lex->last_field->flags|= AUTO_INCREMENT_FLAG | NOT_NULL_FLAG | UNIQUE_KEY_FLAG; lex->alter_info.flags|= Alter_info::ALTER_ADD_INDEX; @@ -15166,6 +15171,7 @@ keyword_sp_not_data_type: | GOTO_SYM {} | HASH_SYM {} | HARD_SYM {} + | INVISIBLE_SYM {} | HOSTS_SYM {} | HOUR_SYM {} | ID_SYM {} diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index 54c27783b0a..1de303baf08 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -537,6 +537,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token HEX_NUM %token HEX_STRING %token HIGH_PRIORITY +%token INVISIBLE_SYM %token HOST_SYM %token HOSTS_SYM %token HOUR_MICROSECOND_SYM @@ -6125,6 +6126,10 @@ vcol_attribute: lex->alter_info.flags|= Alter_info::ALTER_ADD_INDEX; } | COMMENT_SYM TEXT_STRING_sys { Lex->last_field->comment= $2; } + | INVISIBLE_SYM + { + Lex->last_field->field_visibility= USER_DEFINED_INVISIBLE; + } ; parse_vcol_expr: @@ -15046,6 +15051,7 @@ keyword_sp_not_data_type: | GLOBAL_SYM {} | HASH_SYM {} | HARD_SYM {} + | INVISIBLE_SYM {} | HOSTS_SYM {} | HOUR_SYM {} | ID_SYM {} diff --git a/sql/table.cc b/sql/table.cc index e8343903d96..f65d7fa123d 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -43,6 +43,7 @@ #include "rpl_filter.h" #include "sql_cte.h" #include "ha_sequence.h" +#include "sql_show.h" /* For MySQL 5.7 virtual fields */ #define MYSQL57_GENERATED_FIELD 128 @@ -1161,13 +1162,15 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, uint db_create_options, keys, key_parts, n_length; uint com_length, null_bit_pos, mysql57_vcol_null_bit_pos, bitmap_count; uint i; + uint field_additional_property_length= 0; bool use_hash, mysql57_null_bits= 0; char *keynames, *names, *comment_pos; const uchar *forminfo, *extra2; const uchar *frm_image_end = frm_image + frm_length; uchar *record, *null_flags, *null_pos, *mysql57_vcol_null_pos= 0; const uchar *disk_buff, *strpos; - ulong pos, record_offset; + const uchar *field_properties= NULL; + ulong pos, record_offset; ulong rec_buff_length; handler *handler_file= 0; KEY *keyinfo; @@ -1279,6 +1282,10 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, } #endif /*HAVE_SPATIAL*/ break; + case EXTRA2_FIELD_FLAGS: + field_properties = extra2; + field_additional_property_length= length; + break; default: /* abort frm parsing if it's an unknown but important extra2 value */ if (type >= EXTRA2_ENGINE_IMPORTANT) @@ -1595,8 +1602,9 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, memcpy(record, frm_image + record_offset, share->reclength); disk_buff= frm_image + pos + FRM_FORMINFO_SIZE; - share->fields= uint2korr(forminfo+258); + if (field_properties && field_additional_property_length != share->fields) + goto err; pos= uint2korr(forminfo+260); /* Length of all screens */ n_length= uint2korr(forminfo+268); interval_count= uint2korr(forminfo+270); @@ -1607,6 +1615,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, vcol_screen_length= uint2korr(forminfo+286); share->virtual_fields= share->default_expressions= share->field_check_constraints= share->default_fields= 0; + share->visible_fields= 0; share->stored_fields= share->fields; if (forminfo[46] != (uchar)255) { @@ -1978,6 +1987,15 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, reg_field->field_index= i; reg_field->comment=comment; reg_field->vcol_info= vcol_info; + if(field_properties!=NULL) + { + uint temp= *field_properties++; + reg_field->field_visibility= f_visibility(temp); + } + if (reg_field->field_visibility == USER_DEFINED_INVISIBLE) + status_var_increment(thd->status_var.feature_invisible_columns); + if (reg_field->field_visibility == NOT_INVISIBLE) + share->visible_fields++; if (field_type == MYSQL_TYPE_BIT && !f_bit_as_char(pack_flag)) { null_bits_are_used= 1; @@ -2229,6 +2247,8 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, field= key_part->field= share->field[key_part->fieldnr-1]; key_part->type= field->key_type(); + if (field->field_visibility > USER_DEFINED_INVISIBLE) + keyinfo->flags |= HA_INVISIBLE_KEY; if (field->null_ptr) { key_part->null_offset=(uint) ((uchar*) field->null_ptr - @@ -5107,7 +5127,6 @@ int TABLE::verify_constraints(bool ignore_failure) return(VIEW_CHECK_OK); } - /* Find table in underlying tables by mask and check that only this table belong to given mask diff --git a/sql/table.h b/sql/table.h index aded1930a6b..72e70bf0312 100644 --- a/sql/table.h +++ b/sql/table.h @@ -335,6 +335,16 @@ enum enum_vcol_update_mode VCOL_UPDATE_FOR_REPLACE }; +/* Field visibility enums */ + +enum field_visible_type{ + NOT_INVISIBLE= 0, + USER_DEFINED_INVISIBLE, + /* automatically added by the server. Can be queried explicitly + in SELECT, otherwise invisible from anything" */ + SYSTEM_INVISIBLE, + COMPLETELY_INVISIBLE +}; /** Category of table found in the table share. @@ -668,6 +678,7 @@ struct TABLE_SHARE uint blob_fields; /* number of blob fields */ uint varchar_fields; /* number of varchar fields */ uint default_fields; /* number of default fields */ + uint visible_fields; /* number of visible fields */ uint default_expressions; uint table_check_constraints, field_check_constraints; diff --git a/sql/unireg.cc b/sql/unireg.cc index 1ca0233552e..a0f42f68bf0 100644 --- a/sql/unireg.cc +++ b/sql/unireg.cc @@ -87,6 +87,21 @@ static uchar *extra2_write(uchar *pos, enum extra2_frm_value_type type, return extra2_write(pos, type, reinterpret_cast(str)); } +static uchar *extra2_write_additional_field_properties(uchar *pos, + int number_of_fields,List_iterator * it) +{ + *pos++=EXTRA2_FIELD_FLAGS; + /* + always first 2 for field visibility + */ + pos= extra2_write_len(pos, number_of_fields); + Create_field *cf; + while((cf=(*it)++)) + *pos++= cf->field_visibility; + it->rewind(); + return pos; +} + /** Create a frm (table definition) file @@ -121,6 +136,19 @@ LEX_CUSTRING build_frm_image(THD *thd, const char *table, StringBuffer vcols; DBUG_ENTER("build_frm_image"); + List_iterator it(create_fields); + Create_field *field; + bool have_additional_field_properties= false; + while ((field=it++)) + { + if (field->field_visibility != NOT_INVISIBLE) + { + have_additional_field_properties= true; + break; + } + } + it.rewind(); + /* If fixed row records, we need one bit to check for deleted rows */ if (!(create_info->table_options & HA_OPTION_PACK_RECORD)) create_info->null_bits++; @@ -218,7 +246,9 @@ LEX_CUSTRING build_frm_image(THD *thd, const char *table, if (gis_extra2_len) extra2_size+= 1 + (gis_extra2_len > 255 ? 3 : 1) + gis_extra2_len; - + if(have_additional_field_properties) + extra2_size+=1 + (create_fields.elements > 255 ? 3 : 1) + + create_fields.elements; key_buff_length= uint4korr(fileinfo+47); @@ -274,7 +304,8 @@ LEX_CUSTRING build_frm_image(THD *thd, const char *table, pos+= gis_field_options_image(pos, create_fields); } #endif /*HAVE_SPATIAL*/ - + if (have_additional_field_properties) + pos=extra2_write_additional_field_properties(pos,create_fields.elements,&it); int4store(pos, filepos); // end of the extra2 segment pos+= 4; diff --git a/sql/unireg.h b/sql/unireg.h index b0cfb3841ef..36f985ee1bb 100644 --- a/sql/unireg.h +++ b/sql/unireg.h @@ -176,6 +176,7 @@ enum extra2_frm_value_type { #define EXTRA2_ENGINE_IMPORTANT 128 EXTRA2_ENGINE_TABLEOPTS=128, + EXTRA2_FIELD_FLAGS=129 }; int rea_create_table(THD *thd, LEX_CUSTRING *frm, -- cgit v1.2.1 From 022b163ac8c008cb47b0c9d2f3c1d6fe1580651c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 3 Dec 2017 15:37:32 +0100 Subject: Add tests for system and completely invisible columns debug only --- mysql-test/r/invisible_field_debug.result | 285 ++++++++++++++++++++++++++++++ mysql-test/t/invisible_field_debug.test | 202 +++++++++++++++++++++ sql/sql_base.cc | 3 +- sql/sql_show.cc | 3 +- sql/sql_table.cc | 19 +- 5 files changed, 509 insertions(+), 3 deletions(-) create mode 100644 mysql-test/r/invisible_field_debug.result create mode 100644 mysql-test/t/invisible_field_debug.test diff --git a/mysql-test/r/invisible_field_debug.result b/mysql-test/r/invisible_field_debug.result new file mode 100644 index 00000000000..dab95f14d4c --- /dev/null +++ b/mysql-test/r/invisible_field_debug.result @@ -0,0 +1,285 @@ +set @old_debug= @@debug_dbug; +set debug_dbug= "+d,test_pseudo_invisible"; +create table t1(a int); +set debug_dbug=@old_debug; +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +insert into t1 values(1); +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +drop table t1; +set debug_dbug= "+d,test_completely_invisible"; +create table t1(a int); +set debug_dbug=@old_debug; +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +insert into t1 values(1); +select * from t1; +a +1 +select invisible ,a from t1; +ERROR 42S22: Unknown column 'invisible' in 'field list' +set debug_dbug= "+d,test_completely_invisible"; +select invisible ,a from t1; +invisible a +9 1 +set debug_dbug=@old_debug; +drop table t1; +set debug_dbug= "+d,test_pseudo_invisible"; +create table t1(a int); +set debug_dbug=@old_debug; +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +insert into t1 values(1); +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +ALTER table t1 change invisible b int; +ERROR 42S22: Unknown column 'invisible' in 't1' +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +ALTER table t1 modify invisible char; +ERROR 42S22: Unknown column 'invisible' in 't1' +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +ALTER table t1 drop invisible; +ERROR 42000: Can't DROP COLUMN `invisible`; check that it exists +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +ALTER table t1 add invisible int; +ERROR 42S21: Duplicate column name 'invisible' +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +ALTER table t1 add invisible2 int default 2; +select * from t1; +a invisible2 +1 2 +select invisible ,a from t1; +invisible a +9 1 +drop table t1; +set debug_dbug= "+d,test_completely_invisible"; +create table t1(a int); +desc t1; +Field Type Null Key Default Extra +a int(11) YES NULL +insert into t1 values(1); +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +ALTER table t1 change invisible b int; +ERROR 42S22: Unknown column 'invisible' in 't1' +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +ALTER table t1 modify invisible char; +ERROR 42S22: Unknown column 'invisible' in 't1' +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +ALTER table t1 drop invisible; +ERROR 42000: Can't DROP COLUMN `invisible`; check that it exists +select * from t1; +a +1 +select invisible ,a from t1; +invisible a +9 1 +ALTER table t1 add invisible int; +select * from t1; +a invisible +1 NULL +select invisible1, invisible ,a from t1; +invisible1 invisible a +9 NULL 1 +ALTER table t1 add hid int default 2; +set debug_dbug= "+d,test_completely_invisible"; +select * from t1; +a invisible hid +1 NULL 2 +select invisible ,a from t1; +invisible a +NULL 1 +drop table t1; +set debug_dbug=@old_debug; +Create table t1( a int default(99) invisible, b int); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +select * from t1 order by b; +b +1 +2 +3 +4 +alter table t1 add index(a); +alter table t1 add index(a,b); +show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t1 1 a 1 a A NULL NULL NULL YES BTREE +t1 1 a_2 1 a A NULL NULL NULL YES BTREE +t1 1 a_2 2 b A NULL NULL NULL YES BTREE +drop table t1; +set debug_dbug= "+d,test_pseudo_invisible"; +Create table t1( a int default(99) invisible, b int); +Create table t2( a int default(99) invisible, b int, unique(invisible)); +ERROR 42000: Key column 'invisible' doesn't exist in table +set debug_dbug=@old_debug; +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +select * from t1 order by b; +b +1 +2 +3 +4 +select invisible, a, b from t1 order by b; +invisible a b +9 99 1 +9 99 2 +9 99 3 +9 99 4 +alter table t1 add index(invisible); +ERROR 42000: Key column 'invisible' doesn't exist in table +alter table t1 add index(b,invisible); +ERROR 42000: Key column 'invisible' doesn't exist in table +show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +drop table t1; +set debug_dbug= "+d,test_completely_invisible"; +Create table t1( a int default(99) invisible, b int); +Create table t2( a int default(99) invisible, b int, unique(invisible)); +ERROR 42000: Key column 'invisible' doesn't exist in table +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +select * from t1 order by b; +b +1 +2 +3 +4 +select invisible, a, b from t1 order by b; +invisible a b +9 99 1 +9 99 2 +9 99 3 +9 99 4 +set debug_dbug=@old_debug; +alter table t1 add index(invisible); +ERROR 42000: Key column 'invisible' doesn't exist in table +alter table t1 add index(b,invisible); +ERROR 42000: Key column 'invisible' doesn't exist in table +show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +drop table t1; +set debug_dbug= "+d,test_completely_invisible,test_invisible_index"; +Create table t1( a int default(99) , b int,c int, index(b)); +set debug_dbug=@old_debug; +Show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t1 1 b 1 b A NULL NULL NULL YES BTREE +select * from INFORMATION_SCHEMA.STATISTICS where TABLE_SCHEMA ='test' and table_name='t1'; +TABLE_CATALOG TABLE_SCHEMA TABLE_NAME NON_UNIQUE INDEX_SCHEMA INDEX_NAME SEQ_IN_INDEX COLUMN_NAME COLLATION CARDINALITY SUB_PART PACKED NULLABLE INDEX_TYPE COMMENT INDEX_COMMENT +def test t1 1 test b 1 b A NULL NULL NULL YES BTREE +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT 99, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `b` (`b`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +insert into t1 values(1,1,1); +insert into t1 values(2,2,2); +insert into t1 values(3,3,3); +insert into t1 values(4,4,4); +set debug_dbug= "+d,test_completely_invisible,test_invisible_index"; +select invisible, a ,b from t1 order by b; +invisible a b +9 1 1 +9 2 2 +9 3 3 +9 4 4 +explain select * from t1 where invisible =9; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ref invisible invisible 5 const 3 +alter table t1 add x int default 3; +select invisible, a ,b from t1; +invisible a b +9 1 1 +9 2 2 +9 3 3 +9 4 4 +set debug_dbug=@old_debug; +Show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t1 1 b 1 b A NULL NULL NULL YES BTREE +create index a1 on t1(invisible); +ERROR 42000: Key column 'invisible' doesn't exist in table +set debug_dbug= "+d,test_completely_invisible,test_invisible_index"; +drop index invisible on t1; +ERROR 42000: Can't DROP INDEX `invisible`; check that it exists +explain select * from t1 where invisible =9; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ref invisible invisible 5 const 3 +create index invisible on t1(c); +explain select * from t1 where invisible =9; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ref invisible_2 invisible_2 5 const 3 +show indexes in t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t1 1 b 1 b A NULL NULL NULL YES BTREE +t1 1 invisible 1 c A NULL NULL NULL YES BTREE +t1 1 invisible_2 1 invisible A NULL NULL NULL YES BTREE +drop table t1; +set @old_debug= @@debug_dbug; diff --git a/mysql-test/t/invisible_field_debug.test b/mysql-test/t/invisible_field_debug.test new file mode 100644 index 00000000000..89a856fe970 --- /dev/null +++ b/mysql-test/t/invisible_field_debug.test @@ -0,0 +1,202 @@ +--source include/have_debug.inc +##TEST for invisible coloumn level 2 +set @old_debug= @@debug_dbug; +set debug_dbug= "+d,test_pseudo_invisible"; +create table t1(a int); +set debug_dbug=@old_debug; + +desc t1; +show create table t1; +insert into t1 values(1); +select * from t1; +select invisible ,a from t1; +drop table t1; + +##TEST for invisible coloumn level 3 + +set debug_dbug= "+d,test_completely_invisible"; +create table t1(a int); +set debug_dbug=@old_debug; + +desc t1; +show create table t1; +insert into t1 values(1); +select * from t1; + +--error ER_BAD_FIELD_ERROR +select invisible ,a from t1; + +set debug_dbug= "+d,test_completely_invisible"; +select invisible ,a from t1; +set debug_dbug=@old_debug; + +drop table t1; + +##TEST for Alter table for invisibleness level 2 + +set debug_dbug= "+d,test_pseudo_invisible"; +create table t1(a int); +set debug_dbug=@old_debug; + +desc t1; +insert into t1 values(1); +select * from t1; +select invisible ,a from t1; + +## Alter should not be possible + +--error ER_BAD_FIELD_ERROR +ALTER table t1 change invisible b int; +select * from t1; +select invisible ,a from t1; + +--error ER_BAD_FIELD_ERROR +ALTER table t1 modify invisible char; +select * from t1; +select invisible ,a from t1; + +--error ER_CANT_DROP_FIELD_OR_KEY +ALTER table t1 drop invisible; +select * from t1; +select invisible ,a from t1; + +--error ER_DUP_FIELDNAME +ALTER table t1 add invisible int; +select * from t1; +select invisible ,a from t1; + +ALTER table t1 add invisible2 int default 2; +select * from t1; +select invisible ,a from t1; + +drop table t1; + +##TEST for Alter table for invisibleness level 3 + +set debug_dbug= "+d,test_completely_invisible"; +create table t1(a int); + +desc t1; +insert into t1 values(1); +select * from t1; +select invisible ,a from t1; + +## Alter should not be possible + +--error ER_BAD_FIELD_ERROR +ALTER table t1 change invisible b int; +select * from t1; +select invisible ,a from t1; + +--error ER_BAD_FIELD_ERROR +ALTER table t1 modify invisible char; +select * from t1; +select invisible ,a from t1; + +--error ER_CANT_DROP_FIELD_OR_KEY +ALTER table t1 drop invisible; +select * from t1; +select invisible ,a from t1; + +ALTER table t1 add invisible int; +select * from t1; +select invisible1, invisible ,a from t1; + +#set debug_dbug=@old_debug; +ALTER table t1 add hid int default 2; +set debug_dbug= "+d,test_completely_invisible"; +select * from t1; +select invisible ,a from t1; + +drop table t1; +set debug_dbug=@old_debug; + +## Test Index on USER_DEFINED_INVISIBLE + +Create table t1( a int default(99) invisible, b int); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +select * from t1 order by b; +alter table t1 add index(a); +alter table t1 add index(a,b); +show index from t1; +drop table t1; + +## Test Index on PSEUDO_invisible_INVISIBLE + +set debug_dbug= "+d,test_pseudo_invisible"; +Create table t1( a int default(99) invisible, b int); + +--error ER_KEY_COLUMN_DOES_NOT_EXITS +Create table t2( a int default(99) invisible, b int, unique(invisible)); + +set debug_dbug=@old_debug; +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +select * from t1 order by b; +select invisible, a, b from t1 order by b; +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 add index(invisible); +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 add index(b,invisible); +show index from t1; +drop table t1; + +## Test Index on COMPLETELY_INVISIBLE + +set debug_dbug= "+d,test_completely_invisible"; +Create table t1( a int default(99) invisible, b int); + +--error ER_KEY_COLUMN_DOES_NOT_EXITS +Create table t2( a int default(99) invisible, b int, unique(invisible)); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +select * from t1 order by b; +select invisible, a, b from t1 order by b; +set debug_dbug=@old_debug; + +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 add index(invisible); +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 add index(b,invisible); +show index from t1; +drop table t1; +## Sytem Generated index on invisible column +set debug_dbug= "+d,test_completely_invisible,test_invisible_index"; +## index name will be invisible +Create table t1( a int default(99) , b int,c int, index(b)); +set debug_dbug=@old_debug; +Show index from t1; +select * from INFORMATION_SCHEMA.STATISTICS where TABLE_SCHEMA ='test' and table_name='t1'; +show create table t1; +insert into t1 values(1,1,1); +insert into t1 values(2,2,2); +insert into t1 values(3,3,3); +insert into t1 values(4,4,4); +set debug_dbug= "+d,test_completely_invisible,test_invisible_index"; +select invisible, a ,b from t1 order by b; +explain select * from t1 where invisible =9; +alter table t1 add x int default 3; +select invisible, a ,b from t1; +set debug_dbug=@old_debug; +Show index from t1; +## Sytem Generated Index modification +--error ER_KEY_COLUMN_DOES_NOT_EXITS +create index a1 on t1(invisible); +set debug_dbug= "+d,test_completely_invisible,test_invisible_index"; +## index does not exist for user +--error ER_CANT_DROP_FIELD_OR_KEY +drop index invisible on t1; +explain select * from t1 where invisible =9; +## index name will be changed +create index invisible on t1(c); +explain select * from t1 where invisible =9; +show indexes in t1; +drop table t1; +set @old_debug= @@debug_dbug; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index ef8d4afdd3c..b493b7d2b9e 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -5528,7 +5528,8 @@ find_field_in_table(THD *thd, TABLE *table, const char *name, uint length, if (field_ptr && *field_ptr) { - if ((*field_ptr)->field_visibility == COMPLETELY_INVISIBLE) + if ((*field_ptr)->field_visibility == COMPLETELY_INVISIBLE && + DBUG_EVALUATE_IF("test_completely_invisible", 0, 1)) DBUG_RETURN((Field*)0); *cached_field_index_ptr= field_ptr - table->field; diff --git a/sql/sql_show.cc b/sql/sql_show.cc index d0a0293bf1b..850383d697c 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -6389,7 +6389,8 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, } for (uint i=0 ; i < show_table->s->keys ; i++,key_info++) { - if ((key_info->flags & HA_INVISIBLE_KEY)) + if ((key_info->flags & HA_INVISIBLE_KEY) && + DBUG_EVALUATE_IF("test_invisible_index", 0, 1)) continue; KEY_PART_INFO *key_part= key_info->key_part; LEX_CSTRING *str; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 44774d5fcff..5943fe2735a 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -3378,6 +3378,23 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, bool tmp_table= create_table_mode == C_ALTER_TABLE; DBUG_ENTER("mysql_prepare_create_table"); + DBUG_EXECUTE_IF("test_pseudo_invisible",{ + mysql_add_invisible_field(thd, &alter_info->create_list, + "invisible", &type_handler_long, SYSTEM_INVISIBLE, + new (thd->mem_root)Item_int(thd, 9)); + }); + DBUG_EXECUTE_IF("test_completely_invisible",{ + mysql_add_invisible_field(thd, &alter_info->create_list, + "invisible", &type_handler_long, COMPLETELY_INVISIBLE, + new (thd->mem_root)Item_int(thd, 9)); + }); + DBUG_EXECUTE_IF("test_invisible_index",{ + LEX_CSTRING temp; + temp.str= "invisible"; + temp.length= strlen("invisible"); + mysql_add_invisible_index(thd, &alter_info->key_list + , &temp, Key::MULTIPLE); + }); LEX_CSTRING* connect_string = &create_info->connect_string; if (connect_string->length != 0 && connect_string->length > CONNECT_STRING_MAXLEN && @@ -3812,7 +3829,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, DBUG_RETURN(TRUE); } if (sql_field->field_visibility > USER_DEFINED_INVISIBLE && - !key->invisible) + !key->invisible && DBUG_EVALUATE_IF("test_invisible_index", 0, 1)) { my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), column->field_name.str); DBUG_RETURN(TRUE); -- cgit v1.2.1 From c90db2c8beed5aba00a348fb4854d809c560501e Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Thu, 7 Dec 2017 13:58:18 +0530 Subject: BuildBot bug fix for invisible columns --- mysql-test/r/invisible_field.result | 8 ++++---- mysql-test/t/invisible_field.test | 2 +- sql/sql_class.cc | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mysql-test/r/invisible_field.result b/mysql-test/r/invisible_field.result index 22bc1063649..0e6980f31e6 100644 --- a/mysql-test/r/invisible_field.result +++ b/mysql-test/r/invisible_field.result @@ -14,10 +14,10 @@ t1 CREATE TABLE `t1` ( `xyz` int(11) INVISIBLE DEFAULT NULL, PRIMARY KEY (`abc`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 -select * from INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='test' and TABLE_NAME='t1'; -TABLE_CATALOG TABLE_SCHEMA TABLE_NAME COLUMN_NAME ORDINAL_POSITION COLUMN_DEFAULT IS_NULLABLE DATA_TYPE CHARACTER_MAXIMUM_LENGTH CHARACTER_OCTET_LENGTH NUMERIC_PRECISION NUMERIC_SCALE DATETIME_PRECISION CHARACTER_SET_NAME COLLATION_NAME COLUMN_TYPE COLUMN_KEY EXTRA PRIVILEGES COLUMN_COMMENT IS_GENERATED GENERATION_EXPRESSION -def test t1 abc 1 NULL NO int NULL NULL 10 0 NULL NULL NULL int(11) PRI select,insert,update,references NEVER NULL -def test t1 xyz 2 NULL YES int NULL NULL 10 0 NULL NULL NULL int(11) INVISIBLE select,insert,update,references NEVER NULL +select TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,EXTRA from INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='test' and TABLE_NAME='t1'; +TABLE_CATALOG TABLE_SCHEMA TABLE_NAME COLUMN_NAME EXTRA +def test t1 abc +def test t1 xyz INVISIBLE drop table t1; create table t1(a1 int invisible); ERROR 42000: A table must have at least 1 column diff --git a/mysql-test/t/invisible_field.test b/mysql-test/t/invisible_field.test index 55f3463a310..b1d28d882d7 100644 --- a/mysql-test/t/invisible_field.test +++ b/mysql-test/t/invisible_field.test @@ -3,7 +3,7 @@ create table t1(abc int primary key, xyz int invisible); SHOW STATUS LIKE 'Feature_invisible_columns'; desc t1; show create table t1; -select * from INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='test' and TABLE_NAME='t1'; +select TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,EXTRA from INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='test' and TABLE_NAME='t1'; drop table t1; --error ER_TABLE_MUST_HAVE_COLUMNS create table t1(a1 int invisible); diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 30b53807578..35ed9be74f9 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -138,7 +138,7 @@ Key::Key(const Key &rhs, MEM_ROOT *mem_root) columns(rhs.columns, mem_root), name(rhs.name), option_list(rhs.option_list), - generated(rhs.generated) + generated(rhs.generated), invisible(false) { list_copy_and_replace_each_value(columns, mem_root); } -- cgit v1.2.1 From 0bc3c0fbc8d2dde151ab486c204075251a929c35 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Thu, 14 Dec 2017 19:17:27 +0530 Subject: mysqldump fix for invisible column Actually there are 2 issues in the case of invisible columns 1st `select fields from t1` will have more fields then `select * from t1`. So instead of `select * from t1` we are using `select a,b,invisible from t1` these fields are supplied from `select fields from t1`. 2nd We are using --complete-insert when we detect that this table is using invisible columns. --- client/mysqldump.c | 65 +++++++++++---- mysql-test/r/mysqldump.result | 182 ++++++++++++++++++++++++++++++++++++++++- mysql-test/r/show_check.result | 24 +----- mysql-test/t/mysqldump.test | 43 ++++++++++ mysql-test/t/show_check.test | 54 ++++++------ 5 files changed, 300 insertions(+), 68 deletions(-) diff --git a/client/mysqldump.c b/client/mysqldump.c index a260065c64c..72e86db6e43 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -115,10 +115,11 @@ static my_bool verbose= 0, opt_no_create_info= 0, opt_no_data= 0, opt_no_data_m opt_events= 0, opt_comments_used= 0, opt_alltspcs=0, opt_notspcs= 0, opt_logging, opt_drop_trigger= 0 ; -static my_bool insert_pat_inited= 0, debug_info_flag= 0, debug_check_flag= 0; +static my_bool insert_pat_inited= 0, debug_info_flag= 0, debug_check_flag= 0, + select_field_names_inited= 0; static ulong opt_max_allowed_packet, opt_net_buffer_length; static MYSQL mysql_connection,*mysql=0; -static DYNAMIC_STRING insert_pat; +static DYNAMIC_STRING insert_pat, select_field_names; static char *opt_password=0,*current_user=0, *current_host=0,*path=0,*fields_terminated=0, *lines_terminated=0, *enclosed=0, *opt_enclosed=0, *escaped=0, @@ -1641,6 +1642,7 @@ static void free_resources() dynstr_free(&extended_row); dynstr_free(&dynamic_where); dynstr_free(&insert_pat); + dynstr_free(&select_field_names); if (defaults_argv) free_defaults(defaults_argv); mysql_library_end(); @@ -2735,7 +2737,13 @@ static uint get_table_structure(char *table, char *db, char *table_type, else dynstr_set_checked(&insert_pat, ""); } - + if (!select_field_names_inited) + { + select_field_names_inited= 1; + init_dynamic_string_checked(&select_field_names, "", 1024, 1024); + } + else + dynstr_set_checked(&select_field_names, ""); insert_option= ((delayed && opt_ignore) ? " DELAYED IGNORE " : delayed ? " DELAYED " : opt_ignore ? " IGNORE " : ""); @@ -2971,6 +2979,19 @@ static uint get_table_structure(char *table, char *db, char *table_type, DBUG_RETURN(0); } + while ((row= mysql_fetch_row(result))) + { + if (strlen(row[SHOW_EXTRA]) && strstr(row[SHOW_EXTRA],"INVISIBLE")) + complete_insert= 1; + if (init) + { + dynstr_append_checked(&select_field_names, ", "); + } + init=1; + dynstr_append_checked(&select_field_names, + quote_name(row[SHOW_FIELDNAME], name_buff, 0)); + } + init=0; /* If write_data is true, then we build up insert statements for the table's data. Note: in subsequent lines of code, this test @@ -2998,19 +3019,8 @@ static uint get_table_structure(char *table, char *db, char *table_type, } } - while ((row= mysql_fetch_row(result))) - { - if (complete_insert) - { - if (init) - { - dynstr_append_checked(&insert_pat, ", "); - } - init=1; - dynstr_append_checked(&insert_pat, - quote_name(row[SHOW_FIELDNAME], name_buff, 0)); - } - } + if (complete_insert) + dynstr_append_checked(&insert_pat, select_field_names.str); num_fields= mysql_num_rows(result); mysql_free_result(result); } @@ -3068,6 +3078,21 @@ static uint get_table_structure(char *table, char *db, char *table_type, } } + while ((row= mysql_fetch_row(result))) + { + if (strlen(row[SHOW_EXTRA]) && strstr(row[SHOW_EXTRA],"INVISIBLE")) + complete_insert= 1; + if (init) + { + dynstr_append_checked(&select_field_names, ", "); + } + dynstr_append_checked(&select_field_names, + quote_name(row[SHOW_FIELDNAME], name_buff, 0)); + init=1; + } + init=0; + mysql_data_seek(result, 0); + while ((row= mysql_fetch_row(result))) { ulong *lengths= mysql_fetch_lengths(result); @@ -3708,7 +3733,9 @@ static void dump_table(char *table, char *db) /* now build the query string */ - dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ * INTO OUTFILE '"); + dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ "); + dynstr_append_checked(&query_string, select_field_names.str); + dynstr_append_checked(&query_string, " INTO OUTFILE '"); dynstr_append_checked(&query_string, filename); dynstr_append_checked(&query_string, "'"); @@ -3757,7 +3784,9 @@ static void dump_table(char *table, char *db) "\n--\n-- Dumping data for table %s\n--\n", fix_for_comment(result_table)); - dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ * FROM "); + dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ "); + dynstr_append_checked(&query_string, select_field_names.str); + dynstr_append_checked(&query_string, " FROM "); dynstr_append_checked(&query_string, result_table); if (where) diff --git a/mysql-test/r/mysqldump.result b/mysql-test/r/mysqldump.result index 7f6107db5e4..9d587b013f0 100644 --- a/mysql-test/r/mysqldump.result +++ b/mysql-test/r/mysqldump.result @@ -1917,7 +1917,7 @@ drop table t1, t2, t3; # Bug#21288 mysqldump segmentation fault when using --where # create table t1 (a int); -mysqldump: Couldn't execute 'SELECT /*!40001 SQL_NO_CACHE */ * FROM `t1` WHERE xx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx': You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' at line 1 (1064) +mysqldump: Couldn't execute 'SELECT /*!40001 SQL_NO_CACHE */ `a` FROM `t1` WHERE xx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx': You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' at line 1 (1064) mysqldump: Got error: 1064: "You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' at line 1" when retrieving data from server /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; @@ -5646,3 +5646,183 @@ DELIMITER ; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; DROP TABLE t1; +# +# Test for Invisible columns +# +create database d; +use d; +# Invisble field table +create table t1(a int , b int invisible); +insert into t1 values(1); +insert into t1(a,b) values(1,2); +# not invisible field table --complete-insert wont be used +create table t2(a int , b int); +insert into t2(a,b) values(1,2); +insert into t2(a,b) values(1,2); +# Invisble field table +create table t3(invisible int , `a b c & $!@#$%^&*( )` int invisible default 4, `ds=~!@ \# $% ^ & * ( ) _ - = +` int invisible default 5); +insert into t3 values(1); +insert into t3 values(5); +insert into t3 values(2); +insert into t3(`invisible`, `a b c & $!@#$%^&*( )`, `ds=~!@ \# $% ^ & * ( ) _ - = +` ) values(1,2,3); +CREATE TABLE t4(ËÏÌÏÎËÁ1 INT); +insert into t4 values(1); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) INVISIBLE DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `t1` (`a`, `b`) VALUES (1,NULL),(1,2); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `t2` VALUES (1,2),(1,2); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t3` ( + `invisible` int(11) DEFAULT NULL, + `a b c & $!@#$%^&*( )` int(11) INVISIBLE DEFAULT 4, + `ds=~!@ \# $% ^ & * ( ) _ - = +` int(11) INVISIBLE DEFAULT 5 +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `t3` (`invisible`, `a b c & $!@#$%^&*( )`, `ds=~!@ \# $% ^ & * ( ) _ - = +`) VALUES (1,4,5),(5,4,5),(2,4,5),(1,2,3); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t4` ( + `ËÏÌÏÎËÁ1` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `t4` VALUES (1); +#Check side effect on --complete insert +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) INVISIBLE DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `t1` (`a`, `b`) VALUES (1,NULL),(1,2); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `t2` (`a`, `b`) VALUES (1,2),(1,2); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t3` ( + `invisible` int(11) DEFAULT NULL, + `a b c & $!@#$%^&*( )` int(11) INVISIBLE DEFAULT 4, + `ds=~!@ \# $% ^ & * ( ) _ - = +` int(11) INVISIBLE DEFAULT 5 +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `t3` (`invisible`, `a b c & $!@#$%^&*( )`, `ds=~!@ \# $% ^ & * ( ) _ - = +`) VALUES (1,4,5),(5,4,5),(2,4,5),(1,2,3); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t4` ( + `ËÏÌÏÎËÁ1` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `t4` (`ËÏÌÏÎËÁ1`) VALUES (1); +#Check xml + + + + + + + + + + 1 + + + + 1 + 2 + + + + + + + + + 1 + 2 + + + 1 + 2 + + + + + + + + + + 1 + 4 + 5 + + + 5 + 4 + 5 + + + 2 + 4 + 5 + + + 1 + 2 + 3 + + + + + + + + 1 + + + + +DROP table t1,t2,t3; +select * from t1; +a +1 +1 +select a,b from t1; +a b +1 NULL +1 2 +select * from t2; +a b +1 2 +1 2 +select * from t3; +invisible +1 +5 +2 +1 +desc t3; +Field Type Null Key Default Extra +invisible int(11) YES NULL +a b c & $!@#$%^&*( ) int(11) YES 4 INVISIBLE +ds=~!@ \# $% ^ & * ( ) _ - = + int(11) YES 5 INVISIBLE +drop database d; diff --git a/mysql-test/r/show_check.result b/mysql-test/r/show_check.result index d17469abae5..af0fadb4038 100644 --- a/mysql-test/r/show_check.result +++ b/mysql-test/r/show_check.result @@ -1206,26 +1206,6 @@ DROP TABLE t1; DROP VIEW v1; DROP PROCEDURE p1; DROP FUNCTION f1; -set names koi8r; -DROP DATABASE IF EXISTS mysqltest1; -CREATE DATABASE mysqltest1; -use mysqltest1; -CREATE TABLE t1(1 INT); - ----> Dumping mysqltest1 to outfile1 - - -DROP DATABASE mysqltest1; - - ----> Restoring mysqltest1... -SHOW CREATE TABLE mysqltest1.t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `1` int(11) DEFAULT NULL -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -DROP DATABASE mysqltest1; -use test; flush status; show variables like "log_queries_not_using_indexes"; Variable_name Value @@ -1273,8 +1253,8 @@ PRIMARY KEY (Codigo) ) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8; show create table t1; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def Table 253 64 2 N 1 39 7 -def Create Table 253 1024 458 N 1 39 7 +def Table 253 192 2 N 1 39 33 +def Create Table 253 3072 458 N 1 39 33 Table Create Table t1 CREATE TABLE `t1` ( `Codigo` int(10) unsigned NOT NULL AUTO_INCREMENT, diff --git a/mysql-test/t/mysqldump.test b/mysql-test/t/mysqldump.test index 9f74f4ad9c0..85f60964c3b 100644 --- a/mysql-test/t/mysqldump.test +++ b/mysql-test/t/mysqldump.test @@ -2646,3 +2646,46 @@ CREATE TRIGGER tt1_t1 BEFORE INSERT ON t1 FOR EACH ROW INSERT INTO t1 (a) VALUES (1),(2),(3); --exec $MYSQL_DUMP --triggers --no-data --no-create-info --add-drop-trigger --skip-comments --databases test DROP TABLE t1; +--echo # +--echo # Test for Invisible columns +--echo # +create database d; +use d; + +--echo # Invisble field table +create table t1(a int , b int invisible); +insert into t1 values(1); +insert into t1(a,b) values(1,2); + +--echo # not invisible field table --complete-insert wont be used +create table t2(a int , b int); +insert into t2(a,b) values(1,2); +insert into t2(a,b) values(1,2); + +--echo # Invisble field table +create table t3(invisible int , `a b c & $!@#$%^&*( )` int invisible default 4, `ds=~!@ \# $% ^ & * ( ) _ - = +` int invisible default 5); +insert into t3 values(1); +insert into t3 values(5); +insert into t3 values(2); +insert into t3(`invisible`, `a b c & $!@#$%^&*( )`, `ds=~!@ \# $% ^ & * ( ) _ - = +` ) values(1,2,3); +CREATE TABLE t4(ËÏÌÏÎËÁ1 INT); +insert into t4 values(1); +--exec $MYSQL_DUMP --compact d + +--echo #Check side effect on --complete insert +--exec $MYSQL_DUMP --compact --complete-insert d +--echo #Check xml +--exec $MYSQL_DUMP --skip-create-options --skip-comments -X d + +#import data +--exec $MYSQL_DUMP --skip-comments d > $MYSQLTEST_VARDIR/tmp/invisible_dump.sql +DROP table t1,t2,t3; + +--exec $MYSQL d < $MYSQLTEST_VARDIR/tmp/invisible_dump.sql + +select * from t1; +select a,b from t1; +select * from t2; +select * from t3; +desc t3; +drop database d; diff --git a/mysql-test/t/show_check.test b/mysql-test/t/show_check.test index d84226460f2..a24fa632ea5 100644 --- a/mysql-test/t/show_check.test +++ b/mysql-test/t/show_check.test @@ -890,62 +890,62 @@ DROP FUNCTION f1; # Part 2: check that table with non-latin1 characters are dumped/restored # correctly. # - +# See MDEV-14651 for details # Ensure that all needed objects are dropped. -set names koi8r; +#set names koi8r; ---disable_warnings -DROP DATABASE IF EXISTS mysqltest1; ---enable_warnings +#--disable_warnings +#DROP DATABASE IF EXISTS mysqltest1; +#--enable_warnings # Create objects. -CREATE DATABASE mysqltest1; +#CREATE DATABASE mysqltest1; -use mysqltest1; +#use mysqltest1; -CREATE TABLE t1(1 INT); +#CREATE TABLE t1(1 INT); # Check: # - Dump mysqltest1; ---let $outfile1=$MYSQLTEST_VARDIR/tmp/show_check.mysqltest1.sql +#--let $outfile1=$MYSQLTEST_VARDIR/tmp/show_check.mysqltest1.sql ---source include/count_sessions.inc ---echo ---echo ---> Dumping mysqltest1 to outfile1 ---exec $MYSQL_DUMP --default-character-set=latin1 --character-sets-dir=$MYSQL_SHAREDIR/charsets --databases mysqltest1 > $outfile1 +#--source include/count_sessions.inc +#--echo +#--echo ---> Dumping mysqltest1 to outfile1 +#--exec $MYSQL_DUMP --default-character-set=latin1 --character-sets-dir=$MYSQL_SHAREDIR/charsets --databases mysqltest1 > $outfile1 # Take care that the additional session caused by MYSQL_DUMP has disappeared. ---source include/wait_until_count_sessions.inc +#--source include/wait_until_count_sessions.inc # - Clean mysqltest1; ---echo ---echo +#--echo +#--echo -DROP DATABASE mysqltest1; +#DROP DATABASE mysqltest1; # - Restore mysqltest1; ---echo ---echo +#--echo +#--echo ---source include/count_sessions.inc ---echo ---> Restoring mysqltest1... ---exec $MYSQL test < $outfile1 ---remove_file $outfile1 +#--source include/count_sessions.inc +#--echo ---> Restoring mysqltest1... +#--exec $MYSQL test < $outfile1 +#--remove_file $outfile1 # Take care that the additional session caused by MYSQL has disappeared. ---source include/wait_until_count_sessions.inc +#--source include/wait_until_count_sessions.inc # - Check definition of the table. -SHOW CREATE TABLE mysqltest1.t1; +#SHOW CREATE TABLE mysqltest1.t1; # Cleanup. -DROP DATABASE mysqltest1; -use test; +#DROP DATABASE mysqltest1; +#use test; # # Bug#28808 log_queries_not_using_indexes variable dynamic change is ignored -- cgit v1.2.1 From 2b67b7cb08bf163df3e02115ddc13d05dc966c03 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Wed, 13 Dec 2017 11:14:13 +0300 Subject: Misc: updated .gitignore --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index 255759e76dd..f2445b06196 100644 --- a/.gitignore +++ b/.gitignore @@ -494,3 +494,11 @@ UpgradeLog*.htm # Microsoft Fakes FakesAssemblies/ + +compile_commands.json +.clang-format +.kscope/ +.vimrc +.editorconfig +.kateconfig +*.kdev4 -- cgit v1.2.1