diff options
27 files changed, 668 insertions, 66 deletions
diff --git a/mysql-test/main/ctype_utf8.result b/mysql-test/main/ctype_utf8.result index 5356b8d04c8..955530c6b14 100644 --- a/mysql-test/main/ctype_utf8.result +++ b/mysql-test/main/ctype_utf8.result @@ -11379,3 +11379,178 @@ a # # End of 10.3 tests # +# +# Start of 10.4 tests +# +# +# MDEV-27653 long uniques don't work with unicode collations +# +SET NAMES utf8mb3; +CREATE TABLE t1 ( +a CHAR(30) COLLATE utf8mb3_general_ci, +UNIQUE KEY(a) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` char(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 ( +a CHAR(30) COLLATE utf8mb3_general_ci, +UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` char(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`(10)) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 ( +a VARCHAR(30) COLLATE utf8mb3_general_ci, +UNIQUE KEY(a) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 ( +a VARCHAR(30) COLLATE utf8mb3_general_ci, +UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`(10)) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 (a TEXT COLLATE utf8mb3_general_ci UNIQUE); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +CREATE TABLE t1 ( +a LONGTEXT COLLATE utf8mb3_general_ci, +UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` longtext CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`(10)) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t1 VALUES ('a'); +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +SELECT * FROM t1; +a +a +DROP TABLE t1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +ä 2 +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check error Upgrade required. Please do "REPAIR TABLE `t1`" or dump/reload to fix it! +INSERT INTO t1 VALUES ('A'); +ERROR 23000: Duplicate entry 'A' for key 'a' +INSERT INTO t1 VALUES ('Ä'); +ERROR 23000: Duplicate entry 'Ä' for key 'a' +INSERT INTO t1 VALUES ('Ấ'); +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +ä 2 +Ấ 3 +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check error Upgrade required. Please do "REPAIR TABLE `t1`" or dump/reload to fix it! +ALTER TABLE t1 FORCE; +ERROR 23000: Duplicate entry 'ä' for key 'a' +DELETE FROM t1 WHERE OCTET_LENGTH(a)>1; +ALTER TABLE t1 FORCE; +INSERT INTO t1 VALUES ('ä'); +ERROR 23000: Duplicate entry 'ä' for key 'a' +DROP TABLE t1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +ä 2 +ALTER IGNORE TABLE t1 FORCE; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +DROP TABLE t1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, + UNIQUE KEY `a` (`a`) USING HASH +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +ä 2 +REPAIR TABLE t1; +Table Op Msg_type Msg_text +test.t1 repair Warning Number of rows changed from 2 to 1 +test.t1 repair status OK +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +a OCTET_LENGTH(a) +a 1 +DROP TABLE t1; +# +# End of 10.4 tests +# diff --git a/mysql-test/main/ctype_utf8.test b/mysql-test/main/ctype_utf8.test index cc61c2ae0fe..027881bf0eb 100644 --- a/mysql-test/main/ctype_utf8.test +++ b/mysql-test/main/ctype_utf8.test @@ -2310,3 +2310,161 @@ VALUES (_latin1 0xDF) UNION VALUES(_utf8'a' COLLATE utf8_bin); --echo # --echo # End of 10.3 tests --echo # + + +--echo # +--echo # Start of 10.4 tests +--echo # + +--echo # +--echo # MDEV-27653 long uniques don't work with unicode collations +--echo # + +SET NAMES utf8mb3; + +# CHAR + +CREATE TABLE t1 ( + a CHAR(30) COLLATE utf8mb3_general_ci, + UNIQUE KEY(a) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + a CHAR(30) COLLATE utf8mb3_general_ci, + UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + + +# VARCHAR + +CREATE TABLE t1 ( + a VARCHAR(30) COLLATE utf8mb3_general_ci, + UNIQUE KEY(a) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + a VARCHAR(30) COLLATE utf8mb3_general_ci, + UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + + +# TEXT + +CREATE TABLE t1 (a TEXT COLLATE utf8mb3_general_ci UNIQUE); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + a LONGTEXT COLLATE utf8mb3_general_ci, + UNIQUE KEY(a(10)) USING HASH +); +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES ('a'); +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +SELECT * FROM t1; +DROP TABLE t1; + + +# Testing upgrade: +# Prior to MDEV-27653, the UNIQUE HASH function errorneously +# took into account string octet length. +# Old tables should still open and work, but with wrong results. + +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI; +SHOW CREATE TABLE t1; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +CHECK TABLE t1; + +# There is already a one byte value 'a' in the table +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('A'); + +# There is already a two-byte value 'ä' in the table +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('Ä'); + +# There were no three-byte values in the table so far. +# The below value violates UNIQUE, but it gets inserted. +# This is wrong but expected for a pre-MDEV-27653 table. +INSERT INTO t1 VALUES ('Ấ'); +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +CHECK TABLE t1; + +# ALTER FORCE fails: it tries to rebuild the table +# with a correct UNIQUE HASH function, but there are duplicates! +--error ER_DUP_ENTRY +ALTER TABLE t1 FORCE; + +# Let's remove all duplicate values, so only the one-byte 'a' stays. +# ALTER..FORCE should work after that. +DELETE FROM t1 WHERE OCTET_LENGTH(a)>1; +ALTER TABLE t1 FORCE; + +# Make sure that 'a' and 'ä' cannot co-exists any more, +# because the table was recreated with a correct UNIQUE HASH function. +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES ('ä'); +DROP TABLE t1; + +# +# Testing an old table with ALTER IGNORE. +# The table is expected to rebuild with a new hash function, +# duplicates go away. +# +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI; +SHOW CREATE TABLE t1; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +ALTER IGNORE TABLE t1 FORCE; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +DROP TABLE t1; + +# +# Testing an old table with REPAIR. +# The table is expected to rebuild with a new hash function, +# duplicates go away. +# +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD; +copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI; +SHOW CREATE TABLE t1; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +REPAIR TABLE t1; +SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a; +DROP TABLE t1; + +--echo # +--echo # End of 10.4 tests +--echo # diff --git a/mysql-test/main/type_timestamp.result b/mysql-test/main/type_timestamp.result index 306cd77ad25..58cb12ae267 100644 --- a/mysql-test/main/type_timestamp.result +++ b/mysql-test/main/type_timestamp.result @@ -1320,5 +1320,27 @@ CASE WHEN a THEN DEFAULT(a) END DROP TABLE t1; SET timestamp=DEFAULT; # +# MDEV-27653 long uniques don't work with unicode collations +# +CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH); +SET time_zone='+00:00'; +INSERT INTO t1 VALUES ('2001-01-01 10:20:30'); +SET time_zone='+01:00'; +INSERT INTO t1 SELECT MAX(a) FROM t1; +ERROR 23000: Duplicate entry '2001-01-01 11:20:30' for key 'a' +SELECT * FROM t1; +a +2001-01-01 11:20:30 +DROP TABLE t1; +CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH); +SET time_zone='+00:00'; +INSERT INTO t1 VALUES ('2001-01-01 10:20:30'); +SET time_zone='+01:00'; +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +DROP TABLE t1; +SET time_zone=DEFAULT; +# # End of 10.4 tests # diff --git a/mysql-test/main/type_timestamp.test b/mysql-test/main/type_timestamp.test index 485da666661..f12cc2a4bc3 100644 --- a/mysql-test/main/type_timestamp.test +++ b/mysql-test/main/type_timestamp.test @@ -879,5 +879,26 @@ SET timestamp=DEFAULT; --echo # +--echo # MDEV-27653 long uniques don't work with unicode collations +--echo # + +CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH); +SET time_zone='+00:00'; +INSERT INTO t1 VALUES ('2001-01-01 10:20:30'); +SET time_zone='+01:00'; +--error ER_DUP_ENTRY +INSERT INTO t1 SELECT MAX(a) FROM t1; +SELECT * FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH); +SET time_zone='+00:00'; +INSERT INTO t1 VALUES ('2001-01-01 10:20:30'); +SET time_zone='+01:00'; +CHECK TABLE t1; +DROP TABLE t1; +SET time_zone=DEFAULT; + +--echo # --echo # End of 10.4 tests --echo # diff --git a/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD b/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD Binary files differnew file mode 100644 index 00000000000..5f1ad998c00 --- /dev/null +++ b/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD diff --git a/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI b/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI Binary files differnew file mode 100644 index 00000000000..7a748abbae5 --- /dev/null +++ b/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI diff --git a/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm b/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm Binary files differnew file mode 100644 index 00000000000..5a4cec72324 --- /dev/null +++ b/mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm diff --git a/sql/field.cc b/sql/field.cc index 2b93fc225ea..2006f0e3096 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1792,18 +1792,11 @@ Field::Field(uchar *ptr_arg,uint32 length_arg,uchar *null_ptr_arg, } -void Field::hash(ulong *nr, ulong *nr2) +void Field::hash_not_null(Hasher *hasher) { - if (is_null()) - { - *nr^= (*nr << 1) | 1; - } - else - { - uint len= pack_length(); - CHARSET_INFO *cs= sort_charset(); - cs->coll->hash_sort(cs, ptr, len, nr, nr2); - } + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + hasher->add(sort_charset(), ptr, pack_length()); } size_t @@ -8180,18 +8173,12 @@ bool Field_varstring::is_equal(const Column_definition &new_field) const } -void Field_varstring::hash(ulong *nr, ulong *nr2) +void Field_varstring::hash_not_null(Hasher *hasher) { - if (is_null()) - { - *nr^= (*nr << 1) | 1; - } - else - { - uint len= length_bytes == 1 ? (uint) *ptr : uint2korr(ptr); - CHARSET_INFO *cs= charset(); - cs->coll->hash_sort(cs, ptr + length_bytes, len, nr, nr2); - } + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + uint len= length_bytes == 1 ? (uint) *ptr : uint2korr(ptr); + hasher->add(charset(), ptr + length_bytes, len); } @@ -8553,6 +8540,17 @@ oom_error: } +void Field_blob::hash_not_null(Hasher *hasher) +{ + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + char *blob; + memcpy(&blob, ptr + packlength, sizeof(char*)); + if (blob) + hasher->add(Field_blob::charset(), blob, get_length(ptr)); +} + + double Field_blob::val_real(void) { DBUG_ASSERT(marked_for_read()); @@ -9901,20 +9899,27 @@ Field_bit::Field_bit(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, } -void Field_bit::hash(ulong *nr, ulong *nr2) +/* + This method always calculates hash over 8 bytes. + This is different from how the HEAP engine calculate hash: + HEAP takes into account the actual octet size, so say for BIT(18) + it calculates hash over three bytes only: + - the incomplete byte with bits 16..17 + - the two full bytes with bits 0..15 + See hp_rec_hashnr(), hp_hashnr() for details. + + The HEAP way is more efficient, especially for short lengths. + Let's consider fixing Field_bit eventually to do it in the HEAP way, + with proper measures to upgrade partitioned tables easy. +*/ +void Field_bit::hash_not_null(Hasher *hasher) { - if (is_null()) - { - *nr^= (*nr << 1) | 1; - } - else - { - CHARSET_INFO *cs= &my_charset_bin; - longlong value= Field_bit::val_int(); - uchar tmp[8]; - mi_int8store(tmp,value); - cs->coll->hash_sort(cs, tmp, 8, nr, nr2); - } + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + longlong value= Field_bit::val_int(); + uchar tmp[8]; + mi_int8store(tmp,value); + hasher->add(&my_charset_bin, tmp, 8); } diff --git a/sql/field.h b/sql/field.h index d4124ca23a9..6388fa2e08e 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1628,7 +1628,14 @@ public: key_map get_possible_keys(); /* Hash value */ - virtual void hash(ulong *nr, ulong *nr2); + void hash(Hasher *hasher) + { + if (is_null()) + hasher->add_null(); + else + hash_not_null(hasher); + } + virtual void hash_not_null(Hasher *hasher); /** Get the upper limit of the MySQL integral and floating-point type. @@ -3744,7 +3751,7 @@ public: uchar *new_ptr, uint32 length, uchar *new_null_ptr, uint new_null_bit); bool is_equal(const Column_definition &new_field) const; - void hash(ulong *nr, ulong *nr2); + void hash_not_null(Hasher *hasher); uint length_size() const { return length_bytes; } void print_key_value(String *out, uint32 length); private: @@ -3951,6 +3958,7 @@ public: bool make_empty_rec_store_default_value(THD *thd, Item *item); int store(const char *to, size_t length, CHARSET_INFO *charset); using Field_str::store; + void hash_not_null(Hasher *hasher); double val_real(void); longlong val_int(void); String *val_str(String*,String *); @@ -4576,7 +4584,7 @@ public: if (bit_ptr) bit_ptr= ADD_TO_PTR(bit_ptr, ptr_diff, uchar*); } - void hash(ulong *nr, ulong *nr2); + void hash_not_null(Hasher *hasher); SEL_ARG *get_mm_leaf(RANGE_OPT_PARAM *param, KEY_PART *key_part, const Item_bool_func *cond, diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 06ae329ee3a..ca02bf16d5e 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -9850,8 +9850,7 @@ uint8 ha_partition::table_cache_type() uint32 ha_partition::calculate_key_hash_value(Field **field_array) { - ulong nr1= 1; - ulong nr2= 4; + Hasher hasher; bool use_51_hash; use_51_hash= MY_TEST((*field_array)->table->part_info->key_algorithm == partition_info::KEY_ALGORITHM_51); @@ -9878,13 +9877,12 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array) { if (field->is_null()) { - nr1^= (nr1 << 1) | 1; + hasher.add_null(); continue; } /* Force this to my_hash_sort_bin, which was used in 5.1! */ uint len= field->pack_length(); - my_charset_bin.coll->hash_sort(&my_charset_bin, field->ptr, len, - &nr1, &nr2); + hasher.add(&my_charset_bin, field->ptr, len); /* Done with this field, continue with next one. */ continue; } @@ -9902,13 +9900,12 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array) { if (field->is_null()) { - nr1^= (nr1 << 1) | 1; + hasher.add_null(); continue; } /* Force this to my_hash_sort_bin, which was used in 5.1! */ uint len= field->pack_length(); - my_charset_latin1.coll->hash_sort(&my_charset_latin1, field->ptr, - len, &nr1, &nr2); + hasher.add(&my_charset_latin1, field->ptr, len); continue; } /* New types in mysql-5.6. */ @@ -9935,9 +9932,9 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array) } /* fall through, use collation based hashing. */ } - field->hash(&nr1, &nr2); + field->hash(&hasher); } while (*(++field_array)); - return (uint32) nr1; + return (uint32) hasher.finalize(); } diff --git a/sql/handler.cc b/sql/handler.cc index 20bf32f3cdc..42bfec6652f 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4145,6 +4145,35 @@ int handler::check_collation_compatibility() } +int handler::check_long_hash_compatibility() const +{ + if (!table->s->old_long_hash_function()) + return 0; + KEY *key= table->key_info; + KEY *key_end= key + table->s->keys; + for ( ; key < key_end; key++) + { + if (key->algorithm == HA_KEY_ALG_LONG_HASH) + { + /* + The old (pre-MDEV-27653) hash function was wrong. + So the long hash unique constraint can have some + duplicate records. REPAIR TABLE can't fix this, + it will fail on a duplicate key error. + Only "ALTER IGNORE TABLE .. FORCE" can fix this. + So we need to return HA_ADMIN_NEEDS_ALTER here, + (not HA_ADMIN_NEEDS_UPGRADE which is used elsewhere), + to properly send the error message text corresponding + to ER_TABLE_NEEDS_REBUILD (rather than to ER_TABLE_NEEDS_UPGRADE) + to the user. + */ + return HA_ADMIN_NEEDS_ALTER; + } + } + return 0; +} + + int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt) { int error; @@ -4182,6 +4211,9 @@ int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt) if (unlikely((error= check_collation_compatibility()))) return error; + + if (unlikely((error= check_long_hash_compatibility()))) + return error; return check_for_upgrade(check_opt); } diff --git a/sql/handler.h b/sql/handler.h index 3f0fc5c897f..f7a2838b3d9 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -3271,6 +3271,7 @@ public: } int check_collation_compatibility(); + int check_long_hash_compatibility() const; int ha_check_for_upgrade(HA_CHECK_OPT *check_opt); /** to be actually called to get 'check()' functionality*/ int ha_check(THD *thd, HA_CHECK_OPT *check_opt); diff --git a/sql/item.h b/sql/item.h index ac4a42107cb..4fbb7324570 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1271,6 +1271,12 @@ public: */ inline ulonglong val_uint() { return (ulonglong) val_int(); } + virtual bool hash_not_null(Hasher *hasher) + { + DBUG_ASSERT(0); + return true; + } + /* Return string representation of this item object. @@ -3460,6 +3466,13 @@ public: { return Sql_mode_dependency(0, field->value_depends_on_sql_mode()); } + bool hash_not_null(Hasher *hasher) + { + if (field->is_null()) + return true; + field->hash_not_null(hasher); + return false; + } longlong val_int_endpoint(bool left_endp, bool *incl_endp); bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate); bool get_date_result(THD *thd, MYSQL_TIME *ltime,date_mode_t fuzzydate); diff --git a/sql/item_func.cc b/sql/item_func.cc index 9c29280970b..d275dbaa50d 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -1765,7 +1765,7 @@ static void calc_hash_for_unique(ulong &nr1, ulong &nr2, String *str) cs->coll->hash_sort(cs, (uchar *)str->ptr(), str->length(), &nr1, &nr2); } -longlong Item_func_hash::val_int() +longlong Item_func_hash_mariadb_100403::val_int() { DBUG_EXECUTE_IF("same_long_unique_hash", return 9;); unsigned_flag= true; @@ -1786,6 +1786,24 @@ longlong Item_func_hash::val_int() } +longlong Item_func_hash::val_int() +{ + DBUG_EXECUTE_IF("same_long_unique_hash", return 9;); + unsigned_flag= true; + Hasher hasher; + for(uint i= 0;i<arg_count;i++) + { + if (args[i]->hash_not_null(&hasher)) + { + null_value= 1; + return 0; + } + } + null_value= 0; + return (longlong) hasher.finalize(); +} + + bool Item_func_hash::fix_length_and_dec() { decimals= 0; diff --git a/sql/item_func.h b/sql/item_func.h index 7457abd9a39..ef36d5204d2 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -1078,6 +1078,18 @@ public: const char *func_name() const { return "<hash>"; } }; +class Item_func_hash_mariadb_100403: public Item_func_hash +{ +public: + Item_func_hash_mariadb_100403(THD *thd, List<Item> &item) + :Item_func_hash(thd, item) + {} + longlong val_int(); + Item *get_copy(THD *thd) + { return get_item_copy<Item_func_hash_mariadb_100403>(thd, this); } + const char *func_name() const { return "<hash_mariadb_100403>"; } +}; + class Item_longlong_func: public Item_int_func { public: diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 5562e6d3e62..a57fbd7bebc 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -1630,6 +1630,18 @@ bool Item_func_ucase::fix_length_and_dec() } +bool Item_func_left::hash_not_null(Hasher *hasher) +{ + StringBuffer<STRING_BUFFER_USUAL_SIZE> buf; + String *str= val_str(&buf); + DBUG_ASSERT((str == NULL) == null_value); + if (!str) + return true; + hasher->add(collation.collation, str->ptr(), str->length()); + return false; +} + + String *Item_func_left::val_str(String *str) { DBUG_ASSERT(fixed == 1); diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 82b30369734..ee2c59c973f 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -461,6 +461,7 @@ class Item_func_left :public Item_str_func String tmp_value; public: Item_func_left(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} + bool hash_not_null(Hasher *hasher); String *val_str(String *); bool fix_length_and_dec(); const char *func_name() const { return "left"; } diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index 213d77f8237..906458b7b3c 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -35,7 +35,8 @@ #include "wsrep_mysqld.h" /* Prepare, run and cleanup for mysql_recreate_table() */ -static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list) +static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list, + Recreate_info *recreate_info) { bool result_code; DBUG_ENTER("admin_recreate_table"); @@ -56,7 +57,7 @@ static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list) DEBUG_SYNC(thd, "ha_admin_try_alter"); tmp_disable_binlog(thd); // binlogging is done by caller if wanted result_code= (thd->open_temporary_tables(table_list) || - mysql_recreate_table(thd, table_list, false)); + mysql_recreate_table(thd, table_list, recreate_info, false)); reenable_binlog(thd); /* mysql_recreate_table() can push OK or ERROR. @@ -516,6 +517,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, bool open_error; bool collect_eis= FALSE; bool open_for_modify= org_open_for_modify; + Recreate_info recreate_info; DBUG_PRINT("admin", ("table: '%s'.'%s'", db, table->table_name.str)); DEBUG_SYNC(thd, "admin_command_kill_before_modify"); @@ -776,7 +778,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, { /* We use extra_open_options to be able to open crashed tables */ thd->open_options|= extra_open_options; - result_code= admin_recreate_table(thd, table); + result_code= admin_recreate_table(thd, table, &recreate_info) ? + HA_ADMIN_FAILED : HA_ADMIN_OK; thd->open_options&= ~extra_open_options; goto send_result; } @@ -947,12 +950,31 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, repair was not implemented and we need to upgrade the table to a new version so we recreate the table with ALTER TABLE */ - result_code= admin_recreate_table(thd, table); + result_code= admin_recreate_table(thd, table, &recreate_info); } send_result: lex->cleanup_after_one_table_open(); thd->clear_error(); // these errors shouldn't get client + + if (recreate_info.records_duplicate()) + { + protocol->prepare_for_resend(); + protocol->store(table_name, system_charset_info); + protocol->store((char*) operator_name, system_charset_info); + protocol->store(warning_level_names[Sql_condition::WARN_LEVEL_WARN].str, + warning_level_names[Sql_condition::WARN_LEVEL_WARN].length, + system_charset_info); + char buf[80]; + size_t length= my_snprintf(buf, sizeof(buf), + "Number of rows changed from %u to %u", + (uint) recreate_info.records_processed(), + (uint) recreate_info.records_copied()); + protocol->store(buf, length, system_charset_info); + if (protocol->write()) + goto err; + } + { Diagnostics_area::Sql_condition_iterator it= thd->get_stmt_da()->sql_conditions(); @@ -1063,7 +1085,7 @@ send_result_message: table->next_local= table->next_global= 0; tmp_disable_binlog(thd); // binlogging is done by caller if wanted - result_code= admin_recreate_table(thd, table); + result_code= admin_recreate_table(thd, table, &recreate_info); reenable_binlog(thd); trans_commit_stmt(thd); trans_commit(thd); @@ -1409,6 +1431,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd) LEX *m_lex= thd->lex; TABLE_LIST *first_table= m_lex->first_select_lex()->table_list.first; bool res= TRUE; + Recreate_info recreate_info; DBUG_ENTER("Sql_cmd_optimize_table::execute"); if (check_table_access(thd, SELECT_ACL | INSERT_ACL, first_table, @@ -1417,7 +1440,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd) WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table); res= (specialflag & SPECIAL_NO_NEW_FUNC) ? - mysql_recreate_table(thd, first_table, true) : + mysql_recreate_table(thd, first_table, &recreate_info, true) : mysql_admin_table(thd, first_table, &m_lex->check_opt, "optimize", TL_WRITE, 1, 0, 0, 0, &handler::ha_optimize, 0, true); diff --git a/sql/sql_alter.cc b/sql/sql_alter.cc index 67c6a081880..cf44ed67120 100644 --- a/sql/sql_alter.cc +++ b/sql/sql_alter.cc @@ -527,9 +527,11 @@ bool Sql_cmd_alter_table::execute(THD *thd) thd->work_part_info= 0; #endif + Recreate_info recreate_info; result= mysql_alter_table(thd, &select_lex->db, &lex->name, &create_info, first_table, + &recreate_info, &alter_info, select_lex->order_list.elements, select_lex->order_list.first, diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 464d64db73b..300ed902b38 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -7939,3 +7939,16 @@ bool THD::timestamp_to_TIME(MYSQL_TIME *ltime, my_time_t ts, } return 0; } + + +void THD::my_ok_with_recreate_info(const Recreate_info &info, + ulong warn_count) +{ + char buf[80]; + my_snprintf(buf, sizeof(buf), + ER_THD(this, ER_INSERT_INFO), + (ulong) info.records_processed(), + (ulong) info.records_duplicate(), + warn_count); + my_ok(this, info.records_processed(), 0L, buf); +} diff --git a/sql/sql_class.h b/sql/sql_class.h index 136380bbd87..67018c0299a 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -232,6 +232,29 @@ public: }; +class Recreate_info +{ + ha_rows m_records_copied; + ha_rows m_records_duplicate; +public: + Recreate_info() + :m_records_copied(0), + m_records_duplicate(0) + { } + Recreate_info(ha_rows records_copied, + ha_rows records_duplicate) + :m_records_copied(records_copied), + m_records_duplicate(records_duplicate) + { } + ha_rows records_copied() const { return m_records_copied; } + ha_rows records_duplicate() const { return m_records_duplicate; } + ha_rows records_processed() const + { + return m_records_copied + m_records_duplicate; + } +}; + + #define TC_HEURISTIC_RECOVER_COMMIT 1 #define TC_HEURISTIC_RECOVER_ROLLBACK 2 extern ulong tc_heuristic_recover; @@ -3943,6 +3966,8 @@ public: inline bool vio_ok() const { return TRUE; } inline bool is_connected() { return TRUE; } #endif + + void my_ok_with_recreate_info(const Recreate_info &info, ulong warn_count); /** Mark the current error as fatal. Warning: this does not set any error, it sets a property of the error, so must be diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index f8974bb9cc0..bb0b3ebb4da 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4240,8 +4240,10 @@ mysql_execute_command(THD *thd) create_info.row_type= ROW_TYPE_NOT_USED; create_info.default_table_charset= thd->variables.collation_database; + Recreate_info recreate_info; res= mysql_alter_table(thd, &first_table->db, &first_table->table_name, - &create_info, first_table, &alter_info, + &create_info, first_table, + &recreate_info, &alter_info, 0, (ORDER*) 0, 0); break; } diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 725af4adb4e..9f13dcde40f 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -9619,6 +9619,7 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, const LEX_CSTRING *new_name, HA_CREATE_INFO *create_info, TABLE_LIST *table_list, + Recreate_info *recreate_info, Alter_info *alter_info, uint order_num, ORDER *order, bool ignore) { @@ -10687,11 +10688,10 @@ end_inplace: } end_temporary: - my_snprintf(alter_ctx.tmp_buff, sizeof(alter_ctx.tmp_buff), - ER_THD(thd, ER_INSERT_INFO), - (ulong) (copied + deleted), (ulong) deleted, - (ulong) thd->get_stmt_da()->current_statement_warn_count()); - my_ok(thd, copied + deleted, 0L, alter_ctx.tmp_buff); + *recreate_info= Recreate_info(copied, deleted); + thd->my_ok_with_recreate_info(*recreate_info, + (ulong) thd->get_stmt_da()-> + current_statement_warn_count()); DEBUG_SYNC(thd, "alter_table_inplace_trans_commit"); DBUG_RETURN(false); @@ -11208,7 +11208,8 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, Like mysql_alter_table(). */ -bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy) +bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, + Recreate_info *recreate_info, bool table_copy) { HA_CREATE_INFO create_info; Alter_info alter_info; @@ -11233,8 +11234,10 @@ bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy) Alter_info::ALTER_TABLE_ALGORITHM_COPY); bool res= mysql_alter_table(thd, &null_clex_str, &null_clex_str, &create_info, - table_list, &alter_info, 0, - (ORDER *) 0, 0); + table_list, recreate_info, &alter_info, 0, + (ORDER *) 0, + // Ignore duplicate records on REPAIR + thd->lex->sql_command == SQLCOM_REPAIR); table_list->next_global= next_table; DBUG_RETURN(res); } diff --git a/sql/sql_table.h b/sql/sql_table.h index 62b61684286..e51b5ec0f0f 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -220,13 +220,15 @@ bool mysql_trans_commit_alter_copy_data(THD *thd); bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, const LEX_CSTRING *new_name, HA_CREATE_INFO *create_info, TABLE_LIST *table_list, + class Recreate_info *recreate_info, Alter_info *alter_info, uint order_num, ORDER *order, bool ignore); bool mysql_compare_tables(TABLE *table, Alter_info *alter_info, HA_CREATE_INFO *create_info, bool *metadata_equal); -bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy); +bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, + class Recreate_info *recreate_info, bool table_copy); bool mysql_create_like_table(THD *thd, TABLE_LIST *table, TABLE_LIST *src_table, Table_specification_st *create_info); diff --git a/sql/sql_type.h b/sql/sql_type.h index ff6853b4c88..73e7de79cd8 100644 --- a/sql/sql_type.h +++ b/sql/sql_type.h @@ -110,6 +110,32 @@ enum scalar_comparison_op }; +class Hasher +{ + ulong m_nr1; + ulong m_nr2; +public: + Hasher(): m_nr1(1), m_nr2(4) + { } + void add_null() + { + m_nr1^= (m_nr1 << 1) | 1; + } + void add(CHARSET_INFO *cs, const uchar *str, size_t length) + { + cs->coll->hash_sort(cs, str, length, &m_nr1, &m_nr2); + } + void add(CHARSET_INFO *cs, const char *str, size_t length) + { + add(cs, (const uchar *) str, length); + } + uint32 finalize() const + { + return (uint32) m_nr1; + } +}; + + /* A helper class to store column attributes that are inherited by columns (from the table level) when not specified explicitly. diff --git a/sql/table.cc b/sql/table.cc index a06834afd8b..2b5787d1511 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -1056,6 +1056,18 @@ static void mysql57_calculate_null_position(TABLE_SHARE *share, } } + +Item_func_hash *TABLE_SHARE::make_long_hash_func(THD *thd, + MEM_ROOT *mem_root, + List<Item> *field_list) + const +{ + if (old_long_hash_function()) + return new (mem_root) Item_func_hash_mariadb_100403(thd, *field_list); + return new (mem_root) Item_func_hash(thd, *field_list); +} + + /** Parse TABLE_SHARE::vcol_defs unpack_vcol_info_from_frm @@ -1267,7 +1279,10 @@ bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table, list_item= new (mem_root) Item_field(thd, keypart->field); field_list->push_back(list_item, mem_root); } - Item_func_hash *hash_item= new(mem_root)Item_func_hash(thd, *field_list); + + Item_func_hash *hash_item= table->s->make_long_hash_func(thd, mem_root, + field_list); + Virtual_column_info *v= new (mem_root) Virtual_column_info(); field->vcol_info= v; field->vcol_info->expr= hash_item; diff --git a/sql/table.h b/sql/table.h index 1a10566e080..799675ac775 100644 --- a/sql/table.h +++ b/sql/table.h @@ -52,6 +52,7 @@ class Item; /* Needed by ORDER */ typedef Item (*Item_ptr); class Item_subselect; class Item_field; +class Item_func_hash; class GRANT_TABLE; class st_select_lex_unit; class st_select_lex; @@ -1137,6 +1138,21 @@ struct TABLE_SHARE void free_frm_image(const uchar *frm); void set_overlapped_keys(); + + bool old_long_hash_function() const + { + return mysql_version < 100428 || + (mysql_version >= 100500 && mysql_version < 100519) || + (mysql_version >= 100600 && mysql_version < 100612) || + (mysql_version >= 100700 && mysql_version < 100708) || + (mysql_version >= 100800 && mysql_version < 100807) || + (mysql_version >= 100900 && mysql_version < 100905) || + (mysql_version >= 101000 && mysql_version < 101003) || + (mysql_version >= 101100 && mysql_version < 101102); + } + Item_func_hash *make_long_hash_func(THD *thd, + MEM_ROOT *mem_root, + List<Item> *field_list) const; }; /* not NULL, but cannot be dereferenced */ |