MDEV-6255 DUPLICATE KEY Errors on SELECT .. GROUP BY that uses temporary and filesort.

The problem was that my_hash_sort didn't properly delete end-space characters properly, so strings that should compare
identically was seen as different strings.  (Space was handled correctly, but not NBSP)
This caused duplicate key errors when a heap table was converted to Aria as part of overflow in group by.

Fixed by removing all characters that compares as end space when creating a hash.

Other things:
- Fixed that --sorted_results also works for errors in mysqltest.
- Speed up hash by not comparing strings that has different hash.
- Speed up many my_hash_sort functions by using registers to calculate hash instead of pointers.
  This was previously done for some functions, but not for all.
- Made a macro of the hash function, to simplify code and to be able to experiment with new hash functions.







client/mysqltest.cc:
  Fixed that --sorted_results also works for error messages.
mysql-test/r/ctype_partitions.result:
  New test to ensure that partitions on hash works
mysql-test/suite/multi_source/gtid.result:
  Updated result
mysql-test/suite/multi_source/gtid.test:
  Test that --sorted_result works for error messages
mysql-test/suite/multi_source/gtid_ignore_duplicates.result:
  Updated result
mysql-test/suite/multi_source/gtid_ignore_duplicates.test:
  Updated result
mysql-test/suite/multi_source/load_data.result:
  Updated result
mysql-test/suite/multi_source/load_data.test:
  Updated result
mysql-test/t/ctype_partitions.test:
  New test to ensure that partitions on hash works
storage/heap/hp_write.c:
  Speed up hash by not comparing strings that has different hash.
storage/maria/ma_check.c:
  Extra debug
strings/ctype-bin.c:
  Use macro for hash function
strings/ctype-latin1.c:
  Use macro for hash function
  Use registers to calculate hash (speedup)
strings/ctype-mb.c:
  Use macro for hash function
  Use registers to calculate hash (speedup)
strings/ctype-simple.c:
  Use macro for hash function
  Use same variable names as in other my_hash_sort functions.
  Update my_hash_sort_simple() to properly remove end space (patch by Bar)
strings/ctype-uca.c:
  Ignore duplicated space inside strings and end space in my_hash_sort_uca(). This fixed MDEV-6255
  Use macro for hash function
  Use registers to calculate hash (speedup)
strings/ctype-ucs2.c:
  Use macro for hash function
  Use registers to calculate hash (speedup)
strings/ctype-utf8.c:
  Use macro for hash function
  Use registers to calculate hash (speedup)
strings/strings_def.h:
  Made a macro of the hash function, to simplify code and to be able to experiment with new hash functions.

8 files changed, 90 insertions, 3 deletions

diff --git a/mysql-test/r/ctype_partitions.result b/mysql-test/r/ctype_partitions.result
new file mode 100644
index 00000000000..a39ecc11529
--- /dev/null
+++ b/mysql-test/r/ctype_partitions.result
@@ -0,0 +1,51 @@
+#
+# MDEV-6255 DUPLICATE KEY Errors on SELECT .. GROUP BY that uses temporary and filesort
+#
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET cp1251 COLLATE cp1251_ukrainian_ci);
+INSERT INTO t1 VALUES (0x20),(0x60),(0x6060),(0x606060);
+SELECT HEX(a) FROM t1 WHERE a=0x60;
+HEX(a)
+20
+60
+6060
+606060
+ALTER TABLE t1  PARTITION BY KEY(a) PARTITIONS 3;
+SELECT HEX(a) FROM t1 WHERE a=0x60;
+HEX(a)
+20
+60
+6060
+606060
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET koi8u COLLATE koi8u_general_ci);
+INSERT INTO t1 VALUES (0x20),(0x60),(0x6060),(0x606060);
+SELECT HEX(a) FROM t1 WHERE a=0x60;
+HEX(a)
+20
+60
+6060
+606060
+ALTER TABLE t1  PARTITION BY KEY(a) PARTITIONS 3;
+SELECT HEX(a) FROM t1 WHERE a=0x60;
+HEX(a)
+20
+60
+6060
+606060
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET cp1250 COLLATE cp1250_general_ci);
+INSERT INTO t1 VALUES (0x20),(0xA0),(0xA0A0),(0xA0A0A0);
+SELECT HEX(a) FROM t1 WHERE a=0xA0;
+HEX(a)
+20
+A0
+A0A0
+A0A0A0
+ALTER TABLE t1  PARTITION BY KEY(a) PARTITIONS 3;
+SELECT HEX(a) FROM t1 WHERE a=0xA0;
+HEX(a)
+20
+A0
+A0A0
+A0A0A0
+DROP TABLE t1;
diff --git a/mysql-test/suite/multi_source/gtid.result b/mysql-test/suite/multi_source/gtid.result
index 8e6028afa17..ce926ddc995 100644
--- a/mysql-test/suite/multi_source/gtid.result
+++ b/mysql-test/suite/multi_source/gtid.result
@@ -141,8 +141,8 @@ include/reset_master_slave.inc
 SET GLOBAL gtid_domain_id=0;
 STOP ALL SLAVES;
 Warnings:
-Note	1938	SLAVE 'slave2' stopped
 Note	1938	SLAVE 'slave1' stopped
+Note	1938	SLAVE 'slave2' stopped
 include/reset_master_slave.inc
 SET GLOBAL gtid_domain_id=0;
 include/reset_master_slave.inc
diff --git a/mysql-test/suite/multi_source/gtid.test b/mysql-test/suite/multi_source/gtid.test
index 0ab486b1f41..7a085823693 100644
--- a/mysql-test/suite/multi_source/gtid.test
+++ b/mysql-test/suite/multi_source/gtid.test
@@ -141,12 +141,14 @@ DROP TABLE t3;
 SET GLOBAL gtid_domain_id=0;
 --let $wait_condition= SELECT COUNT(*)=0 FROM information_schema.tables WHERE table_name IN ("t1", "t2", "t3") AND table_schema = "test"
 --source include/wait_condition.inc
+--sorted_result
 STOP ALL SLAVES;
 --source reset_master_slave.inc
 --disconnect slave1
 
 --connection slave2
 SET GLOBAL gtid_domain_id=0;
+--sorted_result
 STOP ALL SLAVES;
 --source reset_master_slave.inc
 --disconnect slave2
diff --git a/mysql-test/suite/multi_source/gtid_ignore_duplicates.result b/mysql-test/suite/multi_source/gtid_ignore_duplicates.result
index 9bd09330ae7..5426091b635 100644
--- a/mysql-test/suite/multi_source/gtid_ignore_duplicates.result
+++ b/mysql-test/suite/multi_source/gtid_ignore_duplicates.result
@@ -245,8 +245,8 @@ a
 SET GLOBAL gtid_domain_id=0;
 STOP ALL SLAVES;
 Warnings:
-Note	1938	SLAVE 'c2a' stopped
 Note	1938	SLAVE 'b2a' stopped
+Note	1938	SLAVE 'c2a' stopped
 SET GLOBAL slave_parallel_threads= @old_parallel;
 SET GLOBAL gtid_ignore_duplicates= @old_ignore_duplicates;
 SET GLOBAL gtid_domain_id=0;
diff --git a/mysql-test/suite/multi_source/gtid_ignore_duplicates.test b/mysql-test/suite/multi_source/gtid_ignore_duplicates.test
index 3e98d7c76cb..cf1c750fc19 100644
--- a/mysql-test/suite/multi_source/gtid_ignore_duplicates.test
+++ b/mysql-test/suite/multi_source/gtid_ignore_duplicates.test
@@ -261,24 +261,28 @@ SELECT * FROM t1 WHERE a >= 20 ORDER BY a;
 # Clean up.
 --connection server_1
 SET GLOBAL gtid_domain_id=0;
+--sorted_result
 STOP ALL SLAVES;
 SET GLOBAL slave_parallel_threads= @old_parallel;
 SET GLOBAL gtid_ignore_duplicates= @old_ignore_duplicates;
 
 --connection server_2
 SET GLOBAL gtid_domain_id=0;
+--sorted_result
 STOP ALL SLAVES;
 SET GLOBAL slave_parallel_threads= @old_parallel;
 SET GLOBAL gtid_ignore_duplicates= @old_ignore_duplicates;
 
 --connection server_3
 SET GLOBAL gtid_domain_id=0;
+--sorted_result
 STOP ALL SLAVES;
 SET GLOBAL slave_parallel_threads= @old_parallel;
 SET GLOBAL gtid_ignore_duplicates= @old_ignore_duplicates;
 
 --connection server_4
 SET GLOBAL gtid_domain_id=0;
+--sorted_result
 STOP ALL SLAVES;
 SET GLOBAL slave_parallel_threads= @old_parallel;
 SET GLOBAL gtid_ignore_duplicates= @old_ignore_duplicates;
diff --git a/mysql-test/suite/multi_source/load_data.result b/mysql-test/suite/multi_source/load_data.result
index ef55abc5321..3a04156ce6a 100644
--- a/mysql-test/suite/multi_source/load_data.result
+++ b/mysql-test/suite/multi_source/load_data.result
@@ -23,8 +23,8 @@ drop table t1;
 drop table t2;
 stop all slaves;
 Warnings:
-Note	1938	SLAVE 'master2' stopped
 Note	1938	SLAVE '' stopped
+Note	1938	SLAVE 'master2' stopped
 include/reset_master_slave.inc
 include/reset_master_slave.inc
 include/reset_master_slave.inc
diff --git a/mysql-test/suite/multi_source/load_data.test b/mysql-test/suite/multi_source/load_data.test
index e6e1399cbf3..ca2391a9c8d 100644
--- a/mysql-test/suite/multi_source/load_data.test
+++ b/mysql-test/suite/multi_source/load_data.test
@@ -58,6 +58,7 @@ drop table t2;
 --sync_with_master 0,'master2'
 
 --connection slave
+--sorted_result
 stop all slaves;
 
 --source reset_master_slave.inc
diff --git a/mysql-test/t/ctype_partitions.test b/mysql-test/t/ctype_partitions.test
new file mode 100644
index 00000000000..f80a2c98a1b
--- /dev/null
+++ b/mysql-test/t/ctype_partitions.test
@@ -0,0 +1,29 @@
+--source include/have_partition.inc
+
+--echo #
+--echo # MDEV-6255 DUPLICATE KEY Errors on SELECT .. GROUP BY that uses temporary and filesort
+--echo #
+
+# cp1251_ukrainian_ci: 0x20 SPACE is equal to 0x60 GRAVE ACCENT 
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET cp1251 COLLATE cp1251_ukrainian_ci);
+INSERT INTO t1 VALUES (0x20),(0x60),(0x6060),(0x606060);
+SELECT HEX(a) FROM t1 WHERE a=0x60;
+ALTER TABLE t1  PARTITION BY KEY(a) PARTITIONS 3;
+SELECT HEX(a) FROM t1 WHERE a=0x60;
+DROP TABLE t1;
+
+# koi8u_general_ci: 0x20 SPACE is equal to 0x60 GRAVE ACCENT 
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET koi8u COLLATE koi8u_general_ci);
+INSERT INTO t1 VALUES (0x20),(0x60),(0x6060),(0x606060);
+SELECT HEX(a) FROM t1 WHERE a=0x60;
+ALTER TABLE t1  PARTITION BY KEY(a) PARTITIONS 3;
+SELECT HEX(a) FROM t1 WHERE a=0x60;
+DROP TABLE t1;
+
+# cp1250_general_ci: 0x20 SPACE is equal to 0xA0 NO-BREAK SPACE
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET cp1250 COLLATE cp1250_general_ci);
+INSERT INTO t1 VALUES (0x20),(0xA0),(0xA0A0),(0xA0A0A0);
+SELECT HEX(a) FROM t1 WHERE a=0xA0;
+ALTER TABLE t1  PARTITION BY KEY(a) PARTITIONS 3;
+SELECT HEX(a) FROM t1 WHERE a=0xA0;
+DROP TABLE t1;