Bug#29261: Sort order of the collation wasn't used when comparing trailing

spaces.

When the my_strnncollsp_simple function compares two strings and one is a prefix
of another then this function compares characters in the rest of longer key
with the space character to find whether the longer key is greater or less.
But the sort order of the collation isn't used in this comparison. This may
lead to a wrong comparison result, wrongly created index or wrong order of the
result set of a query with the ORDER BY clause.

Now the my_strnncollsp_simple function uses collation sort order to compare
the characters in the rest of longer key with the space character.


mysql-test/t/ctype_collate.test:
  Added a test case for the bug#29261: Sort order of the collation wasn't used
  when comparing trailing spaces.
mysql-test/r/ctype_collate.result:
  Added a test case for the bug#29261: Sort order of the collation wasn't used
  when comparing trailing spaces.
strings/ctype-simple.c:
  Bug#29261: Sort order of the collation wasn't used when comparing trailing
  spaces.
  Now the my_strnncollsp_simple function uses collation sort order to compare
  the characters in the rest of longer key with the space character.

3 files changed, 20 insertions, 1 deletions

diff --git a/mysql-test/r/ctype_collate.result b/mysql-test/r/ctype_collate.result
index 66266d40fb3..52ee76d1948 100644
--- a/mysql-test/r/ctype_collate.result
+++ b/mysql-test/r/ctype_collate.result
@@ -595,3 +595,11 @@ EXPLAIN SELECT * FROM t1 WHERE s2 LIKE 'a' COLLATE latin1_german1_ci;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ALL	s2	NULL	NULL	NULL	10	Using where
 DROP TABLE t1;
+create table t1(f1 varchar(10) character set latin2 collate latin2_hungarian_ci, key(f1));
+insert into t1 set f1=0x3F3F9DC73F;
+insert into t1 set f1=0x3F3F1E563F;
+insert into t1 set f1=0x3F3F;
+check table t1 extended;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+drop table t1;
diff --git a/mysql-test/t/ctype_collate.test b/mysql-test/t/ctype_collate.test
index aca240b46bc..4bbae42559a 100644
--- a/mysql-test/t/ctype_collate.test
+++ b/mysql-test/t/ctype_collate.test
@@ -207,3 +207,14 @@ EXPLAIN SELECT * FROM t1 WHERE s2 LIKE 'a' COLLATE latin1_german1_ci;
 DROP TABLE t1;
 
 # End of 4.1 tests
+
+#
+# Bug#29261: Sort order of the collation wasn't used when comparing trailing
+#            spaces.
+#
+create table t1(f1 varchar(10) character set latin2 collate latin2_hungarian_ci, key(f1));
+insert into t1 set f1=0x3F3F9DC73F;
+insert into t1 set f1=0x3F3F1E563F;
+insert into t1 set f1=0x3F3F;
+check table t1 extended;
+drop table t1;
diff --git a/strings/ctype-simple.c b/strings/ctype-simple.c
index e57204f8d33..fca5607e152 100644
--- a/strings/ctype-simple.c
+++ b/strings/ctype-simple.c
@@ -180,7 +180,7 @@ int my_strnncollsp_simple(CHARSET_INFO * cs, const uchar *a, uint a_length,
     for (end= a + a_length-length; a < end ; a++)
     {
       if (*a != ' ')
-	return (*a < ' ') ? -swap : swap;
+	return (map[*a] < ' ') ? -swap : swap;
     }
   }
   return res;