bug#3964 and related issues: FTB problems with charsets where one byte can match many

correct prefix compare with my_strnncoll


include/m_ctype.h:
  6th argument to my_strncoll to handle prefix comparison
myisam/ft_boolean_search.c:
  bug#3964 and related issues: problems with charsets where one byte can match many
  *correct* prefix compare with my_strnncoll
  *correct* backup of info->lastkey
mysql-test/r/fulltext.result:
  6th argument to my_strncoll to handle prefix comparison
mysql-test/t/fulltext.test:
  6th argument to my_strncoll to handle prefix comparison
mysys/my_handler.c:
  6th argument to my_strncoll to handle prefix comparison
sql/sql_parse.cc:
  cleanup
strings/ctype-big5.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-bin.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-czech.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-gbk.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-latin1.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-mb.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-simple.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-sjis.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-tis620.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-uca.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-ucs2.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-utf8.c:
  6th argument to my_strncoll to handle prefix comparison
strings/ctype-win1250ch.c:
  6th argument to my_strncoll to handle prefix comparison

13 files changed, 96 insertions, 37 deletions

diff --git a/strings/ctype-big5.c b/strings/ctype-big5.c
index f024fa0cc14..af8f5d49654 100644
--- a/strings/ctype-big5.c
+++ b/strings/ctype-big5.c
@@ -251,11 +251,12 @@ static int my_strnncoll_big5_internal(const uchar **a_res,
 
 static int my_strnncoll_big5(CHARSET_INFO *cs __attribute__((unused)), 
 			     const uchar *a, uint a_length,
-			     const uchar *b, uint b_length)
+                             const uchar *b, uint b_length,
+                             my_bool b_is_prefix)
 {
   uint length= min(a_length, b_length);
   int res= my_strnncoll_big5_internal(&a, &b, length);
-  return res ? res : (int) (a_length - b_length);
+  return res ? res : (int)((b_is_prefix ? length : a_length) - b_length);
 }
 
 
diff --git a/strings/ctype-bin.c b/strings/ctype-bin.c
index 7cac8c7c337..35630eb854f 100644
--- a/strings/ctype-bin.c
+++ b/strings/ctype-bin.c
@@ -91,10 +91,20 @@ static uchar bin_char_array[] =
 
 static int my_strnncoll_binary(CHARSET_INFO * cs __attribute__((unused)),
 				const uchar *s, uint slen,
-				const uchar *t, uint tlen)
+				const uchar *t, uint tlen,
+                                my_bool t_is_prefix)
 {
-  int cmp= memcmp(s,t,min(slen,tlen));
-  return cmp ? cmp : (int) (slen - tlen);
+  uint len=min(slen,tlen);
+  int cmp= memcmp(s,t,len);
+  return cmp ? cmp : (int)((t_is_prefix ? len : slen) - tlen);
+}
+
+
+static int my_strnncollsp_binary(CHARSET_INFO * cs __attribute__((unused)),
+                                const uchar *s, uint slen,
+                                const uchar *t, uint tlen)
+{
+  return my_strnncoll_binary(cs,s,slen,t,tlen,0);
 }
 
 
@@ -334,7 +344,7 @@ skip:
 MY_COLLATION_HANDLER my_collation_8bit_bin_handler =
 {
     my_strnncoll_binary,
-    my_strnncoll_binary,
+    my_strnncollsp_binary,
     my_strnxfrm_bin,
     my_like_range_simple,
     my_wildcmp_bin,
diff --git a/strings/ctype-czech.c b/strings/ctype-czech.c
index 2eb2fac46e9..9175451ceb7 100644
--- a/strings/ctype-czech.c
+++ b/strings/ctype-czech.c
@@ -242,12 +242,16 @@ while (1)						\
 
 static int my_strnncoll_czech(CHARSET_INFO *cs __attribute__((unused)),
 			      const uchar * s1, uint len1, 
-			      const uchar * s2, uint len2)
+			      const uchar * s2, uint len2,
+                              my_bool s2_is_prefix)
 {
   int v1, v2;
   const uchar * p1, * p2, * store1, * store2;
   int pass1 = 0, pass2 = 0;
 
+  if (s2_is_prefix && len1 > len2)
+    len1=len2;
+
   p1 = s1;	p2 = s2;
   store1 = s1;	store2 = s2;
 
@@ -276,7 +280,7 @@ int my_strnncollsp_czech(CHARSET_INFO * cs,
 {
   for ( ; slen && s[slen-1] == ' ' ; slen--);
   for ( ; tlen && t[tlen-1] == ' ' ; tlen--);
-  return my_strnncoll_czech(cs,s,slen,t,tlen);
+  return my_strnncoll_czech(cs,s,slen,t,tlen,0);
 }
 
 
diff --git a/strings/ctype-gbk.c b/strings/ctype-gbk.c
index 0dc00a73fa3..18024db3f35 100644
--- a/strings/ctype-gbk.c
+++ b/strings/ctype-gbk.c
@@ -2614,11 +2614,12 @@ int my_strnncoll_gbk_internal(const uchar **a_res, const uchar **b_res,
 
 int my_strnncoll_gbk(CHARSET_INFO *cs __attribute__((unused)),
 		     const uchar *a, uint a_length,
-		     const uchar *b, uint b_length)
+                     const uchar *b, uint b_length,
+                     my_bool b_is_prefix)
 {
   uint length= min(a_length, b_length);
   int res= my_strnncoll_gbk_internal(&a, &b, length);
-  return res ? res : (int) (a_length - b_length);
+  return res ? res : (int) ((b_is_prefix ? length : a_length) - b_length);
 }
 
 
diff --git a/strings/ctype-latin1.c b/strings/ctype-latin1.c
index 0b439964c7c..257d8e37eee 100644
--- a/strings/ctype-latin1.c
+++ b/strings/ctype-latin1.c
@@ -525,7 +525,8 @@ uchar combo2map[]={
 
 static int my_strnncoll_latin1_de(CHARSET_INFO *cs __attribute__((unused)),
 				  const uchar *a, uint a_length,
-				  const uchar *b, uint b_length)
+				  const uchar *b, uint b_length,
+                                  my_bool b_is_prefix)
 {
   const uchar *a_end= a + a_length;
   const uchar *b_end= b + b_length;
@@ -558,7 +559,7 @@ static int my_strnncoll_latin1_de(CHARSET_INFO *cs __attribute__((unused)),
     A simple test of string lengths won't work -- we test to see
     which string ran out first
   */
-  return ((a < a_end || a_extend) ? 1 :
+  return ((a < a_end || a_extend) ? (b_is_prefix ? 0 : 1) :
 	  (b < b_end || b_extend) ? -1 : 0);
 }
 
diff --git a/strings/ctype-mb.c b/strings/ctype-mb.c
index 9b02cd3b3da..129e2f2d9a9 100644
--- a/strings/ctype-mb.c
+++ b/strings/ctype-mb.c
@@ -322,7 +322,7 @@ uint my_instr_mb(CHARSET_INFO *cs,
       int mblen;
       
       if (!cs->coll->strnncoll(cs, (unsigned char*) b,   s_length, 
-      				   (unsigned char*) s, s_length))
+      				   (unsigned char*) s, s_length, 0))
       {
         if (nmatch)
         {
@@ -352,10 +352,19 @@ uint my_instr_mb(CHARSET_INFO *cs,
 
 static int my_strnncoll_mb_bin(CHARSET_INFO * cs __attribute__((unused)),
 				const uchar *s, uint slen,
-				const uchar *t, uint tlen)
+				const uchar *t, uint tlen,
+                                my_bool t_is_prefix)
 {
-  int cmp= memcmp(s,t,min(slen,tlen));
-  return cmp ? cmp : (int) (slen - tlen);
+  uint len=min(slen,tlen);
+  int cmp= memcmp(s,t,len);
+  return cmp ? cmp : (int) ((t_is_prefix ? len : slen) - tlen);
+}
+
+static int my_strnncollsp_mb_bin(CHARSET_INFO * cs __attribute__((unused)),
+                                 const uchar *s, uint slen,
+                                 const uchar *t, uint tlen)
+{
+  return my_strnncoll_mb_bin(cs,s,slen,t,tlen,0);
 }
 
 
@@ -513,7 +522,7 @@ static int my_wildcmp_mb_bin(CHARSET_INFO *cs,
 MY_COLLATION_HANDLER my_collation_mb_bin_handler =
 {
     my_strnncoll_mb_bin,
-    my_strnncoll_mb_bin,
+    my_strnncollsp_mb_bin,
     my_strnxfrm_mb_bin,
     my_like_range_simple,
     my_wildcmp_mb_bin,
diff --git a/strings/ctype-simple.c b/strings/ctype-simple.c
index ba1fc1c424a..3ef8a9d5997 100644
--- a/strings/ctype-simple.c
+++ b/strings/ctype-simple.c
@@ -47,16 +47,19 @@ int my_strnxfrm_simple(CHARSET_INFO * cs,
 }
 
 int my_strnncoll_simple(CHARSET_INFO * cs, const uchar *s, uint slen, 
-			const uchar *t, uint tlen)
+                        const uchar *t, uint tlen,
+                        my_bool t_is_prefix)
 {
   int len = ( slen > tlen ) ? tlen : slen;
   uchar *map= cs->sort_order;
+  if (t_is_prefix && slen > tlen)
+    slen=tlen;
   while (len--)
   {
     if (map[*s++] != map[*t++])
       return ((int) map[s[-1]] - (int) map[t[-1]]);
   }
-  return (int) (slen-tlen);
+  return (int) (slen - tlen);
 }
 
 
diff --git a/strings/ctype-sjis.c b/strings/ctype-sjis.c
index 72666175a1f..cfe59bb2852 100644
--- a/strings/ctype-sjis.c
+++ b/strings/ctype-sjis.c
@@ -232,9 +232,12 @@ static int my_strnncoll_sjis_internal(CHARSET_INFO *cs,
 
 static int my_strnncoll_sjis(CHARSET_INFO *cs __attribute__((unused)),
 			      const uchar *a, uint a_length, 
-			      const uchar *b, uint b_length)
+			      const uchar *b, uint b_length,
+                              my_bool b_is_prefix)
 {
   int res= my_strnncoll_sjis_internal(cs, &a, a_length, &b, b_length);
+  if (b_is_prefix && a_length > b_length)
+    a_length= b_length;
   return res ? res : (int) (a_length - b_length);
 }
 
diff --git a/strings/ctype-tis620.c b/strings/ctype-tis620.c
index e2a138300c3..b9ee8825eef 100644
--- a/strings/ctype-tis620.c
+++ b/strings/ctype-tis620.c
@@ -529,12 +529,16 @@ static uint thai2sortable(uchar *tstr, uint len)
 static
 int my_strnncoll_tis620(CHARSET_INFO *cs __attribute__((unused)),
                         const uchar * s1, uint len1, 
-                        const uchar * s2, uint len2)
+                        const uchar * s2, uint len2,
+                        my_bool s2_is_prefix)
 {
   uchar	buf[80] ;
   uchar *tc1, *tc2;
   int i;
 
+  if (s2_is_prefix && len1 > len2)
+    len1= len2;
+
   tc1= buf;
   if ((len1 + len2 +2) > (int) sizeof(buf))
     tc1= (uchar*) malloc(len1+len2);
diff --git a/strings/ctype-uca.c b/strings/ctype-uca.c
index 81073d47554..ba7d308ec5c 100644
--- a/strings/ctype-uca.c
+++ b/strings/ctype-uca.c
@@ -6704,7 +6704,8 @@ implicit:
 
 static int my_strnncoll_uca(CHARSET_INFO *cs, 
 			    const uchar *s, uint slen,
-			    const uchar *t, uint tlen)
+                            const uchar *t, uint tlen,
+                            my_bool t_is_prefix)
 {
   my_uca_scanner sscanner;
   my_uca_scanner tscanner;
@@ -6720,7 +6721,7 @@ static int my_strnncoll_uca(CHARSET_INFO *cs,
     t_res= my_uca_scanner_next(&tscanner);
   } while ( s_res == t_res && s_res >0);
   
-  return ( s_res - t_res );
+  return  (t_is_prefix && t_res < 0) ? 0 : (s_res - t_res);
 }
 
 
diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c
index 67340fdd4f4..040fc5f65fd 100644
--- a/strings/ctype-ucs2.c
+++ b/strings/ctype-ucs2.c
@@ -182,7 +182,8 @@ static void my_casedn_str_ucs2(CHARSET_INFO *cs __attribute__((unused)),
 
 static int my_strnncoll_ucs2(CHARSET_INFO *cs, 
 			     const uchar *s, uint slen, 
-			     const uchar *t, uint tlen)
+                             const uchar *t, uint tlen,
+                             my_bool t_is_prefix)
 {
   int s_res,t_res;
   my_wc_t s_wc,t_wc;
@@ -213,7 +214,14 @@ static int my_strnncoll_ucs2(CHARSET_INFO *cs,
     s+=s_res;
     t+=t_res;
   }
-  return ( (se-s) - (te-t) );
+  return t_is_prefix ? t-te : ((se-s) - (te-t));
+}
+
+static int my_strnncollsp_ucs2(CHARSET_INFO *cs, 
+                               const uchar *s, uint slen, 
+                               const uchar *t, uint tlen)
+{
+  return my_strnncoll_ucs2(cs,s,slen,t,tlen,0);
 }
 
 
@@ -1224,8 +1232,9 @@ int my_wildcmp_ucs2_bin(CHARSET_INFO *cs,
 
 static
 int my_strnncoll_ucs2_bin(CHARSET_INFO *cs, 
-				 const uchar *s, uint slen,
-				 const uchar *t, uint tlen)
+                          const uchar *s, uint slen,
+                          const uchar *t, uint tlen,
+                          my_bool t_is_prefix)
 {
   int s_res,t_res;
   my_wc_t s_wc,t_wc;
@@ -1250,7 +1259,14 @@ int my_strnncoll_ucs2_bin(CHARSET_INFO *cs,
     s+=s_res;
     t+=t_res;
   }
-  return ( (se-s) - (te-t) );
+  return t_is_prefix ? t-te : ((se-s) - (te-t));
+}
+
+static int my_strnncollsp_ucs2_bin(CHARSET_INFO *cs, 
+                                   const uchar *s, uint slen, 
+                                   const uchar *t, uint tlen)
+{
+  return my_strnncoll_ucs2_bin(cs,s,slen,t,tlen,0);
 }
 
 
@@ -1374,7 +1390,7 @@ my_bool my_like_range_ucs2(CHARSET_INFO *cs,
 static MY_COLLATION_HANDLER my_collation_ucs2_general_ci_handler =
 {
     my_strnncoll_ucs2,
-    my_strnncoll_ucs2,
+    my_strnncollsp_ucs2,
     my_strnxfrm_ucs2,
     my_like_range_ucs2,
     my_wildcmp_ucs2_ci,
@@ -1387,7 +1403,7 @@ static MY_COLLATION_HANDLER my_collation_ucs2_general_ci_handler =
 static MY_COLLATION_HANDLER my_collation_ucs2_bin_handler =
 {
     my_strnncoll_ucs2_bin,
-    my_strnncoll_ucs2_bin,
+    my_strnncollsp_ucs2_bin,
     my_strnxfrm_ucs2_bin,
     my_like_range_simple,
     my_wildcmp_ucs2_bin,
diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c
index 29d2c5d1358..48882f356fd 100644
--- a/strings/ctype-utf8.c
+++ b/strings/ctype-utf8.c
@@ -1802,7 +1802,8 @@ static void my_casedn_str_utf8(CHARSET_INFO *cs, char * s)
 
 static int my_strnncoll_utf8(CHARSET_INFO *cs, 
 			     const uchar *s, uint slen,
-			     const uchar *t, uint tlen)
+                             const uchar *t, uint tlen,
+                             my_bool t_is_prefix)
 {
   int s_res,t_res;
   my_wc_t s_wc,t_wc;
@@ -1833,7 +1834,7 @@ static int my_strnncoll_utf8(CHARSET_INFO *cs,
     s+=s_res;
     t+=t_res;
   }
-  return ( (se-s) - (te-t) );
+  return t_is_prefix ? t-te : ((se-s) - (te-t));
 }
 
 
diff --git a/strings/ctype-win1250ch.c b/strings/ctype-win1250ch.c
index 2eefb570170..65d20ceedc9 100644
--- a/strings/ctype-win1250ch.c
+++ b/strings/ctype-win1250ch.c
@@ -448,20 +448,25 @@ static struct wordvalue doubles[] = {
 
 static int my_strnncoll_win1250ch(CHARSET_INFO *cs __attribute__((unused)), 
 				  const uchar * s1, uint len1,
-				  const uchar * s2, uint len2)
+                                  const uchar * s2, uint len2,
+                                  my_bool s2_is_prefix)
 {
   int v1, v2;
   const uchar * p1, * p2;
   int pass1 = 0, pass2 = 0;
   int diff;
 
+  if (s2_is_prefix && len1 > len2)
+    len1=len2;
+
   p1 = s1;	p2 = s2;
 
-  do {
+  do
+  {
     NEXT_CMP_VALUE(s1, p1, pass1, v1, (int)len1);
     NEXT_CMP_VALUE(s2, p2, pass2, v2, (int)len2);
-    diff = v1 - v2;
-    if (diff != 0)		return diff;
+    if ((diff = v1 - v2))
+      return diff;
   } while (v1);
   return 0;
 }
@@ -478,7 +483,7 @@ int my_strnncollsp_win1250ch(CHARSET_INFO * cs,
 {
   for ( ; slen && s[slen-1] == ' ' ; slen--);
   for ( ; tlen && t[tlen-1] == ' ' ; tlen--);
-  return my_strnncoll_win1250ch(cs,s,slen,t,tlen);
+  return my_strnncoll_win1250ch(cs,s,slen,t,tlen,0);
 }