MDEV-5745 analyze MySQL fix for bug#12368495

8 files changed, 138 insertions, 44 deletions

diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result
index 2af0b2bb509..f9e9a69de42 100644
--- a/mysql-test/r/ctype_ucs.result
+++ b/mysql-test/r/ctype_ucs.result
@@ -4277,5 +4277,38 @@ COALESCE(c1)
 
 DROP TABLE t1;
 #
+# MDEV-5745 analyze MySQL fix for bug#12368495
+#
+SELECT CHAR_LENGTH(TRIM(LEADING 0x000000 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(LEADING 0x000000 FROM _ucs2 0x0061))
+2
+SELECT CHAR_LENGTH(TRIM(LEADING 0x0001 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(LEADING 0x0001 FROM _ucs2 0x0061))
+2
+SELECT CHAR_LENGTH(TRIM(LEADING 0x00 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(LEADING 0x00 FROM _ucs2 0x0061))
+1
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x000000 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(TRAILING 0x000000 FROM _ucs2 0x0061))
+2
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x0001 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(TRAILING 0x0001 FROM _ucs2 0x0061))
+2
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x61 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(TRAILING 0x61 FROM _ucs2 0x0061))
+1
+SELECT CHAR_LENGTH(TRIM(BOTH 0x000000 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(BOTH 0x000000 FROM _ucs2 0x0061))
+2
+SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _ucs2 0x0061))
+2
+SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(BOTH 0x61 FROM _ucs2 0x0061))
+1
+SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061));
+CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061))
+1
+#
 # End of 5.5 tests
 #
diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result
index 92327821b84..1f316b7b68f 100644
--- a/mysql-test/r/ctype_utf32.result
+++ b/mysql-test/r/ctype_utf32.result
@@ -1237,5 +1237,38 @@ SELECT '2010-10-10 10:10:10' + INTERVAL GeometryType(GeomFromText('POINT(1 1)'))
 '2010-10-10 10:10:10' + INTERVAL GeometryType(GeomFromText('POINT(1 1)')) hour_second
 2010-10-10 10:10:10
 #
+# MDEV-5745 analyze MySQL fix for bug#12368495
+#
+SELECT CHAR_LENGTH(TRIM(LEADING 0x0000000000 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(LEADING 0x0000000000 FROM _utf32 0x00000061))
+4
+SELECT CHAR_LENGTH(TRIM(LEADING 0x0001 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(LEADING 0x0001 FROM _utf32 0x00000061))
+4
+SELECT CHAR_LENGTH(TRIM(LEADING 0x00 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(LEADING 0x00 FROM _utf32 0x00000061))
+1
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x0000000000 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(TRAILING 0x0000000000 FROM _utf32 0x00000061))
+4
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x0001 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(TRAILING 0x0001 FROM _utf32 0x00000061))
+4
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x61 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(TRAILING 0x61 FROM _utf32 0x00000061))
+3
+SELECT CHAR_LENGTH(TRIM(BOTH 0x0000000000 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(BOTH 0x0000000000 FROM _utf32 0x00000061))
+4
+SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _utf32 0x00000061))
+4
+SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(BOTH 0x61 FROM _utf32 0x00000061))
+3
+SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061));
+CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061))
+1
+#
 # End of 5.5 tests
 #
diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test
index 343adcd5712..7fd3768aa5f 100644
--- a/mysql-test/t/ctype_ucs.test
+++ b/mysql-test/t/ctype_ucs.test
@@ -838,5 +838,22 @@ DROP TABLE t1;
 
 
 --echo #
+--echo # MDEV-5745 analyze MySQL fix for bug#12368495
+--echo #
+SELECT CHAR_LENGTH(TRIM(LEADING 0x000000 FROM _ucs2 0x0061));
+SELECT CHAR_LENGTH(TRIM(LEADING 0x0001 FROM _ucs2 0x0061));
+SELECT CHAR_LENGTH(TRIM(LEADING 0x00 FROM _ucs2 0x0061));
+
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x000000 FROM _ucs2 0x0061));
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x0001 FROM _ucs2 0x0061));
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x61 FROM _ucs2 0x0061));
+
+SELECT CHAR_LENGTH(TRIM(BOTH 0x000000 FROM _ucs2 0x0061));
+SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _ucs2 0x0061));
+SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _ucs2 0x0061));
+SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061));
+
+
+--echo #
 --echo # End of 5.5 tests
 --echo #
diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test
index 91221b4f7c7..1be8925873c 100644
--- a/mysql-test/t/ctype_utf32.test
+++ b/mysql-test/t/ctype_utf32.test
@@ -861,5 +861,21 @@ ORDER BY l DESC;
 SELECT '2010-10-10 10:10:10' + INTERVAL GeometryType(GeomFromText('POINT(1 1)')) hour_second;
 
 --echo #
+--echo # MDEV-5745 analyze MySQL fix for bug#12368495
+--echo #
+SELECT CHAR_LENGTH(TRIM(LEADING 0x0000000000 FROM _utf32 0x00000061));
+SELECT CHAR_LENGTH(TRIM(LEADING 0x0001 FROM _utf32 0x00000061));
+SELECT CHAR_LENGTH(TRIM(LEADING 0x00 FROM _utf32 0x00000061));
+
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x0000000000 FROM _utf32 0x00000061));
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x0001 FROM _utf32 0x00000061));
+SELECT CHAR_LENGTH(TRIM(TRAILING 0x61 FROM _utf32 0x00000061));
+
+SELECT CHAR_LENGTH(TRIM(BOTH 0x0000000000 FROM _utf32 0x00000061));
+SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _utf32 0x00000061));
+SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _utf32 0x00000061));
+SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061));
+
+--echo #
 --echo # End of 5.5 tests
 --echo #
diff --git a/sql/item.cc b/sql/item.cc
index b4d7bffae7b..d58e4d285c5 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -2074,7 +2074,7 @@ bool agg_item_collations(DTCollation &c, const char *fname,
   bool unknown_cs= 0;
 
   c.set(av[0]->collation);
-  for (i= 1, arg= &av[item_sep]; i < count; i++, arg++)
+  for (i= 1, arg= &av[item_sep]; i < count; i++, arg+= item_sep)
   {
     if (c.aggregate((*arg)->collation, flags))
     {
diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index 861da69a9be..b0d7a346e83 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -1624,7 +1624,7 @@ String *Item_func_ltrim::val_str(String *str)
 
   if ((remove_length= remove_str->length()) == 0 ||
       remove_length > res->length())
-    return res;
+    return non_trimmed_value(res);
 
   ptr= (char*) res->ptr();
   end= ptr+res->length();
@@ -1643,9 +1643,8 @@ String *Item_func_ltrim::val_str(String *str)
     end+=remove_length;
   }
   if (ptr == res->ptr())
-    return res;
-  tmp_value.set(*res,(uint) (ptr - res->ptr()),(uint) (end-ptr));
-  return &tmp_value;
+    return non_trimmed_value(res);
+  return trimmed_value(res, (uint32) (ptr - res->ptr()), (uint32) (end - ptr));
 }
 
 
@@ -1671,7 +1670,7 @@ String *Item_func_rtrim::val_str(String *str)
 
   if ((remove_length= remove_str->length()) == 0 ||
       remove_length > res->length())
-    return res;
+    return non_trimmed_value(res);
 
   ptr= (char*) res->ptr();
   end= ptr+res->length();
@@ -1683,11 +1682,11 @@ String *Item_func_rtrim::val_str(String *str)
   {
     char chr=(*remove_str)[0];
 #ifdef USE_MB
-    if (use_mb(res->charset()))
+    if (use_mb(collation.collation))
     {
       while (ptr < end)
       {
-	if ((l=my_ismbchar(res->charset(), ptr,end))) ptr+=l,p=ptr;
+	if ((l= my_ismbchar(collation.collation, ptr, end))) ptr+= l, p=ptr;
 	else ++ptr;
       }
       ptr=p;
@@ -1700,12 +1699,12 @@ String *Item_func_rtrim::val_str(String *str)
   {
     const char *r_ptr=remove_str->ptr();
 #ifdef USE_MB
-    if (use_mb(res->charset()))
+    if (use_mb(collation.collation))
     {
   loop:
       while (ptr + remove_length < end)
       {
-	if ((l=my_ismbchar(res->charset(), ptr,end))) ptr+=l;
+	if ((l= my_ismbchar(collation.collation, ptr, end))) ptr+= l;
 	else ++ptr;
       }
       if (ptr + remove_length == end && !memcmp(ptr,r_ptr,remove_length))
@@ -1724,9 +1723,8 @@ String *Item_func_rtrim::val_str(String *str)
     }
   }
   if (end == res->ptr()+res->length())
-    return res;
-  tmp_value.set(*res,0,(uint) (end-res->ptr()));
-  return &tmp_value;
+    return non_trimmed_value(res);
+  return trimmed_value(res, 0, (uint32) (end - res->ptr()));
 }
 
 
@@ -1753,37 +1751,22 @@ String *Item_func_trim::val_str(String *str)
 
   if ((remove_length= remove_str->length()) == 0 ||
       remove_length > res->length())
-    return res;
+    return non_trimmed_value(res);
 
   ptr= (char*) res->ptr();
   end= ptr+res->length();
   r_ptr= remove_str->ptr();
+  while (ptr+remove_length <= end && !memcmp(ptr,r_ptr,remove_length))
+    ptr+=remove_length;
 #ifdef USE_MB
-  if (use_mb(res->charset()))
+  if (use_mb(collation.collation))
   {
-    while (ptr + remove_length <= end)
-    {
-      uint num_bytes= 0;
-      while (num_bytes < remove_length)
-      {
-        uint len;
-        if ((len= my_ismbchar(res->charset(), ptr + num_bytes, end)))
-          num_bytes+= len;
-        else
-          ++num_bytes;
-      }
-      if (num_bytes != remove_length)
-        break;
-      if (memcmp(ptr, r_ptr, remove_length))
-        break;
-      ptr+= remove_length;
-    }
     char *p=ptr;
     register uint32 l;
  loop:
     while (ptr + remove_length < end)
     {
-      if ((l= my_ismbchar(res->charset(), ptr,end)))
+      if ((l= my_ismbchar(collation.collation, ptr, end)))
         ptr+= l;
       else
         ++ptr;
@@ -1799,16 +1782,13 @@ String *Item_func_trim::val_str(String *str)
   else
 #endif /* USE_MB */
   {
-    while (ptr+remove_length <= end && !memcmp(ptr,r_ptr,remove_length))
-      ptr+=remove_length;
     while (ptr + remove_length <= end &&
 	   !memcmp(end-remove_length,r_ptr,remove_length))
       end-=remove_length;
   }
   if (ptr == res->ptr() && end == ptr+res->length())
-    return res;
-  tmp_value.set(*res,(uint) (ptr - res->ptr()),(uint) (end-ptr));
-  return &tmp_value;
+    return non_trimmed_value(res);
+  return trimmed_value(res, (uint32) (ptr - res->ptr()), (uint32) (end - ptr));
 }
 
 void Item_func_trim::fix_length_and_dec()
diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h
index 4579a6949c7..fcc1991774c 100644
--- a/sql/item_strfunc.h
+++ b/sql/item_strfunc.h
@@ -270,6 +270,21 @@ class Item_func_trim :public Item_str_func
 protected:
   String tmp_value;
   String remove;
+  String *trimmed_value(String *res, uint32 offset, uint32 length)
+  {
+    tmp_value.set(*res, offset, length);
+    /*
+      Make sure to return correct charset and collation:
+      TRIM(0x000000 FROM _ucs2 0x0061)
+      should set charset to "binary" rather than to "ucs2".
+    */
+    tmp_value.set_charset(collation.collation);
+    return &tmp_value;
+  }
+  String *non_trimmed_value(String *res)
+  {
+    return trimmed_value(res, 0, res->length());
+  }
 public:
   Item_func_trim(Item *a,Item *b) :Item_str_func(a,b) {}
   Item_func_trim(Item *a) :Item_str_func(a) {}
diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c
index 6e18635f94b..a79f5899ec5 100644
--- a/strings/ctype-ucs2.c
+++ b/strings/ctype-ucs2.c
@@ -1983,10 +1983,10 @@ my_strnxfrmlen_utf32(CHARSET_INFO *cs __attribute__((unused)), size_t len)
 
 static uint
 my_ismbchar_utf32(CHARSET_INFO *cs __attribute__((unused)),
-                  const char *b __attribute__((unused)),
-                  const char *e __attribute__((unused)))
+                  const char *b,
+                  const char *e)
 {
-  return 4;
+  return b + 4 > e ? 0 : 4;
 }
 
 
@@ -2895,10 +2895,10 @@ static int my_strnncollsp_ucs2(CHARSET_INFO *cs __attribute__((unused)),
 
 
 static uint my_ismbchar_ucs2(CHARSET_INFO *cs __attribute__((unused)),
-                             const char *b __attribute__((unused)),
-                             const char *e __attribute__((unused)))
+                             const char *b,
+                             const char *e)
 {
-  return 2;
+  return b + 2 > e ? 0 : 2;
 }