BUG#11766519 (bug#59648): MY_STRTOLL10_MB2: ASSERTION `(*ENDPTR - S) % 2 == 0' FAILED

      
Problem: wrong character set pointer was passed to my_strtoll10_mb2,
which led to DBUG_ASSERT failure in some cases.

  @ mysql-test/r/func_encrypt_ucs2.result
  @ mysql-test/t/func_encrypt_ucs2.test
  @ mysql-test/r/ctype_ucs.result
  @ mysql-test/t/ctype_ucs.test
  Adding tests

  @ sql/item_func.cc
  "cs" initialization was wrong (res does not necessarily point to &str_value)

  @ sql/item_strfunc.cc
  Item_func_dec_encrypt::val_str() and Item_func_des_descrypt::val_str()
  did not set character set for tmp_value (the returned value),
  so the old value, which was previously copied from args[1]->val_str(),
  was incorrectly returned with tmp_value.

6 files changed, 45 insertions, 1 deletions

diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result
index c9dcaf0f09c..c0e6e4fbabd 100644
--- a/mysql-test/r/ctype_ucs.result
+++ b/mysql-test/r/ctype_ucs.result
@@ -1238,4 +1238,10 @@ CREATE VIEW v1 AS SELECT 1 from t1
 WHERE t1.b <=> (SELECT a FROM t1 WHERE a < SOME(SELECT '1'));
 DROP VIEW v1;
 DROP TABLE t1;
+#
+# Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+#
+SELECT HEX(CHAR(COALESCE(NULL, CHAR(COUNT('%s') USING ucs2), 1, @@global.license, NULL) USING cp850));
+HEX(CHAR(COALESCE(NULL, CHAR(COUNT('%s') USING ucs2), 1, @@global.license, NULL) USING cp850))
+00
 End of 5.0 tests
diff --git a/mysql-test/r/func_encrypt_ucs2.result b/mysql-test/r/func_encrypt_ucs2.result
new file mode 100644
index 00000000000..384e931452e
--- /dev/null
+++ b/mysql-test/r/func_encrypt_ucs2.result
@@ -0,0 +1,19 @@
+#
+# Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+#
+SELECT CHAR_LENGTH(DES_ENCRYPT(0, CHAR('1' USING ucs2)));
+CHAR_LENGTH(DES_ENCRYPT(0, CHAR('1' USING ucs2)))
+9
+SELECT CONVERT(DES_ENCRYPT(0, CHAR('1' USING ucs2)),UNSIGNED);
+CONVERT(DES_ENCRYPT(0, CHAR('1' USING ucs2)),UNSIGNED)
+0
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '?T?iK?j??'
+SELECT CHAR_LENGTH(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' USING ucs2)));
+CHAR_LENGTH(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' USING ucs2)))
+4
+SELECT CONVERT(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' using ucs2)), UNSIGNED);
+CONVERT(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' using ucs2)), UNSIGNED)
+0
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: 'test'
diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test
index fd10ee3fdd6..16ee97c5a01 100644
--- a/mysql-test/t/ctype_ucs.test
+++ b/mysql-test/t/ctype_ucs.test
@@ -741,4 +741,9 @@ WHERE t1.b <=> (SELECT a FROM t1 WHERE a < SOME(SELECT '1'));
 DROP VIEW v1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+--echo #
+SELECT HEX(CHAR(COALESCE(NULL, CHAR(COUNT('%s') USING ucs2), 1, @@global.license, NULL) USING cp850));
+
 --echo End of 5.0 tests
diff --git a/mysql-test/t/func_encrypt_ucs2.test b/mysql-test/t/func_encrypt_ucs2.test
new file mode 100644
index 00000000000..920c7a6ad0d
--- /dev/null
+++ b/mysql-test/t/func_encrypt_ucs2.test
@@ -0,0 +1,12 @@
+-- source include/have_ssl.inc
+-- source include/have_ucs2.inc
+
+--echo #
+--echo # Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+--echo #
+
+SELECT CHAR_LENGTH(DES_ENCRYPT(0, CHAR('1' USING ucs2)));
+SELECT CONVERT(DES_ENCRYPT(0, CHAR('1' USING ucs2)),UNSIGNED);
+
+SELECT CHAR_LENGTH(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' USING ucs2)));
+SELECT CONVERT(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' using ucs2)), UNSIGNED);
diff --git a/sql/item_func.cc b/sql/item_func.cc
index bffbd03e15a..efae928a8b6 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -840,7 +840,7 @@ longlong Item_func_numhybrid::val_int()
       return 0;
 
     char *end= (char*) res->ptr() + res->length();
-    CHARSET_INFO *cs= str_value.charset();
+    CHARSET_INFO *cs= res->charset();
     return (*(cs->cset->strtoll10))(cs, res->ptr(), &end, &err_not_used);
   }
   default:
diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index c637c9c29b8..319184002bf 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -519,6 +519,7 @@ String *Item_func_des_encrypt::val_str(String *str)
   tmp_arg[res_length-1]=tail;                   // save extra length
   tmp_value.realloc(res_length+1);
   tmp_value.length(res_length+1);
+  tmp_value.set_charset(&my_charset_bin);
   tmp_value[0]=(char) (128 | key_number);
   // Real encryption
   bzero((char*) &ivec,sizeof(ivec));
@@ -606,6 +607,7 @@ String *Item_func_des_decrypt::val_str(String *str)
   if ((tail=(uint) (uchar) tmp_value[length-2]) > 8)
     goto wrong_key;				     // Wrong key
   tmp_value.length(length-1-tail);
+  tmp_value.set_charset(&my_charset_bin);
   return &tmp_value;
 
 error: