Bug #35087: Inserting duplicate values at one time with DES_ENCRYPT leads

  to wrong results
      
3 problems found with DES_ENCRYPT/DES_DECRYPT :

1. The max length was not calculated properly. Fixed in fix_length_and_dec()
2. DES_ENCRYPT had a side effect of sometimes reallocating and changing 
the value of its argument. Fixed by explicitly pre-allocating the necessary
space to pad the argument with trailing '*' (stars) when calculating the 
DES digest.
3. in DES_ENCRYPT the string buffer for the result value was not 
reallocated to the correct size and only string length was assigned to it. 
Fixed by making sure there's enough space to hold the result.

2 files changed, 20 insertions, 7 deletions

diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index 7edc1a62307..4640929b2bf 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -473,17 +473,21 @@ String *Item_func_des_encrypt::val_str(String *str)
      string marking change of string length.
   */
 
-  tail=  (8-(res_length) % 8);			// 1..8 marking extra length
+  tail= 8 - (res_length % 8);                   // 1..8 marking extra length
   res_length+=tail;
+  tmp_arg.realloc(res_length);
+  tmp_arg.length(0);
+  tmp_arg.append(res->ptr(), res->length());
   code= ER_OUT_OF_RESOURCES;
-  if (tail && res->append(append_str, tail) || tmp_value.alloc(res_length+1))
+  if (tmp_arg.append(append_str, tail) || tmp_value.alloc(res_length+1))
     goto error;
-  (*res)[res_length-1]=tail;			// save extra length
+  tmp_arg[res_length-1]=tail;                   // save extra length
+  tmp_value.realloc(res_length+1);
   tmp_value.length(res_length+1);
   tmp_value[0]=(char) (128 | key_number);
   // Real encryption
   bzero((char*) &ivec,sizeof(ivec));
-  DES_ede3_cbc_encrypt((const uchar*) (res->ptr()),
+  DES_ede3_cbc_encrypt((const uchar*) (tmp_arg.ptr()),
 		       (uchar*) (tmp_value.ptr()+1),
 		       res_length,
 		       &keyschedule.ks1,
diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h
index 9794a092648..1c5346ab074 100644
--- a/sql/item_strfunc.h
+++ b/sql/item_strfunc.h
@@ -306,13 +306,17 @@ public:
 
 class Item_func_des_encrypt :public Item_str_func
 {
-  String tmp_value;
+  String tmp_value,tmp_arg;
 public:
   Item_func_des_encrypt(Item *a) :Item_str_func(a) {}
   Item_func_des_encrypt(Item *a, Item *b): Item_str_func(a,b) {}
   String *val_str(String *);
   void fix_length_and_dec()
-  { maybe_null=1; max_length = args[0]->max_length+8; }
+  {
+    maybe_null=1;
+    /* 9 = MAX ((8- (arg_len % 8)) + 1) */
+    max_length = args[0]->max_length + 9;
+  }
   const char *func_name() const { return "des_encrypt"; }
 };
 
@@ -323,7 +327,12 @@ public:
   Item_func_des_decrypt(Item *a) :Item_str_func(a) {}
   Item_func_des_decrypt(Item *a, Item *b): Item_str_func(a,b) {}
   String *val_str(String *);
-  void fix_length_and_dec() { maybe_null=1; max_length = args[0]->max_length; }
+  void fix_length_and_dec()
+  {
+    maybe_null=1;
+    /* 9 = MAX ((8- (arg_len % 8)) + 1) */
+    max_length = args[0]->max_length - 9;
+  }
   const char *func_name() const { return "des_decrypt"; }
 };