sql_select.cc, item_buff.cc, item.h:

  Fixed bug #11088: a crash for queries with GROUP BY a BLOB column
  + COUNT(DISTINCT...) due to an attempt to allocate a too large
  buffer for the BLOB field.
  Now the size of the buffer is limited by max_sort_length.
group_by.test, group_by.result:
  Added a test case for bug #11088.


mysql-test/r/group_by.result:
  Added a test case for bug #11088.
mysql-test/t/group_by.test:
  Added a test case for bug #11088.
sql/item.h:
  Fixed bug #11088: a crash for queries with GROUP BY a BLOB column
  + COUNT(DISTINCT...) due to an attempt to allocate a too large
  buffer for the BLOB fields.
  Now the size of the buffer is limited by max_sort_length.
sql/item_buff.cc:
  Fixed bug #11088: a crash for queries with GROUP BY a BLOB column
  + COUNT(DISTINCT...) due to an attempt to allocate a too large
  buffer for the BLOB fields.
  Now the size of the buffer is limited by max_sort_length.
sql/sql_select.cc:
  Fixed bug #11088: a crash for queries with GROUP BY a BLOB column
  + COUNT(DISTINCT...) due to an attempt to allocate a too large
  buffer for the BLOB fields.
  Now the size of the buffer is limited by max_sort_length.

3 files changed, 10 insertions, 5 deletions

diff --git a/sql/item.h b/sql/item.h
index 2edbeef400c..8de2adeb730 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -1118,7 +1118,7 @@ class Item_str_buff :public Item_buff
   Item *item;
   String value,tmp_value;
 public:
-  Item_str_buff(Item *arg) :item(arg),value(arg->max_length) {}
+  Item_str_buff(THD *thd, Item *arg);
   bool cmp(void);
   ~Item_str_buff();				// Deallocate String:s
 };
@@ -1385,7 +1385,7 @@ public:
 };
 
 
-extern Item_buff *new_Item_buff(Item *item);
+extern Item_buff *new_Item_buff(THD *thd, Item *item);
 extern Item_result item_cmp_type(Item_result a,Item_result b);
 extern void resolve_const_item(THD *thd, Item **ref, Item *cmp_item);
 extern bool field_is_equal_to_item(Field *field,Item *item);
diff --git a/sql/item_buff.cc b/sql/item_buff.cc
index 1559cfe958e..2324205eb65 100644
--- a/sql/item_buff.cc
+++ b/sql/item_buff.cc
@@ -23,13 +23,13 @@
 ** Create right type of item_buffer for an item
 */
 
-Item_buff *new_Item_buff(Item *item)
+Item_buff *new_Item_buff(THD *thd, Item *item)
 {
   if (item->type() == Item::FIELD_ITEM &&
       !(((Item_field *) item)->field->flags & BLOB_FLAG))
     return new Item_field_buff((Item_field *) item);
   if (item->result_type() == STRING_RESULT)
-    return new Item_str_buff((Item_field *) item);
+    return new Item_str_buff(thd, (Item_field *) item);
   if (item->result_type() == INT_RESULT)
     return new Item_int_buff((Item_field *) item);
   return new Item_real_buff(item);
@@ -42,12 +42,17 @@ Item_buff::~Item_buff() {}
 ** Return true if values have changed
 */
 
+Item_str_buff::Item_str_buff(THD *thd, Item *arg)
+  :item(arg), value(min(arg->max_length, thd->variables. max_sort_length))
+{}
+
 bool Item_str_buff::cmp(void)
 {
   String *res;
   bool tmp;
 
   res=item->val_str(&tmp_value);
+  res->length(min(res->length(), value.alloced_length()));
   if (null_value != item->null_value)
   {
     if ((null_value= item->null_value))
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 0424cd8fdbb..5b1603b44e2 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -8656,7 +8656,7 @@ alloc_group_fields(JOIN *join,ORDER *group)
   {
     for (; group ; group=group->next)
     {
-      Item_buff *tmp=new_Item_buff(*group->item);
+      Item_buff *tmp=new_Item_buff(join->thd, *group->item);
       if (!tmp || join->group_fields.push_front(tmp))
 	return TRUE;
     }