Fixed bug mdev-5721.

Do not define a look-up key for a temporary table if its length
exceeds the maximum length of such keys. 

3 files changed, 90 insertions, 6 deletions

diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 15bd6886682..e7afdef6874 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -8977,6 +8977,25 @@ uint get_next_field_for_derived_key(uchar *arg)
 }
 
 
+static
+uint get_next_field_for_derived_key_simple(uchar *arg)
+{
+  KEYUSE *keyuse= *(KEYUSE **) arg;
+  if (!keyuse)
+    return (uint) (-1);
+  TABLE *table= keyuse->table;
+  uint key= keyuse->key;
+  uint fldno= keyuse->keypart; 
+  for ( ; 
+        keyuse->table == table && keyuse->key == key && keyuse->keypart == fldno;
+        keyuse++)
+    ;
+  if (keyuse->key != key)
+    keyuse= 0;
+  *((KEYUSE **) arg)= keyuse;
+  return fldno;
+}
+
 static 
 bool generate_derived_keys_for_table(KEYUSE *keyuse, uint count, uint keys)
 {
@@ -9007,12 +9026,28 @@ bool generate_derived_keys_for_table(KEYUSE *keyuse, uint count, uint keys)
     }
     else
     {
-      if (table->add_tmp_key(table->s->keys, parts, 
-                             get_next_field_for_derived_key, 
-                             (uchar *) &first_keyuse,
-                             FALSE))
-        return TRUE;
-      table->reginfo.join_tab->keys.set_bit(table->s->keys);
+      KEYUSE *save_first_keyuse= first_keyuse;
+      if (table->check_tmp_key(table->s->keys, parts,
+                               get_next_field_for_derived_key_simple, 
+                               (uchar *) &first_keyuse))
+ 
+      {
+        first_keyuse= save_first_keyuse;
+        if (table->add_tmp_key(table->s->keys, parts, 
+                               get_next_field_for_derived_key, 
+                               (uchar *) &first_keyuse,
+                               FALSE))
+          return TRUE;
+        table->reginfo.join_tab->keys.set_bit(table->s->keys);
+      }
+      else
+      {
+        /* Mark keyuses for this key to be excluded */
+        for (KEYUSE *curr=save_first_keyuse; curr < first_keyuse; curr++)
+	{
+          curr->key= MAX_KEY;
+        }
+      }
       first_keyuse= keyuse;
       key_count++;
       parts= 0;
diff --git a/sql/table.cc b/sql/table.cc
index 419621634b3..71188aec63d 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -5441,6 +5441,52 @@ void TABLE::create_key_part_by_field(KEY *keyinfo,
 
 /**
   @brief
+  Check validity of a possible key for the derived table
+
+  @param key            the number of the key
+  @param key_parts      number of components of the key
+  @param next_field_no  the call-back function that returns the number of
+                        the field used as the next component of the key
+  @param arg            the argument for the above function
+
+  @details
+  The function checks whether a possible key satisfies the constraints
+  imposed on the keys of any temporary table.
+
+  @return TRUE if the key is valid
+  @return FALSE otherwise
+*/
+
+bool TABLE::check_tmp_key(uint key, uint key_parts,
+                          uint (*next_field_no) (uchar *), uchar *arg)
+{
+  Field **reg_field;
+  uint i;
+  uint key_len= 0;
+
+  for (i= 0; i < key_parts; i++)
+  {
+    uint fld_idx= next_field_no(arg);
+    reg_field= field + fld_idx;
+    uint fld_store_len= (uint16) (*reg_field)->key_length();
+    if ((*reg_field)->real_maybe_null())
+      fld_store_len+= HA_KEY_NULL_LENGTH;
+    if ((*reg_field)->type() == MYSQL_TYPE_BLOB ||
+        (*reg_field)->real_type() == MYSQL_TYPE_VARCHAR ||
+        (*reg_field)->type() == MYSQL_TYPE_GEOMETRY)
+      fld_store_len+= HA_KEY_BLOB_LENGTH;
+    key_len+= fld_store_len;
+  }
+  /*
+    We use MI_MAX_KEY_LENGTH (myisam's default) below because it is
+    smaller than MAX_KEY_LENGTH (heap's default) and it's unknown whether
+    myisam or heap will be used for the temporary table.
+  */
+  return key_len <= MI_MAX_KEY_LENGTH;
+}
+
+/**
+  @brief
   Add one key to a temporary table
 
   @param key            the number of the key
@@ -5470,6 +5516,7 @@ bool TABLE::add_tmp_key(uint key, uint key_parts,
   KEY* keyinfo;
   Field **reg_field;
   uint i;
+  
   bool key_start= TRUE;
   KEY_PART_INFO* key_part_info=
       (KEY_PART_INFO*) alloc_root(&mem_root, sizeof(KEY_PART_INFO)*key_parts);
diff --git a/sql/table.h b/sql/table.h
index 95925cb202e..b7d13b03800 100644
--- a/sql/table.h
+++ b/sql/table.h
@@ -986,6 +986,8 @@ struct st_table {
   inline bool needs_reopen_or_name_lock()
   { return s->version != refresh_version; }
   bool alloc_keys(uint key_count);
+  bool check_tmp_key(uint key, uint key_parts,
+                     uint (*next_field_no) (uchar *), uchar *arg);
   bool add_tmp_key(uint key, uint key_parts,
                    uint (*next_field_no) (uchar *), uchar *arg,
                    bool unique);