Merge with new VARCHAR code

configure.in:
  Auto merged
BitKeeper/deleted/.del-acinclude.m4~f4ab416bac5003:
  Auto merged
BitKeeper/deleted/.del-ha_isam.cc~4dce65904db2675e:
  Auto merged
BitKeeper/deleted/.del-ha_isammrg.cc~dc682e4755d77a2e:
  Auto merged
client/mysqldump.c:
  Auto merged
client/mysqltest.c:
  Auto merged
heap/hp_create.c:
  Auto merged
heap/hp_delete.c:
  Auto merged
heap/hp_hash.c:
  Auto merged
heap/hp_write.c:
  Auto merged
include/decimal.h:
  Auto merged
include/m_ctype.h:
  Auto merged
libmysql/libmysql.c:
  Auto merged
libmysqld/Makefile.am:
  Auto merged
myisam/mi_check.c:
  Auto merged
myisam/mi_create.c:
  Auto merged
myisam/mi_write.c:
  Auto merged
mysql-test/r/ctype_tis620.result:
  Auto merged
mysql-test/r/ctype_ucs.result:
  Auto merged
mysql-test/r/myisam.result:
  Auto merged
mysql-test/r/mysqldump.result:
  Auto merged
mysql-test/r/order_by.result:
  Auto merged
mysql-test/r/ps.result:
  Auto merged
mysql-test/r/ps_1general.result:
  Auto merged
mysql-test/r/ps_2myisam.result:
  Auto merged
mysql-test/r/ps_3innodb.result:
  Auto merged
mysql-test/r/ps_4heap.result:
  Auto merged
mysql-test/r/ps_5merge.result:
  Auto merged
mysql-test/r/ps_6bdb.result:
  Auto merged
mysql-test/r/select.result:
  Auto merged
mysql-test/r/strict.result:
  Auto merged
mysql-test/r/subselect.result:
  Auto merged
mysql-test/r/type_blob.result:
  Auto merged
mysql-test/t/ctype_ucs.test:
  Auto merged
mysql-test/t/endspace.test:
  Auto merged
mysql-test/t/myisam.test:
  Auto merged
mysql-test/t/ps_1general.test:
  Auto merged
mysql-test/t/strict.test:
  Auto merged
mysql-test/t/type_blob.test:
  Auto merged
ndb/src/common/util/NdbSqlUtil.cpp:
  Auto merged
scripts/mysql_fix_privilege_tables.sh:
  Auto merged
sql/field.h:
  Auto merged
sql/field_conv.cc:
  Auto merged
sql/ha_heap.cc:
  Auto merged
sql/ha_innodb.cc:
  Auto merged
sql/ha_innodb.h:
  Auto merged
sql/ha_myisam.cc:
  Auto merged
sql/ha_ndbcluster.cc:
  Auto merged
sql/handler.h:
  Auto merged
sql/item_cmpfunc.cc:
  Auto merged
sql/item_sum.cc:
  Auto merged
sql/opt_range.cc:
  Auto merged
sql/opt_sum.cc:
  Auto merged
sql/protocol.cc:
  Auto merged
sql/sql_base.cc:
  Auto merged
sql/sql_help.cc:
  Auto merged
sql/sql_select.cc:
  Auto merged
sql/sql_yacc.yy:
  Auto merged
sql/structs.h:
  Auto merged
sql/table.cc:
  Auto merged
strings/ctype-czech.c:
  Auto merged
strings/ctype-uca.c:
  Auto merged
strings/ctype-utf8.c:
  Auto merged
strings/ctype-win1250ch.c:
  Auto merged
strings/decimal.c:
  Auto merged
tests/client_test.c:
  Auto merged
mysql-test/r/bdb.result:
  Merge with VARCHAR code
mysql-test/r/heap.result:
  Merge with VARCHAR code
mysql-test/r/innodb.result:
  Merge with VARCHAR code
mysql-test/r/select.result.es:
  Merge with VARCHAR code
mysql-test/t/bdb.test:
  Merge with VARCHAR code
mysql-test/t/heap.test:
  Merge with VARCHAR code
mysql-test/t/innodb.test:
  Merge with VARCHAR code
sql/field.cc:
  Merge with VARCHAR code
sql/item.cc:
  Merge with VARCHAR code
sql/sql_acl.cc:
  Merge with VARCHAR code
sql/sql_parse.cc:
  Merge with VARCHAR code
sql/sql_table.cc:
  Merge with VARCHAR code
sql/sql_update.cc:
  Merge with VARCHAR code
sql/table.h:
  Merge with VARCHAR code
strings/ctype-mb.c:
  Don't pad my_like_range with max_str for simple LIKE expression
strings/ctype-tis620.c:
  Merge with VARCHAR code
strings/ctype-ucs2.c:
  Added new argument to my_strnncollsp_ucs2()
  Simply code

7 files changed, 199 insertions, 18 deletions

diff --git a/heap/heapdef.h b/heap/heapdef.h
index 083765334ab..68d9405138f 100644
--- a/heap/heapdef.h
+++ b/heap/heapdef.h
@@ -86,7 +86,8 @@ extern ulong hp_mask(ulong hashnr,ulong buffmax,ulong maxlength);
 extern void hp_movelink(HASH_INFO *pos,HASH_INFO *next_link,
 			 HASH_INFO *newlink);
 extern int hp_rec_key_cmp(HP_KEYDEF *keydef,const byte *rec1,
-			  const byte *rec2);
+			  const byte *rec2,
+                          my_bool diff_if_only_endspace_difference);
 extern int hp_key_cmp(HP_KEYDEF *keydef,const byte *rec,
 		      const byte *key);
 extern void hp_make_key(HP_KEYDEF *keydef,byte *key,const byte *rec);
@@ -94,6 +95,7 @@ extern uint hp_rb_make_key(HP_KEYDEF *keydef, byte *key,
 			   const byte *rec, byte *recpos);
 extern uint hp_rb_key_length(HP_KEYDEF *keydef, const byte *key);
 extern uint hp_rb_null_key_length(HP_KEYDEF *keydef, const byte *key);
+extern uint hp_rb_var_key_length(HP_KEYDEF *keydef, const byte *key);
 extern my_bool hp_if_null_in_key(HP_KEYDEF *keyinfo, const byte *record);
 extern int hp_close(register HP_INFO *info);
 extern void hp_clear(HP_SHARE *info);
diff --git a/heap/hp_create.c b/heap/hp_create.c
index fdfe78a1e09..d296c9db28b 100644
--- a/heap/hp_create.c
+++ b/heap/hp_create.c
@@ -76,9 +76,21 @@ int heap_create(const char *name, uint keys, HP_KEYDEF *keydef,
 	case HA_KEYTYPE_UINT24:
 	case HA_KEYTYPE_INT8:
 	  keyinfo->seg[j].flag|= HA_SWAP_KEY;
+          break;
+        case HA_KEYTYPE_VARBINARY:
+          /* Case-insensitiveness is handled in coll->hash_sort */
+          keyinfo->seg[j].type= HA_KEYTYPE_VARTEXT;
+          /* fall_through */
+        case HA_KEYTYPE_VARTEXT:
+          if (!my_binary_compare(keyinfo->seg[j].charset))
+            keyinfo->flag|= HA_END_SPACE_KEY;
+          keyinfo->flag|= HA_VAR_LENGTH_KEY;
+          break;
 	default:
 	  break;
 	}
+        if (keyinfo->seg[j].flag & HA_END_SPACE_ARE_EQUAL)
+          keyinfo->flag|= HA_END_SPACE_KEY;
       }
       keyinfo->length= length;
       length+= keyinfo->rb_tree.size_of_element + 
@@ -89,7 +101,9 @@ int heap_create(const char *name, uint keys, HP_KEYDEF *keydef,
       if (keyinfo->algorithm == HA_KEY_ALG_BTREE)
       {
         key_segs++; /* additional HA_KEYTYPE_END segment */
-        if (keyinfo->flag & HA_NULL_PART_KEY)
+        if (keyinfo->flag & HA_VAR_LENGTH_KEY)
+          keyinfo->get_key_length= hp_rb_var_key_length;
+        else if (keyinfo->flag & HA_NULL_PART_KEY)
           keyinfo->get_key_length= hp_rb_null_key_length;
         else
           keyinfo->get_key_length= hp_rb_key_length;
diff --git a/heap/hp_delete.c b/heap/hp_delete.c
index 9cf8b8936b6..4adefde1fe9 100644
--- a/heap/hp_delete.c
+++ b/heap/hp_delete.c
@@ -123,7 +123,7 @@ int hp_delete_key(HP_INFO *info, register HP_KEYDEF *keyinfo,
 
   while (pos->ptr_to_rec != recpos)
   {
-    if (flag && !hp_rec_key_cmp(keyinfo, record, pos->ptr_to_rec))
+    if (flag && !hp_rec_key_cmp(keyinfo, record, pos->ptr_to_rec, 0))
       last_ptr=pos;				/* Previous same key */
     gpos=pos;
     if (!(pos=pos->next_key))
diff --git a/heap/hp_hash.c b/heap/hp_hash.c
index ee5b4958e62..7e5f92bc7b8 100644
--- a/heap/hp_hash.c
+++ b/heap/hp_hash.c
@@ -262,14 +262,27 @@ ulong hp_hashnr(register HP_KEYDEF *keydef, register const byte *key)
     if (seg->type == HA_KEYTYPE_TEXT)
     {
        CHARSET_INFO *cs= seg->charset;
-       uint char_length= (uint) ((uchar*) key - pos);
+       uint length= seg->length;
        if (cs->mbmaxlen > 1)
        {
-         uint length= char_length;
+         uint char_length;
          char_length= my_charpos(cs, pos, pos + length, length/cs->mbmaxlen);
-         set_if_smaller(char_length, length);   /* QQ: ok to remove? */
+         set_if_smaller(length, char_length);
        }
-       cs->coll->hash_sort(cs, pos, char_length, &nr, &nr2);
+       cs->coll->hash_sort(cs, pos, length, &nr, &nr2);
+    }
+    else if (seg->type == HA_KEYTYPE_VARTEXT)
+    {
+       CHARSET_INFO *cs= seg->charset;
+       uint length= uint2korr(pos);
+       if (cs->mbmaxlen > 1)
+       {
+         uint char_length;
+         char_length= my_charpos(cs, pos +2, pos +2 + length,
+                                 seg->length/cs->mbmaxlen);
+         set_if_smaller(length, char_length);
+       }
+       cs->coll->hash_sort(cs, pos+2, length, &nr, &nr2);
     }
     else
     {
@@ -314,6 +327,19 @@ ulong hp_rec_hashnr(register HP_KEYDEF *keydef, register const byte *rec)
       }
       cs->coll->hash_sort(cs, pos, char_length, &nr, &nr2);
     }
+    else if (seg->type == HA_KEYTYPE_VARTEXT)
+    {
+      CHARSET_INFO *cs= seg->charset;
+      uint length= uint2korr(pos);
+      if (cs->mbmaxlen > 1)
+      {
+        uint char_length;
+        char_length= my_charpos(cs, pos + 2 , pos + 2 + length,
+                                seg->length/cs->mbmaxlen);
+        set_if_smaller(length, char_length);
+      }
+      cs->coll->hash_sort(cs, pos+2, length, &nr, &nr2);
+    }
     else
     {
       for (; pos < end ; pos++)
@@ -366,6 +392,11 @@ ulong hp_hashnr(register HP_KEYDEF *keydef, register const byte *key)
     {
       seg->charset->hash_sort(seg->charset,pos,((uchar*)key)-pos,&nr,NULL);
     }
+    else if (seg->type == HA_KEYTYPE_VARTEXT)
+    {
+      uint length= uint2korr(pos);
+      seg->charset->hash_sort(seg->charset, pos+2, length, &nr, NULL);
+    }
     else
     {
       for ( ; pos < (uchar*) key ; pos++)
@@ -400,6 +431,11 @@ ulong hp_rec_hashnr(register HP_KEYDEF *keydef, register const byte *rec)
     {
       seg->charset->hash_sort(seg->charset,pos,((uchar*)key)-pos,&nr,NULL);
     }
+    else if (seg->type == HA_KEYTYPE_VARTEXT)
+    {
+      uint length= uint2korr(pos);
+      seg->charset->hash_sort(seg->charset, pos+2, length, &nr, NULL);
+    }
     else
     {
       for ( ; pos < end ; pos++)
@@ -415,9 +451,28 @@ ulong hp_rec_hashnr(register HP_KEYDEF *keydef, register const byte *rec)
 #endif
 
 
-	/* Compare keys for two records. Returns 0 if they are identical */
+/*
+  Compare keys for two records. Returns 0 if they are identical
 
-int hp_rec_key_cmp(HP_KEYDEF *keydef, const byte *rec1, const byte *rec2)
+  SYNOPSIS
+    hp_rec_key_cmp()
+    keydef		Key definition
+    rec1		Record to compare
+    rec2		Other record to compare
+    diff_if_only_endspace_difference
+			Different number of end space is significant    
+
+  NOTES
+    diff_if_only_endspace_difference is used to allow us to insert
+    'a' and 'a ' when there is an an unique key.
+
+  RETURN
+    0		Key is identical
+    <> 0 	Key differes
+*/
+
+int hp_rec_key_cmp(HP_KEYDEF *keydef, const byte *rec1, const byte *rec2,
+                   my_bool diff_if_only_endspace_difference)
 {
   HA_KEYSEG *seg,*endseg;
 
@@ -442,9 +497,9 @@ int hp_rec_key_cmp(HP_KEYDEF *keydef, const byte *rec1, const byte *rec2)
       {
         uint char_length= seg->length / cs->mbmaxlen;
         char_length1= my_charpos(cs, pos1, pos1 + seg->length, char_length);
-        set_if_smaller(char_length1, seg->length); /* QQ: ok to remove? */
+        set_if_smaller(char_length1, seg->length);
         char_length2= my_charpos(cs, pos2, pos2 + seg->length, char_length);
-        set_if_smaller(char_length2, seg->length); /* QQ: ok to remove? */
+        set_if_smaller(char_length2, seg->length);
       }
       else
       {
@@ -452,7 +507,30 @@ int hp_rec_key_cmp(HP_KEYDEF *keydef, const byte *rec1, const byte *rec2)
       }
       if (seg->charset->coll->strnncollsp(seg->charset,
       					  pos1,char_length1,
-					  pos2,char_length2))
+					  pos2,char_length2, 0))
+	return 1;
+    }
+    else if (seg->type == HA_KEYTYPE_VARTEXT)
+    {
+      uchar *pos1= (uchar*)rec1 + seg->start;
+      uchar *pos2= (uchar*)rec2 + seg->start;
+      uint char_length1= uint2korr(pos1);
+      uint char_length2= uint2korr(pos2);
+      CHARSET_INFO *cs= seg->charset;
+      if (cs->mbmaxlen > 1)
+      {
+        uint char_length= seg->length / cs->mbmaxlen;
+        char_length1= my_charpos(cs, pos1, pos1 + char_length1, char_length);
+        set_if_smaller(char_length1, seg->length);
+        char_length2= my_charpos(cs, pos2, pos2 + char_length2, char_length);
+        set_if_smaller(char_length2, seg->length);
+      }
+
+      if (cs->coll->strnncollsp(seg->charset,
+                                pos1+2, char_length1,
+                                pos2+2, char_length2,
+                                seg->flag & HA_END_SPACE_ARE_EQUAL ?
+                                0 : diff_if_only_endspace_difference))
 	return 1;
     }
     else
@@ -504,7 +582,31 @@ int hp_key_cmp(HP_KEYDEF *keydef, const byte *rec, const byte *key)
       
       if (seg->charset->coll->strnncollsp(seg->charset,
 					  (uchar*) pos, char_length_rec,
-					  (uchar*) key, char_length_key))
+					  (uchar*) key, char_length_key, 0))
+	return 1;
+    }
+    else if (seg->type == HA_KEYTYPE_VARTEXT)
+    {
+      uchar *pos= (uchar*) rec + seg->start;
+      CHARSET_INFO *cs= seg->charset;
+      uint char_length_rec= uint2korr(pos);
+      uint char_length_key= uint2korr(key);
+
+      if (cs->mbmaxlen > 1)
+      {
+        uint char_length= seg->length / cs->mbmaxlen;
+        char_length_key= my_charpos(cs, key+2, key +2 + char_length_key,
+                                    char_length);
+        set_if_smaller(char_length_key, seg->length);
+        char_length_rec= my_charpos(cs, pos +2 , pos + 2 + char_length_rec,
+                                    char_length);
+        set_if_smaller(char_length_rec, seg->length);
+      }
+
+
+      if (cs->coll->strnncollsp(seg->charset,
+                                (uchar*) pos+2, char_length_rec,
+                                (uchar*) key+2, char_length_key, 0))
 	return 1;
     }
     else
@@ -541,6 +643,13 @@ void hp_make_key(HP_KEYDEF *keydef, byte *key, const byte *rec)
   }
 }
 
+#define FIX_LENGTH(cs, pos, length, char_length)                        \
+  do {                                                                  \
+    if (length > char_length)                                           \
+      char_length= my_charpos(cs, pos, pos+length, char_length);        \
+    set_if_smaller(char_length,length);                                 \
+  } while(0)
+
 
 uint hp_rb_make_key(HP_KEYDEF *keydef, byte *key, 
 		    const byte *rec, byte *recpos)
@@ -593,6 +702,24 @@ uint hp_rb_make_key(HP_KEYDEF *keydef, byte *key,
       }
       continue;
     }
+
+    if (seg->flag & HA_VAR_LENGTH_PART)
+    {
+      uchar *pos=      (uchar*) rec + seg->start;
+      uint length=     seg->length;
+      uint tmp_length= uint2korr(pos);
+      CHARSET_INFO *cs= seg->charset;
+      char_length= length/cs->mbmaxlen;
+
+      pos+=2;					/* Skip VARCHAR length */
+      set_if_smaller(length,tmp_length);
+      FIX_LENGTH(cs, pos, length, char_length);
+      store_key_length_inc(key,char_length);
+      memcpy((byte*) key,(byte*) pos,(size_t) char_length);
+      key+= char_length;
+      continue;
+    }
+
     char_length= seg->length;
     if (seg->charset->mbmaxlen > 1)
     {
@@ -643,6 +770,23 @@ uint hp_rb_pack_key(HP_KEYDEF *keydef, uchar *key, const uchar *old,
       }
       continue;
     }
+    if (seg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART))
+    {
+      /* Length of key-part used with heap_rkey() always 2 */
+      uint tmp_length=uint2korr(old);
+      uint length= seg->length;
+      CHARSET_INFO *cs= seg->charset;
+      char_length= length/cs->mbmaxlen;
+
+      k_len-= 2+length;
+      old+= 2;
+      set_if_smaller(length,tmp_length);	/* Safety */
+      FIX_LENGTH(cs, old, length, char_length);
+      store_key_length_inc(key,char_length);
+      memcpy((byte*) key, old,(size_t) char_length);
+      key+= char_length;
+      continue;
+    }
     char_length= seg->length;
     if (seg->charset->mbmaxlen > 1)
     {
@@ -682,6 +826,27 @@ uint hp_rb_null_key_length(HP_KEYDEF *keydef, const byte *key)
   return key - start_key;
 }
                   
+
+uint hp_rb_var_key_length(HP_KEYDEF *keydef, const byte *key)
+{
+  const byte *start_key= key;
+  HA_KEYSEG *seg, *endseg;
+  
+  for (seg= keydef->seg, endseg= seg + keydef->keysegs; seg < endseg; seg++)
+  {
+    uint length= seg->length;
+    if (seg->null_bit && !*key++)
+      continue;
+    if (seg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART))
+    {
+      get_key_length(length, key);
+    }
+    key+= length;
+  }
+  return key - start_key;
+}
+
+
 /*
   Test if any of the key parts are NULL.
   Return:
diff --git a/heap/hp_rkey.c b/heap/hp_rkey.c
index 2c23d9d721e..a88139bbdee 100644
--- a/heap/hp_rkey.c
+++ b/heap/hp_rkey.c
@@ -64,7 +64,7 @@ int heap_rkey(HP_INFO *info, byte *record, int inx, const byte *key,
       info->update= 0;
       DBUG_RETURN(my_errno);
     }
-    if (!(keyinfo->flag & HA_NOSAME))
+    if (!(keyinfo->flag & HA_NOSAME) || (keyinfo->flag & HA_END_SPACE_KEY))
       memcpy(info->lastkey, key, (size_t) keyinfo->length);
   }
   memcpy(record, pos, (size_t) share->reclength);
diff --git a/heap/hp_update.c b/heap/hp_update.c
index 2ed0edf08de..2f4ea75f9aa 100644
--- a/heap/hp_update.c
+++ b/heap/hp_update.c
@@ -37,7 +37,7 @@ int heap_update(HP_INFO *info, const byte *old, const byte *heap_new)
   p_lastinx= share->keydef + info->lastinx;
   for (keydef= share->keydef, end= keydef + share->keys; keydef < end; keydef++)
   {
-    if (hp_rec_key_cmp(keydef, old, heap_new))
+    if (hp_rec_key_cmp(keydef, old, heap_new, 0))
     {
       if ((*keydef->delete_key)(info, keydef, old, pos, keydef == p_lastinx) ||
           (*keydef->write_key)(info, keydef, heap_new, pos))
@@ -74,7 +74,7 @@ int heap_update(HP_INFO *info, const byte *old, const byte *heap_new)
     }
     while (keydef >= share->keydef)
     {
-      if (hp_rec_key_cmp(keydef, old, heap_new))
+      if (hp_rec_key_cmp(keydef, old, heap_new, 0))
       {
 	if ((*keydef->delete_key)(info, keydef, heap_new, pos, 0) ||
 	    (*keydef->write_key)(info, keydef, old, pos))
diff --git a/heap/hp_write.c b/heap/hp_write.c
index 43cee67b39c..577c52a007d 100644
--- a/heap/hp_write.c
+++ b/heap/hp_write.c
@@ -105,7 +105,7 @@ int hp_rb_write_key(HP_INFO *info, HP_KEYDEF *keyinfo, const byte *record,
   custom_arg.key_length= hp_rb_make_key(keyinfo, info->recbuf, record, recpos);
   if (keyinfo->flag & HA_NOSAME)
   {
-    custom_arg.search_flag= SEARCH_FIND | SEARCH_SAME;
+    custom_arg.search_flag= SEARCH_FIND | SEARCH_SAME | SEARCH_UPDATE;
     keyinfo->rb_tree.flag= TREE_NO_DUPS;
   }
   else
@@ -369,7 +369,7 @@ int hp_write_key(HP_INFO *info, HP_KEYDEF *keyinfo,
       pos=empty;
       do
       {
-	if (! hp_rec_key_cmp(keyinfo, record, pos->ptr_to_rec))
+	if (! hp_rec_key_cmp(keyinfo, record, pos->ptr_to_rec, 1))
 	{
 	  DBUG_RETURN(my_errno=HA_ERR_FOUND_DUPP_KEY);
 	}