Add 0x before pointers (to help with debugging)

Add support for VARCHAR with 1 or 2 length bytes
Enable VARCHAR packing in MyISAM files (previous patch didn't pack data properly)
Give error if we got problems in temporary tables during a SELECT
Don't use new table generated by ALTER TABLE if index generation fails
Fixed wrong call by range_end() (Could cause an ASSERT in debug mode)


BUILD/SETUP.sh:
  Add flags for Intel 64
dbug/dbug.c:
  Add 0x before pointers (to help with debugging)
heap/_check.c:
  Add 0x before pointers (to help with debugging)
heap/hp_create.c:
  Add support for VARCHAR with 1 or 2 length bytes
heap/hp_delete.c:
  Add 0x before pointers
heap/hp_hash.c:
  Add support for VARCHAR with 1 or 2 length bytes
  Added more debugging
heap/hp_open.c:
  Add 0x before pointers
heap/hp_rkey.c:
  Add 0x before pointers
heap/hp_rrnd.c:
  Add 0x before pointers
heap/hp_write.c:
  Add 0x before pointers
include/my_base.h:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/ft_static.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/ft_test1.c:
  Add support for VARCHAR with 1 or 2 length bytes
  Fixed indentation
  (This file should probably be deleted as it doesn't compile)
myisam/ft_update.c:
  Add support for VARCHAR with 1 or 2 length bytes
  Fixed indentation
  Removed some not needed 'else'
myisam/mi_check.c:
  Don't give an error for tables packed with myisampack
myisam/mi_checksum.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/mi_create.c:
  Add support for VARCHAR with 1 or 2 length bytes
  Store in number of pack-length-bytes in keyseg->bit_start
myisam/mi_dbug.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/mi_dynrec.c:
  Add support for VARCHAR with 1 or 2 length bytes
  (old code in _mi_rec_unpack() didn't really work with VARCHAR's)
myisam/mi_key.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/mi_open.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/mi_packrec.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/mi_search.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/mi_test1.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/mi_test3.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/mi_test_all.res:
  Update results
myisam/mi_unique.c:
  Add support for VARCHAR with 1 or 2 length bytes
myisam/myisampack.c:
  Add support for VARCHAR with 1 or 2 length bytes
mysql-test/include/varchar.inc:
  Added more tests
mysql-test/r/bdb.result:
  Update results after new tests
mysql-test/r/information_schema.result:
  Update results
mysql-test/r/innodb.result:
  Update results
mysql-test/r/myisam.result:
  Update results after new tests
mysql-test/r/ps_1general.result:
  Update results
mysql-test/t/bdb.test:
  Shorter comments
mysys/list.c:
  Add 0x before pointers
mysys/my_handler.c:
  Add support for VARCHAR with 1 or 2 length bytes
mysys/raid.cc:
  Add 0x before pointers
sql/field.cc:
  Add support for VARCHAR with 1 or 2 length bytes
sql/field.h:
  Add support for VARCHAR with 1 or 2 length bytes
sql/field_conv.cc:
  Add support for VARCHAR with 1 or 2 length bytes
sql/ha_berkeley.cc:
  Add support for VARCHAR with 1 or 2 length bytes
sql/ha_heap.cc:
  Add support for VARCHAR with 1 or 2 length bytes
sql/ha_myisam.cc:
  Ensure that enable_indexes() will report an error if it fails
  Enable VARCHAR packing for MyISAM files
sql/item_sum.cc:
  Change key_cmp -> cmp() as we are comparing fields, not key segements
sql/opt_range.cc:
  Add support for VARCHAR with 1 or 2 length bytes
  Change range_end to call ha_index_or_rnd_end() as in some error cases we may be in rnd mode when we abort
sql/sql_base.cc:
  Remove compiler warning
sql/sql_parse.cc:
  Move length checking code to sql_table.cc (as we don't have character set for fields at this stage)
sql/sql_select.cc:
  Add support for VARCHAR with 1 or 2 length bytes
  Ensure that we report an error if we get an error while writing to internal temporary tables
sql/sql_select.h:
  Add support for VARCHAR with 1 or 2 length bytes
sql/sql_show.cc:
  Fix typo in comment
sql/sql_table.cc:
  Don't use new table generated by ALTER TABLE if index generation fails
vio/vio.c:
  Fixed DBUG info
vio/viosocket.c:
  Fixed DBUG info
vio/viossl.c:
  Fixed DBUG info
vio/viosslfactories.c:
  Fixed DBUG info

1 files changed, 23 insertions, 10 deletions

diff --git a/myisam/mi_unique.c b/myisam/mi_unique.c
index c03182456df..f2d5f01be25 100644
--- a/myisam/mi_unique.c
+++ b/myisam/mi_unique.c
@@ -95,8 +95,10 @@ ha_checksum mi_unique_hash(MI_UNIQUEDEF *def, const byte *record)
     pos= record+keyseg->start;
     if (keyseg->flag & HA_VAR_LENGTH_PART)
     {
-      uint tmp_length=uint2korr(pos);
-      pos+=2;					/* Skip VARCHAR length */
+      uint pack_length=  keyseg->bit_start;
+      uint tmp_length= (pack_length == 1 ? (uint) *(uchar*) pos :
+                        uint2korr(pos));
+      pos+= pack_length;			/* Skip VARCHAR length */
       set_if_smaller(length,tmp_length);
     }
     else if (keyseg->flag & HA_BLOB_PART)
@@ -107,7 +109,8 @@ ha_checksum mi_unique_hash(MI_UNIQUEDEF *def, const byte *record)
 	length=tmp_length;			/* The whole blob */
     }
     end= pos+length;
-    if (type == HA_KEYTYPE_TEXT || type == HA_KEYTYPE_VARTEXT)
+    if (type == HA_KEYTYPE_TEXT || type == HA_KEYTYPE_VARTEXT1 ||
+        type == HA_KEYTYPE_VARTEXT2)
     {
       keyseg->charset->coll->hash_sort(keyseg->charset,
                                        (const uchar*) pos, length, &seed1,
@@ -157,12 +160,21 @@ int mi_unique_comp(MI_UNIQUEDEF *def, const byte *a, const byte *b,
     pos_b= b+keyseg->start;
     if (keyseg->flag & HA_VAR_LENGTH_PART)
     {
-      a_length= uint2korr(pos_a);
-      b_length= uint2korr(pos_b);
-      pos_a+= 2;				/* Skip VARCHAR length */
-      pos_b+= 2;
-      set_if_smaller(a_length, keyseg->length);
-      set_if_smaller(b_length, keyseg->length);
+      uint pack_length= keyseg->bit_start;
+      if (pack_length == 1)
+      {
+        a_length= (uint) *(uchar*) pos_a++;
+        b_length= (uint) *(uchar*) pos_b++;
+      }
+      else
+      {
+        a_length= uint2korr(pos_a);
+        b_length= uint2korr(pos_b);
+        pos_a+= 2;				/* Skip VARCHAR length */
+        pos_b+= 2;
+      }
+      set_if_smaller(a_length, keyseg->length); /* Safety */
+      set_if_smaller(b_length, keyseg->length); /* safety */
     }
     else if (keyseg->flag & HA_BLOB_PART)
     {
@@ -182,7 +194,8 @@ int mi_unique_comp(MI_UNIQUEDEF *def, const byte *a, const byte *b,
       memcpy_fixed((byte*) &pos_a,pos_a+keyseg->bit_start,sizeof(char*));
       memcpy_fixed((byte*) &pos_b,pos_b+keyseg->bit_start,sizeof(char*));
     }
-    if (type == HA_KEYTYPE_TEXT || type == HA_KEYTYPE_VARTEXT)
+    if (type == HA_KEYTYPE_TEXT || type == HA_KEYTYPE_VARTEXT1 ||
+        type == HA_KEYTYPE_VARTEXT2)
     {
       if (mi_compare_text(keyseg->charset, (uchar *) pos_a, a_length,
                                            (uchar *) pos_b, b_length, 0, 1))