merge with 4.1

BitKeeper/etc/ignore:
  auto-union
BitKeeper/etc/logging_ok:
  auto-union
BitKeeper/triggers/post-commit:
  Auto merged
Docs/Support/texi2html:
  Auto merged
Makefile.am:
  Auto merged
client/Makefile.am:
  Auto merged
client/mysql.cc:
  Auto merged
client/mysqldump.c:
  Auto merged
include/my_base.h:
  Auto merged
include/my_global.h:
  Auto merged
include/my_pthread.h:
  Auto merged
include/my_sys.h:
  Auto merged
include/my_time.h:
  Auto merged
include/mysql.h:
  Auto merged
include/mysql_com.h:
  Auto merged
innobase/buf/buf0buf.c:
  Auto merged
innobase/include/row0mysql.h:
  Auto merged
innobase/row/row0sel.c:
  Auto merged
libmysql/libmysql.c:
  Auto merged
libmysqld/examples/Makefile.am:
  Auto merged
myisam/mi_check.c:
  Auto merged
mysql-test/include/ps_modify.inc:
  Auto merged
mysql-test/install_test_db.sh:
  Auto merged
mysql-test/r/alter_table.result:
  Auto merged
mysql-test/r/auto_increment.result:
  Auto merged
mysql-test/r/bdb.result:
  Auto merged
mysql-test/r/ctype_latin1_de.result:
  Auto merged
mysql-test/r/ctype_recoding.result:
  Auto merged
mysql-test/r/fulltext.result:
  Auto merged
mysql-test/r/func_gconcat.result:
  Auto merged
mysql-test/r/func_group.result:
  Auto merged
mysql-test/r/func_if.result:
  Auto merged
mysql-test/t/derived.test:
  Auto merged
mysql-test/t/insert.test:
  merge with 4.1
  Fixed test case to not use 'if exists' when it shouldn't
mysql-test/t/range.test:
  merge with 4.1
  Added missing drop table
sql/ha_ndbcluster.cc:
  merge with 4.1
  Simple optimization: use max() instead of ? :
sql/item_func.cc:
  merge with 4.1
  (Added back old variable names for easier merges)
sql/opt_range.cc:
  merge with 4.1
  Removed argument 'parent_alloc' from QUICK_RANGE_SELECT as this was not used
  Added assert if using QUICK_GROUP_MIN_MAX_SELECT with parent_alloc as the init() function can't handle this
  Changed back get_quick_select_for_ref() to use it's own alloc root becasue this function may be called several times for one query
sql/sql_handler.cc:
  merge with 4.1
  change variable 'err' to 'error' as same function had a label named 'err'
sql/sql_update.cc:
  Use multi-update code from 5.0 instead of 4.1
  We will fix the locking code shortly in 5.0 to be faster than in 4.1

13 files changed, 117 insertions, 39 deletions

diff --git a/myisam/ft_nlq_search.c b/myisam/ft_nlq_search.c
index c9caf4f39af..83de6acdab9 100644
--- a/myisam/ft_nlq_search.c
+++ b/myisam/ft_nlq_search.c
@@ -99,6 +99,7 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
 
   info->update|= HA_STATE_AKTIV;              /* for _mi_test_if_changed() */
 
+  /* The following should be safe, even if we compare doubles */
   while (!r && gweight)
   {
 
@@ -127,6 +128,7 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
 #else
 #error
 #endif
+  /* The following should be safe, even if we compare doubles */
     if (tmp_weight==0)
       DBUG_RETURN(doc_cnt); /* stopword, doc_cnt should be 0 */
 
diff --git a/myisam/ft_parser.c b/myisam/ft_parser.c
index b0fe180d0fb..543cf998a82 100644
--- a/myisam/ft_parser.c
+++ b/myisam/ft_parser.c
@@ -174,7 +174,7 @@ byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, byte *end,
                         FT_WORD *word)
 {
   byte *doc= *start;
-  uint mwc, length;
+  uint mwc, length, mbl;
   DBUG_ENTER("ft_simple_get_word");
 
   while (doc<end)
@@ -185,7 +185,7 @@ byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, byte *end,
     }
 
     mwc= length= 0;
-    for (word->pos=doc; doc<end; length++, doc+=my_mbcharlen(cs, *(uchar *)doc))
+    for (word->pos=doc; doc<end; length++, mbl=my_mbcharlen(cs, *(uchar *)doc), doc+=(mbl ? mbl : 1))
       if (true_word_char(cs,*doc))
         mwc= 0;
       else if (!misc_word_char(*doc) || mwc++)
diff --git a/myisam/mi_check.c b/myisam/mi_check.c
index 1f6089d0a3c..4da388af1c7 100644
--- a/myisam/mi_check.c
+++ b/myisam/mi_check.c
@@ -3743,7 +3743,7 @@ int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename)
   create_info.raid_chunksize= share.base.raid_chunksize;
   create_info.language = (param->language ? param->language :
 			  share.state.header.language);
-
+  create_info.key_file_length=  status_info.key_file_length;
   /* We don't have to handle symlinks here because we are using
      HA_DONT_TOUCH_DATA */
   if (mi_create(filename,
diff --git a/myisam/mi_close.c b/myisam/mi_close.c
index deb0ccee8f3..62f5617de1a 100644
--- a/myisam/mi_close.c
+++ b/myisam/mi_close.c
@@ -70,6 +70,12 @@ int mi_close(register MI_INFO *info)
       error=my_errno;
     if (share->kfile >= 0)
     {
+      /*
+        If we are crashed, we can safely flush the current state as it will
+        not change the crashed state.
+        We can NOT write the state in other cases as other threads
+        may be using the file at this point
+      */
       if (share->mode != O_RDONLY && mi_is_crashed(info))
 	mi_state_info_write(share->kfile, &share->state, 1);
       if (my_close(share->kfile,MYF(0)))
diff --git a/myisam/mi_create.c b/myisam/mi_create.c
index 683640630f6..7fc7cc4edf1 100644
--- a/myisam/mi_create.c
+++ b/myisam/mi_create.c
@@ -48,7 +48,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
   ulong reclength, real_reclength,min_pack_length;
   char filename[FN_REFLEN],linkname[FN_REFLEN], *linkname_ptr;
   ulong pack_reclength;
-  ulonglong tot_length,max_rows;
+  ulonglong tot_length,max_rows, tmp;
   enum en_fieldtype type;
   MYISAM_SHARE share;
   MI_KEYDEF *keydef,tmp_keydef;
@@ -464,10 +464,15 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
   share.state.auto_increment=ci->auto_increment;
   share.options=options;
   share.base.rec_reflength=pointer;
+  /* Get estimate for index file length (this may be wrong for FT keys) */
+  tmp= (tot_length + max_key_block_length * keys *
+	MI_INDEX_BLOCK_MARGIN) / MI_MIN_KEY_BLOCK_LENGTH;
+  /*
+    use maximum of key_file_length we calculated and key_file_length value we
+    got from MYI file header (see also myisampack.c:save_state)
+  */
   share.base.key_reflength=
-    mi_get_pointer_length((tot_length + max_key_block_length * keys *
-			   MI_INDEX_BLOCK_MARGIN) / MI_MIN_KEY_BLOCK_LENGTH,
-			  3);
+    mi_get_pointer_length(max(ci->key_file_length,tmp),3);
   share.base.keys= share.state.header.keys= keys;
   share.state.header.uniques= uniques;
   share.state.header.fulltext_keys= fulltext_keys;
diff --git a/myisam/mi_delete.c b/myisam/mi_delete.c
index 3e8ff9db009..b964cb35dd8 100644
--- a/myisam/mi_delete.c
+++ b/myisam/mi_delete.c
@@ -826,7 +826,7 @@ static uint remove_key(MI_KEYDEF *keyinfo, uint nod_flag,
 	  else
 	    get_key_length(rest_length,keypos);
 
-	  if (next_length > prev_length)
+	  if (next_length >= prev_length)
 	  {		/* Key after is based on deleted key */
 	    uint pack_length,tmp;
 	    bmove_upp((char*) keypos,(char*) (lastkey+next_length),
diff --git a/myisam/mi_locking.c b/myisam/mi_locking.c
index b13ebfb4cad..66950f62321 100644
--- a/myisam/mi_locking.c
+++ b/myisam/mi_locking.c
@@ -19,7 +19,7 @@
   reads info from a isam-table. Must be first request before doing any furter
   calls to any isamfunktion.  Is used to allow many process use the same
   isamdatabase.
-  */
+*/
 
 #include "myisamdef.h"
 
@@ -32,12 +32,17 @@ int mi_lock_database(MI_INFO *info, int lock_type)
   MYISAM_SHARE *share=info->s;
   uint flag;
   DBUG_ENTER("mi_lock_database");
-  DBUG_PRINT("info",("lock_type: %d", lock_type));
+  DBUG_PRINT("enter",("lock_type: %d  old lock %d  r_locks: %u  w_locks: %u "
+                      "global_changed:  %d  open_count: %u  name: '%s'",
+                      lock_type, info->lock_type, share->r_locks,
+                      share->w_locks,
+                      share->global_changed, share->state.open_count,
+                      share->index_file_name));
 
   if (share->options & HA_OPTION_READ_ONLY_DATA ||
       info->lock_type == lock_type)
     DBUG_RETURN(0);
-  if (lock_type == F_EXTRA_LCK)
+  if (lock_type == F_EXTRA_LCK)                 /* Used by TMP tables */
   {
     ++share->w_locks;
     ++share->tot_locks;
@@ -51,7 +56,6 @@ int mi_lock_database(MI_INFO *info, int lock_type)
   {
     switch (lock_type) {
     case F_UNLCK:
-      DBUG_PRINT("info", ("old lock: %d", info->lock_type));
       if (info->lock_type == F_RDLCK)
 	count= --share->r_locks;
       else
@@ -81,7 +85,7 @@ int mi_lock_database(MI_INFO *info, int lock_type)
 	  share->state.process= share->last_process=share->this_process;
 	  share->state.unique=   info->last_unique=  info->this_unique;
 	  share->state.update_count= info->last_loop= ++info->this_loop;
-	  if (mi_state_info_write(share->kfile, &share->state, 1))
+          if (mi_state_info_write(share->kfile, &share->state, 1))
 	    error=my_errno;
 	  share->changed=0;
 	  if (myisam_flush)
@@ -119,11 +123,17 @@ int mi_lock_database(MI_INFO *info, int lock_type)
       break;
     case F_RDLCK:
       if (info->lock_type == F_WRLCK)
-      {						/* Change RW to READONLY */
+      {
+        /*
+          Change RW to READONLY
+
+          mysqld does not turn write locks to read locks,
+          so we're never here in mysqld.
+        */
 	if (share->w_locks == 1)
 	{
 	  flag=1;
-	  if (my_lock(share->kfile,lock_type,0L,F_TO_EOF,
+          if (my_lock(share->kfile,lock_type,0L,F_TO_EOF,
 		      MYF(MY_SEEK_NOT_DONE)))
 	  {
 	    error=my_errno;
@@ -346,9 +356,10 @@ int _mi_readinfo(register MI_INFO *info, int lock_type, int check_keybuffer)
 } /* _mi_readinfo */
 
 
-	/* Every isam-function that uppdates the isam-database must! end */
-	/* with this request */
-	/* ARGSUSED */
+/*
+  Every isam-function that uppdates the isam-database MUST end with this
+  request
+*/
 
 int _mi_writeinfo(register MI_INFO *info, uint operation)
 {
@@ -415,12 +426,31 @@ int _mi_test_if_changed(register MI_INFO *info)
 } /* _mi_test_if_changed */
 
 
-/* Put a mark in the .MYI file that someone is updating the table */
+/*
+  Put a mark in the .MYI file that someone is updating the table
+
+
+  DOCUMENTATION
+
+  state.open_count in the .MYI file is used the following way:
+  - For the first change of the .MYI file in this process open_count is
+    incremented by mi_mark_file_change(). (We have a write lock on the file
+    when this happens)
+  - In mi_close() it's decremented by _mi_decrement_open_count() if it
+    was incremented in the same process.
+
+  This mean that if we are the only process using the file, the open_count
+  tells us if the MYISAM file wasn't properly closed. (This is true if
+  my_disable_locking is set).
+*/
+
 
 int _mi_mark_file_changed(MI_INFO *info)
 {
   char buff[3];
   register MYISAM_SHARE *share=info->s;
+  DBUG_ENTER("_mi_mark_file_changed");
+
   if (!(share->state.changed & STATE_CHANGED) || ! share->global_changed)
   {
     share->state.changed|=(STATE_CHANGED | STATE_NOT_ANALYZED |
@@ -434,12 +464,12 @@ int _mi_mark_file_changed(MI_INFO *info)
     {
       mi_int2store(buff,share->state.open_count);
       buff[2]=1;				/* Mark that it's changed */
-      return (my_pwrite(share->kfile,buff,sizeof(buff),
-			sizeof(share->state.header),
-			MYF(MY_NABP)));
+      DBUG_RETURN(my_pwrite(share->kfile,buff,sizeof(buff),
+                            sizeof(share->state.header),
+                            MYF(MY_NABP)));
     }
   }
-  return 0;
+  DBUG_RETURN(0);
 }
 
 
diff --git a/myisam/myisam_ftdump.c b/myisam/myisam_ftdump.c
index 8219c19848a..54b2cc77965 100644
--- a/myisam/myisam_ftdump.c
+++ b/myisam/myisam_ftdump.c
@@ -68,7 +68,7 @@ int main(int argc,char *argv[])
   struct { MI_INFO *info; } aio0, *aio=&aio0; /* for GWS_IN_USE */
 
   MY_INIT(argv[0]);
-  if (error=handle_options(&argc, &argv, my_long_options, get_one_option))
+  if ((error= handle_options(&argc, &argv, my_long_options, get_one_option)))
     exit(error);
   if (count || dump)
     verbose=0;
diff --git a/myisam/myisamchk.c b/myisam/myisamchk.c
index bee248c0c65..cad7be2e802 100644
--- a/myisam/myisamchk.c
+++ b/myisam/myisamchk.c
@@ -354,7 +354,7 @@ static void usage(void)
   puts("Description, check and repair of MyISAM tables.");
   puts("Used without options all tables on the command will be checked for errors");
   printf("Usage: %s [OPTIONS] tables[.MYI]\n", my_progname_short);
-  puts("\nGlobal options:\n\
+  printf("\nGlobal options:\n\
   -#, --debug=...     Output debug log. Often this is 'd:t:o,filename'.\n\
   -?, --help          Display this help and exit.\n\
   -O, --set-variable var=option.\n\
@@ -364,18 +364,18 @@ static void usage(void)
   -t, --tmpdir=path   Path for temporary files. Multiple paths can be\n\
                       specified, separated by ");
 #if defined( __WIN__) || defined(OS2) || defined(__NETWARE__)
-   puts("semicolon (;)");
+   printf("semicolon (;)");
 #else
-   puts("colon (:)");
+   printf("colon (:)");
 #endif
-                      puts(", they will be used\n\
+                      printf(", they will be used\n\
                       in a round-robin fashion.\n\
   -s, --silent	      Only print errors.  One can use two -s to make\n\
 		      myisamchk very silent.\n\
   -v, --verbose       Print more information. This can be used with\n\
                       --description and --check. Use many -v for more verbosity.\n\
   -V, --version       Print version and exit.\n\
-  -w, --wait          Wait if table is locked.\n");
+  -w, --wait          Wait if table is locked.\n\n");
 #ifdef DEBUG
   puts("  --start-check-pos=# Start reading file at given offset.\n");
 #endif
@@ -1695,7 +1695,7 @@ err:
 
 static int not_killed= 0;
 
-volatile int *killed_ptr(MI_CHECK *param)
+volatile int *killed_ptr(MI_CHECK *param __attribute__((unused)))
 {
   return &not_killed;			/* always NULL */
 }
diff --git a/myisam/myisampack.c b/myisam/myisampack.c
index de4bd80805a..eae75b07760 100644
--- a/myisam/myisampack.c
+++ b/myisam/myisampack.c
@@ -111,6 +111,8 @@ typedef struct st_isam_mrg {
   uint	ref_length;
   uint	max_blob_length;
   my_off_t records;
+  /* true if at least one source file has at least one disabled index */
+  my_bool src_file_has_indexes_disabled;
 } PACK_MRG_INFO;
 
 
@@ -413,10 +415,15 @@ static bool open_isam_files(PACK_MRG_INFO *mrg,char **names,uint count)
   mrg->current=0;
   mrg->file=(MI_INFO**) my_malloc(sizeof(MI_INFO*)*count,MYF(MY_FAE));
   mrg->free_file=1;
+  mrg->src_file_has_indexes_disabled= 0;
   for (i=0; i < count ; i++)
   {
     if (!(mrg->file[i]=open_isam_file(names[i],O_RDONLY)))
       goto error;
+
+    mrg->src_file_has_indexes_disabled|= ((mrg->file[i]->s->state.key_map != 
+                                           (((ulonglong) 1) <<
+                                            mrg->file[i]->s->base. keys) - 1));
   }
   /* Check that files are identical */
   for (j=0 ; j < count-1 ; j++)
@@ -2043,8 +2050,21 @@ static int save_state(MI_INFO *isam_file,PACK_MRG_INFO *mrg,my_off_t new_length,
   share->state.dellink= HA_OFFSET_ERROR;
   share->state.split=(ha_rows) mrg->records;
   share->state.version=(ulong) time((time_t*) 0);
+  if (share->state.key_map != (ULL(1) << share->base.keys) - 1)
+  {
+    /*
+      Some indexes are disabled, cannot use current key_file_length value
+      as an estimate of upper bound of index file size. Use packed data file 
+      size instead.
+    */
+    share->state.state.key_file_length= new_length;
+  }
+  /*
+    If there are no disabled indexes, keep key_file_length value from 
+    original file so "myisamchk -rq" can use this value (this is necessary 
+    because index size cannot be easily calculated for fulltext keys)
+  */
   share->state.key_map=0;
-  share->state.state.key_file_length=share->base.keystart;
   for (key=0 ; key < share->base.keys ; key++)
     share->state.key_root[key]= HA_OFFSET_ERROR;
   for (key=0 ; key < share->state.header.max_block_size ; key++)
@@ -2053,8 +2073,7 @@ static int save_state(MI_INFO *isam_file,PACK_MRG_INFO *mrg,my_off_t new_length,
   share->changed=1;			/* Force write of header */
   share->state.open_count=0;
   share->global_changed=0;
-  VOID(my_chsize(share->kfile, share->state.state.key_file_length, 0,
-		 MYF(0)));
+  VOID(my_chsize(share->kfile, share->base.keystart, 0, MYF(0)));
   if (share->base.keys)
     isamchk_neaded=1;
   DBUG_RETURN(mi_state_info_write(share->kfile,&share->state,1+2));
@@ -2077,7 +2096,12 @@ static int save_state_mrg(File file,PACK_MRG_INFO *mrg,my_off_t new_length,
   state.state.del=0;
   state.state.empty=0;
   state.state.records=state.split=(ha_rows) mrg->records;
-  state.state.key_file_length=isam_file->s->base.keystart;
+  /* See comment above in save_state about key_file_length handling. */
+  if (mrg->src_file_has_indexes_disabled)
+  {
+    isam_file->s->state.state.key_file_length=
+      max(isam_file->s->state.state.key_file_length, new_length);
+  }
   state.dellink= HA_OFFSET_ERROR;
   state.version=(ulong) time((time_t*) 0);
   state.key_map=0;
diff --git a/myisam/rt_index.c b/myisam/rt_index.c
index 4fffd848624..cfb2ca877f4 100644
--- a/myisam/rt_index.c
+++ b/myisam/rt_index.c
@@ -494,9 +494,11 @@ static uchar *rtree_pick_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key,
 
   for (; k < last; k = rt_PAGE_NEXT_KEY(k, key_length, nod_flag))
   {
+    /* The following is safe as -1.0 is an exact number */
     if ((increase = rtree_area_increase(keyinfo->seg, k, key, key_length, 
-                                        &area)) == -1)
+                                        &area)) == -1.0)
       return NULL;
+    /* The following should be safe, even if we compare doubles */
     if (increase < best_incr)
     {
       best_key = k;
@@ -505,6 +507,7 @@ static uchar *rtree_pick_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key,
     }
     else
     {
+      /* The following should be safe, even if we compare doubles */
       if ((increase == best_incr) && (area < best_area))
       {
         best_key = k;
@@ -1021,6 +1024,7 @@ ha_rows rtree_estimate(MI_INFO *info, uint keynr, uchar *key,
     {
       double k_area = rtree_rect_volume(keyinfo->seg, k, key_length);
 
+      /* The following should be safe, even if we compare doubles */
       if (k_area == 0)
       {
         if (flag & (MBR_CONTAIN | MBR_INTERSECT))
diff --git a/myisam/rt_mbr.c b/myisam/rt_mbr.c
index 7b556979904..c43daec2f7c 100644
--- a/myisam/rt_mbr.c
+++ b/myisam/rt_mbr.c
@@ -129,6 +129,7 @@ int rtree_key_cmp(HA_KEYSEG *keyseg, uchar *b, uchar *a, uint key_length,
       break;
 #endif
     case HA_KEYTYPE_FLOAT:
+      /* The following should be safe, even if we compare doubles */
       RT_CMP_GET(float, mi_float4get, 4, nextflag);
       break;
     case HA_KEYTYPE_DOUBLE:
@@ -505,7 +506,7 @@ double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar* a, uchar* b,
    amax = korr_func(a+len); \
    bmax = korr_func(b+len); \
    a_area *= (((double)amax) - ((double)amin)); \
-   *ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
+   loc_ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
 }
 
 #define RT_AREA_INC_GET(type, get_func, len)\
@@ -516,7 +517,7 @@ double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar* a, uchar* b,
    get_func(amax, a+len); \
    get_func(bmax, b+len); \
    a_area *= (((double)amax) - ((double)amin)); \
-   *ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
+   loc_ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
 }
 
 /*
@@ -526,6 +527,7 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
                           uint key_length, double *ab_area)
 {
   double a_area= 1.0;
+  double loc_ab_area= 1.0;
   
   *ab_area= 1.0;
   for (; (int)key_length > 0; keyseg += 2)
@@ -575,7 +577,7 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
       RT_AREA_INC_GET(double, mi_float8get, 8);
       break;
     case HA_KEYTYPE_END:
-      return *ab_area - a_area;
+      goto safe_end;
     default:
       return -1;
     }
@@ -584,7 +586,9 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
     a+= keyseg_length;
     b+= keyseg_length;
   }
-  return *ab_area - a_area;
+safe_end:
+  *ab_area= loc_ab_area;
+  return loc_ab_area - a_area;
 }
 
 #define RT_PERIM_INC_KORR(type, korr_func, len) \
diff --git a/myisam/rt_test.c b/myisam/rt_test.c
index 5e883e223b3..4f04aa11fce 100644
--- a/myisam/rt_test.c
+++ b/myisam/rt_test.c
@@ -425,6 +425,7 @@ static void create_record1(char *record,uint rownr)
    }
 }
 
+#ifdef NOT_USED
 
 static void create_record0(char *record,uint rownr)
 {
@@ -447,6 +448,8 @@ static void create_record0(char *record,uint rownr)
    }
 }
 
+#endif
+
 static void create_record(char *record,uint rownr)
 {
    int i;