Fixed some byte order bugs with prepared statements on machines with high-byte-first. (Bug #4173)

Fixed problem with NULL and derived tables (Bug #4097)
Cleanup of new pushed code

15 files changed, 95 insertions, 64 deletions

diff --git a/.bzrignore b/.bzrignore
index 338bc04fabc..996e493de5a 100644
--- a/.bzrignore
+++ b/.bzrignore
@@ -780,3 +780,4 @@ ndb/src/common/mgmcommon/printConfig/*.d
 ndb/src/mgmclient/test_cpcd/*.d
 *.d
 libmysqld/examples/client_test.c
+mysql-test/ndb/ndbcluster
diff --git a/client/mysqltest.c b/client/mysqltest.c
index f638053b515..5ff152bd1c9 100644
--- a/client/mysqltest.c
+++ b/client/mysqltest.c
@@ -59,6 +59,7 @@
 #include <sys/stat.h>
 #include <violite.h>
 
+#define MAX_VAR_NAME	256
 #define MAX_QUERY	65536
 #define MAX_COLUMNS	256
 #define PAD_SIZE	128
@@ -628,6 +629,7 @@ static int check_result(DYNAMIC_STRING* ds, const char* fname,
   return error;
 }
 
+
 VAR* var_get(const char* var_name, const char** var_name_end, my_bool raw,
 	     my_bool ignore_not_existing)
 {
@@ -642,25 +644,26 @@ VAR* var_get(const char* var_name, const char** var_name_end, my_bool raw,
   if (!(digit < 10 && digit >= 0))
   {
     const char* save_var_name = var_name, *end;
+    uint length;
     end = (var_name_end) ? *var_name_end : 0;
     while (my_isvar(charset_info,*var_name) && var_name != end)
-      ++var_name;
+      var_name++;
     if (var_name == save_var_name)
     {
       if (ignore_not_existing)
 	DBUG_RETURN(0);
       die("Empty variable");
     }
+    length= (uint) (var_name - save_var_name);
 
-    if (!(v = (VAR*) hash_search(&var_hash, save_var_name,
-			       var_name - save_var_name)))
+    if (!(v = (VAR*) hash_search(&var_hash, save_var_name, length)) &&
+        length < MAX_VAR_NAME)
     {
-      char c=*var_name, *s=(char*)var_name;;
-      *s=0;
-      v=var_from_env(save_var_name, "");
-      *s=c;
+      char buff[MAX_VAR_NAME+1];
+      strmake(buff, save_var_name, length); 
+      v= var_from_env(buff, "");
     }
-    --var_name;					/* Point at last character */
+    var_name--;					/* Point at last character */
   }
   else
     v = var_reg + digit;
diff --git a/innobase/os/os0file.c b/innobase/os/os0file.c
index 469e7c025b6..d5ca8f927c6 100644
--- a/innobase/os/os0file.c
+++ b/innobase/os/os0file.c
@@ -368,7 +368,7 @@ os_file_handle_error(
 
 #undef USE_FILE_LOCK
 #define USE_FILE_LOCK
-#if defined(UNIV_HOTBACKUP) || defined(__WIN__) || defined(__FreeBSD__)
+#if defined(UNIV_HOTBACKUP) || defined(__WIN__) || defined(__FreeBSD__) || defined(__NETWARE__)
 /* InnoDB Hot Backup does not lock the data files.
  * On Windows, mandatory locking is used.
  * On FreeBSD with LinuxThreads, advisory locking does not work properly.
diff --git a/libmysql/libmysql.c b/libmysql/libmysql.c
index eb8368977e9..4cb6111af32 100644
--- a/libmysql/libmysql.c
+++ b/libmysql/libmysql.c
@@ -3001,6 +3001,7 @@ static uint read_binary_date(MYSQL_TIME *tm, uchar **pos)
   return length;
 }
 
+
 /* Convert Numeric to buffer types */
 static void send_data_long(MYSQL_BIND *param, MYSQL_FIELD *field,
 			   longlong value)
@@ -3015,26 +3016,26 @@ static void send_data_long(MYSQL_BIND *param, MYSQL_FIELD *field,
     *param->buffer= (uchar) value;
     break;
   case MYSQL_TYPE_SHORT:
-    int2store(buffer, value);
+    shortstore(buffer, value);
     break;
   case MYSQL_TYPE_LONG:
-    int4store(buffer, value);
+    longstore(buffer, value);
     break;
   case MYSQL_TYPE_LONGLONG:
-    int8store(buffer, value);
+    longlongstore(buffer, value);
     break;
   case MYSQL_TYPE_FLOAT:
   {
     float data= (field_is_unsigned ? (float) ulonglong2double(value) :
 		 (float) value);
-    float4store(buffer, data);
+    floatstore(buffer, data);
     break;
   }
   case MYSQL_TYPE_DOUBLE:
   {
     double data= (field_is_unsigned ? ulonglong2double(value) :
 		 (double) value);
-    float8store(buffer, data);
+    doublestore(buffer, data);
     break;
   }
   default:
@@ -3070,24 +3071,26 @@ static void send_data_double(MYSQL_BIND *param, double value)
     *buffer= (uchar)value;
     break;
   case MYSQL_TYPE_SHORT:
-    int2store(buffer, (short)value);
+    shortstore(buffer, (short)value);
     break;
   case MYSQL_TYPE_LONG:
-    int4store(buffer, (long)value);
+    longstore(buffer, (long)value);
     break;
   case MYSQL_TYPE_LONGLONG:
-    int8store(buffer, (longlong)value);
+  {
+    longlong val= (longlong) value;
+    longlongstore(buffer, val);
     break;
+  }
   case MYSQL_TYPE_FLOAT:
   {
-    float data= (float)value;
-    float4store(buffer, data);
+    float data= (float) value;
+    floatstore(buffer, data);
     break;
   }
   case MYSQL_TYPE_DOUBLE:
   {
-    double data= (double)value;
-    float8store(buffer, data);
+    doublestore(buffer, value);
     break;
   }
   default:
diff --git a/myisam/ft_boolean_search.c b/myisam/ft_boolean_search.c
index cac4d08f5d6..196cb5c21fb 100644
--- a/myisam/ft_boolean_search.c
+++ b/myisam/ft_boolean_search.c
@@ -53,10 +53,11 @@ static double _nwghts[11]=
  -3.796875000000000};
 static double *nwghts=_nwghts+5; /* nwghts[i] = -0.5*1.5**i */
 
-#define FTB_FLAG_TRUNC 1                  /*                            */
-#define FTB_FLAG_YES   2                  /*  no two from these three   */
-#define FTB_FLAG_NO    4                  /*   YES, NO, WONLY           */
-#define FTB_FLAG_WONLY 8                  /*  should be _ever_ set both */
+#define FTB_FLAG_TRUNC 1
+/* At most one of the following flags can be set */
+#define FTB_FLAG_YES   2
+#define FTB_FLAG_NO    4
+#define FTB_FLAG_WONLY 8
 
 typedef struct st_ftb_expr FTB_EXPR;
 struct st_ftb_expr
diff --git a/myisam/mi_check.c b/myisam/mi_check.c
index 96a26fd90f1..052fa55a559 100644
--- a/myisam/mi_check.c
+++ b/myisam/mi_check.c
@@ -376,8 +376,6 @@ int chk_key(MI_CHECK *param, register MI_INFO *info)
   for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
        rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
   {
-    if (*killed_ptr(param))
-      DBUG_RETURN(-1);
     param->key_crc[key]=0;
     if (!(((ulonglong) 1 << key) & share->state.key_map))
     {
diff --git a/myisam/mi_unique.c b/myisam/mi_unique.c
index 06d50e6905b..77e967e52e2 100644
--- a/myisam/mi_unique.c
+++ b/myisam/mi_unique.c
@@ -69,7 +69,7 @@ my_bool mi_check_unique(MI_INFO *info, MI_UNIQUEDEF *def, byte *record,
 ha_checksum mi_unique_hash(MI_UNIQUEDEF *def, const byte *record)
 {
   const byte *pos, *end;
-  ulong crc=  0;
+  ha_checksum crc= 0;
   ulong seed= 4;
   HA_KEYSEG *keyseg;
 
@@ -109,8 +109,11 @@ ha_checksum mi_unique_hash(MI_UNIQUEDEF *def, const byte *record)
     end= pos+length;
     if (type == HA_KEYTYPE_TEXT || type == HA_KEYTYPE_VARTEXT)
     {
+      ulong tmp= 0;
       keyseg->charset->coll->hash_sort(keyseg->charset,
-                                       (const uchar*) pos, length, &crc, &seed);
+                                       (const uchar*) pos, length, &tmp,
+                                       &seed);
+      crc^= tmp;
     }
     else
       while (pos != end)
@@ -118,7 +121,7 @@ ha_checksum mi_unique_hash(MI_UNIQUEDEF *def, const byte *record)
 	     (((uchar)  *(uchar*) pos++))) +
 	  (crc >> (8*sizeof(ha_checksum)-8));
   }
-  return (ha_checksum)crc;
+  return crc;
 }
 
 	/*
diff --git a/myisam/myisamchk.c b/myisam/myisamchk.c
index ef4b7a5ff87..c4b5acadc92 100644
--- a/myisam/myisamchk.c
+++ b/myisam/myisamchk.c
@@ -362,13 +362,13 @@ static void usage(void)
                       this option is deprecated; you can set variables\n\
                       directly with '--variable-name=value'.\n\
   -t, --tmpdir=path   Path for temporary files. Multiple paths can be\n\
-                      specified, separated by "
+                      specified, separated by ");
 #if defined( __WIN__) || defined(OS2) || defined(__NETWARE__)
-   "semicolon (;)"
+   puts("semicolon (;)");
 #else
-   "colon (:)"
+   puts("colon (:)");
 #endif
-                      ", they will be used\n\
+                      puts(", they will be used\n\
                       in a round-robin fashion.\n\
   -s, --silent	      Only print errors.  One can use two -s to make\n\
 		      myisamchk very silent.\n\
diff --git a/mysql-test/mysql-test-run.sh b/mysql-test/mysql-test-run.sh
index eaa5d0b9da3..241a965e2e9 100644
--- a/mysql-test/mysql-test-run.sh
+++ b/mysql-test/mysql-test-run.sh
@@ -16,7 +16,7 @@ USE_MANAGER=0
 MY_TZ=GMT-3
 TZ=$MY_TZ; export TZ # for UNIX_TIMESTAMP tests to work
 LOCAL_SOCKET=@MYSQL_UNIX_ADDR@
-MYSQL_TCP_PORT=@MYSQL_TCP_PORT@; export MYSQL_TCP_PORT
+MYSQL_TCP_PORT=@MYSQL_TCP_PORT@
 
 # For query_cache test
 case `uname` in
@@ -434,7 +434,7 @@ SLAVE_MYERR="$MYSQL_TEST_DIR/var/log/slave.err"
 CURRENT_TEST="$MYSQL_TEST_DIR/var/log/current_test"
 SMALL_SERVER="--key_buffer_size=1M --sort_buffer=256K --max_heap_table_size=1M"
 
-export MASTER_MYPORT SLAVE_MYPORT
+export MASTER_MYPORT SLAVE_MYPORT MYSQL_TCP_PORT
 
 if [ x$SOURCE_DIST = x1 ] ; then
  MY_BASEDIR=$MYSQL_TEST_DIR
diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index b8b899f4850..f30f047a338 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -1831,3 +1831,13 @@ Warnings:
 Note	1276	Field or reference 'up.a' of SELECT #2 was resolved in SELECT #1
 Note	1003	select test.up.a AS `a`,test.up.b AS `b` from test.t1 up where exists(select 1 AS `Not_used` from test.t1 where (test.t1.a = test.up.a))
 drop table t1;
+CREATE TABLE t1 (id int(11) default NULL,name varchar(10) default NULL);
+INSERT INTO t1 VALUES (1,'Tim'),(2,'Rebecca'),(3,NULL);
+CREATE TABLE t2 (id int(11) default NULL, pet varchar(10) default NULL);
+INSERT INTO t2 VALUES (1,'Fido'),(2,'Spot'),(3,'Felix');
+SELECT a.*, b.* FROM (SELECT * FROM t1) AS a JOIN t2 as b on a.id=b.id;
+id	name	id	pet
+1	Tim	1	Fido
+2	Rebecca	2	Spot
+3	NULL	3	Felix
+drop table t1,t2;
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index 4b2fd33abfd..d68e519e460 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -1167,3 +1167,14 @@ insert into t1 values (1,2),(3,4);
 select * from t1 up where exists (select * from t1 where t1.a=up.a);
 explain extended select * from t1 up where exists (select * from t1 where t1.a=up.a);
 drop table t1;
+
+#
+# Test problem with NULL and derived tables (Bug #4097)
+#
+
+CREATE TABLE t1 (id int(11) default NULL,name varchar(10) default NULL);
+INSERT INTO t1 VALUES (1,'Tim'),(2,'Rebecca'),(3,NULL);
+CREATE TABLE t2 (id int(11) default NULL, pet varchar(10) default NULL);
+INSERT INTO t2 VALUES (1,'Fido'),(2,'Spot'),(3,'Felix');
+SELECT a.*, b.* FROM (SELECT * FROM t1) AS a JOIN t2 as b on a.id=b.id;
+drop table t1,t2;
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 841898ac505..c26d5188fdd 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -47,10 +47,6 @@
 #define ONE_THREAD
 #endif
 
-#define SHUTDOWN_THD
-#define MAIN_THD
-#define SIGNAL_THD
-
 #ifdef HAVE_purify
 #define IF_PURIFY(A,B) (A)
 #else
@@ -827,7 +823,6 @@ static void __cdecl kill_server(int sig_ptr)
 #if defined(USE_ONE_SIGNAL_HAND) || (defined(__NETWARE__) && defined(SIGNALS_DONT_BREAK_READ))
 extern "C" pthread_handler_decl(kill_server_thread,arg __attribute__((unused)))
 {
-  SHUTDOWN_THD;
   my_thread_init();				// Initialize new thread
   kill_server(0);
   my_thread_end();				// Normally never reached
@@ -1716,7 +1711,6 @@ static void init_signals(void)
   signal(SIGALRM, SIG_IGN);
   signal(SIGBREAK,SIG_IGN);
   signal_thread = pthread_self();
-  SIGNAL_THD;
 }
 
 static void start_signal_handler(void)
@@ -2116,7 +2110,6 @@ int uname(struct utsname *a)
 extern "C" pthread_handler_decl(handle_shutdown,arg)
 {
   MSG msg;
-  SHUTDOWN_THD;
   my_thread_init();
 
   /* this call should create the message queue for this thread */
@@ -2145,7 +2138,6 @@ int STDCALL handle_kill(ulong ctrl_type)
 #ifdef OS2
 extern "C" pthread_handler_decl(handle_shutdown,arg)
 {
-  SHUTDOWN_THD;
   my_thread_init();
 
   // wait semaphore
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index f7a0d5259a6..3fa51a98187 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -5137,6 +5137,10 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
     recinfo->length=null_pack_length;
     recinfo++;
     bfill(null_flags,null_pack_length,255);	// Set null fields
+
+    table->null_flags= (uchar*) table->record[0];
+    table->null_fields= null_count+ hidden_null_count;
+    table->null_bytes= null_pack_length;
   }
   null_count= (blob_count == 0) ? 1 : 0;
   hidden_field_count=param->hidden_field_count;
@@ -5200,7 +5204,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
 
   param->copy_field_end=copy;
   param->recinfo=recinfo;
-  store_record(table,default_values);			// Make empty default record
+  store_record(table,default_values);		// Make empty default record
 
   if (thd->variables.tmp_table_size == ~(ulong) 0)		// No limit
     table->max_rows= ~(ha_rows) 0;
@@ -8326,10 +8330,11 @@ calc_group_buffer(JOIN *join,ORDER *group)
   join->tmp_table_param.group_null_parts=null_parts;
 }
 
+
 /*
-  alloc group fields or take prepared (chached)
+  allocate group fields or take prepared (cached)
 
-  SYNOPSYS
+  SYNOPSIS
     make_group_fields()
     main_join - join of current select
     curr_join - current join (join of current select or temporary copy of it)
@@ -8342,22 +8347,21 @@ calc_group_buffer(JOIN *join,ORDER *group)
 static bool
 make_group_fields(JOIN *main_join, JOIN *curr_join)
 {
-    if (main_join->group_fields_cache.elements)
-    {
-      curr_join->group_fields= main_join->group_fields_cache;
-      curr_join->sort_and_group= 1;
-    }
-    else
-    {
-      if (alloc_group_fields(curr_join, curr_join->group_list))
-      {
-	return (1);
-      }
-      main_join->group_fields_cache= curr_join->group_fields;
-    }
-    return (0);
+  if (main_join->group_fields_cache.elements)
+  {
+    curr_join->group_fields= main_join->group_fields_cache;
+    curr_join->sort_and_group= 1;
+  }
+  else
+  {
+    if (alloc_group_fields(curr_join, curr_join->group_list))
+      return (1);
+    main_join->group_fields_cache= curr_join->group_fields;
+  }
+  return (0);
 }
 
+
 /*
   Get a list of buffers for saveing last group
   Groups are saved in reverse order for easyer check loop
@@ -8398,7 +8402,6 @@ test_if_group_changed(List<Item_buff> &list)
 }
 
 
-
 /*
   Setup copy_fields to save fields at start of new group
 
diff --git a/sql/sql_string.cc b/sql/sql_string.cc
index cf4f94ba966..8b2c57db81a 100644
--- a/sql/sql_string.cc
+++ b/sql/sql_string.cc
@@ -370,7 +370,7 @@ bool String::copy(const char *str, uint32 arg_length,
 
 bool String::set_ascii(const char *str, uint32 arg_length)
 {
-  if (!(str_charset->mbminlen > 1))
+  if (str_charset->mbminlen <= 1)
   {
     set(str, arg_length, str_charset);
     return 0;
diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy
index 89915852b9b..e1a4eb1c1a5 100644
--- a/sql/sql_yacc.yy
+++ b/sql/sql_yacc.yy
@@ -803,7 +803,7 @@ verb_clause:
         ;
 
 deallocate:
-        DEALLOCATE_SYM PREPARE_SYM ident
+        deallocate_or_drop PREPARE_SYM ident
         {
           THD *thd=YYTHD;
           LEX *lex= thd->lex;
@@ -816,6 +816,12 @@ deallocate:
           lex->prepared_stmt_name= $3;
         };
 
+deallocate_or_drop:
+	DEALLOCATE_SYM |
+	DROP
+	;
+
+
 prepare:
         PREPARE_SYM ident FROM prepare_src
         {