merging.

88 files changed, 4396 insertions, 2045 deletions

diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
index 96c3643ccbc..599914a8f8f 100644
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@ -37,7 +37,9 @@ SET_SOURCE_FILES_PROPERTIES(${CMAKE_BINARY_DIR}/sql/sql_yacc.h
                               PROPERTIES GENERATED 1)
 
 ADD_DEFINITIONS(-DMYSQL_SERVER -D_CONSOLE -DHAVE_DLOPEN -DHAVE_EVENT_SCHEDULER)
-
+IF(WITH_FEEDBACK_STORAGE_ENGINE)
+  ADD_DEFINITIONS(-DWITH_FEEDBACK_PLUGIN)
+ENDIF()
 
 SET (SQL_SOURCE
                ../sql-common/client.c derror.cc des_key_file.cc
diff --git a/sql/debug_sync.cc b/sql/debug_sync.cc
index 322db38adf2..5e5e5a72850 100644
--- a/sql/debug_sync.cc
+++ b/sql/debug_sync.cc
@@ -1078,7 +1078,7 @@ static bool debug_sync_set_action(THD *thd, st_debug_sync_action *action)
       point decremented it to 0. In this case the following happened:
 
       - an error message was reported with my_error() and
-      - the statement was killed with thd->killed= THD::KILL_QUERY.
+      - the statement was killed with thd->killed= KILL_QUERY.
 
       If a statement reports an error, it must not call send_ok().
       The calling functions will not call send_ok(), if we return TRUE
@@ -1852,7 +1852,7 @@ static void debug_sync_execute(THD *thd, st_debug_sync_action *action)
   {
     if (!--action->hit_limit)
     {
-      thd->killed= THD::KILL_QUERY;
+      thd->killed= KILL_QUERY;
       my_error(ER_DEBUG_SYNC_HIT_LIMIT, MYF(0));
     }
     DBUG_PRINT("debug_sync_exec", ("hit_limit: %lu  at: '%s'",
diff --git a/sql/discover.cc b/sql/discover.cc
index 56dc00cc5c4..92af5d56016 100644
--- a/sql/discover.cc
+++ b/sql/discover.cc
@@ -67,7 +67,7 @@ int readfrm(const char *name, uchar **frmdata, size_t *len)
   error= 2;
   if (my_fstat(file, &state, MYF(0)))
     goto err;
-  read_len= state.st_size;  
+  read_len= (size_t)state.st_size;  
 
   // Read whole frm file
   error= 3;
diff --git a/sql/event_scheduler.cc b/sql/event_scheduler.cc
index ecddcb7ca46..240d2df8e65 100644
--- a/sql/event_scheduler.cc
+++ b/sql/event_scheduler.cc
@@ -642,7 +642,7 @@ Event_scheduler::stop()
     sql_print_information("Event Scheduler: Killing the scheduler thread, "
                           "thread id %lu",
                           scheduler_thd->thread_id);
-    scheduler_thd->awake(THD::KILL_CONNECTION);
+    scheduler_thd->awake(KILL_CONNECTION);
     pthread_mutex_unlock(&scheduler_thd->LOCK_thd_data);
 
     /* thd could be 0x0, when shutting down */
diff --git a/sql/field_conv.cc b/sql/field_conv.cc
index afc4d8252ff..588a99f560f 100644
--- a/sql/field_conv.cc
+++ b/sql/field_conv.cc
@@ -204,6 +204,14 @@ static void do_skip(Copy_field *copy __attribute__((unused)))
 }
 
 
+/* 
+  Copy: (NULLable field) -> (NULLable field) 
+
+  note: if the record we're copying from is NULL-complemetned (i.e. 
+  from_field->table->null_row==1), it will also have all NULLable columns to be
+  set to NULLs, so we dont need to check table->null_row here.
+*/
+
 static void do_copy_null(Copy_field *copy)
 {
   if (*copy->from_null_ptr & copy->from_bit)
@@ -218,6 +226,10 @@ static void do_copy_null(Copy_field *copy)
   }
 }
 
+/*
+  Copy: (not-NULL field in table that can be NULL-complemented) -> (NULLable 
+     field)
+*/
 
 static void do_outer_field_null(Copy_field *copy)
 {
@@ -235,6 +247,7 @@ static void do_outer_field_null(Copy_field *copy)
 }
 
 
+/* Copy: (NULL-able field) -> (not NULL-able field) */
 static void do_copy_not_null(Copy_field *copy)
 {
   if (*copy->from_null_ptr & copy->from_bit)
@@ -248,6 +261,7 @@ static void do_copy_not_null(Copy_field *copy)
 }
 
 
+/* Copy: (non-NULLable field) -> (NULLable field) */
 static void do_copy_maybe_null(Copy_field *copy)
 {
   *copy->to_null_ptr&= ~copy->to_bit;
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 559f4f1dcf6..15020b56e52 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -506,7 +506,7 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
   my_off_t record;
   TABLE *sort_form;
   THD *thd= current_thd;
-  volatile THD::killed_state *killed= &thd->killed;
+  volatile killed_state *killed= &thd->killed;
   handler *file;
   MY_BITMAP *save_read_set, *save_write_set, *save_vcol_set;
   DBUG_ENTER("find_all_keys");
@@ -546,12 +546,11 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
   /* Temporary set for register_used_fields and register_field_in_read_map */
   sort_form->read_set= &sort_form->tmp_set;
   register_used_fields(param);
-  if (select && select->cond)
-    select->cond->walk(&Item::register_field_in_read_map, 1,
-                       (uchar*) sort_form);
-  if (select && select->pre_idx_push_select_cond)
-    select->pre_idx_push_select_cond->walk(&Item::register_field_in_read_map,
-                                           1, (uchar*) sort_form);
+  Item *sort_cond= !select ?  
+                     0 : !select->pre_idx_push_select_cond ? 
+                           select->cond : select->pre_idx_push_select_cond;
+  if (sort_cond)
+    sort_cond->walk(&Item::register_field_in_read_map, 1, (uchar*) sort_form);
   sort_form->column_bitmaps_set(&sort_form->tmp_set, &sort_form->tmp_set, 
                                 &sort_form->tmp_set);
 
@@ -624,15 +623,21 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
           SQL_SELECT::skip_record evaluates this condition. it may include a
           correlated subquery predicate, such that some field in the subquery
           refers to 'sort_form'.
+
+          PSergey-todo: discuss the above with Timour.
         */
+        MY_BITMAP *tmp_read_set= sort_form->read_set;
+        MY_BITMAP *tmp_write_set= sort_form->write_set;
+        MY_BITMAP *tmp_vcol_set= sort_form->vcol_set;
+
         if (select->cond->with_subselect)
           sort_form->column_bitmaps_set(save_read_set, save_write_set,
                                         save_vcol_set);
         write_record= (select->skip_record(thd) > 0);
         if (select->cond->with_subselect)
-          sort_form->column_bitmaps_set(&sort_form->tmp_set,
-                                        &sort_form->tmp_set,
-                                        &sort_form->tmp_set);
+          sort_form->column_bitmaps_set(tmp_read_set,
+                                        tmp_write_set,
+                                        tmp_vcol_set);
       }
       else
         write_record= true;
@@ -1233,9 +1238,9 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
   void *first_cmp_arg;
   element_count dupl_count= 0;
   uchar *src;
-  THD::killed_state not_killable;
+  killed_state not_killable;
   uchar *unique_buff= param->unique_buff;
-  volatile THD::killed_state *killed= &current_thd->killed;
+  volatile killed_state *killed= &current_thd->killed;
   DBUG_ENTER("merge_buffers");
 
   status_var_increment(current_thd->status_var.filesort_merge_passes);
@@ -1243,7 +1248,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
   if (param->not_killable)
   {
     killed= &not_killable;
-    not_killable= THD::NOT_KILLED;
+    not_killable= NOT_KILLED;
   }
 
   error=0;
diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc
index 77851119e34..5a3365b29d2 100644
--- a/sql/ha_ndbcluster_binlog.cc
+++ b/sql/ha_ndbcluster_binlog.cc
@@ -1855,7 +1855,7 @@ static void ndb_binlog_query(THD *thd, Cluster_schema *schema)
   else
     thd->server_id= schema->any_value;
   thd->db= schema->db;
-  int errcode = query_error_code(thd, thd->killed == THD::NOT_KILLED);
+  int errcode = query_error_code(thd, thd->killed == NOT_KILLED);
   thd->binlog_query(THD::STMT_QUERY_TYPE, schema->query,
                     schema->query_length, FALSE,
                     schema->name[0] == 0 || thd->db[0] == 0,
diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc
index 6a72623ce5d..eabb10a0d5a 100644
--- a/sql/ha_partition.cc
+++ b/sql/ha_partition.cc
@@ -1301,7 +1301,7 @@ int ha_partition::prepare_new_partition(TABLE *tbl,
 
   DBUG_RETURN(0);
 error_external_lock:
-  VOID(file->close());
+  VOID(file->ha_close());
 error_open:
   VOID(file->ha_delete_table(part_name));
 error_create:
@@ -1347,7 +1347,7 @@ void ha_partition::cleanup_new_partition(uint part_count)
     while ((part_count > 0) && (*file))
     {
       (*file)->ha_external_lock(thd, F_UNLCK);
-      (*file)->close();
+      (*file)->ha_close();
 
       /* Leave the (*file)->ha_delete_table(part_name) to the ddl-log */
 
@@ -2448,7 +2448,7 @@ bool ha_partition::read_par_file(const char *name)
   fn_format(buff, name, "", ha_par_ext, MY_APPEND_EXT);
 
   /* Following could be done with my_stat to read in whole file */
-  if ((file= my_open(buff, O_RDONLY | O_SHARE, MYF(0))) < 0)
+  if ((file= my_open(buff, O_RDONLY | O_SHARE, MYF(MY_WME))) < 0)
     DBUG_RETURN(true);
   if (my_read(file, (uchar *) & buff[0], PAR_WORD_SIZE, MYF(MY_NABP)))
     goto err1;
@@ -2842,7 +2842,7 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked)
 err_handler:
   DEBUG_SYNC(ha_thd(), "partition_open_error");
   while (file-- != m_file)
-    (*file)->close();
+    (*file)->ha_close();
 err_alloc:
   bitmap_free(&m_bulk_insert_started);
   if (!m_is_clone_of)
@@ -2928,7 +2928,7 @@ int ha_partition::close(void)
 repeat:
   do
   {
-    (*file)->close();
+    (*file)->ha_close();
   } while (*(++file));
 
   if (first && m_added_file && m_added_file[0])
diff --git a/sql/handler.cc b/sql/handler.cc
index 318e2e116e2..6c5d3a580ec 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -1403,9 +1403,13 @@ int ha_rollback_trans(THD *thd, bool all)
     slave SQL thread, it would not stop the thread but just be printed in
     the error log; but we don't want users to wonder why they have this
     message in the error log, so we don't send it.
+
+    We don't have to test for thd->killed == KILL_SYSTEM_THREAD as
+    it doesn't matter if a warning is pushed to a system thread or not:
+    No one will see it...
   */
   if (is_real_trans && thd->transaction.all.modified_non_trans_table &&
-      !thd->slave_thread && thd->killed != THD::KILL_CONNECTION)
+      !thd->slave_thread && thd->killed < KILL_CONNECTION)
     push_warning(thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                  ER_WARNING_NOT_COMPLETE_ROLLBACK,
                  ER(ER_WARNING_NOT_COMPLETE_ROLLBACK));
@@ -2168,7 +2172,7 @@ THD *handler::ha_thd(void) const
     Don't wait for locks if not HA_OPEN_WAIT_IF_LOCKED is set
 */
 int handler::ha_open(TABLE *table_arg, const char *name, int mode,
-                     int test_if_locked)
+                     uint test_if_locked)
 {
   int error;
   DBUG_ENTER("handler::ha_open");
@@ -2212,11 +2216,22 @@ int handler::ha_open(TABLE *table_arg, const char *name, int mode,
       dup_ref=ref+ALIGN_SIZE(ref_length);
     cached_table_flags= table_flags();
   }
-  rows_read= rows_changed= 0;
-  memset(index_rows_read, 0, sizeof(index_rows_read));
+  reset_statistics();
+  internal_tmp_table= test(test_if_locked & HA_OPEN_INTERNAL_TABLE);
   DBUG_RETURN(error);
 }
 
+int handler::ha_close()
+{
+  DBUG_ENTER("ha_close");
+  /*
+    Increment global statistics for temporary tables.
+    In_use is 0 for tables that was closed from the table cache.
+  */
+  if (table->in_use)
+    status_var_add(table->in_use->status_var.rows_tmp_read, rows_tmp_read);
+  DBUG_RETURN(close());
+}
 
 /* Initialize handler for random reading, with error handling */
 
@@ -2548,7 +2563,7 @@ int handler::update_auto_increment()
     /*
       first test if the query was aborted due to strict mode constraints
     */
-    if (thd->killed == THD::KILL_BAD_DATA)
+    if (killed_mask_hard(thd->killed) == KILL_BAD_DATA)
       DBUG_RETURN(HA_ERR_AUTOINC_ERANGE);
 
     /*
@@ -3238,7 +3253,7 @@ int handler::rename_table(const char * from, const char * to)
 
 void handler::drop_table(const char *name)
 {
-  close();
+  ha_close();
   delete_table(name);
 }
 
@@ -3757,6 +3772,7 @@ void handler::update_global_table_stats()
   TABLE_STATS * table_stats;
 
   status_var_add(table->in_use->status_var.rows_read, rows_read);
+  DBUG_ASSERT(rows_tmp_read == 0);
 
   if (!table->in_use->userstat_running)
   {
@@ -4704,7 +4720,7 @@ static bool check_table_binlog_row_based(THD *thd, TABLE *table)
 
 /** @brief
    Write table maps for all (manually or automatically) locked tables
-   to the binary log. Also, if binlog_annotate_rows_events is ON,
+   to the binary log. Also, if binlog_annotate_row_events is ON,
    write Annotate_rows event before the first table map.
 
    SYNOPSIS
@@ -4742,7 +4758,7 @@ static int write_locked_table_maps(THD *thd)
     locks[0]= thd->extra_lock;
     locks[1]= thd->lock;
     locks[2]= thd->locked_tables;
-    my_bool with_annotate= thd->variables.binlog_annotate_rows_events &&
+    my_bool with_annotate= thd->variables.binlog_annotate_row_events &&
                            thd->query() && thd->query_length();
 
     for (uint i= 0 ; i < sizeof(locks)/sizeof(*locks) ; ++i )
@@ -4862,6 +4878,9 @@ int handler::ha_reset()
   /* reset the bitmaps to point to defaults */
   table->default_column_bitmaps();
   pushed_cond= NULL;
+  /* Reset information about pushed engine conditions */
+  cancel_pushed_idx_cond();
+  /* Reset information about pushed index conditions */
   DBUG_RETURN(reset());
 }
 
diff --git a/sql/handler.h b/sql/handler.h
index 4707aabbd52..66320c7e98f 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -1,4 +1,5 @@
 /* Copyright 2000-2008 MySQL AB, 2008 Sun Microsystems, Inc.
+   Copyright 2009-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1389,10 +1390,14 @@ public:
   }
   void add_io(double add_io_cnt, double add_avg_cost)
   {
-    double io_count_sum= io_count + add_io_cnt;
-    avg_io_cost= (io_count * avg_io_cost + 
-                  add_io_cnt * add_avg_cost) / io_count_sum;
-    io_count= io_count_sum;
+    /* In edge cases add_io_cnt may be zero */
+    if (add_io_cnt > 0)
+    {
+      double io_count_sum= io_count + add_io_cnt;
+      avg_io_cost= (io_count * avg_io_cost + 
+                    add_io_cnt * add_avg_cost) / io_count_sum;
+      io_count= io_count_sum;
+    }
   }
 
   /*
@@ -1599,6 +1604,7 @@ public:
   KEY_PART_INFO *range_key_part;
   int key_compare_result_on_equal;
   bool eq_range;
+  bool internal_tmp_table;                      /* If internal tmp table */
 
   /* 
     TRUE <=> the engine guarantees that returned records are within the range
@@ -1643,6 +1649,7 @@ public:
   */
   /* Statistics  variables */
   ulonglong rows_read;
+  ulonglong rows_tmp_read;
   ulonglong rows_changed;
   /* One bigger than needed to avoid to test if key == MAX_KEY */
   ulonglong index_rows_read[MAX_KEY+1];
@@ -1659,7 +1666,7 @@ public:
   handler(handlerton *ht_arg, TABLE_SHARE *share_arg)
     :table_share(share_arg), table(0),
     estimation_rows_to_insert(0), ht(ht_arg),
-    ref(0), in_range_check_pushed_down(FALSE),
+    ref(0), end_range(NULL), in_range_check_pushed_down(FALSE),
     key_used_on_scan(MAX_KEY), active_index(MAX_KEY),
     ref_length(sizeof(my_off_t)),
     ft_handler(0), inited(NONE),
@@ -1685,7 +1692,7 @@ public:
   }
   /* ha_ methods: pubilc wrappers for private virtual API */
 
-  int ha_open(TABLE *table, const char *name, int mode, int test_if_locked);
+  int ha_open(TABLE *table, const char *name, int mode, uint test_if_locked);
   int ha_index_init(uint idx, bool sorted)
   {
     int result;
@@ -1716,6 +1723,7 @@ public:
     DBUG_ENTER("ha_rnd_init");
     DBUG_ASSERT(inited==NONE || (inited==RND && scan));
     inited= (result= rnd_init(scan)) ? NONE: RND;
+    end_range= NULL;
     DBUG_RETURN(result);
   }
   int ha_rnd_end()
@@ -1723,6 +1731,7 @@ public:
     DBUG_ENTER("ha_rnd_end");
     DBUG_ASSERT(inited==RND);
     inited=NONE;
+    end_range= NULL;
     DBUG_RETURN(rnd_end());
   }
   int ha_rnd_init_with_error(bool scan) __attribute__ ((warn_unused_result));
@@ -1809,7 +1818,7 @@ public:
   uint get_dup_key(int error);
   void reset_statistics()
   {
-    rows_read= rows_changed= 0;
+    rows_read= rows_changed= rows_tmp_read= 0;
     bzero(index_rows_read, sizeof(index_rows_read));
   }
   virtual void change_table_ptr(TABLE *table_arg, TABLE_SHARE *share)
@@ -1894,7 +1903,7 @@ public:
   */
   uint get_index(void) const
   { return inited == INDEX ? active_index : MAX_KEY; }
-  virtual int close(void)=0;
+  int ha_close(void);
 
   /**
     @retval  0   Bulk update used by handler
@@ -1970,10 +1979,18 @@ protected:
   virtual int index_last(uchar * buf)
    { return  HA_ERR_WRONG_COMMAND; }
   virtual int index_next_same(uchar *buf, const uchar *key, uint keylen);
+  virtual int close(void)=0;
+  inline void update_rows_read()
+  {
+    if (likely(!internal_tmp_table))
+      rows_read++;
+    else
+      rows_tmp_read++;
+  }
   inline void update_index_statistics()
   {
     index_rows_read[active_index]++;
-    rows_read++;
+    update_rows_read();
   }
 public:
 
@@ -2376,6 +2393,13 @@ public:
  */
  virtual void cond_pop() { return; };
  virtual Item *idx_cond_push(uint keyno, Item* idx_cond) { return idx_cond; }
+ /** Reset information about pushed index conditions */
+ virtual void cancel_pushed_idx_cond()
+ {
+   pushed_idx_cond= NULL;
+   pushed_idx_cond_keyno= MAX_KEY;
+   in_range_check_pushed_down= false;
+ }
  virtual bool check_if_incompatible_data(HA_CREATE_INFO *create_info,
 					 uint table_changes)
  { return COMPATIBLE_DATA_NO; }
@@ -2447,6 +2471,7 @@ private:
   */
 
   virtual int open(const char *name, int mode, uint test_if_locked)=0;
+  /* Note: ha_index_read_idx_map() may buypass index_init() */
   virtual int index_init(uint idx, bool sorted) { return 0; }
   virtual int index_end() { return 0; }
   /**
@@ -2604,6 +2629,7 @@ public:
   virtual handlerton *partition_ht() const
   { return ht; }
   inline int ha_write_tmp_row(uchar *buf);
+  inline int ha_update_tmp_row(const uchar * old_data, uchar * new_data);
 };
 
 #include "multi_range_read.h"
diff --git a/sql/item.cc b/sql/item.cc
index cad3e60e5fc..d321b74c1fc 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -31,6 +31,16 @@ const String my_null_string("NULL", 4, default_charset_info);
 static int save_field_in_field(Field *from, bool *null_value,
                                Field *to, bool no_conversions);
 
+
+/**
+  Compare two Items for List<Item>::add_unique()
+*/
+
+bool cmp_items(Item *a, Item *b)
+{
+  return a->eq(b, FALSE);
+}
+
 /****************************************************************************/
 
 /* Hybrid_type_traits {_real} */
@@ -651,14 +661,14 @@ Item* Item::transform(Item_transformer transformer, uchar *arg)
   A pointer to created wrapper item if successful, NULL - otherwise
 */
 
-Item* Item::set_expr_cache(THD *thd, List<Item *> &depends_on)
+Item* Item::set_expr_cache(THD *thd)
 {
   DBUG_ENTER("Item::set_expr_cache");
   Item_cache_wrapper *wrapper;
   if ((wrapper= new Item_cache_wrapper(this)) &&
       !wrapper->fix_fields(thd, (Item**)&wrapper))
   {
-    if (wrapper->set_cache(thd, depends_on))
+    if (wrapper->set_cache(thd))
       DBUG_RETURN(NULL);
     DBUG_RETURN(wrapper);
   }
@@ -742,6 +752,17 @@ bool Item_ident::remove_dependence_processor(uchar * arg)
 }
 
 
+bool Item_ident::collect_outer_ref_processor(uchar *param)
+{
+  Collect_deps_prm *prm= (Collect_deps_prm *)param;
+  if (depended_from && 
+      depended_from->nest_level_base == prm->nest_level_base &&
+      depended_from->nest_level < prm->nest_level)
+    prm->parameters->add_unique(this, &cmp_items);
+  return FALSE;
+}
+
+
 /**
   Store the pointer to this item field into a list if not already there.
 
@@ -2153,6 +2174,11 @@ bool Item_field::enumerate_field_refs_processor(uchar *arg)
   return FALSE;
 }
 
+bool Item_field::update_table_bitmaps_processor(uchar *arg)
+{
+  update_table_bitmaps();
+  return FALSE;
+}
 
 const char *Item_ident::full_name() const
 {
@@ -4357,9 +4383,6 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
                             ((ref_type == REF_ITEM || ref_type == FIELD_ITEM) ?
                              (Item_ident*) (*reference) :
                              0));
-          context->select_lex->
-              register_dependency_item(last_checked_context->select_lex,
-                                       reference);
           /*
             A reference to a view field had been found and we
             substituted it instead of this Item (find_field_in_tables
@@ -4460,9 +4483,6 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
     mark_as_dependent(thd, last_checked_context->select_lex,
                       context->select_lex, rf,
                       rf);
-    context->select_lex->
-              register_dependency_item(last_checked_context->select_lex,
-                                       reference);
 
     return 0;
   }
@@ -4471,9 +4491,6 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
     mark_as_dependent(thd, last_checked_context->select_lex,
                       context->select_lex,
                       this, (Item_ident*)*reference);
-    context->select_lex->
-              register_dependency_item(last_checked_context->select_lex,
-                                       reference);
     if (last_checked_context->select_lex->having_fix_field)
     {
       Item_ref *rf;
@@ -4638,6 +4655,7 @@ bool Item_field::fix_fields(THD *thd, Item **reference)
     if (!outer_fixed && cached_table && cached_table->select_lex &&
         context->select_lex &&
         cached_table->select_lex != context->select_lex &&
+        !context->select_lex->is_merged_child_of(cached_table->select_lex) &&
         is_outer_table(cached_table, context->select_lex))
     {
       int ret;
@@ -5181,6 +5199,10 @@ Field *Item::make_string_field(TABLE *table)
 {
   Field *field;
   DBUG_ASSERT(collation.collation);
+  /* 
+    Note: the following check is repeated in 
+    subquery_types_allow_materialization():
+  */
   if (max_length/collation.collation->mbmaxlen > CONVERT_IF_BIGGER_TO_BLOB)
     field= new Field_blob(max_length, maybe_null, name,
                           collation.collation);
@@ -5967,7 +5989,7 @@ bool Item::send(Protocol *protocol, String *buffer)
   case MYSQL_TYPE_TIMESTAMP:
   {
     MYSQL_TIME tm;
-    get_date(&tm, TIME_FUZZY_DATE);
+    get_date(&tm, TIME_FUZZY_DATE | sql_mode_for_dates());
     if (!null_value)
     {
       if (f_type == MYSQL_TYPE_DATE)
@@ -6304,9 +6326,6 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
                                 refer_type == FIELD_ITEM) ?
                                (Item_ident*) (*reference) :
                                0));
-           context->select_lex->
-              register_dependency_item(last_checked_context->select_lex,
-                                       reference);
             /*
               view reference found, we substituted it instead of this
               Item, so can quit
@@ -6357,9 +6376,6 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
         thd->change_item_tree(reference, fld);
         mark_as_dependent(thd, last_checked_context->select_lex,
                           thd->lex->current_select, fld, fld);
-        context->select_lex->
-              register_dependency_item(last_checked_context->select_lex,
-                                       reference);
         /*
           A reference is resolved to a nest level that's outer or the same as
           the nest level of the enclosing set function : adjust the value of
@@ -6383,9 +6399,6 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
       DBUG_ASSERT(*ref && (*ref)->fixed);
       mark_as_dependent(thd, last_checked_context->select_lex,
                         context->select_lex, this, this);
-      context->select_lex->
-              register_dependency_item(last_checked_context->select_lex,
-                                       reference);
       /*
         A reference is resolved to a nest level that's outer or the same as
         the nest level of the enclosing set function : adjust the value of
@@ -6400,12 +6413,6 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
   }
   else if (ref_type() != VIEW_REF)
   {
-    if (depended_from && reference)
-    {
-      DBUG_ASSERT(context->select_lex != get_depended_from());
-      context->select_lex->register_dependency_item(get_depended_from(),
-                                                    reference);
-    }
     /*
       It could be that we're referring to something that's in ancestor selects.
       We must make an appropriate mark_as_dependent() call for each such
@@ -6768,7 +6775,19 @@ my_decimal *Item_ref::val_decimal(my_decimal *decimal_value)
 int Item_ref::save_in_field(Field *to, bool no_conversions)
 {
   int res;
-  DBUG_ASSERT(!result_field);
+  if (result_field)
+  {
+    if (result_field->is_null())
+    {
+      null_value= 1;
+      res= set_field_to_null_with_conversions(to, no_conversions);
+      return res;
+    }
+    to->set_notnull();
+    res= field_conv(to, result_field);
+    null_value= 0;
+    return res;
+  }
   res= (*ref)->save_in_field(to, no_conversions);
   null_value= (*ref)->null_value;
   return res;
@@ -6883,40 +6902,6 @@ bool Item_direct_ref::get_date(MYSQL_TIME *ltime,uint fuzzydate)
 }
 
 
-Item* Item_direct_ref_to_ident::transform(Item_transformer transformer,
-                                          uchar *argument)
-{
-  DBUG_ASSERT(!current_thd->is_stmt_prepare());
-
-  Item *new_item= ident->transform(transformer, argument);
-  if (!new_item)
-    return 0;
-  DBUG_ASSERT(new_item->type() == FIELD_ITEM || new_item->type() == REF_ITEM);
-
-  if (ident != new_item)
-    current_thd->change_item_tree((Item**)&ident, new_item);
-  return (this->*transformer)(argument);
-}
-
-
-Item* Item_direct_ref_to_ident::compile(Item_analyzer analyzer, uchar **arg_p,
-                                        Item_transformer transformer,
-                                        uchar *arg_t)
-{
-  if (!(this->*analyzer)(arg_p))
-    return 0;
-
-  uchar *arg_v= *arg_p;
-  Item *new_item= ident->compile(analyzer, &arg_v, transformer, arg_t);
-  if (new_item && ident != new_item)
-  {
-    DBUG_ASSERT(new_item->type() == FIELD_ITEM || new_item->type() == REF_ITEM);
-    current_thd->change_item_tree((Item**)&ident, new_item);
-  }
-  return (this->*transformer)(arg_t);
-}
-
-
 Item_cache_wrapper::~Item_cache_wrapper()
 {
   DBUG_ASSERT(expr_cache == 0);
@@ -6936,6 +6921,7 @@ Item_cache_wrapper::Item_cache_wrapper(Item *item_arg)
   unsigned_flag= orig_item->unsigned_flag;
   name= item_arg->name;
   name_length= item_arg->name_length;
+  with_subselect=  orig_item->with_subselect;
 
   if ((expr_value= Item_cache::get_cache(orig_item)))
     expr_value->setup(orig_item);
@@ -6944,11 +6930,28 @@ Item_cache_wrapper::Item_cache_wrapper(Item *item_arg)
 }
 
 
+/**
+  Initialize the cache if it is needed
+*/
+
+void Item_cache_wrapper::init_on_demand()
+{
+    if (!expr_cache->is_inited())
+    {
+      orig_item->get_cache_parameters(parameters);
+      expr_cache->init();
+    }
+}
+
+
 void Item_cache_wrapper::print(String *str, enum_query_type query_type)
 {
   str->append(func_name());
   if (expr_cache)
+  {
+    init_on_demand();
     expr_cache->print(str, query_type);
+  }
   else
     str->append(STRING_WITH_LEN("<<DISABLED>>"));
   str->append('(');
@@ -6984,6 +6987,7 @@ void Item_cache_wrapper::cleanup()
   expr_cache= 0;
   /* expr_value is Item so it will be destroyed from list of Items */
   expr_value= 0;
+  parameters.empty();
   DBUG_VOID_RETURN;
 }
 
@@ -7004,11 +7008,11 @@ void Item_cache_wrapper::cleanup()
   @retval TRUE  Error
 */
 
-bool Item_cache_wrapper::set_cache(THD *thd, List<Item*> &depends_on)
+bool Item_cache_wrapper::set_cache(THD *thd)
 {
   DBUG_ENTER("Item_cache_wrapper::set_cache");
   DBUG_ASSERT(expr_cache == 0);
-  expr_cache= new Expression_cache_tmptable(thd, depends_on, expr_value);
+  expr_cache= new Expression_cache_tmptable(thd, parameters, expr_value);
   DBUG_RETURN(expr_cache == NULL);
 }
 
@@ -7033,6 +7037,7 @@ Item *Item_cache_wrapper::check_cache()
   {
     Expression_cache_tmptable::result res;
     Item *cached_value;
+    init_on_demand();
     res= expr_cache->check_value(&cached_value);
     if (res == Expression_cache_tmptable::HIT)
       DBUG_RETURN(cached_value);
@@ -7318,7 +7323,11 @@ bool Item_direct_view_ref::fix_fields(THD *thd, Item **reference)
            ((*ref)->fix_fields(thd, ref)))
     return TRUE;
 
-  return Item_direct_ref::fix_fields(thd, reference);
+  if (Item_direct_ref::fix_fields(thd, reference))
+    return TRUE;
+  if (view->table && view->table->maybe_null)
+    maybe_null= TRUE;
+  return FALSE;
 }
 
 /*
@@ -7561,7 +7570,7 @@ Item *Item_direct_view_ref::replace_equal_field(uchar *arg)
   field_item->set_item_equal(item_equal);
   Item *item= field_item->replace_equal_field(arg);
   field_item->set_item_equal(0);
-  return item;
+  return item != field_item ? item : this;
 }
 
 
@@ -9109,6 +9118,12 @@ table_map Item_direct_view_ref::used_tables() const
          (view->merged ? (*ref)->used_tables() : view->table->map); 
 }
 
+table_map Item_direct_view_ref::not_null_tables() const		
+{
+  return get_depended_from() ? 
+         0 :
+         (view->merged ? (*ref)->not_null_tables() : view->table->map); 
+}
 
 /*
   we add RAND_TABLE_BIT to prevent moving this item from HAVING to WHERE
@@ -9121,7 +9136,22 @@ table_map Item_ref_null_helper::used_tables() const
 }
 
 
+/* Debugger help function */
+static char dbug_item_print_buf[256];
 
+const char *dbug_print_item(Item *item)
+{
+  char *buf= dbug_item_print_buf;
+  String str(buf, sizeof(dbug_item_print_buf), &my_charset_bin);
+  str.length(0);
+  if (!item)
+    return "(Item*)NULL";
+  item->print(&str ,QT_ORDINARY);
+  if (str.c_ptr() == buf)
+    return buf;
+  else
+    return "Couldn't fit into buffer";
+}
 /*****************************************************************************
 ** Instantiate templates
 *****************************************************************************/
diff --git a/sql/item.h b/sql/item.h
index 1829f45d00b..fa6918d8484 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -505,6 +505,7 @@ typedef void (*Cond_traverser) (const Item *item, void *arg);
 class Item_equal;
 class COND_EQUAL;
 
+class st_select_lex_unit;
 
 class Item {
   Item(const Item &);			/* Prevent use of these */
@@ -1018,8 +1019,10 @@ public:
   virtual bool mark_as_eliminated_processor(uchar *arg) { return 0; }
   virtual bool eliminate_subselect_processor(uchar *arg) { return 0; }
   virtual bool set_fake_select_as_master_processor(uchar *arg) { return 0; }
+  virtual bool update_table_bitmaps_processor(uchar *arg) { return 0; }
   virtual bool view_used_tables_processor(uchar *arg) { return 0; }
   virtual bool eval_not_null_tables(uchar *opt_arg) { return 0; }
+  virtual bool clear_sum_processor(uchar *opt_arg) { return 0; }
 
   /* To call bool function for all arguments */
   struct bool_func_call_args
@@ -1155,6 +1158,17 @@ public:
   {
     return FALSE;
   }
+  struct Collect_deps_prm
+  {
+    List<Item> *parameters;
+    /* unit from which we count nest_level */
+    st_select_lex_unit *nest_level_base;
+    int nest_level;
+  };
+  /**
+    Collect outer references
+  */
+  virtual bool collect_outer_ref_processor(uchar *arg) {return FALSE; }
 
   /**
     Find a function of a given type
@@ -1249,7 +1263,7 @@ public:
     { return Field::GEOM_GEOMETRY; };
   String *check_well_formed_result(String *str, bool send_error= 0);
   bool eq_by_collation(Item *item, bool binary_cmp, CHARSET_INFO *cs); 
-  Item* set_expr_cache(THD *thd, List<Item*> &depends_on);
+  Item* set_expr_cache(THD *thd);
   virtual Item *get_cached_item() { return NULL; }
 
   virtual Item_equal *get_item_equal() { return NULL; }
@@ -1273,9 +1287,27 @@ public:
     walk(&Item::view_used_tables_processor, 0, (uchar *) view);
     return view->view_used_tables;
   }
+
+  /**
+    Collect and add to the list cache parameters for this Item.
+
+    @note Now implemented only for subqueries and in_optimizer,
+    if we need it for general function then this method should
+    be defined for Item_func.
+  */
+  virtual void get_cache_parameters(List<Item> &parameters) { };
+
+  virtual void mark_as_condition_AND_part(TABLE_LIST *embedding) {};
 };
 
 
+/**
+  Compare two Items for List<Item>::add_unique()
+*/
+
+bool cmp_items(Item *a, Item *b);
+
+
 /*
   Class to be used to enumerate all field references in an item tree. This
   includes references to outside but not fields of the tables within a
@@ -1677,6 +1709,10 @@ public:
   virtual void print(String *str, enum_query_type query_type);
   virtual bool change_context_processor(uchar *cntx)
     { context= (Name_resolution_context *)cntx; return FALSE; }
+  /**
+    Collect outer references
+  */
+  virtual bool collect_outer_ref_processor(uchar *arg);
   friend bool insert_fields(THD *thd, Name_resolution_context *context,
                             const char *db_name,
                             const char *table_name, List_iterator<Item> *it,
@@ -1784,6 +1820,20 @@ public:
   bool get_date_result(MYSQL_TIME *ltime,uint fuzzydate);
   bool is_null() { return field->is_null(); }
   void update_null_value();
+  void update_table_bitmaps()
+  {
+    if (field && field->table)
+    {
+      TABLE *tab= field->table;
+      tab->covering_keys.intersect(field->part_of_key);
+      tab->merge_keys.merge(field->part_of_key);
+      if (tab->read_set)
+        bitmap_fast_test_and_set(tab->read_set, field->field_index);
+      if (field->vcol_info)
+        tab->mark_virtual_col(field);
+    }  
+  }
+  void update_used_tables() { update_table_bitmaps(); }
   Item *get_tmp_table_item(THD *thd);
   bool collect_item_field_processor(uchar * arg);
   bool add_field_to_set_processor(uchar * arg);
@@ -1795,6 +1845,7 @@ public:
   bool vcol_in_partition_func_processor(uchar *bool_arg);
   bool check_vcol_func_processor(uchar *arg) { return FALSE;}
   bool enumerate_field_refs_processor(uchar *arg);
+  bool update_table_bitmaps_processor(uchar *arg);
   void cleanup();
   Item_equal *get_item_equal() { return item_equal; }
   void set_item_equal(Item_equal *item_eq) { item_equal= item_eq; }
@@ -2570,13 +2621,16 @@ public:
   Field *get_tmp_table_field()
   { return result_field ? result_field : (*ref)->get_tmp_table_field(); }
   Item *get_tmp_table_item(THD *thd);
-  inline table_map used_tables() const;		
-  inline void update_used_tables(); 
+  table_map used_tables() const;		
+  void update_used_tables(); 
   bool const_item() const 
   {
     return (*ref)->const_item();
   }
-  table_map not_null_tables() const { return (*ref)->not_null_tables(); }
+  table_map not_null_tables() const 
+  { 
+    return depended_from ? 0 : (*ref)->not_null_tables();
+  }
   void set_result_field(Field *field)	{ result_field= field; }
   bool is_result_field() { return 1; }
   void save_in_result_field(bool no_conversions)
@@ -2713,9 +2767,6 @@ public:
   virtual void print(String *str, enum_query_type query_type)
   { ident->print(str, query_type); }
 
-  virtual Item* transform(Item_transformer transformer, uchar *arg);
-  virtual Item* compile(Item_analyzer analyzer, uchar **arg_p,
-                        Item_transformer transformer, uchar *arg_t);
 };
 
 
@@ -2742,8 +2793,11 @@ private:
   */
   Item_cache *expr_value;
 
+  List<Item> parameters;
+
   Item *check_cache();
-  inline void cache();
+  void cache();
+  void init_on_demand();
 
 public:
   Item_cache_wrapper(Item *item_arg);
@@ -2753,7 +2807,7 @@ public:
   enum Type type() const { return EXPR_CACHE_ITEM; }
   enum Type real_type() const { return orig_item->type(); }
 
-  bool set_cache(THD *thd, List<Item*> &depends_on);
+  bool set_cache(THD *thd);
 
   bool fix_fields(THD *thd, Item **it);
   void fix_length_and_dec() {}
@@ -2835,6 +2889,9 @@ public:
     if (result_type() == ROW_RESULT)
       orig_item->bring_value();
   }
+  virtual bool is_expensive() { return orig_item->is_expensive(); }
+  bool is_expensive_processor(uchar *arg)
+  { return orig_item->is_expensive_processor(arg); }
   bool check_vcol_func_processor(uchar *arg)
   {
     return trace_unsupported_by_check_vcol_func_processor("cache");
@@ -2881,6 +2938,7 @@ public:
   Item *equal_fields_propagator(uchar *arg);
   Item *replace_equal_field(uchar *arg);
   table_map used_tables() const;	
+  table_map not_null_tables() const;
   bool walk(Item_processor processor, bool walk_subquery, uchar *arg)
   { 
     return (*ref)->walk(processor, walk_subquery, arg) ||
diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
index dc25cf5942c..a33dd090f14 100644
--- a/sql/item_cmpfunc.cc
+++ b/sql/item_cmpfunc.cc
@@ -29,8 +29,6 @@
 #include <m_ctype.h>
 #include "sql_select.h"
 
-static bool convert_const_to_int(THD *, Item_field *, Item **);
-
 static Item_result item_store_type(Item_result a, Item *item,
                                    my_bool unsigned_flag)
 {
@@ -266,36 +264,61 @@ Item_bool_func2* Eq_creator::create(Item *a, Item *b) const
   return new Item_func_eq(a, b);
 }
 
+Item_bool_func2* Eq_creator::create_swap(Item *a, Item *b) const
+{
+  return new Item_func_eq(b, a);
+}
 
 Item_bool_func2* Ne_creator::create(Item *a, Item *b) const
 {
   return new Item_func_ne(a, b);
 }
 
+Item_bool_func2* Ne_creator::create_swap(Item *a, Item *b) const
+{
+  return new Item_func_ne(b, a);
+}
 
 Item_bool_func2* Gt_creator::create(Item *a, Item *b) const
 {
   return new Item_func_gt(a, b);
 }
 
+Item_bool_func2* Gt_creator::create_swap(Item *a, Item *b) const
+{
+  return new Item_func_lt(b, a);
+}
 
 Item_bool_func2* Lt_creator::create(Item *a, Item *b) const
 {
   return new Item_func_lt(a, b);
 }
 
+Item_bool_func2* Lt_creator::create_swap(Item *a, Item *b) const
+{
+  return new Item_func_gt(b, a);
+}
 
 Item_bool_func2* Ge_creator::create(Item *a, Item *b) const
 {
   return new Item_func_ge(a, b);
 }
 
+Item_bool_func2* Ge_creator::create_swap(Item *a, Item *b) const
+{
+  return new Item_func_le(b, a);
+}
 
 Item_bool_func2* Le_creator::create(Item *a, Item *b) const
 {
   return new Item_func_le(a, b);
 }
 
+Item_bool_func2* Le_creator::create_swap(Item *a, Item *b) const
+{
+  return new Item_func_ge(b, a);
+}
+
 /*
   Test functions
   Most of these  returns 0LL if false and 1LL if true and
@@ -434,7 +457,7 @@ static bool convert_const_to_int(THD *thd, Item_field *field_item,
       field_item->field_type() != MYSQL_TYPE_YEAR)
     return 1;
 
-  if ((*item)->const_item())
+  if ((*item)->const_item() && !(*item)->is_expensive())
   {
     TABLE *table= field->table;
     ulong orig_sql_mode= thd->variables.sql_mode;
@@ -490,7 +513,6 @@ static bool convert_const_to_int(THD *thd, Item_field *field_item,
 void Item_bool_func2::fix_length_and_dec()
 {
   max_length= 1;				     // Function returns 0 or 1
-  THD *thd;
 
   /*
     As some compare functions are generated after sql_yacc,
@@ -522,14 +544,14 @@ void Item_bool_func2::fix_length_and_dec()
 
   /*
     Make a special case of compare with fields to get nicer comparisons
-    of numbers with constant string.
+    of bigint numbers with constant string.
     This directly contradicts the manual (number and a string should
     be compared as doubles), but seems to provide more
     "intuitive" behavior in some cases (but less intuitive in others).
 
     But disable conversion in case of LIKE function.
   */
-  thd= current_thd;
+  THD *thd= current_thd;
   if (functype() != LIKE_FUNC && !thd->lex->is_ps_or_view_context_analysis())
   {
     int field;
@@ -537,7 +559,8 @@ void Item_bool_func2::fix_length_and_dec()
         args[field= 1]->real_item()->type() == FIELD_ITEM)
     {
       Item_field *field_item= (Item_field*) (args[field]->real_item());
-      if (field_item->cmp_type() == INT_RESULT &&
+      if ((field_item->field_type() ==  MYSQL_TYPE_LONGLONG ||
+           field_item->field_type() ==  MYSQL_TYPE_YEAR) &&
           convert_const_to_int(thd, field_item, &args[!field]))
         args[0]->cmp_context= args[1]->cmp_context= INT_RESULT;
     }
@@ -1374,6 +1397,36 @@ longlong Item_func_truth::val_int()
 }
 
 
+bool Item_in_optimizer::is_top_level_item()
+{
+  return ((Item_in_subselect *)args[1])->is_top_level_item();
+}
+
+
+void Item_in_optimizer::fix_after_pullout(st_select_lex *new_parent, Item **ref)
+{
+  /* This will re-calculate attributes of our Item_in_subselect: */
+  Item_bool_func::fix_after_pullout(new_parent, ref);
+
+  /* Then, re-calculate not_null_tables_cache: */
+  eval_not_null_tables(NULL);
+}
+
+
+bool Item_in_optimizer::eval_not_null_tables(uchar *opt_arg)
+{
+  not_null_tables_cache= 0;
+  if (is_top_level_item())
+  {
+    /*
+      It is possible to determine NULL-rejectedness of the left arguments
+      of IN only if it is a top-level predicate.
+    */
+    not_null_tables_cache= args[0]->not_null_tables();
+  }
+  return FALSE;
+}
+
 bool Item_in_optimizer::fix_left(THD *thd, Item **ref)
 {
   if ((!args[0]->fixed && args[0]->fix_fields(thd, args)) ||
@@ -1400,7 +1453,7 @@ bool Item_in_optimizer::fix_left(THD *thd, Item **ref)
     }
     used_tables_cache= args[0]->used_tables();
   }
-  not_null_tables_cache= args[0]->not_null_tables();
+  eval_not_null_tables(NULL);
   with_sum_func= args[0]->with_sum_func;
   with_field= args[0]->with_field;
   if ((const_item_cache= args[0]->const_item()))
@@ -1433,7 +1486,6 @@ bool Item_in_optimizer::fix_fields(THD *thd, Item **ref)
   with_sum_func= with_sum_func || args[1]->with_sum_func;
   with_field= with_field || args[1]->with_field;
   used_tables_cache|= args[1]->used_tables();
-  not_null_tables_cache|= args[1]->not_null_tables();
   const_item_cache&= args[1]->const_item();
   fixed= 1;
   return FALSE;
@@ -1464,32 +1516,41 @@ Item *Item_in_optimizer::expr_cache_insert_transformer(uchar *thd_arg)
   DBUG_ENTER("Item_in_optimizer::expr_cache_insert_transformer");
   if (args[1]->type() != Item::SUBSELECT_ITEM)
     DBUG_RETURN(this); // MAX/MIN transformed => do nothing
-  List<Item*> &depends_on= ((Item_subselect *)args[1])->depends_on;
 
   if (expr_cache)
     DBUG_RETURN(expr_cache);
 
+  if (args[1]->expr_cache_is_needed(thd) &&
+      (expr_cache= set_expr_cache(thd)))
+    DBUG_RETURN(expr_cache);
+
+  DBUG_RETURN(this);
+}
+
+
+
+/**
+    Collect and add to the list cache parameters for this Item.
+
+    @param parameters    The list where to add parameters
+*/
+
+void Item_in_optimizer::get_cache_parameters(List<Item> &parameters)
+{
   /* Add left expression to the list of the parameters of the subquery */
   if (args[0]->cols() == 1)
-    depends_on.push_front((Item**)args);
+    parameters.add_unique(args[0], &cmp_items);
   else
   {
     for (uint i= 0; i < args[0]->cols(); i++)
     {
-      depends_on.push_front(args[0]->addr(i));
+      parameters.add_unique(args[0]->element_index(i), &cmp_items);
     }
   }
-
-  if (args[1]->expr_cache_is_needed(thd) &&
-      (expr_cache= set_expr_cache(thd, depends_on)))
-    DBUG_RETURN(expr_cache);
-
-  /* no cache => return list in original state just to be safe */
-  for (uint i= 0; i < args[0]->cols(); i++)
-    depends_on.pop();
-  DBUG_RETURN(this);
+  args[1]->get_cache_parameters(parameters);
 }
 
+
 /*
    The implementation of optimized \<outer expression\> [NOT] IN \<subquery\>
    predicates. The implementation works as follows.
@@ -1731,7 +1792,7 @@ Item *Item_in_optimizer::transform(Item_transformer transformer, uchar *argument
     if (!new_item)
       return 0;
     if (args[1] != new_item)
-      current_thd->change_item_tree(args, new_item);
+      current_thd->change_item_tree(args + 1, new_item);
   }
   else
   {
@@ -1751,7 +1812,7 @@ Item *Item_in_optimizer::transform(Item_transformer transformer, uchar *argument
                  Item_subselect::ANY_SUBS));
 
     Item_in_subselect *in_arg= (Item_in_subselect*)args[1];
-    in_arg->left_expr= args[0];
+    current_thd->change_item_tree(&in_arg->left_expr, args[0]);
   }
   return (this->*transformer)(argument);
 }
@@ -2083,6 +2144,14 @@ bool Item_func_between::eval_not_null_tables(uchar *opt_arg)
 }  
 
 
+void Item_func_between::fix_after_pullout(st_select_lex *new_parent, Item **ref)
+{
+  /* This will re-calculate attributes of the arguments */
+  Item_func_opt_neg::fix_after_pullout(new_parent, ref);
+  /* Then, re-calculate not_null_tables_cache according to our special rules */
+  eval_not_null_tables(NULL);
+}
+
 void Item_func_between::fix_length_and_dec()
 {
   THD *thd= current_thd;
@@ -2117,7 +2186,8 @@ void Item_func_between::fix_length_and_dec()
       !thd->lex->is_ps_or_view_context_analysis())
   {
     Item_field *field_item= (Item_field*) (args[0]->real_item());
-    if (field_item->cmp_type() == INT_RESULT)
+    if (field_item->field_type() ==  MYSQL_TYPE_LONGLONG ||
+        field_item->field_type() ==  MYSQL_TYPE_YEAR)
     {
       /*
         The following can't be recoded with || as convert_const_to_int
@@ -2463,6 +2533,16 @@ Item_func_if::eval_not_null_tables(uchar *opt_arg)
   return 0;
 }
 
+
+void Item_func_if::fix_after_pullout(st_select_lex *new_parent, Item **ref)
+{
+  /* This will re-calculate attributes of the arguments */
+  Item_func::fix_after_pullout(new_parent, ref);
+  /* Then, re-calculate not_null_tables_cache according to our special rules */
+  eval_not_null_tables(NULL);
+}
+
+
 void
 Item_func_if::fix_length_and_dec()
 {
@@ -2710,7 +2790,7 @@ Item *Item_func_case::find_item(String *str)
     {
       if (args[i]->real_item()->type() == NULL_ITEM)
         continue;
-      cmp_type= item_cmp_type(left_result_type, args[i]->result_type());
+      cmp_type= item_cmp_type(left_result_type, args[i]->cmp_type());
       DBUG_ASSERT(cmp_type != ROW_RESULT);
       DBUG_ASSERT(cmp_items[(uint)cmp_type]);
       if (!(value_added_map & (1<<(uint)cmp_type)))
@@ -2875,7 +2955,7 @@ void Item_func_case::fix_length_and_dec()
   {
     uint i;
     agg[0]= args[first_expr_num];
-    left_result_type= agg[0]->result_type();
+    left_result_type= agg[0]->cmp_type();
 
     for (nagg= 0; nagg < ncases/2 ; nagg++)
       agg[nagg+1]= args[nagg*2];
@@ -2893,17 +2973,21 @@ void Item_func_case::fix_length_and_dec()
       found_types |= (1 << item_cmp_type(left_result_type, STRING_RESULT));
     }
 
+    Item *date_arg= 0;
     for (i= 0; i <= (uint)TIME_RESULT; i++)
     {
       if (found_types & (1 << i) && !cmp_items[i])
       {
         DBUG_ASSERT((Item_result)i != ROW_RESULT);
-        DBUG_ASSERT((Item_result)i != TIME_RESULT);
         if ((Item_result)i == STRING_RESULT &&
             agg_arg_charsets(cmp_collation, agg, nagg, MY_COLL_CMP_CONV, 1))
           return;
+
+        if ((Item_result)i == TIME_RESULT)
+          date_arg= find_date_time_item(args, arg_count, 0);
+
         if (!(cmp_items[i]=
-            cmp_item::get_comparator((Item_result)i, 0,
+            cmp_item::get_comparator((Item_result)i, date_arg,
                                      cmp_collation.collation)))
           return;
       }
@@ -3494,10 +3578,13 @@ void cmp_item_row::store_value(Item *item)
     for (uint i=0; i < n; i++)
     {
       if (!comparators[i])
+      {
+        DBUG_ASSERT(item->element_index(i)->cmp_type() != TIME_RESULT);
         if (!(comparators[i]=
               cmp_item::get_comparator(item->element_index(i)->result_type(), 0,
                                        item->element_index(i)->collation.collation)))
 	  break;					// new failed
+      }
       comparators[i]->store_value(item->element_index(i));
       item->null_value|= item->element_index(i)->null_value;
     }
@@ -3700,6 +3787,14 @@ Item_func_in::eval_not_null_tables(uchar *opt_arg)
 }
 
 
+void Item_func_in::fix_after_pullout(st_select_lex *new_parent, Item **ref)
+{
+  /* This will re-calculate attributes of the arguments */
+  Item_func_opt_neg::fix_after_pullout(new_parent, ref);
+  /* Then, re-calculate not_null_tables_cache according to our special rules */
+  eval_not_null_tables(NULL);
+}
+
 static int srtcmp_in(CHARSET_INFO *cs, const String *x,const String *y)
 {
   return cs->coll->strnncollsp(cs,
@@ -3798,7 +3893,8 @@ void Item_func_in::fix_length_and_dec()
         !thd->lex->is_ps_or_view_context_analysis() && cmp_type != INT_RESULT)
     {
       Item_field *field_item= (Item_field*) (args[0]->real_item());
-      if (field_item->cmp_type() == INT_RESULT)
+      if (field_item->field_type() ==  MYSQL_TYPE_LONGLONG ||
+          field_item->field_type() ==  MYSQL_TYPE_YEAR)
       {
         bool all_converted= TRUE;
         for (arg=args+1, arg_end=args+arg_count; arg != arg_end ; arg++)
@@ -4023,15 +4119,12 @@ Item_cond::fix_fields(THD *thd, Item **ref)
   DBUG_ASSERT(fixed == 0);
   List_iterator<Item> li(list);
   Item *item;
-  TABLE_LIST *save_emb_on_expr_nest= thd->thd_marker.emb_on_expr_nest;
 #ifndef EMBEDDED_LIBRARY
   uchar buff[sizeof(char*)];			// Max local vars in function
 #endif
   not_null_tables_cache= used_tables_cache= 0;
   const_item_cache= 1;
 
-  if (functype() != COND_AND_FUNC)
-    thd->thd_marker.emb_on_expr_nest= NULL;
   /*
     and_table_cache is the value that Item_cond_or() returns for
     not_null_tables()
@@ -4091,7 +4184,6 @@ Item_cond::fix_fields(THD *thd, Item **ref)
       maybe_null=1;
   }
   thd->lex->current_select->cond_count+= list.elements;
-  thd->thd_marker.emb_on_expr_nest= save_emb_on_expr_nest;
   fix_length_and_dec();
   fixed= 1;
   return FALSE;
@@ -4103,6 +4195,7 @@ Item_cond::eval_not_null_tables(uchar *opt_arg)
 {
   Item *item;
   List_iterator<Item> li(list);
+  not_null_tables_cache= (table_map) 0;
   and_tables_cache= ~(table_map) 0;
   while ((item=li++))
   {
@@ -4361,6 +4454,17 @@ void Item_cond::neg_arguments(THD *thd)
 }
 
 
+void Item_cond_and::mark_as_condition_AND_part(TABLE_LIST *embedding)
+{
+  List_iterator<Item> li(list);
+  Item *item;
+  while ((item=li++))
+  {
+    item->mark_as_condition_AND_part(embedding);
+  }
+}
+
+
 /**
   Evaluation of AND(expr, expr, expr ...).
 
@@ -5065,23 +5169,21 @@ bool Item_func_like::turboBM_matches(const char* text, int text_len) const
     very fast to use.
 */
 
-longlong Item_cond_xor::val_int()
+longlong Item_func_xor::val_int()
 {
   DBUG_ASSERT(fixed == 1);
-  List_iterator<Item> li(list);
-  Item *item;
-  int result=0;	
-  null_value=0;
-  while ((item=li++))
+  int result= 0;
+  null_value= false;
+  for (uint i= 0; i < arg_count; i++)
   {
-    result^= (item->val_int() != 0);
-    if (item->null_value)
+    result^= (args[i]->val_int() != 0);
+    if (args[i]->null_value)
     {
-      null_value=1;
+      null_value= true;
       return 0;
     }
   }
-  return (longlong) result;
+  return result;
 }
 
 /**
@@ -5122,6 +5224,33 @@ Item *Item_bool_rowready_func2::neg_transformer(THD *thd)
   return item;
 }
 
+/**
+  XOR can be negated by negating one of the operands:
+
+  NOT (a XOR b)  => (NOT a) XOR b
+                 => a       XOR (NOT b)
+
+  @param thd     Thread handle
+  @return        New negated item
+*/
+Item *Item_func_xor::neg_transformer(THD *thd)
+{
+  Item *neg_operand;
+  Item_func_xor *new_item;
+  if ((neg_operand= args[0]->neg_transformer(thd)))
+    // args[0] has neg_tranformer
+    new_item= new(thd->mem_root) Item_func_xor(neg_operand, args[1]);
+  else if ((neg_operand= args[1]->neg_transformer(thd)))
+    // args[1] has neg_tranformer
+    new_item= new(thd->mem_root) Item_func_xor(args[0], neg_operand);
+  else
+  {
+    neg_operand= new(thd->mem_root) Item_func_not(args[0]);
+    new_item= new(thd->mem_root) Item_func_xor(neg_operand, args[1]);
+  }
+  return new_item;
+}
+
 
 /**
   a IS NULL  ->  a IS NOT NULL.
@@ -5579,7 +5708,7 @@ longlong Item_equal::val_int()
 void Item_equal::fix_length_and_dec()
 {
   Item *item= get_first(NULL);
-  eval_item= cmp_item::get_comparator(item->result_type(), 0,
+  eval_item= cmp_item::get_comparator(item->cmp_type(), item,
                                       item->collation.collation);
 }
 
@@ -5682,7 +5811,6 @@ Item* Item_equal::get_first(Item *field_item)
 {
   Item_equal_fields_iterator it(*this);
   Item *item;
-  JOIN_TAB *field_tab;
   if (!field_item)
     return (it++);
   Field *field= ((Item_field *) (field_item->real_item()))->field;
@@ -5707,8 +5835,6 @@ Item* Item_equal::get_first(Item *field_item)
     in presense of SJM nests.
   */
 
-  field_tab= field->table->reginfo.join_tab;
-
   TABLE_LIST *emb_nest= field->table->pos_in_table_list->embedding;
 
   if (emb_nest && emb_nest->sj_mat_info && emb_nest->sj_mat_info->is_used)
diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h
index 285367ba9d6..81a18bb594e 100644
--- a/sql/item_cmpfunc.h
+++ b/sql/item_cmpfunc.h
@@ -259,6 +259,10 @@ public:
   bool is_expensive();
   void set_join_tab_idx(uint join_tab_idx_arg)
   { args[1]->set_join_tab_idx(join_tab_idx_arg); }
+  virtual void get_cache_parameters(List<Item> &parameters);
+  bool is_top_level_item();
+  bool eval_not_null_tables(uchar *opt_arg);
+  void fix_after_pullout(st_select_lex *new_parent, Item **ref);
 };
 
 class Comp_creator
@@ -266,7 +270,14 @@ class Comp_creator
 public:
   Comp_creator() {}                           /* Remove gcc warning */
   virtual ~Comp_creator() {}                  /* Remove gcc warning */
+  /**
+    Create operation with given arguments.
+  */
   virtual Item_bool_func2* create(Item *a, Item *b) const = 0;
+  /**
+    Create operation with given arguments in swap order.
+  */
+  virtual Item_bool_func2* create_swap(Item *a, Item *b) const = 0;
   virtual const char* symbol(bool invert) const = 0;
   virtual bool eqne_op() const = 0;
   virtual bool l_op() const = 0;
@@ -278,6 +289,7 @@ public:
   Eq_creator() {}                             /* Remove gcc warning */
   virtual ~Eq_creator() {}                    /* Remove gcc warning */
   virtual Item_bool_func2* create(Item *a, Item *b) const;
+  virtual Item_bool_func2* create_swap(Item *a, Item *b) const;
   virtual const char* symbol(bool invert) const { return invert? "<>" : "="; }
   virtual bool eqne_op() const { return 1; }
   virtual bool l_op() const { return 0; }
@@ -289,6 +301,7 @@ public:
   Ne_creator() {}                             /* Remove gcc warning */
   virtual ~Ne_creator() {}                    /* Remove gcc warning */
   virtual Item_bool_func2* create(Item *a, Item *b) const;
+  virtual Item_bool_func2* create_swap(Item *a, Item *b) const;
   virtual const char* symbol(bool invert) const { return invert? "=" : "<>"; }
   virtual bool eqne_op() const { return 1; }
   virtual bool l_op() const { return 0; }
@@ -300,6 +313,7 @@ public:
   Gt_creator() {}                             /* Remove gcc warning */
   virtual ~Gt_creator() {}                    /* Remove gcc warning */
   virtual Item_bool_func2* create(Item *a, Item *b) const;
+  virtual Item_bool_func2* create_swap(Item *a, Item *b) const;
   virtual const char* symbol(bool invert) const { return invert? "<=" : ">"; }
   virtual bool eqne_op() const { return 0; }
   virtual bool l_op() const { return 0; }
@@ -311,6 +325,7 @@ public:
   Lt_creator() {}                             /* Remove gcc warning */
   virtual ~Lt_creator() {}                    /* Remove gcc warning */
   virtual Item_bool_func2* create(Item *a, Item *b) const;
+  virtual Item_bool_func2* create_swap(Item *a, Item *b) const;
   virtual const char* symbol(bool invert) const { return invert? ">=" : "<"; }
   virtual bool eqne_op() const { return 0; }
   virtual bool l_op() const { return 1; }
@@ -322,6 +337,7 @@ public:
   Ge_creator() {}                             /* Remove gcc warning */
   virtual ~Ge_creator() {}                    /* Remove gcc warning */
   virtual Item_bool_func2* create(Item *a, Item *b) const;
+  virtual Item_bool_func2* create_swap(Item *a, Item *b) const;
   virtual const char* symbol(bool invert) const { return invert? "<" : ">="; }
   virtual bool eqne_op() const { return 0; }
   virtual bool l_op() const { return 0; }
@@ -333,6 +349,7 @@ public:
   Le_creator() {}                             /* Remove gcc warning */
   virtual ~Le_creator() {}                    /* Remove gcc warning */
   virtual Item_bool_func2* create(Item *a, Item *b) const;
+  virtual Item_bool_func2* create_swap(Item *a, Item *b) const;
   virtual const char* symbol(bool invert) const { return invert? ">" : "<="; }
   virtual bool eqne_op() const { return 0; }
   virtual bool l_op() const { return 1; }
@@ -391,6 +408,26 @@ public:
   }
 };
 
+/**
+  XOR inherits from Item_bool_func2 because it is not optimized yet.
+  Later, when XOR is optimized, it needs to inherit from
+  Item_cond instead. See WL#5800. 
+*/
+class Item_func_xor :public Item_bool_func2
+{
+public:
+  Item_func_xor(Item *i1, Item *i2) :Item_bool_func2(i1, i2) {}
+  enum Functype functype() const { return XOR_FUNC; }
+  const char *func_name() const { return "xor"; }
+  longlong val_int();
+  void top_level_item() {}
+  Item *neg_transformer(THD *thd);
+  bool subst_argument_checker(uchar **arg)
+  {
+    return (*arg != NULL);     
+  }
+};
+
 class Item_func_not :public Item_bool_func
 {
 public:
@@ -460,7 +497,7 @@ public:
      show(0)
     {}
   virtual void top_level_item() { abort_on_null= 1; }
-  bool top_level() { return abort_on_null; }
+  bool is_top_level_item() { return abort_on_null; }
   longlong val_int();
   enum Functype functype() const { return NOT_ALL_FUNC; }
   const char *func_name() const { return "<not>"; }
@@ -638,6 +675,7 @@ public:
   CHARSET_INFO *compare_collation() { return cmp_collation.collation; }
   uint decimal_precision() const { return 1; }
   bool eval_not_null_tables(uchar *opt_arg);
+  void fix_after_pullout(st_select_lex *new_parent, Item **ref);
 };
 
 
@@ -739,6 +777,7 @@ public:
   uint decimal_precision() const;
   const char *func_name() const { return "if"; }
   bool eval_not_null_tables(uchar *opt_arg);
+  void fix_after_pullout(st_select_lex *new_parent, Item **ref);
 };
 
 
@@ -1277,6 +1316,7 @@ public:
   bool is_bool_func() { return 1; }
   CHARSET_INFO *compare_collation() { return cmp_collation.collation; }
   bool eval_not_null_tables(uchar *opt_arg);
+  void fix_after_pullout(st_select_lex *new_parent, Item **ref);
 };
 
 class cmp_item_row :public cmp_item
@@ -1331,7 +1371,11 @@ public:
       const_item_cache= 1;
     }
     else
+    {
       args[0]->update_used_tables();
+      used_tables_cache= args[0]->used_tables();
+      const_item_cache= args[0]->const_item();
+    }
   }
   table_map not_null_tables() const { return 0; }
   optimize_type select_optimize() const { return OPTIMIZE_NULL; }
@@ -1688,7 +1732,8 @@ public:
   friend class Item_equal_fields_iterator;
   friend Item *eliminate_item_equal(COND *cond, COND_EQUAL *upper_levels,
                            Item_equal *item_equal);
-  friend bool setup_sj_materialization(struct st_join_table *tab);
+  friend bool setup_sj_materialization_part1(struct st_join_table *tab);
+  friend bool setup_sj_materialization_part2(struct st_join_table *tab);
 }; 
 
 class COND_EQUAL: public Sql_alloc
@@ -1775,6 +1820,7 @@ public:
     return item;
   }
   Item *neg_transformer(THD *thd);
+  void mark_as_condition_AND_part(TABLE_LIST *embedding);
 };
 
 inline bool is_cond_and(Item *item)
@@ -1816,45 +1862,6 @@ inline bool is_cond_or(Item *item)
   return (cond_item->functype() == Item_func::COND_OR_FUNC);
 }
 
-/*
-  XOR is Item_cond, not an Item_int_func because we could like to
-  optimize (a XOR b) later on. It's low prio, though
-*/
-
-class Item_cond_xor :public Item_cond
-{
-public:
-  Item_cond_xor(Item *i1,Item *i2) :Item_cond(i1,i2) 
-  {
-    /* 
-      Items must be stored in args[] as well because this Item_cond is
-      treated as a FUNC_ITEM (see type()). I.e., users of it will get
-      it's children by calling arguments(), not argument_list(). This
-      is a temporary solution until XOR is optimized and treated like
-      a full Item_cond citizen.
-     */
-    arg_count= 2;
-    args= tmp_arg;
-    args[0]= i1; 
-    args[1]= i2;
-  }
-  enum Functype functype() const { return COND_XOR_FUNC; }
-  /* TODO: remove the next line when implementing XOR optimization */
-  enum Type type() const { return FUNC_ITEM; }
-  longlong val_int();
-  const char *func_name() const { return "xor"; }
-  void top_level_item() {}
-  /* Since child Items are stored in args[], Items cannot be added.
-     However, since Item_cond_xor is treated as a FUNC_ITEM (see
-     type()), the methods below should never be called. 
-  */
-  bool add(Item *item) { DBUG_ASSERT(FALSE); return FALSE; }
-  bool add_at_head(Item *item) { DBUG_ASSERT(FALSE); return FALSE; }
-  bool add_at_head(List<Item> *nlist) { DBUG_ASSERT(FALSE); return FALSE; }
-  void copy_andor_arguments(THD *thd, Item_cond *item) { DBUG_ASSERT(FALSE); }
-};
-
-
 /* Some useful inline functions */
 
 inline Item *and_conds(Item *a, Item *b)
diff --git a/sql/item_func.cc b/sql/item_func.cc
index 54c60ea03e1..ce43f90be8d 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -152,11 +152,9 @@ Item_func::fix_fields(THD *thd, Item **ref)
 {
   DBUG_ASSERT(fixed == 0);
   Item **arg,**arg_end;
-  TABLE_LIST *save_emb_on_expr_nest= thd->thd_marker.emb_on_expr_nest;
 #ifndef EMBEDDED_LIBRARY			// Avoid compiler warning
   uchar buff[STACK_BUFF_ALLOC];			// Max argument in function
 #endif
-  thd->thd_marker.emb_on_expr_nest= NULL;
 
   used_tables_cache= not_null_tables_cache= 0;
   const_item_cache=1;
@@ -210,7 +208,6 @@ Item_func::fix_fields(THD *thd, Item **ref)
   if (thd->is_error()) // An error inside fix_length_and_dec occured
     return TRUE;
   fixed= 1;
-  thd->thd_marker.emb_on_expr_nest= save_emb_on_expr_nest;
   return FALSE;
 }
 
@@ -234,6 +231,7 @@ bool
 Item_func::eval_not_null_tables(uchar *opt_arg)
 {
   Item **arg,**arg_end;
+  not_null_tables_cache= 0;
   if (arg_count)
   {		
     for (arg=args, arg_end=args+arg_count; arg != arg_end ; arg++)
@@ -5065,12 +5063,16 @@ void Item_func_get_system_var::fix_length_and_dec()
   switch (var->show_type())
   {
     case SHOW_LONG:
-    case SHOW_INT:
     case SHOW_HA_ROWS:
       unsigned_flag= TRUE;
       max_length= MY_INT64_NUM_DECIMAL_DIGITS;
       decimals=0;
       break;
+    case SHOW_INT:
+      unsigned_flag= FALSE;
+      max_length= MY_INT64_NUM_DECIMAL_DIGITS;
+      decimals=0;
+      break;
     case SHOW_LONGLONG:
       unsigned_flag= TRUE;
       max_length= MY_INT64_NUM_DECIMAL_DIGITS;
@@ -5211,7 +5213,7 @@ longlong Item_func_get_system_var::val_int()
 
   switch (var->show_type())
   {
-    case SHOW_INT:      get_sys_var_safe (uint);
+    case SHOW_INT:      get_sys_var_safe (int);
     case SHOW_LONG:     get_sys_var_safe (ulong);
     case SHOW_LONGLONG: get_sys_var_safe (ulonglong);
     case SHOW_HA_ROWS:  get_sys_var_safe (ha_rows);
diff --git a/sql/item_func.h b/sql/item_func.h
index 63056f9079e..4ff7492bd89 100644
--- a/sql/item_func.h
+++ b/sql/item_func.h
@@ -44,7 +44,7 @@ public:
   enum Functype { UNKNOWN_FUNC,EQ_FUNC,EQUAL_FUNC,NE_FUNC,LT_FUNC,LE_FUNC,
 		  GE_FUNC,GT_FUNC,FT_FUNC,
 		  LIKE_FUNC,ISNULL_FUNC,ISNOTNULL_FUNC,
-		  COND_AND_FUNC, COND_OR_FUNC, COND_XOR_FUNC,
+		  COND_AND_FUNC, COND_OR_FUNC, XOR_FUNC,
                   BETWEEN, IN_FUNC, MULT_EQUAL_FUNC,
 		  INTERVAL_FUNC, ISNOTNULLTEST_FUNC,
 		  SP_EQUALS_FUNC, SP_DISJOINT_FUNC,SP_INTERSECTS_FUNC,
diff --git a/sql/item_geofunc.h b/sql/item_geofunc.h
index f637167d6d2..f3208c5b4d4 100644
--- a/sql/item_geofunc.h
+++ b/sql/item_geofunc.h
@@ -185,6 +185,7 @@ public:
   String *val_str(String *);
   void fix_length_and_dec()
   {
+    Item_geometry_func::fix_length_and_dec();
     for (unsigned int i= 0; i < arg_count; ++i)
     {
       if (args[i]->fixed && args[i]->field_type() != MYSQL_TYPE_GEOMETRY)
diff --git a/sql/item_row.cc b/sql/item_row.cc
index 99a1644cc48..46d5f13f6fa 100644
--- a/sql/item_row.cc
+++ b/sql/item_row.cc
@@ -93,6 +93,22 @@ bool Item_row::fix_fields(THD *thd, Item **ref)
 }
 
 
+bool
+Item_row::eval_not_null_tables(uchar *opt_arg)
+{
+  Item **arg,**arg_end;
+  not_null_tables_cache= 0;
+  if (arg_count)
+  {		
+    for (arg= items, arg_end= items+arg_count; arg != arg_end ; arg++)
+    {
+      not_null_tables_cache|= (*arg)->not_null_tables();
+    }
+  }
+  return FALSE;
+}
+
+
 void Item_row::cleanup()
 {
   DBUG_ENTER("Item_row::cleanup");
@@ -133,11 +149,13 @@ void Item_row::fix_after_pullout(st_select_lex *new_parent, Item **ref)
 {
   used_tables_cache= 0;
   const_item_cache= 1;
+  not_null_tables_cache= 0;
   for (uint i= 0; i < arg_count; i++)
   {
     items[i]->fix_after_pullout(new_parent, &items[i]);
     used_tables_cache|= items[i]->used_tables();
     const_item_cache&= items[i]->const_item();
+    not_null_tables_cache|= items[i]->not_null_tables();
   }
 }
 
diff --git a/sql/item_row.h b/sql/item_row.h
index 0b43309391d..1572ef77b79 100644
--- a/sql/item_row.h
+++ b/sql/item_row.h
@@ -73,6 +73,7 @@ public:
 
   bool walk(Item_processor processor, bool walk_subquery, uchar *arg);
   Item *transform(Item_transformer transformer, uchar *arg);
+  bool eval_not_null_tables(uchar *opt_arg);
 
   uint cols() { return arg_count; }
   Item* element_index(uint i) { return items[i]; }
diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index 453773478d8..9abbdfcec4f 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -3026,16 +3026,16 @@ String *Item_load_file::val_str(String *str)
 			func_name(), current_thd->variables.max_allowed_packet);
     goto err;
   }
-  if (tmp_value.alloc(stat_info.st_size))
+  if (tmp_value.alloc((size_t)stat_info.st_size))
     goto err;
   if ((file = my_open(file_name->ptr(), O_RDONLY, MYF(0))) < 0)
     goto err;
-  if (my_read(file, (uchar*) tmp_value.ptr(), stat_info.st_size, MYF(MY_NABP)))
+  if (my_read(file, (uchar*) tmp_value.ptr(), (size_t)stat_info.st_size, MYF(MY_NABP)))
   {
     my_close(file, MYF(0));
     goto err;
   }
-  tmp_value.length(stat_info.st_size);
+  tmp_value.length((uint32)stat_info.st_size);
   my_close(file, MYF(0));
   null_value = 0;
   DBUG_RETURN(&tmp_value);
@@ -3606,40 +3606,40 @@ void Item_func_dyncol_create::prepare_arguments()
       DBUG_ASSERT(args[valpos]->field_type() == MYSQL_TYPE_NULL);
       break;
     case DYN_COL_INT:
-      vals[i].long_value= args[valpos]->val_int();
+      vals[i].x.long_value= args[valpos]->val_int();
       break;
     case DYN_COL_UINT:
-      vals[i].ulong_value= args[valpos]->val_int();
+      vals[i].x.ulong_value= args[valpos]->val_int();
       break;
     case DYN_COL_DOUBLE:
-      vals[i].double_value= args[valpos]->val_real();
+      vals[i].x.double_value= args[valpos]->val_real();
       break;
     case DYN_COL_STRING:
       res= args[valpos]->val_str(&tmp);
       if (res &&
-          (vals[i].string_value.str= my_strndup(res->ptr(), res->length(),
+          (vals[i].x.string.value.str= my_strndup(res->ptr(), res->length(),
                                                 MYF(MY_WME))))
       {
-	vals[i].string_value.length= res->length();
-	vals[i].charset= res->charset();
+	vals[i].x.string.value.length= res->length();
+	vals[i].x.string.charset= res->charset();
       }
       else
       {
         args[valpos]->null_value= 1;            // In case of out of memory
-        vals[i].string_value.str= NULL;
-        vals[i].string_value.length= 0;         // just to be safe
+        vals[i].x.string.value.str= NULL;
+        vals[i].x.string.value.length= 0;         // just to be safe
       }
       break;
     case DYN_COL_DECIMAL:
       if ((dres= args[valpos]->val_decimal(&dtmp)))
       {
 	dynamic_column_prepare_decimal(&vals[i]);
-        DBUG_ASSERT(vals[i].decimal_value.len == dres->len);
-        vals[i].decimal_value.intg= dres->intg;
-        vals[i].decimal_value.frac= dres->frac;
-        vals[i].decimal_value.sign= dres->sign();
-        memcpy(vals[i].decimal_buffer, dres->buf,
-               sizeof(vals[i].decimal_buffer));
+        DBUG_ASSERT(vals[i].x.decimal.value.len == dres->len);
+        vals[i].x.decimal.value.intg= dres->intg;
+        vals[i].x.decimal.value.frac= dres->frac;
+        vals[i].x.decimal.value.sign= dres->sign();
+        memcpy(vals[i].x.decimal.buffer, dres->buf,
+               sizeof(vals[i].x.decimal.buffer));
       }
       else
       {
@@ -3648,13 +3648,13 @@ void Item_func_dyncol_create::prepare_arguments()
       }
       break;
     case DYN_COL_DATETIME:
-      args[valpos]->get_date(&vals[i].time_value, TIME_FUZZY_DATE);
+      args[valpos]->get_date(&vals[i].x.time_value, TIME_FUZZY_DATE);
       break;
     case DYN_COL_DATE:
-      args[valpos]->get_date(&vals[i].time_value, TIME_FUZZY_DATE);
+      args[valpos]->get_date(&vals[i].x.time_value, TIME_FUZZY_DATE);
       break;
     case DYN_COL_TIME:
-      args[valpos]->get_time(&vals[i].time_value);
+      args[valpos]->get_time(&vals[i].x.time_value);
       break;
     default:
       DBUG_ASSERT(0);
@@ -3663,7 +3663,7 @@ void Item_func_dyncol_create::prepare_arguments()
     if (vals[i].type != DYN_COL_NULL && args[valpos]->null_value)
     {
       if (vals[i].type == DYN_COL_STRING)
-        my_free(vals[i].string_value.str, MYF(MY_ALLOW_ZERO_PTR));
+        my_free(vals[i].x.string.value.str, MYF(MY_ALLOW_ZERO_PTR));
       vals[i].type= DYN_COL_NULL;
     }
   }
@@ -3677,7 +3677,7 @@ void Item_func_dyncol_create::cleanup_arguments()
   for (i= 0; i < column_count; i++)
   {
     if (vals[i].type == DYN_COL_STRING)
-      my_free(vals[i].string_value.str, MYF(MY_ALLOW_ZERO_PTR));
+      my_free(vals[i].x.string.value.str, MYF(MY_ALLOW_ZERO_PTR));
   }
 }
 
@@ -3894,19 +3894,19 @@ String *Item_dyncol_get::val_str(String *str_result)
     goto null;
   case DYN_COL_INT:
   case DYN_COL_UINT:
-    str_result->set_int(val.long_value, test(val.type == DYN_COL_UINT),
+    str_result->set_int(val.x.long_value, test(val.type == DYN_COL_UINT),
                        &my_charset_latin1);
     break;
   case DYN_COL_DOUBLE:
-    str_result->set_real(val.double_value, NOT_FIXED_DEC, &my_charset_latin1);
+    str_result->set_real(val.x.double_value, NOT_FIXED_DEC, &my_charset_latin1);
     break;
   case DYN_COL_STRING:
-    if ((char*) tmp.ptr() <= val.string_value.str &&
-        (char*) tmp.ptr() + tmp.length() >= val.string_value.str)
+    if ((char*) tmp.ptr() <= val.x.string.value.str &&
+        (char*) tmp.ptr() + tmp.length() >= val.x.string.value.str)
     {
       /* value is allocated in tmp buffer; We have to make a copy */
-      str_result->copy(val.string_value.str, val.string_value.length,
-                      val.charset);
+      str_result->copy(val.x.string.value.str, val.x.string.value.length,
+                      val.x.string.charset);
     }
     else
     {
@@ -3915,24 +3915,24 @@ String *Item_dyncol_get::val_str(String *str_result)
         into a field or in a buffer for another item and this buffer
         is not going to be deleted during expression evaluation
       */
-      str_result->set(val.string_value.str, val.string_value.length,
-                      val.charset);
+      str_result->set(val.x.string.value.str, val.x.string.value.length,
+                      val.x.string.charset);
     }
     break;
   case DYN_COL_DECIMAL:
   {
     int res;
     int length=
-      my_decimal_string_length((const my_decimal*)&val.decimal_value);
+      my_decimal_string_length((const my_decimal*)&val.x.decimal.value);
     if (str_result->alloc(length))
       goto null;
-    if ((res= decimal2string(&val.decimal_value, (char*) str_result->ptr(),
+    if ((res= decimal2string(&val.x.decimal.value, (char*) str_result->ptr(),
                              &length, 0, 0, ' ')) != E_DEC_OK)
     {
       char buff[40];
       int len= sizeof(buff);
       DBUG_ASSERT(length < (int)sizeof(buff));
-      decimal2string(&val.decimal_value, buff, &len, 0, 0, ' ');
+      decimal2string(&val.x.decimal.value, buff, &len, 0, 0, ' ');
       decimal_operation_results(res, buff, "CHAR");
     }
     str_result->set_charset(&my_charset_latin1);
@@ -3950,7 +3950,7 @@ String *Item_dyncol_get::val_str(String *str_result)
       asked to return the time argument as a string.
     */
     if (str_result->alloc(MAX_DATE_STRING_REP_LENGTH) ||
-        !(length= my_TIME_to_str(&val.time_value, (char*) str_result->ptr(),
+        !(length= my_TIME_to_str(&val.x.time_value, (char*) str_result->ptr(),
                                  AUTO_SEC_PART_DIGITS)))
       goto null;
     str_result->set_charset(&my_charset_latin1);
@@ -3980,20 +3980,20 @@ longlong Item_dyncol_get::val_int()
     goto null;
   case DYN_COL_UINT:
     unsigned_flag= 1;            // Make it possible for caller to detect sign
-    return val.long_value;
+    return val.x.long_value;
   case DYN_COL_INT:
     unsigned_flag= 0;            // Make it possible for caller to detect sign
-    return val.long_value;
+    return val.x.long_value;
   case DYN_COL_DOUBLE:
   {
     bool error;
     longlong num;
 
-    num= double_to_longlong(val.double_value, unsigned_flag, &error);
+    num= double_to_longlong(val.x.double_value, unsigned_flag, &error);
     if (error)
     {
       char buff[30];
-      sprintf(buff, "%lg", val.double_value);
+      sprintf(buff, "%lg", val.x.double_value);
       push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                           ER_DATA_OVERFLOW,
                           ER(ER_DATA_OVERFLOW),
@@ -4006,14 +4006,14 @@ longlong Item_dyncol_get::val_int()
   {
     int error;
     longlong num;
-    char *end= val.string_value.str + val.string_value.length, *org_end= end;
+    char *end= val.x.string.value.str + val.x.string.value.length, *org_end= end;
 
-    num= my_strtoll10(val.string_value.str, &end, &error);
+    num= my_strtoll10(val.x.string.value.str, &end, &error);
     if (end != org_end || error > 0)
     {
       char buff[80];
-      strmake(buff, val.string_value.str, min(sizeof(buff)-1,
-                                              val.string_value.length));
+      strmake(buff, val.x.string.value.str, min(sizeof(buff)-1,
+                                              val.x.string.value.length));
       push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                           ER_BAD_DATA,
                           ER(ER_BAD_DATA),
@@ -4026,18 +4026,18 @@ longlong Item_dyncol_get::val_int()
   case DYN_COL_DECIMAL:
   {
     longlong num;
-    my_decimal2int(E_DEC_FATAL_ERROR, &val.decimal_value, unsigned_flag,
+    my_decimal2int(E_DEC_FATAL_ERROR, &val.x.decimal.value, unsigned_flag,
                    &num);
     return num;
   }
   case DYN_COL_DATETIME:
   case DYN_COL_DATE:
   case DYN_COL_TIME:
-    unsigned_flag= !val.time_value.neg;
+    unsigned_flag= !val.x.time_value.neg;
     if (unsigned_flag)
-      return TIME_to_ulonglong(&val.time_value);
+      return TIME_to_ulonglong(&val.x.time_value);
     else
-      return -(longlong)TIME_to_ulonglong(&val.time_value);
+      return -(longlong)TIME_to_ulonglong(&val.x.time_value);
   }
 
 null:
@@ -4059,24 +4059,24 @@ double Item_dyncol_get::val_real()
   case DYN_COL_NULL:
     goto null;
   case DYN_COL_UINT:
-    return ulonglong2double(val.ulong_value);
+    return ulonglong2double(val.x.ulong_value);
   case DYN_COL_INT:
-    return (double) val.long_value;
+    return (double) val.x.long_value;
   case DYN_COL_DOUBLE:
-    return (double) val.double_value;
+    return (double) val.x.double_value;
   case DYN_COL_STRING:
   {
     int error;
     char *end;
-    double res= my_strntod(val.charset, (char*) val.string_value.str,
-                           val.string_value.length, &end, &error);
+    double res= my_strntod(val.x.string.charset, (char*) val.x.string.value.str,
+                           val.x.string.value.length, &end, &error);
 
-    if (end != (char*) val.string_value.str + val.string_value.length ||
+    if (end != (char*) val.x.string.value.str + val.x.string.value.length ||
         error)
     {
       char buff[80];
-      strmake(buff, val.string_value.str, min(sizeof(buff)-1,
-                                              val.string_value.length));
+      strmake(buff, val.x.string.value.str, min(sizeof(buff)-1,
+                                              val.x.string.value.length));
       push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                           ER_BAD_DATA,
                           ER(ER_BAD_DATA),
@@ -4088,13 +4088,13 @@ double Item_dyncol_get::val_real()
   {
     double res;
     /* This will always succeed */
-    decimal2double(&val.decimal_value, &res);
+    decimal2double(&val.x.decimal.value, &res);
     return res;
   }
   case DYN_COL_DATETIME:
   case DYN_COL_DATE:
   case DYN_COL_TIME:
-    return TIME_to_double(&val.time_value);
+    return TIME_to_double(&val.x.time_value);
   }
 
 null:
@@ -4116,22 +4116,22 @@ my_decimal *Item_dyncol_get::val_decimal(my_decimal *decimal_value)
   case DYN_COL_NULL:
     goto null;
   case DYN_COL_UINT:
-    int2my_decimal(E_DEC_FATAL_ERROR, val.long_value, TRUE, decimal_value);
+    int2my_decimal(E_DEC_FATAL_ERROR, val.x.long_value, TRUE, decimal_value);
     break;
   case DYN_COL_INT:
-    int2my_decimal(E_DEC_FATAL_ERROR, val.long_value, FALSE, decimal_value);
+    int2my_decimal(E_DEC_FATAL_ERROR, val.x.long_value, FALSE, decimal_value);
     break;
   case DYN_COL_DOUBLE:
-    double2my_decimal(E_DEC_FATAL_ERROR, val.double_value, decimal_value);
+    double2my_decimal(E_DEC_FATAL_ERROR, val.x.double_value, decimal_value);
     break;
   case DYN_COL_STRING:
   {
     int rc;
-    rc= str2my_decimal(0, val.string_value.str, val.string_value.length,
-                       val.charset, decimal_value);
+    rc= str2my_decimal(0, val.x.string.value.str, val.x.string.value.length,
+                       val.x.string.charset, decimal_value);
     char buff[80];
-    strmake(buff, val.string_value.str, min(sizeof(buff)-1,
-                                            val.string_value.length));
+    strmake(buff, val.x.string.value.str, min(sizeof(buff)-1,
+                                            val.x.string.value.length));
     if (rc != E_DEC_OK)
     {
       push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
@@ -4142,14 +4142,14 @@ my_decimal *Item_dyncol_get::val_decimal(my_decimal *decimal_value)
     break;
   }
   case DYN_COL_DECIMAL:
-    decimal2my_decimal(&val.decimal_value, decimal_value);
+    decimal2my_decimal(&val.x.decimal.value, decimal_value);
     break;
   case DYN_COL_DATETIME:
   case DYN_COL_DATE:
   case DYN_COL_TIME:
-    decimal_value= seconds2my_decimal(val.time_value.neg,
-                                      TIME_to_ulonglong(&val.time_value),
-                                      val.time_value.second_part,
+    decimal_value= seconds2my_decimal(val.x.time_value.neg,
+                                      TIME_to_ulonglong(&val.x.time_value),
+                                      val.x.time_value.second_part,
                                       decimal_value);
     break;
   }
@@ -4178,36 +4178,36 @@ bool Item_dyncol_get::get_date(MYSQL_TIME *ltime, uint fuzzy_date)
     signed_value= 1;                                  // For error message
     /* fall_trough */
   case DYN_COL_UINT:
-    if (signed_value || val.ulong_value <= LONGLONG_MAX)
+    if (signed_value || val.x.ulong_value <= LONGLONG_MAX)
     {
-      if (int_to_datetime_with_warn(val.ulong_value, ltime, fuzzy_date,
+      if (int_to_datetime_with_warn(val.x.ulong_value, ltime, fuzzy_date,
                                    0 /* TODO */))
         goto null;
       return 0;
     }
     /* let double_to_datetime_with_warn() issue the warning message */
-    val.double_value= static_cast<double>(ULONGLONG_MAX);
+    val.x.double_value= static_cast<double>(ULONGLONG_MAX);
     /* fall_trough */
   case DYN_COL_DOUBLE:
-    if (double_to_datetime_with_warn(val.double_value, ltime, fuzzy_date,
+    if (double_to_datetime_with_warn(val.x.double_value, ltime, fuzzy_date,
                                      0 /* TODO */))
       goto null;
     return 0;
   case DYN_COL_DECIMAL:
-    if (decimal_to_datetime_with_warn((my_decimal*)&val.decimal_value, ltime,
+    if (decimal_to_datetime_with_warn((my_decimal*)&val.x.decimal.value, ltime,
                                       fuzzy_date, 0 /* TODO */))
       goto null;
     return 0;
   case DYN_COL_STRING:
-    if (str_to_datetime_with_warn(val.string_value.str,
-                                  val.string_value.length,
+    if (str_to_datetime_with_warn(val.x.string.value.str,
+                                  val.x.string.value.length,
                                   ltime, fuzzy_date) <= MYSQL_TIMESTAMP_ERROR)
       goto null;
     return 0;
   case DYN_COL_DATETIME:
   case DYN_COL_DATE:
   case DYN_COL_TIME:
-    *ltime= val.time_value;
+    *ltime= val.x.time_value;
     return 0;
   }
 
diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc
index 32e94d3226f..6134903fee6 100644
--- a/sql/item_subselect.cc
+++ b/sql/item_subselect.cc
@@ -38,11 +38,14 @@ Item_subselect::Item_subselect():
   Item_result_field(), value_assigned(0), own_engine(0), thd(0), old_engine(0), 
   used_tables_cache(0), have_to_be_excluded(0), const_item_cache(1),
   inside_first_fix_fields(0), done_first_fix_fields(FALSE), 
-  substitution(0), expr_cache(0), engine(0), forced_const(FALSE), eliminated(FALSE),
+  expr_cache(0), forced_const(FALSE), substitution(0), engine(0), eliminated(FALSE),
   engine_changed(0), changed(0), is_correlated(FALSE)
 {
   DBUG_ENTER("Item_subselect::Item_subselect");
   DBUG_PRINT("enter", ("this: 0x%lx", (ulong) this));
+#ifndef DBUG_OFF
+  exec_counter= 0;
+#endif
   with_subselect= 1;
   reset();
   /*
@@ -125,11 +128,14 @@ void Item_subselect::cleanup()
   }
   if (engine)
     engine->cleanup();
-  depends_on.empty();
   reset();
   value_assigned= 0;
   expr_cache= 0;
   forced_const= FALSE;
+  DBUG_PRINT("info", ("exec_counter: %d", exec_counter));
+#ifndef DBUG_OFF
+  exec_counter= 0;
+#endif
   DBUG_VOID_RETURN;
 }
 
@@ -152,14 +158,32 @@ void Item_in_subselect::cleanup()
     delete left_expr_cache;
     left_expr_cache= NULL;
   }
+  /*
+    TODO: This breaks the commented assert in add_strategy().
+    in_strategy&= ~SUBS_STRATEGY_CHOSEN;
+  */
   first_execution= TRUE;
-  if (in_strategy & SUBS_MATERIALIZATION)
-    in_strategy= 0;
   pushed_cond_guards= NULL;
   Item_subselect::cleanup();
   DBUG_VOID_RETURN;
 }
 
+
+void Item_allany_subselect::cleanup()
+{
+  /*
+    The MAX/MIN transformation through injection is reverted through the
+    change_item_tree() mechanism. Revert the select_lex object of the
+    query to its initial state.
+  */
+  for (SELECT_LEX *sl= unit->first_select();
+       sl; sl= sl->next_select())
+    if (test_strategy(SUBS_MAXMIN_INJECTED))
+      sl->with_sum_func= false;
+  Item_in_subselect::cleanup();
+}
+
+
 Item_subselect::~Item_subselect()
 {
   DBUG_ENTER("Item_subselect::~Item_subselect");
@@ -206,7 +230,7 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref)
   if (check_stack_overrun(thd, STACK_MIN_SIZE, (uchar*)&res))
     return TRUE;
   
-
+  
   if (!(res= engine->prepare()))
   {
     // all transformation is done (used by prepared statements)
@@ -469,12 +493,12 @@ void Item_subselect::recalc_used_tables(st_select_lex *new_parent,
           upper->item->walk(&Item::enumerate_field_refs_processor, FALSE,
                             (uchar*)&fixer);
           used_tables_cache |= fixer.used_tables;
-          /*
+          upper->item->walk(&Item::update_table_bitmaps_processor, FALSE, NULL);
+/*
           if (after_pullout)
             upper->item->fix_after_pullout(new_parent, &(upper->item));
           upper->item->update_used_tables();
-          used_tables_cache |= upper->item->used_tables();
-          */
+*/          
         }
       }
     }
@@ -492,6 +516,20 @@ void Item_subselect::recalc_used_tables(st_select_lex *new_parent,
 bool Item_subselect::walk(Item_processor processor, bool walk_subquery,
                           uchar *argument)
 {
+  if (!(unit->uncacheable & ~UNCACHEABLE_DEPENDENT) && engine->is_executed() &&
+      !unit->describe)
+  {
+    /*
+      The subquery has already been executed (for real, it wasn't EXPLAIN's
+      fake execution) so it should not matter what it has inside.
+      
+      The actual reason for not walking inside is that parts of the subquery
+      (e.g. JTBM join nests and their IN-equality conditions may have been 
+       invalidated by irreversible cleanups (those happen after an uncorrelated 
+       subquery has been executed).
+    */
+    return FALSE;
+  }
 
   if (walk_subquery)
   {
@@ -548,7 +586,9 @@ bool Item_subselect::exec()
   DBUG_EXECUTE_IF("subselect_exec_fail", return 1;);
 
   res= engine->exec();
-
+#ifndef DBUG_OFF
+  ++exec_counter;
+#endif
   if (engine_changed)
   {
     engine_changed= 0;
@@ -558,6 +598,14 @@ bool Item_subselect::exec()
 }
 
 
+void Item_subselect::get_cache_parameters(List<Item> &parameters)
+{
+  Collect_deps_prm prm= {&parameters,
+    unit->first_select()->nest_level_base,
+    unit->first_select()->nest_level};
+  walk(&Item::collect_outer_ref_processor, TRUE, (uchar*)&prm);
+}
+
 int Item_in_subselect::optimize(double *out_rows, double *cost)
 {
   int res;
@@ -622,7 +670,7 @@ int Item_in_subselect::optimize(double *out_rows, double *cost)
 
 bool Item_subselect::expr_cache_is_needed(THD *thd)
 {
-  return (depends_on.elements &&
+  return ((engine->uncacheable() & UNCACHEABLE_DEPENDENT) &&
           engine->cols() == 1 &&
           optimizer_flag(thd, OPTIMIZER_SWITCH_SUBQUERY_CACHE) &&
           !(engine->uncacheable() & (UNCACHEABLE_RAND |
@@ -650,8 +698,7 @@ bool Item_subselect::expr_cache_is_needed(THD *thd)
 
 bool Item_in_subselect::expr_cache_is_needed(THD *thd)
 {
-  return (depends_on.elements &&
-          optimizer_flag(thd, OPTIMIZER_SWITCH_SUBQUERY_CACHE) &&
+  return (optimizer_flag(thd, OPTIMIZER_SWITCH_SUBQUERY_CACHE) &&
           !(engine->uncacheable() & (UNCACHEABLE_RAND |
                                      UNCACHEABLE_SIDEEFFECT)));
 }
@@ -676,7 +723,7 @@ bool Item_in_subselect::exec()
     - on a cost-based basis, that takes into account the cost of a cache
       lookup, the cache hit rate, and the savings per cache hit.
   */
-  if (!left_expr_cache && (in_strategy & SUBS_MATERIALIZATION))
+  if (!left_expr_cache && (test_strategy(SUBS_MATERIALIZATION)))
     init_left_expr_cache();
 
   /*
@@ -793,7 +840,10 @@ Item_maxmin_subselect::Item_maxmin_subselect(THD *thd_param,
 {
   DBUG_ENTER("Item_maxmin_subselect::Item_maxmin_subselect");
   max= max_arg;
-  init(select_lex, new select_max_min_finder_subselect(this, max_arg));
+  init(select_lex,
+       new select_max_min_finder_subselect(this, max_arg,
+                                           parent->substype() ==
+                                           Item_subselect::ALL_SUBS));
   max_columns= 1;
   maybe_null= 1;
   max_columns= 1;
@@ -984,7 +1034,7 @@ Item* Item_singlerow_subselect::expr_cache_insert_transformer(uchar *thd_arg)
     DBUG_RETURN(expr_cache);
 
   if (expr_cache_is_needed(thd) &&
-      (expr_cache= set_expr_cache(thd, depends_on)))
+      (expr_cache= set_expr_cache(thd)))
     DBUG_RETURN(expr_cache);
   DBUG_RETURN(this);
 }
@@ -1136,9 +1186,11 @@ bool Item_in_subselect::test_limit(st_select_lex_unit *unit_arg)
 
 Item_in_subselect::Item_in_subselect(Item * left_exp,
 				     st_select_lex *select_lex):
-  Item_exists_subselect(), left_expr_cache(0), first_execution(TRUE),
-  optimizer(0), pushed_cond_guards(NULL), in_strategy(0),
-  is_jtbm_merged(FALSE), is_flattenable_semijoin(FALSE), 
+  Item_exists_subselect(), 
+  left_expr_cache(0), first_execution(TRUE), in_strategy(SUBS_NOT_TRANSFORMED),
+  optimizer(0), pushed_cond_guards(NULL), emb_on_expr_nest(NULL),
+  is_jtbm_merged(FALSE), is_flattenable_semijoin(FALSE),
+  is_registered_semijoin(FALSE), 
   upper_item(0)
 {
   DBUG_ENTER("Item_in_subselect::Item_in_subselect");
@@ -1244,7 +1296,7 @@ Item* Item_exists_subselect::expr_cache_insert_transformer(uchar *thd_arg)
     DBUG_RETURN(expr_cache);
 
   if (substype() == EXISTS_SUBS && expr_cache_is_needed(thd) &&
-      (expr_cache= set_expr_cache(thd, depends_on)))
+      (expr_cache= set_expr_cache(thd)))
     DBUG_RETURN(expr_cache);
   DBUG_RETURN(this);
 }
@@ -1398,9 +1450,9 @@ String *Item_in_subselect::val_str(String *str)
 bool Item_in_subselect::val_bool()
 {
   DBUG_ASSERT(fixed == 1);
-  null_value= was_null= FALSE;
   if (forced_const)
     return value;
+  null_value= was_null= FALSE;
   if (exec())
   {
     reset();
@@ -1557,7 +1609,7 @@ Item_in_subselect::single_value_transformer(JOIN *join)
 bool Item_allany_subselect::transform_into_max_min(JOIN *join)
 {
   DBUG_ENTER("Item_allany_subselect::transform_into_max_min");
-  if (!(in_strategy & SUBS_MAXMIN))
+  if (!test_strategy(SUBS_MAXMIN_INJECTED | SUBS_MAXMIN_ENGINE))
     DBUG_RETURN(false);
   Item **place= optimizer->arguments() + 1;
   THD *thd= join->thd;
@@ -1568,11 +1620,20 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
   */
   DBUG_ASSERT(!substitution);
 
-  if (!select_lex->group_list.elements &&
-      !select_lex->having &&
-      !select_lex->with_sum_func &&
-      !(select_lex->next_select()) &&
-      select_lex->table_list.elements)
+  /*
+    Check if optimization with aggregate min/max possible
+    1 There is no aggregate in the subquery
+    2 It is not UNION
+    3 There is tables
+    4 It is not ALL subquery with possible NULLs in the SELECT list
+  */
+  if (!select_lex->group_list.elements &&                /*1*/
+      !select_lex->having &&                             /*1*/
+      !select_lex->with_sum_func &&                      /*1*/
+      !(select_lex->next_select()) &&                    /*2*/
+      select_lex->table_list.elements &&                 /*3*/
+      (!select_lex->ref_pointer_array[0]->maybe_null ||  /*4*/
+       substype() != Item_subselect::ALL_SUBS))          /*4*/
   {
     Item_sum_hybrid *item;
     nesting_map save_allow_sum_func;
@@ -1617,6 +1678,12 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
     if (join->prepare_stage2())
       DBUG_RETURN(true);
     subs= new Item_singlerow_subselect(select_lex);
+
+    /*
+      Remove other strategies if any (we already changed the query and
+      can't apply other strategy).
+    */
+    set_strategy(SUBS_MAXMIN_INJECTED);
   }
   else
   {
@@ -1624,9 +1691,17 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
     subs= item= new Item_maxmin_subselect(thd, this, select_lex, func->l_op());
     if (upper_item)
       upper_item->set_sub_test(item);
+    /*
+      Remove other strategies if any (we already changed the query and
+      can't apply other strategy).
+    */
+    set_strategy(SUBS_MAXMIN_ENGINE);
   }
-  /* fix fields is already called for  left expression */
-  subs= func->create(left_expr, subs);
+  /*
+    The swap is needed for expressions of type 'f1 < ALL ( SELECT ....)'
+    where we want to evaluate the sub query even if f1 would be null.
+  */
+  subs= func->create_swap(*(optimizer->get_cache()), subs);
   thd->change_item_tree(place, subs);
   if (subs->fix_fields(thd, &subs))
     DBUG_RETURN(true);
@@ -1634,11 +1709,6 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
 
   select_lex->master_unit()->uncacheable&= ~UNCACHEABLE_DEPENDENT_INJECTED;
   select_lex->uncacheable&= ~UNCACHEABLE_DEPENDENT_INJECTED;
-  /*
-    Remove other strategies if there was (we already changed the query and
-    can't apply other strategy).
-  */
-  in_strategy= SUBS_MAXMIN;
 
   DBUG_RETURN(false);
 }
@@ -1662,7 +1732,7 @@ bool Item_allany_subselect::is_maxmin_applicable(JOIN *join)
     Check if max/min optimization applicable: It is top item of
     WHERE condition.
   */
-  return (abort_on_null || (upper_item && upper_item->top_level())) &&
+  return (abort_on_null || (upper_item && upper_item->is_top_level_item())) &&
       !join->select_lex->master_unit()->uncacheable && !func->eqne_op();
 }
 
@@ -2212,7 +2282,12 @@ bool Item_in_subselect::inject_in_to_exists_cond(JOIN *join_arg)
     {
       /* The argument list of the top-level AND may change after fix fields. */
       and_args= ((Item_cond*) join_arg->conds)->argument_list();
-      and_args->concat((List<Item> *) &join_arg->cond_equal->current_level);
+      List_iterator<Item_equal> li(join_arg->cond_equal->current_level);
+      Item_equal *elem;
+      while ((elem= li++))
+      {
+        and_args->push_back(elem);
+      }
     }
   }
 
@@ -2340,7 +2415,7 @@ err:
 
 void Item_in_subselect::print(String *str, enum_query_type query_type)
 {
-  if (in_strategy & SUBS_IN_TO_EXISTS)
+  if (test_strategy(SUBS_IN_TO_EXISTS))
     str->append(STRING_WITH_LEN("<exists>"));
   else
   {
@@ -2356,7 +2431,7 @@ bool Item_in_subselect::fix_fields(THD *thd_arg, Item **ref)
   uint outer_cols_num;
   List<Item> *inner_cols;
 
-  if (in_strategy & SUBS_SEMI_JOIN)
+  if (test_strategy(SUBS_SEMI_JOIN))
     return !( (*ref)= new Item_int(1));
 
   /*
@@ -2410,7 +2485,6 @@ bool Item_in_subselect::fix_fields(THD *thd_arg, Item **ref)
     return TRUE;
   if (Item_subselect::fix_fields(thd_arg, ref))
     return TRUE;
-
   fixed= TRUE;
   return FALSE;
 }
@@ -2420,6 +2494,7 @@ void Item_in_subselect::fix_after_pullout(st_select_lex *new_parent, Item **ref)
 {
   left_expr->fix_after_pullout(new_parent, &left_expr);
   Item_subselect::fix_after_pullout(new_parent, ref);
+  used_tables_cache |= left_expr->used_tables();
 }
 
 void Item_in_subselect::update_used_tables()
@@ -2531,8 +2606,8 @@ bool
 Item_allany_subselect::select_transformer(JOIN *join)
 {
   DBUG_ENTER("Item_allany_subselect::select_transformer");
-  DBUG_ASSERT((in_strategy & ~(SUBS_MAXMIN | SUBS_IN_TO_EXISTS)) == 0);
-  in_strategy|= SUBS_IN_TO_EXISTS;
+  DBUG_ASSERT((in_strategy & ~(SUBS_MAXMIN_INJECTED | SUBS_MAXMIN_ENGINE |
+                               SUBS_IN_TO_EXISTS | SUBS_STRATEGY_CHOSEN)) == 0);
   if (upper_item)
     upper_item->show= 1;
   DBUG_RETURN(select_in_like_transformer(join));
@@ -2541,7 +2616,7 @@ Item_allany_subselect::select_transformer(JOIN *join)
 
 void Item_allany_subselect::print(String *str, enum_query_type query_type)
 {
-  if (in_strategy & SUBS_IN_TO_EXISTS)
+  if (test_strategy(SUBS_IN_TO_EXISTS))
     str->append(STRING_WITH_LEN("<exists>"));
   else
   {
@@ -2920,7 +2995,7 @@ int subselect_single_select_engine::exec()
     executed= 1;
     thd->where= save_where;
     thd->lex->current_select= save_select;
-    DBUG_RETURN(join->error||thd->is_fatal_error);
+    DBUG_RETURN(join->error || thd->is_fatal_error || thd->is_error());
   }
   thd->where= save_where;
   thd->lex->current_select= save_select;
@@ -3856,9 +3931,10 @@ subselect_hash_sj_engine::get_strategy_using_data()
       bitmap_set_bit(&non_null_key_parts, i);
       --count_partial_match_columns;
     }
-    if (result_sink->get_null_count_of_col(i) ==
-               tmp_table->file->stats.records)
+    if (result_sink->get_null_count_of_col(i) == tmp_table->file->stats.records)
       ++count_null_only_columns;
+    if (result_sink->get_null_count_of_col(i))
+      ++count_columns_with_nulls;
   }
 
   /* If no column contains NULLs use regular hash index lookups. */
@@ -4334,7 +4410,13 @@ double get_fanout_with_deps(JOIN *join, table_map tset)
   for (JOIN_TAB *tab= first_top_level_tab(join, WITHOUT_CONST_TABLES); tab;
        tab= next_top_level_tab(join, tab))
   {
-    if ((tab->table->map & checked_deps) && !tab->emb_sj_nest && 
+    /* 
+      Ignore SJM nests. They have tab->table==NULL. There is no point to walk
+      inside them, because GROUP BY clause cannot refer to tables from within
+      subquery.
+    */
+    if (!tab->is_sjm_nest() && (tab->table->map & checked_deps) && 
+        !tab->emb_sj_nest && 
         tab->records_read != 0)
     {
       fanout *= rows2double(tab->records_read);
@@ -4524,7 +4606,8 @@ int subselect_hash_sj_engine::exec()
   /* The subquery should be optimized, and materialized only once. */
   DBUG_ASSERT(materialize_join->optimized && !is_materialized);
   materialize_join->exec();
-  if ((res= test(materialize_join->error || thd->is_fatal_error)))
+  if ((res= test(materialize_join->error || thd->is_fatal_error ||
+                 thd->is_error())))
     goto err;
 
   /*
@@ -4565,29 +4648,50 @@ int subselect_hash_sj_engine::exec()
   if (strategy == PARTIAL_MATCH)
   {
     uint count_pm_keys; /* Total number of keys needed for partial matching. */
-    MY_BITMAP *nn_key_parts; /* The key parts of the only non-NULL index. */
-    uint covering_null_row_width;
+    MY_BITMAP *nn_key_parts= NULL; /* Key parts of the only non-NULL index. */
+    uint count_non_null_columns= 0; /* Number of columns in nn_key_parts. */
+    bool has_covering_null_row;
+    bool has_covering_null_columns;
     select_materialize_with_stats *result_sink=
       (select_materialize_with_stats *) result;
+    uint field_count= tmp_table->s->fields;
 
-    nn_key_parts= (count_partial_match_columns < tmp_table->s->fields) ?
-                  &non_null_key_parts : NULL;
+    if (count_partial_match_columns < field_count)
+    {
+      nn_key_parts= &non_null_key_parts;
+      count_non_null_columns= bitmap_bits_set(nn_key_parts);
+    }
+    has_covering_null_row= (result_sink->get_max_nulls_in_row() == field_count);
+    has_covering_null_columns= (count_non_null_columns +
+                                count_null_only_columns == field_count);
 
-    if (result_sink->get_max_nulls_in_row() ==
-        tmp_table->s->fields -
-        (nn_key_parts ? bitmap_bits_set(nn_key_parts) : 0))
-      covering_null_row_width= result_sink->get_max_nulls_in_row();
-    else
-      covering_null_row_width= 0;
+    if (has_covering_null_row && has_covering_null_columns)
+    {
+      /*
+        The whole table consist of only NULL values. The result of IN is
+        a constant UNKNOWN.
+      */
+      DBUG_ASSERT(tmp_table->file->stats.records == 1);
+      item_in->value= 0;
+      item_in->null_value= 1;
+      item_in->make_const();
+      item_in->set_first_execution();
+      DBUG_RETURN(FALSE);
+    }
 
-    if (covering_null_row_width)
-      count_pm_keys= nn_key_parts ? 1 : 0;
+    if (has_covering_null_row)
+    {
+      DBUG_ASSERT(count_partial_match_columns = field_count);
+      count_pm_keys= 0;
+    }
+    else if (has_covering_null_columns)
+      count_pm_keys= 1;
     else
       count_pm_keys= count_partial_match_columns - count_null_only_columns +
-        (nn_key_parts ? 1 : 0);
+                     (nn_key_parts ? 1 : 0);
 
     choose_partial_match_strategy(test(nn_key_parts),
-                                  test(covering_null_row_width),
+                                  has_covering_null_row,
                                   &partial_match_key_parts);
     DBUG_ASSERT(strategy == PARTIAL_MATCH_MERGE ||
                 strategy == PARTIAL_MATCH_SCAN);
@@ -4597,7 +4701,9 @@ int subselect_hash_sj_engine::exec()
         new subselect_rowid_merge_engine(thd, (subselect_uniquesubquery_engine*)
                                          lookup_engine, tmp_table,
                                          count_pm_keys,
-                                         covering_null_row_width,
+                                         has_covering_null_row,
+                                         has_covering_null_columns,
+                                         count_columns_with_nulls,
                                          item, result,
                                          semi_join_conds->argument_list());
       if (!pm_engine ||
@@ -4622,7 +4728,9 @@ int subselect_hash_sj_engine::exec()
                                             lookup_engine, tmp_table,
                                             item, result,
                                             semi_join_conds->argument_list(),
-                                            covering_null_row_width)))
+                                            has_covering_null_row,
+                                            has_covering_null_columns,
+                                            count_columns_with_nulls)))
       {
         /* This is an irrecoverable error. */
         res= 1;
@@ -5058,49 +5166,56 @@ subselect_partial_match_engine::subselect_partial_match_engine(
   TABLE *tmp_table_arg, Item_subselect *item_arg,
   select_result_interceptor *result_arg,
   List<Item> *equi_join_conds_arg,
-  uint covering_null_row_width_arg)
+  bool has_covering_null_row_arg,
+  bool has_covering_null_columns_arg,
+  uint count_columns_with_nulls_arg)
   :subselect_engine(thd_arg, item_arg, result_arg),
    tmp_table(tmp_table_arg), lookup_engine(engine_arg),
    equi_join_conds(equi_join_conds_arg),
-   covering_null_row_width(covering_null_row_width_arg)
+   has_covering_null_row(has_covering_null_row_arg),
+   has_covering_null_columns(has_covering_null_columns_arg),
+   count_columns_with_nulls(count_columns_with_nulls_arg)
 {}
 
 
 int subselect_partial_match_engine::exec()
 {
   Item_in_subselect *item_in= (Item_in_subselect *) item;
-  int res;
+  int copy_res, lookup_res;
 
   /* Try to find a matching row by index lookup. */
-  res= lookup_engine->copy_ref_key_simple();
-  if (res == -1)
+  copy_res= lookup_engine->copy_ref_key_simple();
+  if (copy_res == -1)
   {
     /* The result is FALSE based on the outer reference. */
     item_in->value= 0;
     item_in->null_value= 0;
     return 0;
   }
-  else if (res == 0)
+  else if (copy_res == 0)
   {
     /* Search for a complete match. */
-    if ((res= lookup_engine->index_lookup()))
+    if ((lookup_res= lookup_engine->index_lookup()))
     {
       /* An error occured during lookup(). */
       item_in->value= 0;
       item_in->null_value= 0;
-      return res;
+      return lookup_res;
     }
-    else if (item_in->value)
+    else if (item_in->value || !count_columns_with_nulls)
     {
       /*
         A complete match was found, the result of IN is TRUE.
+        If no match was found, and there are no NULLs in the materialized
+        subquery, then the result is guaranteed to be false because this
+        branch is executed when the outer reference has no NULLs as well.
         Notice: (this->item == lookup_engine->item)
       */
       return 0;
     }
   }
 
-  if (covering_null_row_width == tmp_table->s->fields)
+  if (has_covering_null_row)
   {
     /*
       If there is a NULL-only row that coveres all columns the result of IN
@@ -5164,7 +5279,6 @@ void subselect_partial_match_engine::print(String *str,
 /*
   @param non_null_key_parts  
   @param partial_match_key_parts  A union of all single-column NULL key parts.
-  @param count_partial_match_columns Number of NULL keyparts (set bits above).
 
   @retval FALSE  the engine was initialized successfully
   @retval TRUE   there was some (memory allocation) error during initialization,
@@ -5185,20 +5299,26 @@ subselect_rowid_merge_engine::init(MY_BITMAP *non_null_key_parts,
   Item_in_subselect *item_in= (Item_in_subselect*) item;
   int error;
 
-  if (keys_count == 0)
+  if (merge_keys_count == 0)
   {
+    DBUG_ASSERT(bitmap_bits_set(partial_match_key_parts) == 0 ||
+                has_covering_null_row);
     /* There is nothing to initialize, we will only do regular lookups. */
     return FALSE;
   }
 
-  DBUG_ASSERT(!covering_null_row_width || (covering_null_row_width &&
-                                           keys_count == 1 &&
-                                           non_null_key_parts));
+  /*
+    If all nullable columns contain only NULLs, there must be one index
+    over all non-null columns.
+  */
+  DBUG_ASSERT(!has_covering_null_columns ||
+              (has_covering_null_columns &&
+               merge_keys_count == 1 && non_null_key_parts));
   /*
     Allocate buffers to hold the merged keys and the mapping between rowids and
     row numbers.
   */
-  if (!(merge_keys= (Ordered_key**) thd->alloc(keys_count *
+  if (!(merge_keys= (Ordered_key**) thd->alloc(merge_keys_count *
                                                sizeof(Ordered_key*))) ||
       !(row_num_to_rowid= (uchar*) my_malloc((size_t)(row_count * rowid_length),
         MYF(MY_WME))))
@@ -5217,15 +5337,13 @@ subselect_rowid_merge_engine::init(MY_BITMAP *non_null_key_parts,
   }
 
   /*
-    If there is a covering NULL row, the only key that is needed is the
-    only non-NULL key that is already created above. We create keys on
-    NULL-able columns only if there is no covering NULL row.
+    If all nullable columns contain NULLs, the only key that is needed is the
+    only non-NULL key that is already created above.
   */
-  if (!covering_null_row_width)
+  if (!has_covering_null_columns)
   {
-    if (bitmap_init_memroot(&matching_keys, keys_count, thd->mem_root) ||
-        bitmap_init_memroot(&matching_outer_cols, keys_count, thd->mem_root) ||
-        bitmap_init_memroot(&null_only_columns, keys_count, thd->mem_root))
+    if (bitmap_init_memroot(&matching_keys, merge_keys_count, thd->mem_root) ||
+        bitmap_init_memroot(&matching_outer_cols, merge_keys_count, thd->mem_root))
       return TRUE;
 
     /*
@@ -5234,31 +5352,25 @@ subselect_rowid_merge_engine::init(MY_BITMAP *non_null_key_parts,
     */
     for (uint i= 0; i < partial_match_key_parts->n_bits; i++)
     {
-      if (!bitmap_is_set(partial_match_key_parts, i))
+      /* Skip columns that have no NULLs, or contain only NULLs. */
+      if (!bitmap_is_set(partial_match_key_parts, i) ||
+          result_sink->get_null_count_of_col(i) == row_count)
         continue;
 
-      if (result_sink->get_null_count_of_col(i) == row_count)
-      {
-        bitmap_set_bit(&null_only_columns, cur_keyid);
-        continue;
-      }
-      else
-      {
-        merge_keys[cur_keyid]= new Ordered_key(
+      merge_keys[cur_keyid]= new Ordered_key(
                                      cur_keyid, tmp_table,
                                      item_in->left_expr->element_index(i),
                                      result_sink->get_null_count_of_col(i),
                                      result_sink->get_min_null_of_col(i),
                                      result_sink->get_max_null_of_col(i),
                                      row_num_to_rowid);
-        if (merge_keys[cur_keyid]->init(i))
-          return TRUE;
-        merge_keys[cur_keyid]->first();
-      }
+      if (merge_keys[cur_keyid]->init(i))
+        return TRUE;
+      merge_keys[cur_keyid]->first();
       ++cur_keyid;
     }
   }
-  DBUG_ASSERT(cur_keyid == keys_count);
+  DBUG_ASSERT(cur_keyid == merge_keys_count);
 
   /* Populate the indexes with data from the temporary table. */
   if (tmp_table->file->ha_rnd_init_with_error(1))
@@ -5299,7 +5411,7 @@ subselect_rowid_merge_engine::init(MY_BITMAP *non_null_key_parts,
       non_null_key->add_key(cur_rownum);
     }
 
-    for (uint i= (non_null_key ? 1 : 0); i < keys_count; i++)
+    for (uint i= (non_null_key ? 1 : 0); i < merge_keys_count; i++)
     {
       /*
         Check if the first and only indexed column contains NULL in the curent
@@ -5316,14 +5428,14 @@ subselect_rowid_merge_engine::init(MY_BITMAP *non_null_key_parts,
   tmp_table->file->ha_rnd_end();
 
   /* Sort all the keys by their NULL selectivity. */
-  my_qsort(merge_keys, keys_count, sizeof(Ordered_key*),
+  my_qsort(merge_keys, merge_keys_count, sizeof(Ordered_key*),
            (qsort_cmp) cmp_keys_by_null_selectivity);
 
   /* Sort the keys in each of the indexes. */
-  for (uint i= 0; i < keys_count; i++)
+  for (uint i= 0; i < merge_keys_count; i++)
     merge_keys[i]->sort_keys();
 
-  if (init_queue(&pq, keys_count, 0, FALSE,
+  if (init_queue(&pq, merge_keys_count, 0, FALSE,
                  subselect_rowid_merge_engine::cmp_keys_by_cur_rownum, NULL,
                  0, 0))
     return TRUE;
@@ -5335,10 +5447,10 @@ subselect_rowid_merge_engine::init(MY_BITMAP *non_null_key_parts,
 subselect_rowid_merge_engine::~subselect_rowid_merge_engine()
 {
   /* None of the resources below is allocated if there are no ordered keys. */
-  if (keys_count)
+  if (merge_keys_count)
   {
     my_free((char*) row_num_to_rowid, MYF(0));
-    for (uint i= 0; i < keys_count; i++)
+    for (uint i= 0; i < merge_keys_count; i++)
       delete merge_keys[i];
     delete_queue(&pq);
     if (tmp_table->file->inited == handler::RND)
@@ -5396,6 +5508,10 @@ subselect_rowid_merge_engine::cmp_keys_by_cur_rownum(void *arg,
   Check if certain table row contains a NULL in all columns for which there is
   no match in the corresponding value index.
 
+  @note
+  There is no need to check the columns that contain only NULLs, because
+  those are guaranteed to match.
+
   @retval TRUE if a NULL row exists
   @retval FALSE otherwise
 */
@@ -5403,16 +5519,14 @@ subselect_rowid_merge_engine::cmp_keys_by_cur_rownum(void *arg,
 bool subselect_rowid_merge_engine::test_null_row(rownum_t row_num)
 {
   Ordered_key *cur_key;
-  uint cur_id;
-  for (uint i = 0; i < keys_count; i++)
+  for (uint i = 0; i < merge_keys_count; i++)
   {
     cur_key= merge_keys[i];
-    cur_id= cur_key->get_keyid();
-    if (bitmap_is_set(&matching_keys, cur_id))
+    if (bitmap_is_set(&matching_keys, cur_key->get_keyid()))
     {
       /*
-        The key 'i' (with id 'cur_keyid') already matches a value in row 'row_num',
-        thus we skip it as it can't possibly match a NULL.
+        The key 'i' (with id 'cur_keyid') already matches a value in row
+        'row_num', thus we skip it as it can't possibly match a NULL.
       */
       continue;
     }
@@ -5435,6 +5549,8 @@ bool subselect_rowid_merge_engine::partial_match()
   Ordered_key *cur_key;
   rownum_t cur_row_num;
   uint count_nulls_in_search_key= 0;
+  uint max_covering_null_row_len=
+    ((select_materialize_with_stats *) result)->get_max_nulls_in_row();
   bool res= FALSE;
 
   /* If there is a non-NULL key, it must be the first key in the keys array. */
@@ -5457,11 +5573,10 @@ bool subselect_rowid_merge_engine::partial_match()
   }
 
   /*
-    If there is a NULL (sub)row that covers all NULL-able columns,
-    then there is a guranteed partial match, and we don't need to search
-    for the matching row.
-   */
-  if (covering_null_row_width)
+    If all nullable columns contain only NULLs, then there is a guranteed
+    partial match, and we don't need to search for a matching row.
+  */
+  if (has_covering_null_columns)
   {
     res= TRUE;
     goto end;
@@ -5473,7 +5588,7 @@ bool subselect_rowid_merge_engine::partial_match()
     Do not add the non_null_key, since it was already processed above.
   */
   bitmap_clear_all(&matching_outer_cols);
-  for (uint i= test(non_null_key); i < keys_count; i++)
+  for (uint i= test(non_null_key); i < merge_keys_count; i++)
   {
     DBUG_ASSERT(merge_keys[i]->get_column_count() == 1);
     if (merge_keys[i]->get_search_key(0)->null_value)
@@ -5501,8 +5616,16 @@ bool subselect_rowid_merge_engine::partial_match()
     If there is no NULL (sub)row that covers all NULL columns, and there is no
     single match for any of the NULL columns, the result is FALSE.
   */
-  if (pq.elements - test(non_null_key) == 0)
+  if ((pq.elements == 1 && non_null_key &&
+       max_covering_null_row_len < merge_keys_count - 1) ||
+      pq.elements == 0)
   {
+    if (pq.elements == 0)
+    {
+      DBUG_ASSERT(!non_null_key); /* Must follow from the logic of this method */
+      /* This case must be handled by subselect_partial_match_engine::exec() */
+      DBUG_ASSERT(max_covering_null_row_len != tmp_table->s->fields);
+    }
     res= FALSE;
     goto end;
   }
@@ -5511,7 +5634,6 @@ bool subselect_rowid_merge_engine::partial_match()
 
   min_key= (Ordered_key*) queue_remove_top(&pq);
   min_row_num= min_key->current();
-  bitmap_copy(&matching_keys, &null_only_columns);
   bitmap_set_bit(&matching_keys, min_key->get_keyid());
   bitmap_union(&matching_keys, &matching_outer_cols);
   if (min_key->next_same())
@@ -5547,7 +5669,7 @@ bool subselect_rowid_merge_engine::partial_match()
       {
         min_key= cur_key;
         min_row_num= cur_row_num;
-        bitmap_copy(&matching_keys, &null_only_columns);
+        bitmap_clear_all(&matching_keys);
         bitmap_set_bit(&matching_keys, min_key->get_keyid());
         bitmap_union(&matching_keys, &matching_outer_cols);
       }
@@ -5568,6 +5690,8 @@ bool subselect_rowid_merge_engine::partial_match()
   DBUG_ASSERT(FALSE);
 
 end:
+  if (!has_covering_null_columns)
+    bitmap_clear_all(&matching_keys);
   queue_remove_all(&pq);
   tmp_table->file->ha_rnd_end();
   return res;
@@ -5580,10 +5704,14 @@ subselect_table_scan_engine::subselect_table_scan_engine(
   Item_subselect *item_arg,
   select_result_interceptor *result_arg,
   List<Item> *equi_join_conds_arg,
-  uint covering_null_row_width_arg)
+  bool has_covering_null_row_arg,
+  bool has_covering_null_columns_arg,
+  uint count_columns_with_nulls_arg)
   :subselect_partial_match_engine(thd_arg, engine_arg, tmp_table_arg, item_arg,
                                   result_arg, equi_join_conds_arg,
-                                  covering_null_row_width_arg)
+                                  has_covering_null_row_arg,
+                                  has_covering_null_columns_arg,
+                                  count_columns_with_nulls_arg)
 {}
 
 
@@ -5622,10 +5750,6 @@ bool subselect_table_scan_engine::partial_match()
 
   tmp_table->file->extra_opt(HA_EXTRA_CACHE,
                              current_thd->variables.read_buff_size);
-  /*
-  TIMOUR:
-  scan_table() also calls "table->null_row= 0;", why, do we need it?
-  */
   for (;;)
   {
     error= tmp_table->file->ha_rnd_next(tmp_table->record[0]);
@@ -5674,3 +5798,4 @@ end:
 void subselect_table_scan_engine::cleanup()
 {
 }
+
diff --git a/sql/item_subselect.h b/sql/item_subselect.h
index 5342e4e754e..0ec0969e0ae 100644
--- a/sql/item_subselect.h
+++ b/sql/item_subselect.h
@@ -52,6 +52,17 @@ protected:
   
   bool inside_first_fix_fields;
   bool done_first_fix_fields;
+  Item *expr_cache;
+  /*
+    Set to TRUE if at optimization or execution time we determine that this
+    item's value is a constant. We need this member because it is not possible
+    to substitute 'this' with a constant item.
+  */
+  bool forced_const;
+#ifndef DBUG_OFF
+  /* Count the number of times this subquery predicate has been executed. */
+  uint exec_counter;
+#endif
 public:
   /* 
     Used inside Item_subselect::fix_fields() according to this scenario:
@@ -66,19 +77,13 @@ public:
         substitution= NULL;
       < Item_subselect::fix_fields
   */
+  /* TODO make this protected member again. */
   Item *substitution;
-  /* unit of subquery */
-  st_select_lex_unit *unit;
-  Item *expr_cache;
   /* engine that perform execution of subselect (single select or union) */
+  /* TODO make this protected member again. */
   subselect_engine *engine;
-  /*
-    Set to TRUE if at optimization or execution time we determine that this
-    item's value is a constant. We need this member because it is not possible
-    to substitute 'this' with a constant item.
-  */
-  bool forced_const;
-
+  /* unit of subquery */
+  st_select_lex_unit *unit;
   /* A reference from inside subquery predicate to somewhere outside of it */
   class Ref_to_outside : public Sql_alloc
   {
@@ -97,14 +102,6 @@ public:
   List<Ref_to_outside> upper_refs;
   st_select_lex *parent_select;
 
-  /**
-     List of references on items subquery depends on (externally resolved);
-
-     @note We can't store direct links on Items because it could be
-           substituted with other item (for example for grouping).
-   */
-  List<Item*> depends_on;
-
   /*
    TRUE<=>Table Elimination has made it redundant to evaluate this select
           (and so it is not part of QEP, etc)
@@ -220,6 +217,7 @@ public:
     @retval FALSE otherwise
   */
   bool is_expensive_processor(uchar *arg) { return TRUE; }
+
   /**
     Get the SELECT_LEX structure associated with this Item.
     @return the SELECT_LEX structure associated with this Item
@@ -227,6 +225,7 @@ public:
   st_select_lex* get_select_lex();
   const char *func_name() const { DBUG_ASSERT(0); return "subselect"; }
   virtual bool expr_cache_is_needed(THD *);
+  virtual void get_cache_parameters(List<Item> &parameters);
 
   friend class select_result_interceptor;
   friend class Item_in_optimizer;
@@ -354,14 +353,19 @@ TABLE_LIST * const NO_JOIN_NEST=(TABLE_LIST*)0x1;
   based on user-set optimizer switches, semantic analysis and cost comparison.
 */
 #define SUBS_NOT_TRANSFORMED 0 /* No execution method was chosen for this IN. */
-#define SUBS_SEMI_JOIN 1       /* IN was converted to semi-join. */
-#define SUBS_IN_TO_EXISTS 2    /* IN was converted to correlated EXISTS. */
-#define SUBS_MATERIALIZATION 4 /* Execute IN via subquery materialization. */
+/* The Final decision about the strategy is made. */
+#define SUBS_STRATEGY_CHOSEN 1
+#define SUBS_SEMI_JOIN 2       /* IN was converted to semi-join. */
+#define SUBS_IN_TO_EXISTS 4    /* IN was converted to correlated EXISTS. */
+#define SUBS_MATERIALIZATION 8 /* Execute IN via subquery materialization. */
 /* Partial matching substrategies of MATERIALIZATION. */
-#define SUBS_PARTIAL_MATCH_ROWID_MERGE 8
-#define SUBS_PARTIAL_MATCH_TABLE_SCAN 16
+#define SUBS_PARTIAL_MATCH_ROWID_MERGE 16
+#define SUBS_PARTIAL_MATCH_TABLE_SCAN 32
 /* ALL/ANY will be transformed with max/min optimization */
-#define SUBS_MAXMIN 32
+/*   The subquery has not aggregates, transform it into a MAX/MIN query. */
+#define SUBS_MAXMIN_INJECTED 64
+/*   The subquery has aggregates, use a special max/min subselect engine. */
+#define SUBS_MAXMIN_ENGINE 128
 
 
 /**
@@ -396,6 +400,8 @@ protected:
   Item *expr;
   bool was_null;
   bool abort_on_null;
+  /* A bitmap of possible execution strategies for an IN predicate. */
+  uchar in_strategy;
 public:
   Item_in_optimizer *optimizer;
 protected:
@@ -426,7 +432,6 @@ public:
       join nest pointer - the predicate is an AND-part of ON expression
                           of a join nest   
       NULL              - for all other locations
-    See also THD::emb_on_expr_nest.
   */
   TABLE_LIST *emb_on_expr_nest;
   /*
@@ -441,11 +446,7 @@ public:
   */
   bool sjm_scan_allowed;
   double jtbm_read_time;
-  double jtbm_record_count;
-
-  /* A bitmap of possible execution strategies for an IN predicate. */
-  uchar in_strategy;
-
+  double jtbm_record_count;   
   bool is_jtbm_merged;
 
   /*
@@ -454,6 +455,11 @@ public:
   bool is_flattenable_semijoin;
 
   /*
+    TRUE<=>registered in the list of semijoins in outer select
+  */
+  bool is_registered_semijoin;
+  
+  /*
     Used to determine how this subselect item is represented in the item tree,
     in case there is a need to locate it there and replace with something else.
     Two options are possible:
@@ -481,8 +487,8 @@ public:
   Item_in_subselect(Item * left_expr, st_select_lex *select_lex);
   Item_in_subselect()
     :Item_exists_subselect(), left_expr_cache(0), first_execution(TRUE),
-     abort_on_null(0), optimizer(0),
-    pushed_cond_guards(NULL), func(NULL), in_strategy(SUBS_NOT_TRANSFORMED),
+     abort_on_null(0), in_strategy(SUBS_NOT_TRANSFORMED), optimizer(0),
+    pushed_cond_guards(NULL), func(NULL), emb_on_expr_nest(NULL), 
     is_jtbm_merged(FALSE),
     upper_item(0)
     {}
@@ -526,6 +532,70 @@ public:
     user.
   */
   int get_identifier();
+
+  void mark_as_condition_AND_part(TABLE_LIST *embedding)
+  {
+    emb_on_expr_nest= embedding;
+  }
+
+  bool test_strategy(uchar strategy)
+  { return test(in_strategy & strategy); }
+
+  /**
+    Test that the IN strategy was chosen for execution. This is so
+    when the CHOSEN flag is ON, and there is no other strategy.
+  */
+  bool test_set_strategy(uchar strategy)
+  {
+    DBUG_ASSERT(strategy == SUBS_SEMI_JOIN ||
+                strategy == SUBS_IN_TO_EXISTS ||
+                strategy == SUBS_MATERIALIZATION ||
+                strategy == SUBS_PARTIAL_MATCH_ROWID_MERGE ||
+                strategy == SUBS_PARTIAL_MATCH_TABLE_SCAN ||
+                strategy == SUBS_MAXMIN_INJECTED ||
+                strategy == SUBS_MAXMIN_ENGINE);
+    return ((in_strategy & SUBS_STRATEGY_CHOSEN) &&
+            (in_strategy & ~SUBS_STRATEGY_CHOSEN) == strategy);
+  }
+
+  bool is_set_strategy()
+  { return test(in_strategy & SUBS_STRATEGY_CHOSEN); }
+
+  bool has_strategy()
+  { return in_strategy != SUBS_NOT_TRANSFORMED; }
+
+  void add_strategy (uchar strategy)
+  {
+    DBUG_ASSERT(strategy != SUBS_NOT_TRANSFORMED);
+    DBUG_ASSERT(!(strategy & SUBS_STRATEGY_CHOSEN));
+    /*
+      TODO: PS re-execution breaks this condition, because
+      check_and_do_in_subquery_rewrites() is called for each reexecution
+      and re-adds the same strategies.
+      DBUG_ASSERT(!(in_strategy & SUBS_STRATEGY_CHOSEN));
+    */
+    in_strategy|= strategy;
+  }
+
+  void reset_strategy(uchar strategy)
+  {
+    DBUG_ASSERT(strategy != SUBS_NOT_TRANSFORMED);
+    in_strategy= strategy;
+  }
+
+  void set_strategy(uchar strategy)
+  {
+    /* Check that only one strategy is set for execution. */
+    DBUG_ASSERT(strategy == SUBS_SEMI_JOIN ||
+                strategy == SUBS_IN_TO_EXISTS ||
+                strategy == SUBS_MATERIALIZATION ||
+                strategy == SUBS_PARTIAL_MATCH_ROWID_MERGE ||
+                strategy == SUBS_PARTIAL_MATCH_TABLE_SCAN ||
+                strategy == SUBS_MAXMIN_INJECTED ||
+                strategy == SUBS_MAXMIN_ENGINE);
+    in_strategy= (SUBS_STRATEGY_CHOSEN | strategy);
+  }
+
   friend class Item_ref_null_helper;
   friend class Item_is_not_null_test;
   friend class Item_in_optimizer;
@@ -545,6 +615,7 @@ public:
   Item_allany_subselect(Item * left_expr, chooser_compare_func_creator fc,
                         st_select_lex *select_lex, bool all);
 
+  void cleanup();
   // only ALL subquery has upper not
   subs_type substype() { return all?ALL_SUBS:ANY_SUBS; }
   bool select_transformer(JOIN *join);
@@ -862,7 +933,7 @@ public:
       tmp_table(NULL), is_materialized(FALSE), materialize_engine(old_engine),
       materialize_join(NULL),  semi_join_conds(NULL), lookup_engine(NULL),
       count_partial_match_columns(0), count_null_only_columns(0),
-      strategy(UNDEFINED)
+      count_columns_with_nulls(0), strategy(UNDEFINED)
   {}
   ~subselect_hash_sj_engine();
 
@@ -900,6 +971,7 @@ protected:
   MY_BITMAP partial_match_key_parts;
   uint count_partial_match_columns;
   uint count_null_only_columns;
+  uint count_columns_with_nulls;
   /* Possible execution strategies that can be used to compute hash semi-join.*/
   enum exec_strategy {
     UNDEFINED,
@@ -1126,11 +1198,19 @@ protected:
   /* A list of equalities between each pair of IN operands. */
   List<Item> *equi_join_conds;
   /*
-    If there is a row, such that all its NULL-able components are NULL, this
-    member is set to the number of covered columns. If there is no covering
-    row, then this is 0.
+    True if there is an all NULL row in tmp_table. If so, then if there is
+    no complete match, there is a guaranteed partial match.
+  */
+  bool has_covering_null_row;
+
+  /*
+    True if all nullable columns of tmp_table consist of only NULL values.
+    If so, then if there is a match in the non-null columns, there is a
+    guaranteed partial match.
   */
-  uint covering_null_row_width;
+  bool has_covering_null_columns;
+  uint count_columns_with_nulls;
+
 protected:
   virtual bool partial_match()= 0;
 public:
@@ -1139,7 +1219,9 @@ public:
                                  TABLE *tmp_table_arg, Item_subselect *item_arg,
                                  select_result_interceptor *result_arg,
                                  List<Item> *equi_join_conds_arg,
-                                 uint covering_null_row_width_arg);
+                                 bool has_covering_null_row_arg,
+                                 bool has_covering_null_columns_arg,
+                                 uint count_columns_with_nulls_arg);
   int prepare() { return 0; }
   int exec();
   void fix_length_and_dec(Item_cache**) {}
@@ -1187,11 +1269,6 @@ protected:
   */
   MY_BITMAP matching_outer_cols;
   /*
-    Columns that consist of only NULLs. Such columns match any value.
-    Computed once per query execution.
-  */
-  MY_BITMAP null_only_columns;
-  /*
     Indexes of row numbers, sorted by <column_value, row_number>. If an
     index may contain NULLs, the NULLs are stored efficiently in a bitmap.
 
@@ -1200,13 +1277,13 @@ protected:
     non-NULL columns, it is contained in keys[0].
   */
   Ordered_key **merge_keys;
-  /* The number of elements in keys. */
-  uint keys_count;
+  /* The number of elements in merge_keys. */
+  uint merge_keys_count;
   /*
     An index on all non-NULL columns of 'tmp_table'. The index has the
     logical form: <[v_i1 | ... | v_ik], rownum>. It allows to find the row
     number where the columns c_i1,...,c1_k contain the values v_i1,...,v_ik.
-    If such an index exists, it is always the first element of 'keys'.
+    If such an index exists, it is always the first element of 'merge_keys'.
   */
   Ordered_key *non_null_key;
   /*
@@ -1231,15 +1308,19 @@ protected:
 public:
   subselect_rowid_merge_engine(THD *thd_arg,
                                subselect_uniquesubquery_engine *engine_arg,
-                               TABLE *tmp_table_arg, uint keys_count_arg,
-                               uint covering_null_row_width_arg,
+                               TABLE *tmp_table_arg, uint merge_keys_count_arg,
+                               bool has_covering_null_row_arg,
+                               bool has_covering_null_columns_arg,
+                               uint count_columns_with_nulls_arg,
                                Item_subselect *item_arg,
                                select_result_interceptor *result_arg,
                                List<Item> *equi_join_conds_arg)
     :subselect_partial_match_engine(thd_arg, engine_arg, tmp_table_arg,
                                     item_arg, result_arg, equi_join_conds_arg,
-                                    covering_null_row_width_arg),
-    keys_count(keys_count_arg), non_null_key(NULL)
+                                    has_covering_null_row_arg,
+                                    has_covering_null_columns_arg,
+                                    count_columns_with_nulls_arg),
+    merge_keys_count(merge_keys_count_arg), non_null_key(NULL)
   {}
   ~subselect_rowid_merge_engine();
   bool init(MY_BITMAP *non_null_key_parts, MY_BITMAP *partial_match_key_parts);
@@ -1258,7 +1339,9 @@ public:
                               TABLE *tmp_table_arg, Item_subselect *item_arg,
                               select_result_interceptor *result_arg,
                               List<Item> *equi_join_conds_arg,
-                              uint covering_null_row_width_arg);
+                              bool has_covering_null_row_arg,
+                              bool has_covering_null_columns_arg,
+                              uint count_columns_with_nulls_arg);
   void cleanup();
   virtual enum_engine_type engine_type() { return TABLE_SCAN_ENGINE; }
 };
diff --git a/sql/item_sum.cc b/sql/item_sum.cc
index e7fe2095481..063406990cb 100644
--- a/sql/item_sum.cc
+++ b/sql/item_sum.cc
@@ -319,7 +319,6 @@ bool Item_sum::register_sum_func(THD *thd, Item **ref)
   if (aggr_level >= 0)
   {
     ref_by= ref;
-    thd->lex->current_select->register_dependency_item(aggr_sel, ref);
     /* Add the object to the list of registered objects assigned to aggr_sel */
     if (!aggr_sel->inner_sum_func_list)
       next= this;
@@ -356,6 +355,18 @@ bool Item_sum::register_sum_func(THD *thd, Item **ref)
 }
 
 
+bool Item_sum::collect_outer_ref_processor(uchar *param)
+{
+  Collect_deps_prm *prm= (Collect_deps_prm *)param;
+  SELECT_LEX *ds;
+  if ((ds= depended_from()) &&
+      ds->nest_level_base == prm->nest_level_base &&
+      ds->nest_level < prm->nest_level)
+    prm->parameters->add_unique(this, &cmp_items);
+  return FALSE;
+}
+
+
 Item_sum::Item_sum(List<Item> &list) :arg_count(list.elements), 
   forced_const(FALSE)
 {
@@ -2587,7 +2598,7 @@ bool Item_sum_count_distinct::add()
     */
     return tree->unique_add(table->record[0] + table->s->null_bytes);
   }
-  if ((error= table->file->ha_write_row(table->record[0])) &&
+  if ((error= table->file->ha_write_tmp_row(table->record[0])) &&
       table->file->is_fatal_error(error, HA_CHECK_DUP))
     return TRUE;
   return FALSE;
diff --git a/sql/item_sum.h b/sql/item_sum.h
index 851b77ddeae..0d557a030d8 100644
--- a/sql/item_sum.h
+++ b/sql/item_sum.h
@@ -374,6 +374,7 @@ public:
   virtual Field *create_tmp_field(bool group, TABLE *table,
                                   uint convert_blob_length);
   bool walk(Item_processor processor, bool walk_subquery, uchar *argument);
+  virtual bool collect_outer_ref_processor(uchar *param);
   bool init_sum_func_check(THD *thd);
   bool check_sum_func(THD *thd, Item **ref);
   bool register_sum_func(THD *thd, Item **ref);
@@ -387,6 +388,7 @@ public:
   {
     return trace_unsupported_by_check_vcol_func_processor(func_name()); 
   }
+  bool clear_sum_processor(uchar *arg) { clear(); return 0; }
 };
 
 
diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc
index b080f2e9707..3a1e3a93f72 100644
--- a/sql/item_timefunc.cc
+++ b/sql/item_timefunc.cc
@@ -1370,7 +1370,8 @@ longlong Item_temporal_func::val_int()
   MYSQL_TIME ltime;
   if (get_date(&ltime, TIME_FUZZY_DATE))
     return 0;
-  return (longlong)TIME_to_ulonglong(&ltime);
+  longlong v= TIME_to_ulonglong(&ltime);
+  return ltime.neg ? -v : v;
 }
 
 
@@ -2303,7 +2304,17 @@ bool Item_date_typecast::get_date(MYSQL_TIME *ltime, uint fuzzy_date)
     return 1;
   ltime->hour= ltime->minute= ltime->second= ltime->second_part= 0;
   ltime->time_type= MYSQL_TIMESTAMP_DATE;
-  return 0;
+
+  int unused;
+  if (check_date(ltime, ltime->year || ltime->month || ltime->day,
+                 fuzzy_date, &unused))
+  {
+    Lazy_string_time str(ltime);
+    make_truncated_value_warning(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
+                                 &str, MYSQL_TIMESTAMP_DATE, 0);
+    return (null_value= 1);
+  }
+  return (null_value= 0);
 }
 
 
@@ -2425,7 +2436,6 @@ bool Item_func_add_time::get_date(MYSQL_TIME *ltime, uint fuzzy_date)
   long days, microseconds;
   longlong seconds;
   int l_sign= sign, was_cut= 0;
-  uint dec= decimals;
 
   if (is_date)                        // TIMESTAMP function
   {
@@ -2467,10 +2477,6 @@ bool Item_func_add_time::get_date(MYSQL_TIME *ltime, uint fuzzy_date)
 
   ltime->time_type= is_time ? MYSQL_TIMESTAMP_TIME : MYSQL_TIMESTAMP_DATETIME;
 
-  if (cached_field_type == MYSQL_TYPE_STRING &&
-      (l_time1.second_part || l_time2.second_part))
-    dec= TIME_SECOND_PART_DIGITS;
-
   if (!is_time)
   {
     get_date_from_daynr(days,&ltime->year,&ltime->month,&ltime->day);
diff --git a/sql/lex.h b/sql/lex.h
index 6ab234f8d81..3041168e4fb 100644
--- a/sql/lex.h
+++ b/sql/lex.h
@@ -238,6 +238,7 @@ static SYMBOL symbols[] = {
   { "GRANTS",	        SYM(GRANTS)},
   { "GROUP",		SYM(GROUP_SYM)},
   { "HANDLER",		SYM(HANDLER_SYM)},
+  { "HARD",		SYM(HARD_SYM)},
   { "HASH",		SYM(HASH_SYM)},
   { "HAVING",		SYM(HAVING)},
   { "HELP",		SYM(HELP_SYM)},
@@ -496,6 +497,7 @@ static SYMBOL symbols[] = {
   { "SNAPSHOT",         SYM(SNAPSHOT_SYM)},
   { "SMALLINT",		SYM(SMALLINT)},
   { "SOCKET",		SYM(SOCKET_SYM)},
+  { "SOFT",             SYM(SOFT_SYM)},
   { "SOME",             SYM(ANY_SYM)},
   { "SONAME",		SYM(SONAME_SYM)},
   { "SOUNDS",		SYM(SOUNDS_SYM)},
diff --git a/sql/log.cc b/sql/log.cc
index d345d0f6102..e99493945bf 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -70,7 +70,7 @@ static LEX_STRING const write_error_msg=
 static my_bool opt_optimize_thread_scheduling= TRUE;
 ulong binlog_checksum_options;
 #ifndef DBUG_OFF
-static ulong opt_binlog_dbug_fsync_sleep= 0;
+ulong opt_binlog_dbug_fsync_sleep= 0;
 #endif
 
 static my_bool mutexes_inited;
@@ -1062,17 +1062,6 @@ bool LOGGER::slow_log_print(THD *thd, const char *query, uint query_length,
       query_length= command_name[thd->command].length;
     }
 
-    if (!query_length) 
-    {
-      /*
-        Not a real query; Reset counts for slow query logging
-        (QQ: Wonder if this is really needed)
-      */
-      thd->sent_row_count= thd->examined_row_count= 0;
-      thd->query_plan_flags= QPLAN_INIT;
-      thd->query_plan_fsort_passes= 0;
-    }
-
     for (current_handler= slow_log_handler_list; *current_handler ;)
       error= (*current_handler++)->log_slow(thd, current_time,
                                             user_host_buff, user_host_len,
@@ -1809,7 +1798,7 @@ static int binlog_savepoint_set(handlerton *hton, THD *thd, void *sv)
       log_query.append(thd->lex->ident.str, thd->lex->ident.length) ||
       log_query.append("`"))
     DBUG_RETURN(1);
-  int errcode= query_error_code(thd, thd->killed == THD::NOT_KILLED);
+  int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
   Query_log_event qinfo(thd, log_query.ptr(), log_query.length(),
                         TRUE, TRUE, errcode);
   DBUG_RETURN(mysql_bin_log.write(&qinfo));
@@ -1833,7 +1822,7 @@ static int binlog_savepoint_rollback(handlerton *hton, THD *thd, void *sv)
         log_query.append(thd->lex->ident.str, thd->lex->ident.length) ||
         log_query.append("`"))
       DBUG_RETURN(1);
-    int errcode= query_error_code(thd, thd->killed == THD::NOT_KILLED);
+    int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
     Query_log_event qinfo(thd, log_query.ptr(), log_query.length(),
                           TRUE, TRUE, errcode);
     DBUG_RETURN(mysql_bin_log.write(&qinfo));
@@ -5166,7 +5155,7 @@ int query_error_code(THD *thd, bool not_killed)
 {
   int error;
   
-  if (not_killed || (thd->killed == THD::KILL_BAD_DATA))
+  if (not_killed || (killed_mask_hard(thd->killed) == KILL_BAD_DATA))
   {
     error= thd->is_error() ? thd->main_da.sql_errno() : 0;
 
@@ -5175,7 +5164,8 @@ int query_error_code(THD *thd, bool not_killed)
        is not set to these errors when specified not_killed by the
        caller.
     */
-    if (error == ER_SERVER_SHUTDOWN || error == ER_QUERY_INTERRUPTED)
+    if (error == ER_SERVER_SHUTDOWN || error == ER_QUERY_INTERRUPTED ||
+        error == ER_NEW_ABORTING_CONNECTION || error == ER_CONNECTION_KILLED)
       error= 0;
   }
   else
@@ -7077,27 +7067,10 @@ static MYSQL_SYSVAR_ENUM(
   BINLOG_CHECKSUM_ALG_OFF,
   &binlog_checksum_typelib);
 
-#ifndef DBUG_OFF
-static MYSQL_SYSVAR_ULONG(
-  dbug_fsync_sleep,
-  opt_binlog_dbug_fsync_sleep,
-  PLUGIN_VAR_RQCMDARG,
-  "Extra sleep (in microseconds) to add to binlog fsync(), for debugging",
-  NULL,
-  NULL,
-  0,
-  0,
-  ULONG_MAX,
-  0);
-#endif
-
 static struct st_mysql_sys_var *binlog_sys_vars[]=
 {
   MYSQL_SYSVAR(optimize_thread_scheduling),
   MYSQL_SYSVAR(checksum),
-#ifndef DBUG_OFF
-  MYSQL_SYSVAR(dbug_fsync_sleep),
-#endif
   NULL
 };
 
diff --git a/sql/log_event.cc b/sql/log_event.cc
index 3dad0c87e00..ae7a37f2b3d 100644
--- a/sql/log_event.cc
+++ b/sql/log_event.cc
@@ -437,6 +437,7 @@ inline bool unexpected_error_code(int unexpected_error)
   case ER_NET_READ_ERROR:
   case ER_NET_ERROR_ON_WRITE:
   case ER_QUERY_INTERRUPTED:
+  case ER_CONNECTION_KILLED:
   case ER_SERVER_SHUTDOWN:
   case ER_NEW_ABORTING_CONNECTION:
     return(TRUE);
@@ -3686,7 +3687,7 @@ Default database: '%s'. Query: '%s'",
     {
       DBUG_PRINT("info",("error ignored"));
       clear_all_errors(thd, const_cast<Relay_log_info*>(rli));
-      thd->killed= THD::NOT_KILLED;
+      thd->killed= NOT_KILLED;
       /*
         When an error is expected and matches the actual error the
         slave does not report any error and by consequence changes
diff --git a/sql/log_event.h b/sql/log_event.h
index 4053db4e3e7..ab58aac5fd0 100644
--- a/sql/log_event.h
+++ b/sql/log_event.h
@@ -3082,7 +3082,7 @@ char *str_to_hex(char *to, const char *from, uint len);
 /**
   @class Annotate_rows_log_event
 
-  In row-based mode, if binlog_annotate_rows_events = ON, each group of
+  In row-based mode, if binlog_annotate_row_events = ON, each group of
   Table_map_log_events is preceded by an Annotate_rows_log_event which
   contains the query which caused the subsequent rows operations.
 
diff --git a/sql/multi_range_read.cc b/sql/multi_range_read.cc
index c666c571787..055a9268417 100644
--- a/sql/multi_range_read.cc
+++ b/sql/multi_range_read.cc
@@ -424,10 +424,10 @@ void Mrr_ordered_index_reader::interrupt_read()
 {
   DBUG_ASSERT(support_scan_interruptions);
   TABLE *table= file->get_table();
+  KEY *used_index= &table->key_info[file->active_index];
   /* Save the current key value */
   key_copy(saved_key_tuple, table->record[0],
-           &table->key_info[file->active_index],
-           keypar.key_tuple_length);
+           used_index, used_index->key_length);
   
   if (saved_primary_key)
   {
@@ -452,9 +452,9 @@ void Mrr_ordered_index_reader::position()
 void Mrr_ordered_index_reader::resume_read()
 {
   TABLE *table= file->get_table();
+  KEY *used_index= &table->key_info[file->active_index];
   key_restore(table->record[0], saved_key_tuple, 
-              &table->key_info[file->active_index],
-              keypar.key_tuple_length);
+              used_index, used_index->key_length);
   if (saved_primary_key)
   {
     key_restore(table->record[0], saved_primary_key, 
@@ -531,7 +531,7 @@ int Mrr_ordered_index_reader::init(handler *h_arg, RANGE_SEQ_IF *seq_funcs,
   mrr_funcs= *seq_funcs;
   source_exhausted= FALSE;
   if (support_scan_interruptions)
-    bzero(saved_key_tuple, keypar.key_tuple_length);
+    bzero(saved_key_tuple, key_info->key_length);
   have_saved_rowid= FALSE;
   return 0;
 }
@@ -848,12 +848,14 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs,
       if (h_idx->primary_key_is_clustered())
       {
         uint pk= h_idx->get_table()->s->primary_key;
-        saved_pk_length= h_idx->get_table()->key_info[pk].key_length;
+        if (pk != MAX_KEY)
+          saved_pk_length= h_idx->get_table()->key_info[pk].key_length;
       }
-
+      
+      KEY *used_index= &h_idx->get_table()->key_info[h_idx->active_index];
       if (reader_factory.ordered_index_reader.
             set_interruption_temp_buffer(primary_file->ref_length,
-                                         keypar.key_tuple_length,
+                                         used_index->key_length,
                                          saved_pk_length,
                                          &full_buf, full_buf_end))
         goto use_default_impl;
@@ -1075,7 +1077,7 @@ void DsMrr_impl::close_second_handler()
   {
     secondary_file->ha_index_or_rnd_end();
     secondary_file->ha_external_lock(current_thd, F_UNLCK);
-    secondary_file->close();
+    secondary_file->ha_close();
     delete secondary_file;
     secondary_file= NULL;
   }
@@ -1635,7 +1637,8 @@ bool DsMrr_impl::get_disk_sweep_mrr_cost(uint keynr, ha_rows rows, uint flags,
                                          uint *buffer_size, COST_VECT *cost)
 {
   ulong max_buff_entries, elem_size;
-  ha_rows rows_in_full_step, rows_in_last_step;
+  ha_rows rows_in_full_step;
+  ha_rows rows_in_last_step;
   uint n_full_steps;
   double index_read_cost;
 
@@ -1660,7 +1663,7 @@ bool DsMrr_impl::get_disk_sweep_mrr_cost(uint keynr, ha_rows rows, uint flags,
   /* Adjust buffer size if we expect to use only part of the buffer */
   if (n_full_steps)
   {
-    get_sort_and_sweep_cost(table, rows, cost);
+    get_sort_and_sweep_cost(table, rows_in_full_step, cost);
     cost->multiply(n_full_steps);
   }
   else
diff --git a/sql/mysql_priv.h b/sql/mysql_priv.h
index f12874be1ea..e335406493c 100644
--- a/sql/mysql_priv.h
+++ b/sql/mysql_priv.h
@@ -118,8 +118,6 @@ char *sql_strmake_with_convert(const char *str, size_t arg_length,
 			       CHARSET_INFO *from_cs,
 			       size_t max_res_length,
 			       CHARSET_INFO *to_cs, size_t *result_length);
-uint kill_one_thread(THD *thd, ulong id, bool only_kill_query);
-void sql_kill(THD *thd, ulong id, bool only_kill_query);
 bool net_request_file(NET* net, const char* fname);
 char* query_table_status(THD *thd,const char *db,const char *table_name);
 
@@ -413,7 +411,6 @@ protected:
 #define DELAYED_LIMIT		100		/**< pause after xxx inserts */
 #define DELAYED_QUEUE_SIZE	1000
 #define DELAYED_WAIT_TIMEOUT	5*60		/**< Wait for delayed insert */
-#define FLUSH_TIME		0		/**< Don't flush tables */
 #define MAX_CONNECT_ERRORS	10		///< errors before disabling host
 
 #ifdef __NETWARE__
@@ -423,8 +420,6 @@ protected:
 #endif
 
 #if defined(__WIN__)
-#undef	FLUSH_TIME
-#define FLUSH_TIME	1800			/**< Flush every half hour */
 
 #define INTERRUPT_PRIOR -2
 #define CONNECT_PRIOR	-1
@@ -594,64 +589,30 @@ protected:
 #define OPTIMIZER_SWITCH_JOIN_CACHE_HASHED         (1ULL << 22)
 #define OPTIMIZER_SWITCH_JOIN_CACHE_BKA            (1ULL << 23)
 #define OPTIMIZER_SWITCH_OPTIMIZE_JOIN_BUFFER_SIZE (1ULL << 24)
-#ifdef DBUG_OFF
-#  define OPTIMIZER_SWITCH_LAST                    (1ULL << 25)
-#else
-#  define OPTIMIZER_SWITCH_TABLE_ELIMINATION       (1ULL << 25)
-#  define OPTIMIZER_SWITCH_LAST                    (1ULL << 26)
-#endif
+#define OPTIMIZER_SWITCH_TABLE_ELIMINATION         (1ULL << 25)
+#define OPTIMIZER_SWITCH_LAST                      (1ULL << 26)
 
-#ifdef DBUG_OFF 
 /* The following must be kept in sync with optimizer_switch_str in mysqld.cc */
 /*
 TODO: Materialization is off by default to mimic 5.1/5.2 behavior.
 Once cost based choice between materialization and in-to-exists should be
 enabled by default, add OPTIMIZER_SWITCH_MATERIALIZATION
 */
-#  define OPTIMIZER_SWITCH_DEFAULT (OPTIMIZER_SWITCH_INDEX_MERGE | \
-                                    OPTIMIZER_SWITCH_INDEX_MERGE_UNION | \
-                                    OPTIMIZER_SWITCH_INDEX_MERGE_SORT_UNION | \
-                                    OPTIMIZER_SWITCH_INDEX_MERGE_INTERSECT | \
-                                    OPTIMIZER_SWITCH_INDEX_COND_PUSHDOWN | \
-                                    OPTIMIZER_SWITCH_DERIVED_MERGE | \
-                                    OPTIMIZER_SWITCH_DERIVED_WITH_KEYS | \
-                                    OPTIMIZER_SWITCH_FIRSTMATCH | \
-                                    OPTIMIZER_SWITCH_LOOSE_SCAN | \
-                                    OPTIMIZER_SWITCH_IN_TO_EXISTS | \
-                                    OPTIMIZER_SWITCH_SEMIJOIN | \
-                                    OPTIMIZER_SWITCH_PARTIAL_MATCH_ROWID_MERGE|\
-                                    OPTIMIZER_SWITCH_PARTIAL_MATCH_TABLE_SCAN|\
-                                    OPTIMIZER_SWITCH_SUBQUERY_CACHE|\
-                                    OPTIMIZER_SWITCH_MRR|\
-                                    OPTIMIZER_SWITCH_MRR_SORT_KEYS|\
-                                    OPTIMIZER_SWITCH_SUBQUERY_CACHE | \
-                                    OPTIMIZER_SWITCH_JOIN_CACHE_INCREMENTAL | \
-                                    OPTIMIZER_SWITCH_JOIN_CACHE_HASHED | \
-                                    OPTIMIZER_SWITCH_JOIN_CACHE_BKA | \
-                                    OPTIMIZER_SWITCH_OPTIMIZE_JOIN_BUFFER_SIZE)
-#else
-#  define OPTIMIZER_SWITCH_DEFAULT (OPTIMIZER_SWITCH_INDEX_MERGE | \
+#define OPTIMIZER_SWITCH_DEFAULT   (OPTIMIZER_SWITCH_INDEX_MERGE | \
                                     OPTIMIZER_SWITCH_INDEX_MERGE_UNION | \
                                     OPTIMIZER_SWITCH_INDEX_MERGE_SORT_UNION | \
                                     OPTIMIZER_SWITCH_INDEX_MERGE_INTERSECT | \
-                                    OPTIMIZER_SWITCH_INDEX_COND_PUSHDOWN | \
-                                    OPTIMIZER_SWITCH_DERIVED_MERGE | \
-                                    OPTIMIZER_SWITCH_DERIVED_WITH_KEYS | \
                                     OPTIMIZER_SWITCH_TABLE_ELIMINATION | \
-                                    OPTIMIZER_SWITCH_FIRSTMATCH | \
-                                    OPTIMIZER_SWITCH_LOOSE_SCAN | \
                                     OPTIMIZER_SWITCH_IN_TO_EXISTS | \
-                                    OPTIMIZER_SWITCH_SEMIJOIN | \
                                     OPTIMIZER_SWITCH_PARTIAL_MATCH_ROWID_MERGE|\
                                     OPTIMIZER_SWITCH_PARTIAL_MATCH_TABLE_SCAN|\
-                                    OPTIMIZER_SWITCH_SUBQUERY_CACHE|\
-                                    OPTIMIZER_SWITCH_MRR|\
-                                    OPTIMIZER_SWITCH_MRR_SORT_KEYS|\
                                     OPTIMIZER_SWITCH_JOIN_CACHE_INCREMENTAL | \
                                     OPTIMIZER_SWITCH_JOIN_CACHE_HASHED | \
                                     OPTIMIZER_SWITCH_JOIN_CACHE_BKA | \
-                                    OPTIMIZER_SWITCH_OPTIMIZE_JOIN_BUFFER_SIZE)
-#endif
+                                    OPTIMIZER_SWITCH_SUBQUERY_CACHE |\
+                                    OPTIMIZER_SWITCH_SEMIJOIN | \
+                                    OPTIMIZER_SWITCH_FIRSTMATCH | \
+                                    OPTIMIZER_SWITCH_LOOSE_SCAN )
 
 /*
   Replication uses 8 bytes to store SQL_MODE in the binary log. The day you
@@ -808,17 +769,6 @@ inline THD *_current_thd(void)
 #endif
 #define current_thd _current_thd()
 
-
-/** 
-  The meat of thd_proc_info(THD*, char*), a macro that packs the last
-  three calling-info parameters. 
-*/
-extern "C"
-const char *set_thd_proc_info(THD *thd, const char *info, 
-                              const char *calling_func, 
-                              const char *calling_file, 
-                              const unsigned int calling_line);
-
 /**
   Enumerate possible types of a table from re-execution
   standpoint.
@@ -995,6 +945,7 @@ inline bool check_merge_table_access(THD *thd, char *db, TABLE_LIST *table_list)
 inline bool check_some_routine_access(THD *thd, const char *db,
                                       const char *name, bool is_proc)
 { return false; }
+#define decrease_user_connections(X) do { } while(0)       /* nothing */
 #endif /*NO_EMBEDDED_ACCESS_CHECKS*/
 
 bool multi_update_precheck(THD *thd, TABLE_LIST *tables);
@@ -1125,6 +1076,10 @@ struct Query_cache_query_flags
 #define query_cache_is_cacheable_query(L) 0
 #endif /*HAVE_QUERY_CACHE*/
 
+uint kill_one_thread(THD *thd, ulong id, killed_state kill_signal);
+void sql_kill(THD *thd, ulong id, killed_state kill_signal);
+void sql_kill_user(THD *thd, LEX_USER *str, killed_state kill_signal);
+
 /*
   Error injector Macros to enable easy testing of recovery after failures
   in various error cases.
@@ -1228,7 +1183,9 @@ bool init_new_connection_handler_thread();
 void reset_mqh(LEX_USER *lu, bool get_them);
 bool check_mqh(THD *thd, uint check_command);
 void time_out_user_resource_limits(THD *thd, USER_CONN *uc);
+#ifndef NO_EMBEDDED_ACCESS_CHECKS
 void decrease_user_connections(USER_CONN *uc);
+#endif
 bool thd_init_client_charset(THD *thd, uint cs_number);
 inline bool is_supported_parser_charset(CHARSET_INFO *cs)
 {
@@ -1690,6 +1647,7 @@ bool add_to_list(THD *thd, SQL_I_List<ORDER> &list, Item *group,bool asc);
 bool push_new_name_resolution_context(THD *thd,
                                       TABLE_LIST *left_op,
                                       TABLE_LIST *right_op);
+Item *normalize_cond(Item *cond);
 void add_join_on(TABLE_LIST *b,Item *expr);
 void add_join_natural(TABLE_LIST *a,TABLE_LIST *b,List<String> *using_fields,
                       SELECT_LEX *lex);
@@ -2184,6 +2142,7 @@ extern ulonglong thd_startup_options;
 extern ulong thread_id;
 extern ulong binlog_cache_use, binlog_cache_disk_use;
 extern ulong aborted_threads,aborted_connects;
+extern ulong opt_progress_report_time;
 extern ulong delayed_insert_timeout;
 extern ulong delayed_insert_limit, delayed_queue_size;
 extern ulong delayed_insert_threads, delayed_insert_writes;
@@ -2196,14 +2155,15 @@ extern MYSQL_PLUGIN_IMPORT ulong max_connections;
 extern ulong max_connect_errors, connect_timeout;
 extern ulong extra_max_connections;
 extern ulong slave_net_timeout, slave_trans_retries;
-extern uint max_user_connections;
+extern int max_user_connections;
+extern bool max_user_connections_checking;
 extern ulonglong denied_connections;
 extern ulong what_to_log,flush_time;
 extern ulong query_buff_size;
 extern ulong max_prepared_stmt_count, prepared_stmt_count;
 extern ulong binlog_cache_size, open_files_limit;
 extern ulonglong max_binlog_cache_size;
-extern ulong max_binlog_size, max_relay_log_size;
+extern ulong max_binlog_size, max_relay_log_size, opt_binlog_dbug_fsync_sleep;
 extern ulong opt_binlog_rows_event_max_size;
 extern my_bool opt_master_verify_checksum;
 extern my_bool opt_slave_sql_verify_checksum;
@@ -2224,6 +2184,9 @@ extern uint test_flags,select_errors,ha_open_options;
 extern uint protocol_version, mysqld_port, mysqld_extra_port, dropping_tables;
 extern uint delay_key_write_options;
 extern ulong max_long_data_size;
+extern uint internal_tmp_table_max_key_length;
+extern uint internal_tmp_table_max_key_segments;
+
 #endif /* MYSQL_SERVER */
 #if defined MYSQL_SERVER || defined INNODB_COMPATIBILITY_HOOKS
 extern MYSQL_PLUGIN_IMPORT uint lower_case_table_names;
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 08794b6ab68..1d8a352b97e 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -350,9 +350,7 @@ static const char *optimizer_switch_names[]=
   "join_cache_hashed",
   "join_cache_bka",
   "optimize_join_buffer_size",
-#ifndef DBUG_OFF
   "table_elimination",
-#endif
   "default", NullS
 };
 
@@ -384,9 +382,7 @@ static const unsigned int optimizer_switch_names_len[]=
   sizeof("join_cache_hashed") - 1,
   sizeof("join_cache_bka") - 1,
   sizeof("optimize_join_buffer_size") - 1,
-#ifndef DBUG_OFF
   sizeof("table_elimination") - 1,
-#endif
   sizeof("default") - 1
 };
 TYPELIB optimizer_switch_typelib= { array_elements(optimizer_switch_names)-1,"",
@@ -485,9 +481,9 @@ static const char *optimizer_switch_str="index_merge=on,index_merge_union=on,"
                                         "index_merge_sort_union=on,"
                                         "index_merge_intersection=on,"
                                         "index_merge_sort_intersection=off,"
-                                        "index_condition_pushdown=on,"
-                                        "derived_merge=on,"
-                                        "derived_with_keys=on,"
+                                        "index_condition_pushdown=off,"
+                                        "derived_merge=off,"
+                                        "derived_with_keys=off,"
                                         "firstmatch=on,"
                                         "loosescan=on,"
                                         "materialization=off,"
@@ -496,17 +492,17 @@ static const char *optimizer_switch_str="index_merge=on,index_merge_union=on,"
                                         "partial_match_rowid_merge=on,"
                                         "partial_match_table_scan=on,"
                                         "subquery_cache=on,"
-                                        "mrr=on,"
+                                        "mrr=off,"
                                         "mrr_cost_based=off,"
-                                        "mrr_sort_keys=on,"
+                                        "mrr_sort_keys=off,"
                                         "join_cache_incremental=on,"
                                         "join_cache_hashed=on,"
                                         "join_cache_bka=on,"
-                                        "optimize_join_buffer_size=on"
-#ifndef DBUG_OFF
-                                        ",table_elimination=on";
-#else
+                                        "optimize_join_buffer_size=off,"
+                                        "table_elimination=on";
                                         ;
+#ifdef SAFEMALLOC
+my_bool sf_malloc_trough_check= 0;
 #endif
 static char *mysqld_user, *mysqld_chroot, *log_error_file_ptr;
 static char *opt_init_slave, *language_ptr, *opt_init_connect;
@@ -571,7 +567,7 @@ my_bool opt_local_infile, opt_slave_compressed_protocol;
 my_bool opt_safe_user_create = 0, opt_no_mix_types = 0;
 my_bool opt_show_slave_auth_info, opt_sql_bin_update = 0;
 my_bool opt_log_slave_updates= 0;
-my_bool opt_replicate_annotate_rows_events= 0;
+my_bool opt_replicate_annotate_row_events= 0;
 bool slave_warning_issued = false;
 
 /*
@@ -673,7 +669,12 @@ ulong extra_max_connections;
 */
 ulong max_long_data_size;
 
-uint  max_user_connections= 0;
+/* Limits for internal temporary tables (MyISAM or Aria) */
+uint internal_tmp_table_max_key_length;
+uint internal_tmp_table_max_key_segments;
+
+int  max_user_connections= 0;
+bool max_user_connections_checking=0;
 ulonglong denied_connections;
 /**
   Limit of the total number of prepared statements in the server.
@@ -706,6 +707,7 @@ char mysql_real_data_home[FN_REFLEN],
      language[FN_REFLEN], reg_ext[FN_EXTLEN], mysql_charsets_dir[FN_REFLEN],
      *opt_init_file, *opt_tc_log_file,
      def_ft_boolean_syntax[sizeof(ft_boolean_syntax)];
+const char *opt_basename;
 char mysql_unpacked_real_data_home[FN_REFLEN];
 int mysql_unpacked_real_data_home_len;
 uint reg_ext_length;
@@ -1086,7 +1088,7 @@ static void close_connections(void)
     if (tmp->slave_thread)
       continue;
 
-    tmp->killed= THD::KILL_CONNECTION;
+    tmp->killed= KILL_SERVER_HARD;
     thread_scheduler.post_kill_notification(tmp);
     pthread_mutex_lock(&tmp->LOCK_thd_data);
     if (tmp->mysys_var)
@@ -1146,7 +1148,7 @@ static void close_connections(void)
                           tmp->thread_id,
                           (tmp->main_security_ctx.user ?
                            tmp->main_security_ctx.user : ""));
-      close_connection(tmp,0,0);
+      close_connection(tmp,ER_SERVER_SHUTDOWN,0);
     }
 #endif
     DBUG_PRINT("quit",("Unlocking LOCK_thread_count"));
@@ -2037,7 +2039,18 @@ void close_connection(THD *thd, uint errcode, bool lock)
 		      errcode ? ER(errcode) : ""));
   if (lock)
     (void) pthread_mutex_lock(&LOCK_thread_count);
-  thd->killed= THD::KILL_CONNECTION;
+  thd->killed= KILL_CONNECTION;
+
+  if (global_system_variables.log_warnings > 3)
+  {
+    Security_context *sctx= &thd->main_security_ctx;
+    sql_print_warning(ER(ER_NEW_ABORTING_CONNECTION),
+                      thd->thread_id,(thd->db ? thd->db : "unconnected"),
+                      sctx->user ? sctx->user : "unauthenticated",
+                      sctx->host_or_ip,
+                      (errcode ? ER(errcode) : "CLOSE_CONNECTION"));
+  }
+
   if ((vio= thd->net.vio) != 0)
   {
     if (errcode)
@@ -2135,6 +2148,7 @@ static bool cache_thread()
       */
       thd->mysys_var->abort= 0;
       thd->thr_create_utime= microsecond_interval_timer();
+      thd->start_utime= thd->thr_create_utime;
       threads.append(thd);
       return(1);
     }
@@ -2199,24 +2213,6 @@ void flush_thread_cache()
 }
 
 
-#ifdef THREAD_SPECIFIC_SIGPIPE
-/**
-  Aborts a thread nicely. Comes here on SIGPIPE.
-
-  @todo
-    One should have to fix that thr_alarm know about this thread too.
-*/
-extern "C" sig_handler abort_thread(int sig __attribute__((unused)))
-{
-  THD *thd=current_thd;
-  DBUG_ENTER("abort_thread");
-  if (thd)
-    thd->killed= THD::KILL_CONNECTION;
-  DBUG_VOID_RETURN;
-}
-#endif
-
-
 /******************************************************************************
   Setup a signal thread with handles all signals.
   Because Linux doesn't support schemas use a mutex to check that
@@ -2694,6 +2690,8 @@ or misconfigured. This error can also be caused by malfunctioning hardware.\n",
 We will try our best to scrape up some info that will hopefully help diagnose\n\
 the problem, but since we have already crashed, something is definitely wrong\n\
 and this may fail.\n\n");
+  set_server_version();
+  fprintf(stderr, "Server version: %s\n", server_version);
   fprintf(stderr, "key_buffer_size=%lu\n",
           (ulong) dflt_key_cache->key_cache_mem_size);
   fprintf(stderr, "read_buffer_size=%ld\n", (long) global_system_variables.read_buff_size);
@@ -2739,29 +2737,47 @@ the thread stack. Please read http://dev.mysql.com/doc/mysql/en/linux.html\n\n",
   {
     const char *kreason= "UNKNOWN";
     switch (thd->killed) {
-    case THD::NOT_KILLED:
+    case NOT_KILLED:
+    case KILL_HARD_BIT:
       kreason= "NOT_KILLED";
       break;
-    case THD::KILL_BAD_DATA:
+    case KILL_BAD_DATA:
+    case KILL_BAD_DATA_HARD:
       kreason= "KILL_BAD_DATA";
       break;
-    case THD::KILL_CONNECTION:
+    case KILL_CONNECTION:
+    case KILL_CONNECTION_HARD:
       kreason= "KILL_CONNECTION";
       break;
-    case THD::KILL_QUERY:
+    case KILL_QUERY:
+    case KILL_QUERY_HARD:
       kreason= "KILL_QUERY";
       break;
-    case THD::KILLED_NO_VALUE:
-      kreason= "KILLED_NO_VALUE";
+    case KILL_SYSTEM_THREAD:
+    case KILL_SYSTEM_THREAD_HARD:
+      kreason= "KILL_SYSTEM_THREAD";
+      break;
+    case KILL_SERVER:
+    case KILL_SERVER_HARD:
+      kreason= "KILL_SERVER";
       break;
     }
     fprintf(stderr, "\nTrying to get some variables.\n"
                     "Some pointers may be invalid and cause the dump to abort.\n");
     fprintf(stderr, "Query (%p): ", thd->query());
-    my_safe_print_str(thd->query(), min(1024, thd->query_length()));
+    my_safe_print_str(thd->query(), min(65536,thd->query_length()));
     fprintf(stderr, "Connection ID (thread ID): %lu\n", (ulong) thd->thread_id);
     fprintf(stderr, "Status: %s\n", kreason);
-    fputc('\n', stderr);
+    fprintf(stderr, "Optimizer switch: ");
+    ulonglong optsw= thd->variables.optimizer_switch;
+    for (uint i= 0; optimizer_switch_names[i+1]; i++, optsw >>= 1)
+    {
+      if (i)
+        fputc(',', stderr);
+      fprintf(stderr, "%s=%s",
+        optimizer_switch_names[i], optsw & 1 ? "on" : "off");
+    }
+    fprintf(stderr, "\n\n");
   }
   fprintf(stderr, "\
 The manual page at http://dev.mysql.com/doc/mysql/en/crashing.html contains\n\
@@ -3102,7 +3118,7 @@ int my_message_sql(uint error, const char *str, myf MyFlags)
   MYSQL_ERROR::enum_warning_level level;
   sql_print_message_func func;
   DBUG_ENTER("my_message_sql");
-  DBUG_PRINT("error", ("error: %u  message: '%s'", error, str));
+  DBUG_PRINT("error", ("error: %u  message: '%s'  Flag: %d", error, str, MyFlags));
 
   DBUG_ASSERT(str != NULL);
   /*
@@ -3118,7 +3134,8 @@ int my_message_sql(uint error, const char *str, myf MyFlags)
   {
     /* At least, prevent new abuse ... */
     DBUG_ASSERT(strncmp(str, "MyISAM table", 12) == 0 ||
-                strncmp(str, "Aria table", 11) == 0);
+                strncmp(str, "Aria table", 11) == 0 ||
+                (MyFlags & ME_JUST_INFO));
     error= ER_UNKNOWN_ERROR;
   }
 
@@ -3145,7 +3162,10 @@ int my_message_sql(uint error, const char *str, myf MyFlags)
       this could be improved by having a common stack of handlers.
     */
     if (thd->handle_error(error, str, level))
+    {
+      DBUG_PRINT("info", ("error handled by handle_error()"));
       DBUG_RETURN(0);
+    }
 
     if (level == MYSQL_ERROR::WARN_LEVEL_WARN)
       push_warning(thd, MYSQL_ERROR::WARN_LEVEL_WARN, error, str);
@@ -3224,7 +3244,7 @@ to_error_log:
   /* When simulating OOM, skip writing to error log to avoid mtr errors */
   DBUG_EXECUTE_IF("simulate_out_of_memory", DBUG_RETURN(0););
 
-  if (!thd || (MyFlags & ME_NOREFRESH))
+  if (!thd || thd->log_all_errors || (MyFlags & ME_NOREFRESH))
     (*func)("%s: %s", my_progname_short, str); /* purecov: inspected */
   DBUG_RETURN(0);
 }
@@ -3270,6 +3290,7 @@ const char *load_default_groups[]= {
 #endif
 "mysqld", "server", MYSQL_BASE_VERSION,
 "mariadb", MARIADB_BASE_VERSION,
+"client-server",
 0, 0};
 
 #if defined(__WIN__) && !defined(EMBEDDED_LIBRARY)
@@ -3489,7 +3510,6 @@ static int init_common_variables(const char *conf_file_name, int argc,
 				 char **argv, const char **groups)
 {
   char buff[FN_REFLEN], *s;
-  const char *basename;
   umask(((~my_umask) & 0666));
   tzset();			// Set tzname
 
@@ -3549,13 +3569,13 @@ static int init_common_variables(const char *conf_file_name, int argc,
     strmake(glob_hostname, STRING_WITH_LEN("localhost"));
     sql_print_warning("gethostname failed, using '%s' as hostname",
                         glob_hostname);
-    basename= "mysql";
+    opt_basename= "mysql";
   }
   else
   {
-    basename= glob_hostname;
+    opt_basename= glob_hostname;
   }
-  strmake(pidfile_name, basename, sizeof(pidfile_name)-5);
+  strmake(pidfile_name, opt_basename, sizeof(pidfile_name)-5);
   strmov(fn_ext(pidfile_name),".pid");		// Add proper extension
 
   /*
@@ -4285,11 +4305,13 @@ a file name for --log-bin-index option", opt_binlog_index_name);
         require a name. But as we don't want to break many existing setups, we
         only give warning, not error.
       */
-      sql_print_warning("No argument was provided to --log-bin, and "
-                        "--log-bin-index was not used; so replication "
-                        "may break when this MySQL server acts as a "
-                        "master and has his hostname changed!! Please "
-                        "use '--log-bin=%s' to avoid this problem.", ln);
+      sql_print_warning("No argument was provided to --log-bin and "
+                        "neither --log-basename or --log-bin-index where "
+                        "used;  This may cause repliction to break when this "
+                        "server acts as a master and has its hostname "
+                        "changed!! Please use '--log-basename=%s' or "
+                        "'--log-bin=%s' to avoid this problem.",
+                        opt_basename, ln);
     }
     if (ln == buf)
     {
@@ -4448,6 +4470,11 @@ a file name for --log-bin-index option", opt_binlog_index_name);
     sql_print_error("Aria engine is not enabled or did not start. The Aria engine must be enabled to continue as mysqld was configured with --with-aria-tmp-tables");
     unireg_abort(1);
   }
+  internal_tmp_table_max_key_length=   maria_max_key_length();
+  internal_tmp_table_max_key_segments= maria_max_key_segments();
+#else
+  internal_tmp_table_max_key_length=   myisam_max_key_length();
+  internal_tmp_table_max_key_segments= myisam_max_key_segments();
 #endif
 
   tc_log= (total_ha_2pc > 1 ? (opt_bin_log  ?
@@ -5259,7 +5286,7 @@ void create_thread_to_handle_connection(THD *thd)
     thread_created++;
     threads.append(thd);
     DBUG_PRINT("info",(("creating thread %lu"), thd->thread_id));
-    thd->prior_thr_create_utime= thd->start_utime= microsecond_interval_timer();
+    thd->prior_thr_create_utime= microsecond_interval_timer();
     if ((error=pthread_create(&thd->real_id,&connection_attrib,
                               handle_one_connection,
                               (void*) thd)))
@@ -5269,7 +5296,7 @@ void create_thread_to_handle_connection(THD *thd)
                  ("Can't create thread to handle request (error %d)",
                   error));
       thread_count--;
-      thd->killed= THD::KILL_CONNECTION;			// Safety
+      thd->killed= KILL_CONNECTION;             // Safety
       (void) pthread_mutex_unlock(&LOCK_thread_count);
 
       pthread_mutex_lock(&LOCK_connection_count);
@@ -5282,7 +5309,7 @@ void create_thread_to_handle_connection(THD *thd)
                   ER(ER_CANT_CREATE_THREAD), error);
       net_send_error(thd, ER_CANT_CREATE_THREAD, error_message_buff);
       (void) pthread_mutex_lock(&LOCK_thread_count);
-      close_connection(thd,0,0);
+      close_connection(thd,ER_OUT_OF_RESOURCES,0);
       delete thd;
       (void) pthread_mutex_unlock(&LOCK_thread_count);
       return;
@@ -5990,7 +6017,7 @@ enum options_mysqld
   OPT_SQL_BIN_UPDATE_SAME,     OPT_REPLICATE_DO_DB,
   OPT_REPLICATE_IGNORE_DB,     OPT_LOG_SLAVE_UPDATES,
   OPT_BINLOG_DO_DB,            OPT_BINLOG_IGNORE_DB,
-  OPT_BINLOG_FORMAT,
+  OPT_BINLOG_FORMAT,           OPT_DEBUG_BINLOG_FSYNC_SLEEP,
   OPT_BINLOG_ANNOTATE_ROWS_EVENTS,
   OPT_REPLICATE_ANNOTATE_ROWS_EVENTS,
 #ifndef DBUG_OFF
@@ -6019,7 +6046,7 @@ enum options_mysqld
   OPT_NDB_REPORT_THRESH_BINLOG_EPOCH_SLIP,
   OPT_NDB_REPORT_THRESH_BINLOG_MEM_USAGE,
   OPT_NDB_USE_COPYING_ALTER_TABLE,
-  OPT_SKIP_SAFEMALLOC, OPT_MUTEX_DEADLOCK_DETECTOR,
+  OPT_SAFEMALLOC, OPT_MUTEX_DEADLOCK_DETECTOR,
   OPT_TEMP_POOL, OPT_TX_ISOLATION, OPT_COMPLETION_TYPE,
   OPT_SKIP_SYMLINKS,
   OPT_MAX_BINLOG_DUMP_EVENTS, OPT_SPORADIC_BINLOG_DUMP_FAIL,
@@ -6081,7 +6108,7 @@ enum options_mysqld
   OPT_SORT_BUFFER, OPT_TABLE_OPEN_CACHE, OPT_TABLE_DEF_CACHE,
   OPT_THREAD_CONCURRENCY, OPT_THREAD_CACHE_SIZE,
   OPT_TMP_TABLE_SIZE, OPT_THREAD_STACK,
-  OPT_WAIT_TIMEOUT,
+  OPT_WAIT_TIMEOUT, OPT_PROGRESS_REPORT_TIME,
   OPT_ERROR_LOG_FILE,
   OPT_DEFAULT_WEEK_FORMAT,
   OPT_RANGE_ALLOC_BLOCK_SIZE, OPT_ALLOW_SUSPICIOUS_UDFS,
@@ -6140,7 +6167,9 @@ enum options_mysqld
   OPT_SECURE_FILE_PRIV,
   OPT_MIN_EXAMINED_ROW_LIMIT,
   OPT_LOG_SLOW_SLAVE_STATEMENTS,
-  OPT_DEBUG_CRC, OPT_DEBUG_ON, OPT_DEBUG_ASSERT_IF_CRASHED_TABLE, OPT_OLD_MODE,
+  OPT_DEBUG_CRC, OPT_DEBUG_ON, OPT_DEBUG_ASSERT_IF_CRASHED_TABLE,
+  OPT_DEBUG_ASSERT_ON_ERROR,
+  OPT_OLD_MODE,
   OPT_TEST_IGNORE_WRONG_OPTIONS, OPT_TEST_RESTART,
 #if defined(ENABLED_DEBUG_SYNC)
   OPT_DEBUG_SYNC_TIMEOUT,
@@ -6214,6 +6243,12 @@ struct my_option my_long_options[] =
   {"bind-address", OPT_BIND_ADDRESS, "IP address to bind to.",
    &my_bind_addr_str, &my_bind_addr_str, 0, GET_STR,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+#ifndef DBUG_OFF
+  {"debug-binlog-fsync-sleep", OPT_DEBUG_BINLOG_FSYNC_SLEEP,
+    "Extra sleep (in microseconds) to add to binlog fsync(), for debugging",
+    &opt_binlog_dbug_fsync_sleep, &opt_binlog_dbug_fsync_sleep,
+   0, GET_ULONG, REQUIRED_ARG, 0, 0, ULONG_MAX, 0, 1, 0},
+#endif
   {"binlog_format", OPT_BINLOG_FORMAT,
    "Does not have any effect without '--log-bin'. "
    "Tell the master the form of binary logging to use: either 'row' for "
@@ -6227,17 +6262,17 @@ struct my_option my_long_options[] =
 #endif
    , &opt_binlog_format, &opt_binlog_format,
    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  {"binlog-annotate-rows-events", OPT_BINLOG_ANNOTATE_ROWS_EVENTS,
+  {"binlog-annotate-row-events", OPT_BINLOG_ANNOTATE_ROWS_EVENTS,
    "Tells the master to annotate RBR events with the statement that "
    "caused these events.",
-   (uchar**) &global_system_variables.binlog_annotate_rows_events,
-   (uchar**) &max_system_variables.binlog_annotate_rows_events,
+   (uchar**) &global_system_variables.binlog_annotate_row_events,
+   (uchar**) &max_system_variables.binlog_annotate_row_events,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
-  {"replicate-annotate-rows-events", OPT_REPLICATE_ANNOTATE_ROWS_EVENTS,
+  {"replicate-annotate-row-events", OPT_REPLICATE_ANNOTATE_ROWS_EVENTS,
    "Tells the slave to write annotate rows events recieved from the master "
    "to its own binary log. Sensible only in pair with log-slave-updates option.",
-   (uchar**) &opt_replicate_annotate_rows_events,
-   (uchar**) &opt_replicate_annotate_rows_events,
+   (uchar**) &opt_replicate_annotate_row_events,
+   (uchar**) &opt_replicate_annotate_row_events,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"binlog-do-db", OPT_BINLOG_DO_DB,
    "Tells the master it should log updates for the specified database, "
@@ -6330,6 +6365,10 @@ struct my_option my_long_options[] =
    "Do an assert in handler::print_error() if we get a crashed table",
    &debug_assert_if_crashed_table, &debug_assert_if_crashed_table,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"debug-assert-on-error", OPT_DEBUG_ASSERT_ON_ERROR,
+   "Do an assert in various functions if we get a fatal error",
+   &my_assert_on_error, &my_assert_on_error,
+   0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 #endif
   {"default-character-set", OPT_DEFAULT_CHARACTER_SET_OLD, 
    "Set the default character set (deprecated option, use --character-set-server instead).",
@@ -6417,8 +6456,8 @@ struct my_option my_long_options[] =
    &opt_debugging, &opt_debugging,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"general_log", OPT_GENERAL_LOG,
-   "Enable/disable general log.  Filename can be specified with --general-log-file or --log-basename.  Is 'hostname.err' by default.",
-   &opt_log, &opt_log, 0, GET_BOOL, OPT_ARG, 0, 0, 0, 0, 0, 0},
+   "Enable/disable general log.  Filename can be specified with --general-log-file or --log-basename.  Is 'hostname.log' by default.",
+   &opt_log, &opt_log, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 #ifdef HAVE_LARGE_PAGES
   {"large-pages", OPT_ENABLE_LARGE_PAGES, "Enable support for large pages. "
    "Disable with --skip-large-pages.", &opt_large_pages, &opt_large_pages,
@@ -6585,8 +6624,7 @@ each time the SQL thread starts.",
    "log and this option just turns on --log-bin instead.",
    &opt_update_logname, &opt_update_logname, 0, GET_STR,
    OPT_ARG, 0, 0, 0, 0, 0, 0},
-  {"log-warnings", 'W', "Log some not critical warnings to the general log "
-   "file.",
+  {"log-warnings", 'W', "Log some not critical warnings to the general log file.  Value can be between 0-11; The higher value, the more warnings",
    &global_system_variables.log_warnings,
    &max_system_variables.log_warnings, 0, GET_ULONG, OPT_ARG, 1, 0, 0,
    0, 0, 0},
@@ -6992,12 +7030,11 @@ each time the SQL thread starts.",
   {"skip-networking", OPT_SKIP_NETWORKING,
    "Don't allow connection with TCP/IP.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0,
    0, 0, 0},
-#ifndef DBUG_OFF
 #ifdef SAFEMALLOC
-  {"skip-safemalloc", OPT_SKIP_SAFEMALLOC,
-   "Don't use the memory allocation checking.", 0, 0, 0, GET_NO_ARG, NO_ARG,
-   0, 0, 0, 0, 0, 0},
-#endif
+  {"safemalloc", OPT_SAFEMALLOC,
+   "Check all memory allocation for every malloc/free call.",
+   &sf_malloc_trough_check, &sf_malloc_trough_check, 0,
+   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
 #endif
   {"skip-show-database", OPT_SKIP_SHOW_DB,
    "Don't allow 'SHOW DATABASE' commands.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0,
@@ -7211,7 +7248,7 @@ each time the SQL thread starts.",
   { "flush_time", OPT_FLUSH_TIME,
     "A dedicated thread is created to flush all tables at the given interval.",
     &flush_time, &flush_time, 0, GET_ULONG, REQUIRED_ARG,
-    FLUSH_TIME, 0, LONG_TIMEOUT, 0, 1, 0},
+    0 , 0, LONG_TIMEOUT, 0, 1, 0},
   { "ft_boolean_syntax", OPT_FT_BOOLEAN_SYNTAX,
     "List of operators for MATCH ... AGAINST ( ... IN BOOLEAN MODE).",
     0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -7424,9 +7461,9 @@ each time the SQL thread starts.",
    &max_system_variables.max_tmp_tables, 0, GET_ULONG,
    REQUIRED_ARG, 32, 1, (longlong) ULONG_MAX, 0, 1, 0},
   {"max_user_connections", OPT_MAX_USER_CONNECTIONS,
-   "The maximum number of active connections for a single user (0 = no limit).",
-   &max_user_connections, &max_user_connections, 0, GET_UINT,
-   REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0},
+   "The maximum number of active connections for a single user (0 = no limit. In addition global max_user_connections counting and checking is permanently disabled).",
+   &max_user_connections, &max_user_connections, 0, GET_INT,
+   REQUIRED_ARG, 0, 0, INT_MAX, 0, 1, 0},
   {"max_write_lock_count", OPT_MAX_WRITE_LOCK_COUNT,
    "After this many write locks, allow some read locks to run in between.",
    &max_write_lock_count, &max_write_lock_count, 0, GET_ULONG,
@@ -7555,9 +7592,7 @@ each time the SQL thread starts.",
    "subquery_cache, outer_join_with_cache, semijoin_with_cache, "
    "join_cache_incremental, join_cache_hashed, join_cache_bka, "
    "optimize_join_buffer_size"
-#ifndef DBUG_OFF
    ", table_elimination"
-#endif 
    "} and val={on, off, default}.",
    &optimizer_switch_str, &optimizer_switch_str, 0, GET_STR, REQUIRED_ARG, 
    /*OPTIMIZER_SWITCH_DEFAULT*/0, 0, 0, 0, 0, 0},
@@ -7580,6 +7615,11 @@ each time the SQL thread starts.",
    &global_system_variables.preload_buff_size,
    &max_system_variables.preload_buff_size, 0, GET_ULONG,
    REQUIRED_ARG, 32*1024L, 1024, 1024*1024*1024L, 0, 1, 0},
+  {"progress_report_time", OPT_PROGRESS_REPORT_TIME,
+   "Seconds between sending progress reports to the client for slow commands. Set to 0 to disable progress reporting.",
+   &global_system_variables.progress_report_time,
+   &max_system_variables.progress_report_time,
+   0, GET_ULONG, REQUIRED_ARG, 5, 0, ULONG_MAX, 0, 1, 0},
   {"query_alloc_block_size", OPT_QUERY_ALLOC_BLOCK_SIZE,
    "Allocation block size for query parsing and execution.",
    &global_system_variables.query_alloc_block_size,
@@ -7784,7 +7824,7 @@ each time the SQL thread starts.",
    "Define threads usage for handling queries: "
    "one-thread-per-connection"
 #if HAVE_POOL_OF_THREADS == 1
-  ", pool-of-threads"
+  ", pool-of-threads "
 #endif
    "or no-threads.",
    &opt_thread_handling, &opt_thread_handling,
@@ -8262,6 +8302,8 @@ SHOW_VAR status_vars[]= {
   {"Handler_savepoint_rollback",(char*) offsetof(STATUS_VAR, ha_savepoint_rollback_count), SHOW_LONG_STATUS},
   {"Handler_update",           (char*) offsetof(STATUS_VAR, ha_update_count), SHOW_LONG_STATUS},
   {"Handler_write",            (char*) offsetof(STATUS_VAR, ha_write_count), SHOW_LONG_STATUS},
+  {"Handler_tmp_update",       (char*) offsetof(STATUS_VAR, ha_tmp_update_count), SHOW_LONG_STATUS},
+  {"Handler_tmp_write",        (char*) offsetof(STATUS_VAR, ha_tmp_write_count), SHOW_LONG_STATUS},
   {"Key",                      (char*) &show_default_keycache, SHOW_FUNC},
   {"Last_query_cost",          (char*) offsetof(STATUS_VAR, last_query_cost), SHOW_DOUBLE_STATUS},
   {"Max_used_connections",     (char*) &max_used_connections,  SHOW_LONG},
@@ -8276,6 +8318,7 @@ SHOW_VAR status_vars[]= {
   {"Prepared_stmt_count",      (char*) &show_prepared_stmt_count, SHOW_FUNC},
   {"Rows_sent",                (char*) offsetof(STATUS_VAR, rows_sent), SHOW_LONGLONG_STATUS},
   {"Rows_read",                (char*) offsetof(STATUS_VAR, rows_read), SHOW_LONGLONG_STATUS},
+  {"Rows_tmp_read",            (char*) offsetof(STATUS_VAR, rows_tmp_read), SHOW_LONGLONG_STATUS},
 #ifdef HAVE_QUERY_CACHE
   {"Qcache_free_blocks",       (char*) &query_cache.free_memory_blocks, SHOW_LONG_NOFLUSH},
   {"Qcache_free_memory",       (char*) &query_cache.free_memory, SHOW_LONG_NOFLUSH},
@@ -9394,11 +9437,6 @@ mysqld_get_one_option(int optid,
     }
     strmake(ft_boolean_syntax, argument, sizeof(ft_boolean_syntax)-1);
     break;
-  case OPT_SKIP_SAFEMALLOC:
-#ifdef SAFEMALLOC
-    sf_malloc_quick=1;
-#endif
-    break;
   case OPT_LOWER_CASE_TABLE_NAMES:
     lower_case_table_names= argument ? atoi(argument) : 1;
     lower_case_table_names_used= 1;
@@ -9578,6 +9616,19 @@ static int get_options(int *argc,char **argv)
   myisam_block_size=(uint) 1 << my_bit_log2(opt_myisam_block_size);
   my_crc_dbug_check= opt_my_crc_dbug_check;
 
+  /*
+    Log mysys errors when we don't have a thd or thd->log_all_errors is set
+    (recovery) to the log.  This is mainly useful for debugging strange system
+    errors.
+  */
+  if (global_system_variables.log_warnings >= 10)
+    my_global_flags= MY_WME | ME_JUST_INFO;
+  /* Log all errors not handled by thd->handle_error() to my_message_sql() */
+  if (global_system_variables.log_warnings >= 11)
+    my_global_flags|= ME_NOREFRESH;
+  if (my_assert_on_error)
+    debug_assert_if_crashed_table= 1;
+
   /* long_query_time is in microseconds */
   global_system_variables.long_query_time= max_system_variables.long_query_time=
     (longlong) (long_query_time * 1000000.0);
@@ -9593,6 +9644,9 @@ static int get_options(int *argc,char **argv)
 				  &global_system_variables.datetime_format))
     return 1;
 
+#ifdef SAFEMALLOC
+  sf_malloc_quick= !sf_malloc_trough_check;
+#endif
 #ifdef EMBEDDED_LIBRARY
   one_thread_scheduler(&thread_scheduler);
   one_thread_scheduler(&extra_thread_scheduler);
@@ -9616,6 +9670,8 @@ static int get_options(int *argc,char **argv)
   if (!max_long_data_size_used)
     max_long_data_size= global_system_variables.max_allowed_packet;
 
+  /* Rember if max_user_connections was 0 at startup */
+  max_user_connections_checking= max_user_connections != 0;
   return 0;
 }
 
diff --git a/sql/opt_index_cond_pushdown.cc b/sql/opt_index_cond_pushdown.cc
index e0a2d3b1f30..3a9c813b93c 100644
--- a/sql/opt_index_cond_pushdown.cc
+++ b/sql/opt_index_cond_pushdown.cc
@@ -83,15 +83,44 @@ bool uses_index_fields_only(Item *item, TABLE *tbl, uint keyno,
   case Item::FIELD_ITEM:
     {
       Item_field *item_field= (Item_field*)item;
-      if (item_field->field->table != tbl) 
+      Field *field= item_field->field;
+      if (field->table != tbl)
         return TRUE;
       /*
         The below is probably a repetition - the first part checks the
         other two, but let's play it safe:
       */
-      return item_field->field->part_of_key.is_set(keyno) &&
-             item_field->field->type() != MYSQL_TYPE_GEOMETRY &&
-             item_field->field->type() != MYSQL_TYPE_BLOB;
+      if(!field->part_of_key.is_set(keyno) ||
+         field->type() == MYSQL_TYPE_GEOMETRY ||
+         field->type() == MYSQL_TYPE_BLOB)
+        return FALSE;
+      KEY *key_info= tbl->key_info + keyno;
+      KEY_PART_INFO *key_part= key_info->key_part;
+      KEY_PART_INFO *key_part_end= key_part + key_info->key_parts;
+      for ( ; key_part < key_part_end; key_part++)
+      {
+        if (field->eq(key_part->field))
+	  return !(key_part->key_part_flag & HA_PART_KEY_SEG);          
+      }
+      if ((tbl->file->ha_table_flags() & HA_PRIMARY_KEY_IN_READ_INDEX) &&
+          tbl->s->primary_key != MAX_KEY &&
+	  tbl->s->primary_key != keyno)
+      {
+        key_info= tbl->key_info + tbl->s->primary_key;
+        key_part= key_info->key_part;
+        key_part_end= key_part + key_info->key_parts;
+        for ( ; key_part < key_part_end; key_part++)
+        {
+          /* 
+            It does not make sense to use the fact that the engine can read in
+            a full field if the key if the index is built only over a part
+            of this field.
+	  */
+          if (field->eq(key_part->field))
+	    return !(key_part->key_part_flag & HA_PART_KEY_SEG);          
+        }
+      }  
+      return FALSE;
     }
   case Item::REF_ITEM:
     return uses_index_fields_only(item->real_item(), tbl, keyno,
@@ -210,11 +239,9 @@ Item *make_cond_for_index(Item *cond, TABLE *table, uint keyno,
 }
 
 
-Item *make_cond_remainder(Item *cond, bool exclude_index)
+Item *make_cond_remainder(Item *cond, TABLE *table, uint keyno,
+                          bool other_tbls_ok, bool exclude_index)
 {
-  if (exclude_index && cond->marker == ICP_COND_USES_INDEX_ONLY)
-    return 0; /* Already checked */
-
   if (cond->type() == Item::COND_ITEM)
   {
     table_map tbl_map= 0;
@@ -228,7 +255,8 @@ Item *make_cond_remainder(Item *cond, bool exclude_index)
       Item *item;
       while ((item=li++))
       {
-	Item *fix= make_cond_remainder(item, exclude_index);
+	Item *fix= make_cond_remainder(item, table, keyno,
+                                       other_tbls_ok, exclude_index);
 	if (fix)
         {
 	  new_cond->argument_list()->push_back(fix);
@@ -255,7 +283,8 @@ Item *make_cond_remainder(Item *cond, bool exclude_index)
       Item *item;
       while ((item=li++))
       {
-	Item *fix= make_cond_remainder(item, FALSE);
+	Item *fix= make_cond_remainder(item, table, keyno, 
+                                       other_tbls_ok, FALSE);
 	if (!fix)
 	  return (COND*) 0;
 	new_cond->argument_list()->push_back(fix);
@@ -267,7 +296,14 @@ Item *make_cond_remainder(Item *cond, bool exclude_index)
       return new_cond;
     }
   }
-  return cond;
+  else
+  {
+    if (exclude_index && 
+        uses_index_fields_only(cond, table, keyno, other_tbls_ok))
+      return 0;
+    else
+      return cond;
+  }
 }
 
 
@@ -288,30 +324,13 @@ void push_index_cond(JOIN_TAB *tab, uint keyno)
 {
   DBUG_ENTER("push_index_cond");
   Item *idx_cond;
-  bool do_index_cond_pushdown=
-    ((tab->table->file->index_flags(keyno, 0, 1) &
-      HA_DO_INDEX_COND_PUSHDOWN) &&
-     optimizer_flag(tab->join->thd, OPTIMIZER_SWITCH_INDEX_COND_PUSHDOWN));
-
-  /*
-    Do not try index condition pushdown on indexes which have partially-covered
-    columns. Unpacking from a column prefix into index tuple is not a supported 
-    operation in some engines, see e.g. MySQL BUG#42991.
-    TODO: a better solution would be not to consider partially-covered columns
-    as parts of the index and still produce/check index condition for
-    fully-covered index columns.
-  */
-  KEY *key_info= tab->table->key_info + keyno;
-  for (uint kp= 0; kp < key_info->key_parts; kp++)
-  {
-    if ((key_info->key_part[kp].key_part_flag & HA_PART_KEY_SEG))
-    {
-      do_index_cond_pushdown= FALSE;
-      break;
-    }
-  }
 
-  if (do_index_cond_pushdown)
+  if ((tab->table->file->index_flags(keyno, 0, 1) &
+      HA_DO_INDEX_COND_PUSHDOWN) &&
+     optimizer_flag(tab->join->thd, OPTIMIZER_SWITCH_INDEX_COND_PUSHDOWN) &&
+     tab->join->thd->lex->sql_command != SQLCOM_UPDATE_MULTI &&
+     tab->join->thd->lex->sql_command != SQLCOM_DELETE_MULTI &&
+     tab->type != JT_CONST && tab->type != JT_SYSTEM)
   {
     DBUG_EXECUTE("where",
                  print_where(tab->select_cond, "full cond", QT_ORDINARY););
@@ -356,7 +375,8 @@ void push_index_cond(JOIN_TAB *tab, uint keyno)
         tab->ref.disable_cache= TRUE;
 
       Item *row_cond= tab->idx_cond_fact_out ? 
-                        make_cond_remainder(tab->select_cond, TRUE) :
+	                make_cond_remainder(tab->select_cond, tab->table, keyno,
+			                    tab->icp_other_tables_ok, TRUE) :
 	                tab->pre_idx_push_select_cond;
 
       DBUG_EXECUTE("where",
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index dc987e196f4..121c923845f 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -1136,10 +1136,10 @@ int SEL_IMERGE::and_sel_tree(RANGE_OPT_PARAM *param, SEL_TREE *tree,
 
     2. In the second mode, when is_first_check_pass==FALSE :
       2.1. For each rt_j in the imerge that can be ored (see the function
-           sel_trees_can_be_ored), but not must be ored, with rt the function
-           replaces rt_j for a range tree such that for each index for which
-           ranges are defined in both in rt_j and rt  the tree contains the
-           result of oring of these ranges.
+           sel_trees_can_be_ored) with rt the function replaces rt_j for a
+           range tree such that for each index for which ranges are defined
+           in both in rt_j and rt  the tree contains the  result of oring of
+           these ranges.
       2.2. In other cases the function does not produce any imerge.
 
     When is_first_check==TRUE the function returns FALSE in the parameter
@@ -1163,7 +1163,7 @@ int SEL_IMERGE::or_sel_tree_with_checks(RANGE_OPT_PARAM *param,
                                         bool *is_last_check_pass)
 {
   bool was_ored= FALSE;
-  *is_last_check_pass= TRUE;
+  *is_last_check_pass= is_first_check_pass;
   SEL_TREE** or_tree = trees;
   for (uint i= 0; i < n_trees; i++, or_tree++)
   {
@@ -1174,7 +1174,7 @@ int SEL_IMERGE::or_sel_tree_with_checks(RANGE_OPT_PARAM *param,
     {
       bool must_be_ored= sel_trees_must_be_ored(param, *or_tree, tree,
                                                 ored_keys);
-      if (must_be_ored || !is_first_check_pass) 
+      if (must_be_ored || !is_first_check_pass)
       {
         result_keys.clear_all();
         result= *or_tree;
@@ -1210,22 +1210,19 @@ int SEL_IMERGE::or_sel_tree_with_checks(RANGE_OPT_PARAM *param,
     {
       if (result_keys.is_clear_all())
         result->type= SEL_TREE::ALWAYS;
-      *is_last_check_pass= TRUE;
       if ((result->type == SEL_TREE::MAYBE) ||
           (result->type == SEL_TREE::ALWAYS))
         return 1;
       /* SEL_TREE::IMPOSSIBLE is impossible here */
       result->keys_map= result_keys; 
       *or_tree= result;
-      if (is_first_check_pass)
-        return 0;
       was_ored= TRUE;
     }
   }
   if (was_ored)
     return 0;
 
-  if (!*is_last_check_pass && 
+  if (is_first_check_pass && !*is_last_check_pass &&
       !(tree= new SEL_TREE(tree, FALSE, param)))
     return (-1);
   return or_sel_tree(param, tree);
@@ -1804,7 +1801,7 @@ QUICK_RANGE_SELECT::~QUICK_RANGE_SELECT()
         DBUG_PRINT("info", ("Freeing separate handler 0x%lx (free: %d)", (long) file,
                             free_file));
         file->ha_external_lock(current_thd, F_UNLCK);
-        file->close();
+        file->ha_close();
         delete file;
       }
     }
@@ -1999,7 +1996,7 @@ int QUICK_RANGE_SELECT::init_ror_merged_scan(bool reuse_handler)
   if (init() || reset())
   {
     file->ha_external_lock(thd, F_UNLCK);
-    file->close();
+    file->ha_close();
     goto failure;
   }
   free_file= TRUE;
@@ -7292,7 +7289,7 @@ static SEL_TREE *get_mm_tree(RANGE_OPT_PARAM *param,COND *cond)
   case Item_func::MULT_EQUAL_FUNC:
   {
     Item_equal *item_equal= (Item_equal *) cond;    
-    if (!(value= item_equal->get_const()))
+    if (!(value= item_equal->get_const()) || value->is_expensive())
       DBUG_RETURN(0);
     Item_equal_fields_iterator it(*item_equal);
     ref_tables= value->used_tables();
@@ -7325,6 +7322,9 @@ static SEL_TREE *get_mm_tree(RANGE_OPT_PARAM *param,COND *cond)
     }
     else
       DBUG_RETURN(0);
+    if (value && value->is_expensive())
+      DBUG_RETURN(0);
+
     ftree= get_full_func_mm_tree(param, cond_func, field_item, value, inv);
   }
 
@@ -8379,9 +8379,9 @@ tree_or(RANGE_OPT_PARAM *param,SEL_TREE *tree1,SEL_TREE *tree2)
     /* Build the imerge part of the tree for the formula (1) */
     SEL_TREE *rt1= tree1;
     SEL_TREE *rt2= tree2;
-    if (!no_merges1)
+    if (no_merges1)
       rt1= new SEL_TREE(tree1, TRUE, param);
-    if (!no_merges2)
+    if (no_merges2)
       rt2= new SEL_TREE(tree2, TRUE, param);
     if (!rt1 || !rt2 ||
         result->merges.push_back(imerge_from_ranges) ||
@@ -8695,7 +8695,7 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2)
   {
     key1->free_tree();
     key2->free_tree();
-    return 0;					// Can't optimize this
+    return 0;                                   // Can't optimize this
   }
 
   // If one of the key is MAYBE_KEY then the found region may be bigger
@@ -8719,248 +8719,553 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2)
       swap_variables(SEL_ARG *,key1,key2);
     }
     if (key1->use_count > 0 && !(key1=key1->clone_tree(param)))
-      return 0;					// OOM
+      return 0;                                 // OOM
   }
 
   // Add tree at key2 to tree at key1
   bool key2_shared=key2->use_count != 0;
   key1->maybe_flag|=key2->maybe_flag;
 
+  /*
+    Notation for illustrations used in the rest of this function: 
+
+      Range: [--------]
+             ^        ^
+             start    stop
+
+      Two overlapping ranges:
+        [-----]               [----]            [--]
+            [---]     or    [---]       or   [-------]
+
+      Ambiguity: *** 
+        The range starts or stops somewhere in the "***" range.
+        Example: a starts before b and may end before/the same plase/after b
+        a: [----***]
+        b:   [---]
+
+      Adjacent ranges:
+        Ranges that meet but do not overlap. Example: a = "x < 3", b = "x >= 3"
+        a: ----]
+        b:      [----
+   */
+
   uint max_part_no= max(key1->max_part_no, key2->max_part_no);
 
   for (key2=key2->first(); key2; )
   {
-    SEL_ARG *tmp=key1->find_range(key2);	// Find key1.min <= key2.min
-    int cmp;
+    /*
+      key1 consists of one or more ranges. tmp is the range currently
+      being handled.
+
+      initialize tmp to the latest range in key1 that starts the same
+      place or before the range in key2 starts
+
+      key2:           [------]
+      key1: [---] [-----] [----]
+                  ^
+                  tmp
+    */
+    SEL_ARG *tmp=key1->find_range(key2);
+
+    /*
+      Used to describe how two key values are positioned compared to
+      each other. Consider key_value_a.<cmp_func>(key_value_b):
+
+        -2: key_value_a is smaller than key_value_b, and they are adjacent
+        -1: key_value_a is smaller than key_value_b (not adjacent)
+         0: the key values are equal
+         1: key_value_a is bigger than key_value_b (not adjacent)
+        -2: key_value_a is bigger than key_value_b, and they are adjacent
+
+      Example: "cmp= tmp->cmp_max_to_min(key2)"
+
+      key2:         [--------            (10 <= x ...)
+      tmp:    -----]                      (... x <  10) => cmp==-2
+      tmp:    ----]                       (... x <=  9) => cmp==-1
+      tmp:    ------]                     (... x  = 10) => cmp== 0
+      tmp:    --------]                   (... x <= 12) => cmp== 1
+      (cmp == 2 does not make sense for cmp_max_to_min())
+     */
+    int cmp= 0;
 
     if (!tmp)
     {
-      tmp=key1->first();			// tmp.min > key2.min
+      /*
+        The range in key2 starts before the first range in key1. Use
+        the first range in key1 as tmp.
+
+        key2:     [--------]
+        key1:            [****--] [----]   [-------]
+                         ^
+                         tmp
+      */
+      tmp=key1->first();
       cmp= -1;
     }
-    else if ((cmp=tmp->cmp_max_to_min(key2)) < 0)
-    {						// Found tmp.max < key2.min
+    else if ((cmp= tmp->cmp_max_to_min(key2)) < 0)
+    {
+      /*
+        This is the case:
+        key2:          [-------]
+        tmp:   [----**]
+       */
       SEL_ARG *next=tmp->next;
-      /* key1 on the left of key2 non-overlapping */
       if (cmp == -2 && eq_tree(tmp->next_key_part,key2->next_key_part))
       {
-	// Join near ranges like tmp.max < 0 and key2.min >= 0
-	SEL_ARG *key2_next=key2->next;
-	if (key2_shared)
-	{
-	  if (!(key2=new SEL_ARG(*key2)))
-	    return 0;		// out of memory
-	  key2->increment_use_count(key1->use_count+1);
-	  key2->next=key2_next;			// New copy of key2
-	}
-	key2->copy_min(tmp);
-	if (!(key1=key1->tree_delete(tmp)))
-	{					// Only one key in tree
-	  key1=key2;
-	  key1->make_root();
-	  key2=key2_next;
-	  break;
-	}
+        /*
+          Adjacent (cmp==-2) and equal next_key_parts => ranges can be merged
+
+          This is the case:
+          key2:          [-------]
+          tmp:     [----]
+
+          Result:
+          key2:    [-------------]     => inserted into key1 below
+          tmp:                         => deleted
+        */
+        SEL_ARG *key2_next=key2->next;
+        if (key2_shared)
+        {
+          if (!(key2=new SEL_ARG(*key2)))
+            return 0;           // out of memory
+          key2->increment_use_count(key1->use_count+1);
+          key2->next=key2_next;                 // New copy of key2
+        }
+
+        key2->copy_min(tmp);
+        if (!(key1=key1->tree_delete(tmp)))
+        {                                       // Only one key in tree
+          key1=key2;
+          key1->make_root();
+          key2=key2_next;
+          break;
+        }
       }
-      if (!(tmp=next))				// tmp.min > key2.min
-	break;					// Copy rest of key2
+      if (!(tmp=next)) // Move to next range in key1. Now tmp.min > key2.min
+        break;         // No more ranges in key1. Copy rest of key2
     }
+
     if (cmp < 0)
-    {						// tmp.min > key2.min
+    {
+      /*
+        This is the case:
+        key2:  [--***]
+        tmp:       [----]
+      */
       int tmp_cmp;
-      if ((tmp_cmp=tmp->cmp_min_to_max(key2)) > 0) // if tmp.min > key2.max
+      if ((tmp_cmp=tmp->cmp_min_to_max(key2)) > 0)
       {
-        /* tmp is on the right of key2 non-overlapping */
-	if (tmp_cmp == 2 && eq_tree(tmp->next_key_part,key2->next_key_part))
-	{					// ranges are connected
-	  tmp->copy_min_to_min(key2);
-	  key1->merge_flags(key2);
-	  if (tmp->min_flag & NO_MIN_RANGE &&
-	      tmp->max_flag & NO_MAX_RANGE)
-	  {
-	    if (key1->maybe_flag)
-	      return new SEL_ARG(SEL_ARG::MAYBE_KEY);
-	    return 0;
-	  }
-	  key2->increment_use_count(-1);	// Free not used tree
-	  key2=key2->next;
-	  continue;
-	}
-	else
-	{
-	  SEL_ARG *next=key2->next;		// Keys are not overlapping
-	  if (key2_shared)
-	  {
-	    SEL_ARG *cpy= new SEL_ARG(*key2);	// Must make copy
-	    if (!cpy)
-	      return 0;				// OOM
-	    key1=key1->insert(cpy);
-	    key2->increment_use_count(key1->use_count+1);
-	  }
-	  else
-	    key1=key1->insert(key2);		// Will destroy key2_root
-	  key2=next;
-	  continue;
-	}
+        /*
+          This is the case:
+          key2:  [------**]
+          tmp:             [----]
+        */
+        if (tmp_cmp == 2 && eq_tree(tmp->next_key_part,key2->next_key_part))
+        {
+          /*
+            Adjacent ranges with equal next_key_part. Merge like this:
+
+            This is the case:
+            key2:    [------]
+            tmp:             [-----]
+
+            Result:
+            key2:    [------]
+            tmp:     [-------------]
+
+            Then move on to next key2 range.
+          */
+          tmp->copy_min_to_min(key2);
+          key1->merge_flags(key2);
+          if (tmp->min_flag & NO_MIN_RANGE &&
+              tmp->max_flag & NO_MAX_RANGE)
+          {
+            if (key1->maybe_flag)
+              return new SEL_ARG(SEL_ARG::MAYBE_KEY);
+            return 0;
+          }
+          key2->increment_use_count(-1);        // Free not used tree
+          key2=key2->next;
+          continue;
+        }
+        else
+        {
+          /*
+            key2 not adjacent to tmp or has different next_key_part.
+            Insert into key1 and move to next range in key2
+            
+            This is the case:
+            key2:  [------**]
+            tmp:             [----]
+
+            Result:
+            key1_  [------**][----]
+                   ^         ^
+                   insert    tmp
+          */
+          SEL_ARG *next=key2->next;
+          if (key2_shared)
+          {
+            SEL_ARG *cpy= new SEL_ARG(*key2);   // Must make copy
+            if (!cpy)
+              return 0;                         // OOM
+            key1=key1->insert(cpy);
+            key2->increment_use_count(key1->use_count+1);
+          }
+          else
+            key1=key1->insert(key2);            // Will destroy key2_root
+          key2=next;
+          continue;
+        }
       }
     }
 
-    /* 
-      tmp.min >= key2.min && tmp.min <= key.max  (overlapping ranges)
-      key2.min <= tmp.min <= key2.max 
-    */  
+    /*
+      The ranges in tmp and key2 are overlapping:
+
+      key2:          [----------] 
+      tmp:        [*****-----*****]
+
+      Corollary: tmp.min <= key2.max
+    */
     if (eq_tree(tmp->next_key_part,key2->next_key_part))
     {
+      // Merge overlapping ranges with equal next_key_part
       if (tmp->is_same(key2))
       {
-        /* 
-          Found exact match of key2 inside key1. 
+        /*
+          Found exact match of key2 inside key1.
           Use the relevant range in key1.
         */
-	tmp->merge_flags(key2);			// Copy maybe flags
-	key2->increment_use_count(-1);		// Free not used tree
+        tmp->merge_flags(key2);                 // Copy maybe flags
+        key2->increment_use_count(-1);          // Free not used tree
       }
       else
       {
-	SEL_ARG *last=tmp;
-        SEL_ARG *first=tmp;
-        /* 
-          Find the last range in tmp that overlaps key2 and has the same 
-          condition on the rest of the keyparts.
+        SEL_ARG *last= tmp;
+        SEL_ARG *first= tmp;
+
+        /*
+          Find the last range in key1 that overlaps key2 and
+          where all ranges first...last have the same next_key_part as
+          key2.
+
+          key2:  [****----------------------*******]
+          key1:     [--]  [----] [---]  [-----] [xxxx]
+                    ^                   ^       ^
+                    first               last    different next_key_part
+
+          Since key2 covers them, the ranges between first and last
+          are merged into one range by deleting first...last-1 from
+          the key1 tree. In the figure, this applies to first and the
+          two consecutive ranges. The range of last is then extended:
+            * last.min: Set to min(key2.min, first.min)
+            * last.max: If there is a last->next that overlaps key2 (i.e.,
+                        last->next has a different next_key_part):
+                                        Set adjacent to last->next.min
+                        Otherwise:      Set to max(key2.max, last.max)
+
+          Result:
+          key2:  [****----------------------*******]
+                    [--]  [----] [---]                   => deleted from key1
+          key1:  [**------------------------***][xxxx]
+                 ^                              ^
+                 tmp=last                       different next_key_part
         */
-	while (last->next && last->next->cmp_min_to_max(key2) <= 0 &&
-	       eq_tree(last->next->next_key_part,key2->next_key_part))
-	{
+        while (last->next && last->next->cmp_min_to_max(key2) <= 0 &&
+               eq_tree(last->next->next_key_part,key2->next_key_part))
+        {
           /*
-            We've found the last overlapping key1 range in last.
-            This means that the ranges between (and including) the 
-            first overlapping range (tmp) and the last overlapping range
-            (last) are fully nested into the current range of key2 
-            and can safely be discarded. We just need the minimum endpoint
-            of the first overlapping range (tmp) so we can compare it with
-            the minimum endpoint of the enclosing key2 range.
+            last->next is covered by key2 and has same next_key_part.
+            last can be deleted
           */
-	  SEL_ARG *save=last;
-	  last=last->next;
-	  key1=key1->tree_delete(save);
-	}
+          SEL_ARG *save=last;
+          last=last->next;
+          key1=key1->tree_delete(save);
+        }
+        // Redirect tmp to last which will cover the entire range
+        tmp= last;
+
         /*
-          The tmp range (the first overlapping range) could have been discarded
-          by the previous loop. We should re-direct tmp to the new united range 
-          that's taking its place.
+          We need the minimum endpoint of first so we can compare it
+          with the minimum endpoint of the enclosing key2 range.
         */
-        tmp= last;
         last->copy_min(first);
         bool full_range= last->copy_min(key2);
         if (!full_range)
         {
           if (last->next && key2->cmp_max_to_min(last->next) >= 0)
           {
-            last->max_value= last->next->min_value;
-            if (last->next->min_flag & NEAR_MIN)
-              last->max_flag&= ~NEAR_MAX;
-            else
-              last->max_flag|= NEAR_MAX;
+            /*
+              This is the case:
+              key2:    [-------------]
+              key1:  [***------]  [xxxx]
+                     ^            ^
+                     last         different next_key_part
+
+              Extend range of last up to last->next:
+              key2:    [-------------]
+              key1:  [***--------][xxxx]
+            */
+            last->copy_min_to_max(last->next);
           }
           else
+            /*
+              This is the case:
+              key2:    [--------*****]
+              key1:  [***---------]    [xxxx]
+                     ^                 ^
+                     last              different next_key_part
+
+              Extend range of last up to max(last.max, key2.max):
+              key2:    [--------*****]
+              key1:  [***----------**] [xxxx]
+             */
             full_range= last->copy_max(key2);
         }
-	if (full_range)
-	{					// Full range
-	  key1->free_tree();
-	  for (; key2 ; key2=key2->next)
-	    key2->increment_use_count(-1);	// Free not used tree
-	  if (key1->maybe_flag)
-	    return new SEL_ARG(SEL_ARG::MAYBE_KEY);
-	  return 0;
-	}
+        if (full_range)
+        {                                       // Full range
+          key1->free_tree();
+          for (; key2 ; key2=key2->next)
+            key2->increment_use_count(-1);      // Free not used tree
+          if (key1->maybe_flag)
+            return new SEL_ARG(SEL_ARG::MAYBE_KEY);
+          return 0;
+        }
       }
     }
 
     if (cmp >= 0 && tmp->cmp_min_to_min(key2) < 0)
-    {						// tmp.min <= x < key2.min
+    {
+      /*
+        This is the case ("cmp>=0" means that tmp.max >= key2.min):
+        key2:              [----]
+        tmp:     [------------*****]
+      */
+
+      if (!tmp->next_key_part)
+      {
+        /*
+          tmp->next_key_part is empty: cut the range that is covered
+          by tmp from key2. 
+          Reason: (key2->next_key_part OR tmp->next_key_part) will be
+          empty and therefore equal to tmp->next_key_part. Thus, this
+          part of the key2 range is completely covered by tmp.
+        */
+        if (tmp->cmp_max_to_max(key2) >= 0)
+        {
+          /*
+            tmp covers the entire range in key2. 
+            key2:              [----]
+            tmp:     [-----------------]
+
+            Move on to next range in key2
+          */
+          key2->increment_use_count(-1); // Free not used tree
+          key2=key2->next;
+          continue;
+        }
+        else
+        {
+          /*
+            This is the case:
+            key2:           [-------]
+            tmp:     [---------]
+
+            Result:
+            key2:               [---]
+            tmp:     [---------]
+          */
+          if (key2->use_count)
+	  {
+	    SEL_ARG *key2_cpy= new SEL_ARG(*key2);
+            if (key2_cpy)
+              return 0;
+            key2= key2_cpy;
+	  }
+          key2->copy_max_to_min(tmp);
+          continue;
+        }
+      }
+
+      /*
+        The ranges are overlapping but have not been merged because
+        next_key_part of tmp and key2 differ. 
+        key2:              [----]
+        tmp:     [------------*****]
+
+        Split tmp in two where key2 starts:
+        key2:              [----]
+        key1:    [--------][--*****]
+                 ^         ^
+                 insert    tmp
+      */
       SEL_ARG *new_arg=tmp->clone_first(key2);
       if (!new_arg)
-	return 0;				// OOM
-      if ((new_arg->next_key_part= key1->next_key_part))
-	new_arg->increment_use_count(key1->use_count+1);
+        return 0;                               // OOM
+      if ((new_arg->next_key_part= tmp->next_key_part))
+        new_arg->increment_use_count(key1->use_count+1);
       tmp->copy_min_to_min(key2);
       key1=key1->insert(new_arg);
-    }
+    } // tmp.min >= key2.min due to this if()
 
-    // tmp.min >= key2.min && tmp.min <= key2.max
-    SEL_ARG key(*key2);				// Get copy we can modify
+    /*
+      Now key2.min <= tmp.min <= key2.max:
+      key2:   [---------]
+      tmp:    [****---*****]
+     */
+    SEL_ARG key2_cpy(*key2); // Get copy we can modify
     for (;;)
     {
-      if (tmp->cmp_min_to_min(&key) > 0)
-      {						// key.min <= x < tmp.min
-	SEL_ARG *new_arg=key.clone_first(tmp);
-	if (!new_arg)
-	  return 0;				// OOM
-	if ((new_arg->next_key_part=key.next_key_part))
-	  new_arg->increment_use_count(key1->use_count+1);
-	key1=key1->insert(new_arg);
-      }
-      if ((cmp=tmp->cmp_max_to_max(&key)) <= 0)
-      {						// tmp.min. <= x <= tmp.max
-	tmp->maybe_flag|= key.maybe_flag;
-	key.increment_use_count(key1->use_count+1);
-	tmp->next_key_part= key_or(param, tmp->next_key_part, key.next_key_part);
-	if (!cmp)				// Key2 is ready
-	  break;
-	key.copy_max_to_min(tmp);
-	if (!(tmp=tmp->next))
-	{
-	  SEL_ARG *tmp2= new SEL_ARG(key);
-	  if (!tmp2)
-	    return 0;				// OOM
-	  key1=key1->insert(tmp2);
-	  key2=key2->next;
-	  goto end;
-	}
-	if (tmp->cmp_min_to_max(&key) > 0)
-	{
-	  SEL_ARG *tmp2= new SEL_ARG(key);
-	  if (!tmp2)
-	    return 0;				// OOM
-	  key1=key1->insert(tmp2);
-	  break;
-	}
+      if (tmp->cmp_min_to_min(&key2_cpy) > 0)
+      {
+        /*
+          This is the case:
+          key2_cpy:    [------------]
+          key1:                 [-*****]
+                                ^
+                                tmp
+                             
+          Result:
+          key2_cpy:             [---]
+          key1:        [-------][-*****]
+                       ^        ^
+                       insert   tmp
+         */
+        SEL_ARG *new_arg=key2_cpy.clone_first(tmp);
+        if (!new_arg)
+          return 0; // OOM
+        if ((new_arg->next_key_part=key2_cpy.next_key_part))
+          new_arg->increment_use_count(key1->use_count+1);
+        key1=key1->insert(new_arg);
+        key2_cpy.copy_min_to_min(tmp);
+      } 
+      // Now key2_cpy.min == tmp.min
+
+      if ((cmp= tmp->cmp_max_to_max(&key2_cpy)) <= 0)
+      {
+        /*
+          tmp.max <= key2_cpy.max:
+          key2_cpy:   a)  [-------]    or b)     [----]
+          tmp:            [----]                 [----]
+
+          Steps:
+           1) Update next_key_part of tmp: OR it with key2_cpy->next_key_part.
+           2) If case a: Insert range [tmp.max, key2_cpy.max] into key1 using
+                         next_key_part of key2_cpy
+
+           Result:
+           key1:      a)  [----][-]    or b)     [----]
+         */
+        tmp->maybe_flag|= key2_cpy.maybe_flag;
+        key2_cpy.increment_use_count(key1->use_count+1);
+        tmp->next_key_part= key_or(param, tmp->next_key_part,
+                                   key2_cpy.next_key_part);
+
+        if (!cmp)
+          break;                     // case b: done with this key2 range
+
+        // Make key2_cpy the range [tmp.max, key2_cpy.max]
+        key2_cpy.copy_max_to_min(tmp);
+        if (!(tmp=tmp->next))
+        {
+          /*
+            No more ranges in key1. Insert key2_cpy and go to "end"
+            label to insert remaining ranges in key2 if any.
+          */
+          SEL_ARG *tmp2= new SEL_ARG(key2_cpy);
+          if (!tmp2)
+            return 0; // OOM
+          key1=key1->insert(tmp2);
+          key2=key2->next;
+          goto end;
+        }
+        if (tmp->cmp_min_to_max(&key2_cpy) > 0)
+        {
+          /*
+            The next range in key1 does not overlap with key2_cpy.
+            Insert this range into key1 and move on to the next range
+            in key2.
+          */
+          SEL_ARG *tmp2= new SEL_ARG(key2_cpy);
+          if (!tmp2)
+            return 0;                           // OOM
+          key1=key1->insert(tmp2);
+          break;
+        }
+        /*
+          key2_cpy overlaps with the next range in key1 and the case
+          is now "key2.min <= tmp.min <= key2.max". Go back to for(;;)
+          to handle this situation.
+        */
+        continue;
       }
       else
       {
-	SEL_ARG *new_arg=tmp->clone_last(&key); // tmp.min <= x <= key.max
-	if (!new_arg)
-	  return 0;				// OOM
-	tmp->copy_max_to_min(&key);
-	tmp->increment_use_count(key1->use_count+1);
-	/* Increment key count as it may be used for next loop */
-	key.increment_use_count(1);
-	new_arg->next_key_part= key_or(param, tmp->next_key_part, key.next_key_part);
-	key1=key1->insert(new_arg);
-	break;
+        /*
+          This is the case:
+          key2_cpy:   [-------]
+          tmp:        [------------]
+
+          Result:
+          key1:       [-------][---]
+                      ^        ^
+                      new_arg  tmp
+          Steps:
+           0) If tmp->next_key_part is empty: do nothing. Reason:
+              (key2_cpy->next_key_part OR tmp->next_key_part) will be
+              empty and therefore equal to tmp->next_key_part. Thus,
+              the range in key2_cpy is completely covered by tmp
+           1) Make new_arg with range [tmp.min, key2_cpy.max].
+              new_arg->next_key_part is OR between next_key_part
+              of tmp and key2_cpy
+           2) Make tmp the range [key2.max, tmp.max]
+           3) Insert new_arg into key1
+        */
+        if (!tmp->next_key_part) // Step 0
+        {
+          key2_cpy.increment_use_count(-1);     // Free not used tree
+          break;
+        }
+        SEL_ARG *new_arg=tmp->clone_last(&key2_cpy);
+        if (!new_arg)
+          return 0; // OOM
+        tmp->copy_max_to_min(&key2_cpy);
+        tmp->increment_use_count(key1->use_count+1);
+        /* Increment key count as it may be used for next loop */
+        key2_cpy.increment_use_count(1);
+        new_arg->next_key_part= key_or(param, tmp->next_key_part,
+                                       key2_cpy.next_key_part);
+        key1=key1->insert(new_arg);
+        break;
       }
     }
-    key2=key2->next;
+    // Move on to next range in key2
+    key2=key2->next;                            
   }
 
 end:
+  /*
+    Add key2 ranges that are non-overlapping with and higher than the
+    highest range in key1.
+  */
   while (key2)
   {
     SEL_ARG *next=key2->next;
     if (key2_shared)
     {
-      SEL_ARG *tmp=new SEL_ARG(*key2);		// Must make copy
+      SEL_ARG *tmp=new SEL_ARG(*key2);          // Must make copy
       if (!tmp)
-	return 0;
+        return 0;
       key2->increment_use_count(key1->use_count+1);
       key1=key1->insert(tmp);
     }
     else
-      key1=key1->insert(key2);			// Will destroy key2_root
+      key1=key1->insert(key2);                  // Will destroy key2_root
     key2=next;
   }
   key1->use_count++;
+
   key1->max_part_no= max_part_no;
   return key1;
 }
@@ -11728,17 +12033,19 @@ check_group_min_max_predicates(Item *cond, Item_field *min_max_arg_item,
     Disallow loose index scan if the MIN/MAX argument field is referenced by
     a subquery in the WHERE clause.
   */
+
   if (cond_type == Item::SUBSELECT_ITEM)
   {
     Item_subselect *subs_cond= (Item_subselect*) cond;
     if (subs_cond->is_correlated)
     {
-      DBUG_ASSERT(subs_cond->depends_on.elements > 0);
-      List_iterator_fast<Item*> li(subs_cond->depends_on);
-      Item **dep;
+      DBUG_ASSERT(subs_cond->upper_refs.elements > 0);
+      List_iterator_fast<Item_subselect::Ref_to_outside>
+        li(subs_cond->upper_refs);
+      Item_subselect::Ref_to_outside *dep;
       while ((dep= li++))
       {
-        if ((*dep)->eq(min_max_arg_item, FALSE))
+        if (dep->item->eq(min_max_arg_item, FALSE))
           DBUG_RETURN(FALSE);
       }
     }
@@ -13341,7 +13648,6 @@ print_key(KEY_PART *key_part, const uchar *key, uint used_length)
 {
   char buff[1024];
   const uchar *key_end= key+used_length;
-  String tmp(buff,sizeof(buff),&my_charset_bin);
   uint store_length;
   TABLE *table= key_part->field->table;
   my_bitmap_map *old_sets[2];
@@ -13350,6 +13656,7 @@ print_key(KEY_PART *key_part, const uchar *key, uint used_length)
 
   for (; key < key_end; key+=store_length, key_part++)
   {
+    String tmp(buff,sizeof(buff),&my_charset_bin);
     Field *field=      key_part->field;
     store_length= key_part->store_length;
 
diff --git a/sql/opt_range.h b/sql/opt_range.h
index d7a0c1e2f8f..e498d229dde 100644
--- a/sql/opt_range.h
+++ b/sql/opt_range.h
@@ -274,7 +274,6 @@ public:
 
   virtual bool reverse_sorted() = 0;
   virtual bool unique_key_range() { return false; }
-  virtual bool clustered_pk_range() { return false; }
 
   /*
     Request that this quick select produces sorted output. Not all quick
@@ -355,6 +354,12 @@ public:
     Table record buffer used by this quick select.
   */
   uchar    *record;
+
+  virtual void replace_handler(handler *new_file)
+  {
+    DBUG_ASSERT(0); /* Only supported in QUICK_RANGE_SELECT */
+  }
+
 #ifndef DBUG_OFF
   /*
     Print quick select information to DBUG_FILE. Caller is responsible
@@ -450,6 +455,7 @@ public:
 #ifndef DBUG_OFF
   void dbug_dump(int indent, bool verbose);
 #endif
+  virtual void replace_handler(handler *new_file) { file= new_file; }
 private:
   /* Default copy ctor used by QUICK_SELECT_DESC */
   friend class TRP_ROR_INTERSECT;
@@ -593,9 +599,6 @@ public:
   MEM_ROOT alloc;
   THD *thd;
   virtual int read_keys_and_merge()= 0;
-
-  bool clustered_pk_range() { return test(pk_quick_select); }
-
   /* used to get rows collected in Unique */
   READ_RECORD read_record;
 };
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index 8ff45e0306e..56815a624e2 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -177,13 +177,13 @@ bool subquery_types_allow_materialization(Item_in_subselect *in_subs);
 static bool replace_where_subcondition(JOIN *join, Item **expr, 
                                        Item *old_cond, Item *new_cond,
                                        bool do_fix_fields);
-static int subq_sj_candidate_cmp(Item_in_subselect* const *el1, 
-                                 Item_in_subselect* const *el2);
+static int subq_sj_candidate_cmp(Item_in_subselect* el1, Item_in_subselect* el2,
+                                 void *arg);
 static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred);
 static bool convert_subq_to_jtbm(JOIN *parent_join, 
                                  Item_in_subselect *subq_pred, bool *remove);
 static TABLE_LIST *alloc_join_nest(THD *thd);
-static uint get_tmp_table_rec_length(List<Item> &items);
+static uint get_tmp_table_rec_length(Item **p_list, uint elements);
 static double get_tmp_table_lookup_cost(THD *thd, double row_count,
                                         uint row_size);
 static double get_tmp_table_write_cost(THD *thd, double row_count,
@@ -210,6 +210,74 @@ end_sj_materialize(JOIN *join, JOIN_TAB *join_tab, bool end_of_records);
 
 
 /*
+  Check if Materialization strategy is allowed for given subquery predicate.
+
+  @param thd           Thread handle
+  @param in_subs       The subquery predicate
+  @param child_select  The select inside predicate (the function will
+                       check it is the only one)
+
+  @return TRUE  - Materialization is applicable 
+          FALSE - Otherwise
+*/
+
+bool is_materialization_applicable(THD *thd, Item_in_subselect *in_subs,
+                                   st_select_lex *child_select)
+{
+  st_select_lex_unit* parent_unit= child_select->master_unit();
+  /*
+    Check if the subquery predicate can be executed via materialization.
+    The required conditions are:
+    0. The materialization optimizer switch was set.
+    1. Subquery is a single SELECT (not a UNION).
+       TODO: this is a limitation that can be fixed
+    2. Subquery is not a table-less query. In this case there is no
+       point in materializing.
+    2A The upper query is not a table-less SELECT ... FROM DUAL. We
+       can't do materialization for SELECT .. FROM DUAL because it
+       does not call setup_subquery_materialization(). We could make 
+       SELECT ... FROM DUAL call that function but that doesn't seem
+       to be the case that is worth handling.
+    3. Either the subquery predicate is a top-level predicate, or at
+       least one partial match strategy is enabled. If no partial match
+       strategy is enabled, then materialization cannot be used for
+       non-top-level queries because it cannot handle NULLs correctly.
+    4. Subquery is non-correlated
+       TODO:
+       This condition is too restrictive (limitation). It can be extended to:
+       (Subquery is non-correlated ||
+        Subquery is correlated to any query outer to IN predicate ||
+        (Subquery is correlated to the immediate outer query &&
+         Subquery !contains {GROUP BY, ORDER BY [LIMIT],
+         aggregate functions}) && subquery predicate is not under "NOT IN"))
+
+    (*) The subquery must be part of a SELECT statement. The current
+         condition also excludes multi-table update statements.
+  A note about prepared statements: we want the if-branch to be taken on
+  PREPARE and each EXECUTE. The rewrites are only done once, but we need 
+  select_lex->sj_subselects list to be populated for every EXECUTE. 
+
+  */
+  if (optimizer_flag(thd, OPTIMIZER_SWITCH_MATERIALIZATION) &&      // 0
+        !child_select->is_part_of_union() &&                          // 1
+        parent_unit->first_select()->leaf_tables.elements &&          // 2
+        thd->lex->sql_command == SQLCOM_SELECT &&                     // *
+        child_select->outer_select()->leaf_tables.elements &&           // 2A
+        subquery_types_allow_materialization(in_subs) &&
+        (in_subs->is_top_level_item() ||                               //3
+         optimizer_flag(thd,
+                        OPTIMIZER_SWITCH_PARTIAL_MATCH_ROWID_MERGE) || //3
+         optimizer_flag(thd,
+                        OPTIMIZER_SWITCH_PARTIAL_MATCH_TABLE_SCAN)) && //3
+        !in_subs->is_correlated)                                       //4
+   {
+     return TRUE;
+   }
+  return FALSE;
+}
+
+
+/*
   Check if we need JOIN::prepare()-phase subquery rewrites and if yes, do them
 
   SYNOPSIS
@@ -340,10 +408,10 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         !select_lex->is_part_of_union() &&                            // 2
         !select_lex->group_list.elements && !join->order &&           // 3
         !join->having && !select_lex->with_sum_func &&                // 4
-        thd->thd_marker.emb_on_expr_nest &&                           // 5
+        in_subs->emb_on_expr_nest &&                                  // 5
         select_lex->outer_select()->join &&                           // 6
         parent_unit->first_select()->leaf_tables.elements &&          // 7
-        !in_subs->in_strategy &&                                      // 8
+        !in_subs->has_strategy() &&                                   // 8
         select_lex->outer_select()->leaf_tables.elements &&           // 9
         !((join->select_options |                                     // 10
            select_lex->outer_select()->join->select_options)          // 10
@@ -353,12 +421,18 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
 
       (void)subquery_types_allow_materialization(in_subs);
 
-      in_subs->emb_on_expr_nest= thd->thd_marker.emb_on_expr_nest;
       in_subs->is_flattenable_semijoin= TRUE;
 
       /* Register the subquery for further processing in flatten_subqueries() */
-      select_lex->
-        outer_select()->join->sj_subselects.append(thd->mem_root, in_subs);
+      if (!in_subs->is_registered_semijoin)
+      {
+        Query_arena *arena, backup;
+        arena= thd->activate_stmt_arena_if_needed(&backup);
+        select_lex->outer_select()->sj_subselects.push_back(in_subs);
+        if (arena)
+          thd->restore_active_arena(arena, &backup);
+        in_subs->is_registered_semijoin= TRUE;
+      }
     }
     else
     {
@@ -375,65 +449,27 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
       */
       if (in_subs)
       {
-        /*
-          Check if the subquery predicate can be executed via materialization.
-          The required conditions are:
-          0. The materialization optimizer switch was set.
-          1. Subquery is a single SELECT (not a UNION).
-             TODO: this is a limitation that can be fixed
-          2. Subquery is not a table-less query. In this case there is no
-             point in materializing.
-          2A The upper query is not a table-less SELECT ... FROM DUAL. We
-             can't do materialization for SELECT .. FROM DUAL because it
-             does not call setup_subquery_materialization(). We could make 
-             SELECT ... FROM DUAL call that function but that doesn't seem
-             to be the case that is worth handling.
-          3. Either the subquery predicate is a top-level predicate, or at
-             least one partial match strategy is enabled. If no partial match
-             strategy is enabled, then materialization cannot be used for
-             non-top-level queries because it cannot handle NULLs correctly.
-          4. Subquery is non-correlated
-             TODO:
-             This condition is too restrictive (limitation). It can be extended to:
-             (Subquery is non-correlated ||
-              Subquery is correlated to any query outer to IN predicate ||
-              (Subquery is correlated to the immediate outer query &&
-               Subquery !contains {GROUP BY, ORDER BY [LIMIT],
-               aggregate functions}) && subquery predicate is not under "NOT IN"))
-
-          (*) The subquery must be part of a SELECT statement. The current
-               condition also excludes multi-table update statements.
-        A note about prepared statements: we want the if-branch to be taken on
-        PREPARE and each EXECUTE. The rewrites are only done once, but we need 
-        join->sj_subselects list to be populated for every EXECUTE. 
-
-        */
-        if (optimizer_flag(thd, OPTIMIZER_SWITCH_MATERIALIZATION) &&      // 0
-            !select_lex->is_part_of_union() &&                            // 1
-            parent_unit->first_select()->leaf_tables.elements &&          // 2
-            thd->lex->sql_command == SQLCOM_SELECT &&                     // *
-            select_lex->outer_select()->leaf_tables.elements &&           // 2A
-            subquery_types_allow_materialization(in_subs) &&
-            (in_subs->is_top_level_item() ||                               //3
-             optimizer_flag(thd,
-                            OPTIMIZER_SWITCH_PARTIAL_MATCH_ROWID_MERGE) || //3
-             optimizer_flag(thd,
-                            OPTIMIZER_SWITCH_PARTIAL_MATCH_TABLE_SCAN)) && //3
-            !in_subs->is_correlated)                                       //4
-       {
-          in_subs->in_strategy|= SUBS_MATERIALIZATION;
+        if (is_materialization_applicable(thd, in_subs, select_lex))
+        {
+          in_subs->add_strategy(SUBS_MATERIALIZATION);
 
           /*
             If the subquery is an AND-part of WHERE register for being processed
             with jtbm strategy
           */
-          if (thd->thd_marker.emb_on_expr_nest == NO_JOIN_NEST &&
+          if (in_subs->emb_on_expr_nest == NO_JOIN_NEST &&
               optimizer_flag(thd, OPTIMIZER_SWITCH_SEMIJOIN))
           {
-            in_subs->emb_on_expr_nest= thd->thd_marker.emb_on_expr_nest;
             in_subs->is_flattenable_semijoin= FALSE;
-            select_lex->outer_select()->
-              join->sj_subselects.append(thd->mem_root, in_subs);
+            if (!in_subs->is_registered_semijoin)
+	    {
+              Query_arena *arena, backup;
+              arena= thd->activate_stmt_arena_if_needed(&backup);
+              select_lex->outer_select()->sj_subselects.push_back(in_subs);
+              if (arena)
+                thd->restore_active_arena(arena, &backup);
+              in_subs->is_registered_semijoin= TRUE;
+            }
           }
         }
 
@@ -443,17 +479,18 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
           possible.
         */
         if (optimizer_flag(thd, OPTIMIZER_SWITCH_IN_TO_EXISTS) ||
-            !in_subs->in_strategy)
-        {
-          in_subs->in_strategy|= SUBS_IN_TO_EXISTS;
-        }
+            !in_subs->has_strategy())
+          in_subs->add_strategy(SUBS_IN_TO_EXISTS);
       }
 
       /* Check if max/min optimization applicable */
-      if (allany_subs)
-        allany_subs->in_strategy|= (allany_subs->is_maxmin_applicable(join) ?
-                                    SUBS_MAXMIN :
-                                    SUBS_IN_TO_EXISTS);
+      if (allany_subs && !allany_subs->is_set_strategy())
+      {
+        uchar strategy= (allany_subs->is_maxmin_applicable(join) ?
+                         (SUBS_MAXMIN_INJECTED | SUBS_MAXMIN_ENGINE) :
+                         SUBS_IN_TO_EXISTS);
+        allany_subs->add_strategy(strategy);
+      }
 
       /*
         Transform each subquery predicate according to its overloaded
@@ -547,6 +584,16 @@ bool subquery_types_allow_materialization(Item_in_subselect *in_subs)
       if (inner->field_type() == MYSQL_TYPE_BLOB || 
           inner->field_type() == MYSQL_TYPE_GEOMETRY)
         DBUG_RETURN(FALSE);
+      /* 
+        Materialization also is unable to work when create_tmp_table() will
+        create a blob column because item->max_length is too big.
+        The following check is copied from Item::make_string_field():
+      */ 
+      if (inner->max_length / inner->collation.collation->mbmaxlen > 
+          CONVERT_IF_BIGGER_TO_BLOB)
+      {
+        DBUG_RETURN(FALSE);
+      }
       break;
     case TIME_RESULT:
       if (mysql_type_to_time_type(outer->field_type()) !=
@@ -731,21 +778,19 @@ bool check_for_outer_joins(List<TABLE_LIST> *join_list)
 bool convert_join_subqueries_to_semijoins(JOIN *join)
 {
   Query_arena *arena, backup;
-  Item_in_subselect **in_subq;
-  Item_in_subselect **in_subq_end;
+  Item_in_subselect *in_subq;
   THD *thd= join->thd;
   List_iterator<TABLE_LIST> ti(join->select_lex->leaf_tables);
   DBUG_ENTER("convert_join_subqueries_to_semijoins");
 
-  if (join->sj_subselects.elements() == 0)
+  if (join->select_lex->sj_subselects.is_empty())
     DBUG_RETURN(FALSE);
 
-  for (in_subq= join->sj_subselects.front(), 
-       in_subq_end= join->sj_subselects.back(); 
-       in_subq != in_subq_end; 
-       in_subq++)
+  List_iterator_fast<Item_in_subselect> li(join->select_lex->sj_subselects);
+
+  while ((in_subq= li++))
   {
-    SELECT_LEX *subq_sel= (*in_subq)->get_select_lex();
+    SELECT_LEX *subq_sel= in_subq->get_select_lex();
     if (subq_sel->handle_derived(thd->lex, DT_OPTIMIZE))
       DBUG_RETURN(1);
     if (subq_sel->handle_derived(thd->lex, DT_MERGE))
@@ -753,13 +798,11 @@ bool convert_join_subqueries_to_semijoins(JOIN *join)
     subq_sel->update_used_tables();
   }
 
+  li.rewind();
   /* First, convert child join's subqueries. We proceed bottom-up here */
-  for (in_subq= join->sj_subselects.front(), 
-       in_subq_end= join->sj_subselects.back(); 
-       in_subq != in_subq_end; 
-       in_subq++)
+  while ((in_subq= li++)) 
   {
-    st_select_lex *child_select= (*in_subq)->get_select_lex();
+    st_select_lex *child_select= in_subq->get_select_lex();
     JOIN *child_join= child_select->join;
     child_join->outer_tables = child_join->table_count;
 
@@ -773,19 +816,21 @@ bool convert_join_subqueries_to_semijoins(JOIN *join)
 
     if (convert_join_subqueries_to_semijoins(child_join))
       DBUG_RETURN(TRUE);
-    (*in_subq)->sj_convert_priority= 
-      (*in_subq)->is_correlated * MAX_TABLES + child_join->outer_tables;
+    in_subq->sj_convert_priority= 
+      test(in_subq->emb_on_expr_nest != NO_JOIN_NEST) * MAX_TABLES * 2 +
+      in_subq->is_correlated * MAX_TABLES + child_join->outer_tables;
   }
   
   // Temporary measure: disable semi-joins when they are together with outer
   // joins.
+#if 0  
   if (check_for_outer_joins(join->join_list))
   {
-    in_subq= join->sj_subselects.front();
+    in_subq= join->select_lex->sj_subselects.head();
     arena= thd->activate_stmt_arena_if_needed(&backup);
     goto skip_conversion;
   }
-
+#endif
   //dump_TABLE_LIST_struct(select_lex, select_lex->leaf_tables);
   /* 
     2. Pick which subqueries to convert:
@@ -793,75 +838,79 @@ bool convert_join_subqueries_to_semijoins(JOIN *join)
       - prefer correlated subqueries over uncorrelated;
       - prefer subqueries that have greater number of outer tables;
   */
-  join->sj_subselects.sort(subq_sj_candidate_cmp);
+  bubble_sort<Item_in_subselect>(&join->select_lex->sj_subselects,
+				 subq_sj_candidate_cmp, NULL);
   // #tables-in-parent-query + #tables-in-subquery < MAX_TABLES
   /* Replace all subqueries to be flattened with Item_int(1) */
   arena= thd->activate_stmt_arena_if_needed(&backup);
  
-  for (in_subq= join->sj_subselects.front(); 
-       in_subq != in_subq_end;
-       in_subq++)
+  li.rewind();
+  while ((in_subq= li++))
   {
     bool remove_item= TRUE;
-    if ((*in_subq)->is_flattenable_semijoin) 
+
+    /* Stop processing if we've reached a subquery that's attached to the ON clause */
+    if (in_subq->emb_on_expr_nest != NO_JOIN_NEST)
+      break;
+
+    if (in_subq->is_flattenable_semijoin) 
     {
       if (join->table_count + 
-          (*in_subq)->unit->first_select()->join->table_count >= MAX_TABLES)
+          in_subq->unit->first_select()->join->table_count >= MAX_TABLES)
         break;
-      if (convert_subq_to_sj(join, *in_subq))
-        DBUG_RETURN(TRUE);
+      if (convert_subq_to_sj(join, in_subq))
+        goto restore_arena_and_fail;
     }
     else
     {
       if (join->table_count + 1 >= MAX_TABLES)
         break;
-      if (convert_subq_to_jtbm(join, *in_subq, &remove_item))
-        DBUG_RETURN(TRUE);
+      if (convert_subq_to_jtbm(join, in_subq, &remove_item))
+        goto restore_arena_and_fail;
     }
     if (remove_item)
     {
-      Item **tree= ((*in_subq)->emb_on_expr_nest == NO_JOIN_NEST)?
-                     &join->conds : &((*in_subq)->emb_on_expr_nest->on_expr);
-      Item *replace_me= (*in_subq)->original_item();
+      Item **tree= (in_subq->emb_on_expr_nest == NO_JOIN_NEST)?
+                     &join->conds : &(in_subq->emb_on_expr_nest->on_expr);
+      Item *replace_me= in_subq->original_item();
       if (replace_where_subcondition(join, tree, replace_me, new Item_int(1),
                                      FALSE))
-        DBUG_RETURN(TRUE); /* purecov: inspected */
+        goto restore_arena_and_fail;
     }
   }
-skip_conversion:
+//skip_conversion:
   /* 
     3. Finalize (perform IN->EXISTS rewrite) the subqueries that we didn't
     convert:
   */
-  for (; in_subq!= in_subq_end; in_subq++)
+  while (in_subq)
   {
-    JOIN *child_join= (*in_subq)->unit->first_select()->join;
-    (*in_subq)->changed= 0;
-    (*in_subq)->fixed= 0;
+    JOIN *child_join= in_subq->unit->first_select()->join;
+    in_subq->changed= 0;
+    in_subq->fixed= 0;
 
     SELECT_LEX *save_select_lex= thd->lex->current_select;
-    thd->lex->current_select= (*in_subq)->unit->first_select();
+    thd->lex->current_select= in_subq->unit->first_select();
 
-    bool res= (*in_subq)->select_transformer(child_join);
+    bool res= in_subq->select_transformer(child_join);
 
     thd->lex->current_select= save_select_lex;
 
     if (res)
       DBUG_RETURN(TRUE);
 
-    (*in_subq)->changed= 1;
-    (*in_subq)->fixed= 1;
+    in_subq->changed= 1;
+    in_subq->fixed= 1;
 
-    Item *substitute= (*in_subq)->substitution;
-    bool do_fix_fields= !(*in_subq)->substitution->fixed;
-    Item **tree= ((*in_subq)->emb_on_expr_nest == NO_JOIN_NEST)?
-                   &join->conds : &((*in_subq)->emb_on_expr_nest->on_expr);
-    Item *replace_me= (*in_subq)->original_item();
+    Item *substitute= in_subq->substitution;
+    bool do_fix_fields= !in_subq->substitution->fixed;
+    Item **tree= (in_subq->emb_on_expr_nest == NO_JOIN_NEST)?
+                   &join->conds : &(in_subq->emb_on_expr_nest->on_expr);
+    Item *replace_me= in_subq->original_item();
     if (replace_where_subcondition(join, tree, replace_me, substitute, 
                                    do_fix_fields))
       DBUG_RETURN(TRUE);
-    (*in_subq)->substitution= NULL;
-    
+    in_subq->substitution= NULL;
     /*
       If this is a prepared statement, repeat the above operation for
       prep_where (or prep_on_expr). Subquery-to-semijoin conversion is 
@@ -869,29 +918,41 @@ skip_conversion:
     */
     if (!thd->stmt_arena->is_conventional())
     {
-      tree= ((*in_subq)->emb_on_expr_nest == NO_JOIN_NEST)?
+      tree= (in_subq->emb_on_expr_nest == NO_JOIN_NEST)?
              &join->select_lex->prep_where : 
-             &((*in_subq)->emb_on_expr_nest->prep_on_expr);
-
-      if (replace_where_subcondition(join, tree, replace_me, substitute, 
+             &(in_subq->emb_on_expr_nest->prep_on_expr);
+      /* 
+        prep_on_expr/ prep_where may be NULL in some cases. 
+        If that is the case, do nothing - simplify_joins() will copy 
+        ON/WHERE expression into prep_on_expr/prep_where.
+      */
+      if (*tree && replace_where_subcondition(join, tree, replace_me, substitute, 
                                      FALSE))
         DBUG_RETURN(TRUE);
     }
     /*
       Revert to the IN->EXISTS strategy in the rare case when the subquery could
       not be flattened.
-      TODO: This is a limitation done for simplicity. Such subqueries could also
-      be executed via materialization. In order to determine this, we should
-      re-run the test for materialization that was done in
-      check_and_do_in_subquery_rewrites.
     */
-    (*in_subq)->in_strategy= SUBS_IN_TO_EXISTS;
+    in_subq->reset_strategy(SUBS_IN_TO_EXISTS);
+    if (is_materialization_applicable(thd, in_subq, 
+                                      in_subq->unit->first_select()))
+    {
+      in_subq->add_strategy(SUBS_MATERIALIZATION);
+    }
+
+    in_subq= li++;
   }
 
   if (arena)
     thd->restore_active_arena(arena, &backup);
-  join->sj_subselects.clear();
+  join->select_lex->sj_subselects.empty();
   DBUG_RETURN(FALSE);
+
+restore_arena_and_fail:
+  if (arena)
+    thd->restore_active_arena(arena, &backup);
+  DBUG_RETURN(TRUE);
 }
 
 
@@ -962,7 +1023,6 @@ static bool replace_where_subcondition(JOIN *join, Item **expr,
                                        Item *old_cond, Item *new_cond,
                                        bool do_fix_fields)
 {
-  //Item **expr= (emb_nest == (TABLE_LIST*)1)? &join->conds : &emb_nest->on_expr;
   if (*expr == old_cond)
   {
     *expr= new_cond;
@@ -986,16 +1046,22 @@ static bool replace_where_subcondition(JOIN *join, Item **expr,
       }
     }
   }
-  // If we came here it means there were an error during prerequisites check.
-  DBUG_ASSERT(0);
-  return TRUE;
+  /* 
+    We can come to here when 
+     - we're doing replace operations on both on_expr and prep_on_expr
+     - on_expr is the same as prep_on_expr, or they share a sub-tree 
+       (so, when we do replace in on_expr, we replace in prep_on_expr, too,
+        and when we try doing a replace in prep_on_expr, the item we wanted 
+        to replace there has already been replaced)
+  */
+  return FALSE;
 }
 
-static int subq_sj_candidate_cmp(Item_in_subselect* const *el1, 
-                                 Item_in_subselect* const *el2)
+static int subq_sj_candidate_cmp(Item_in_subselect* el1, Item_in_subselect* el2,
+                                 void *arg)
 {
-  return ((*el1)->sj_convert_priority < (*el2)->sj_convert_priority) ? 1 : 
-         ( ((*el1)->sj_convert_priority == (*el2)->sj_convert_priority)? 0 : -1);
+  return (el1->sj_convert_priority > el2->sj_convert_priority) ? 1 : 
+         ( (el1->sj_convert_priority == el2->sj_convert_priority)? 0 : -1);
 }
 
 
@@ -1191,7 +1257,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
   /* 3. Remove the original subquery predicate from the WHERE/ON */
 
   // The subqueries were replaced for Item_int(1) earlier
-  subq_pred->in_strategy= SUBS_SEMI_JOIN;         // for subsequent executions
+  subq_pred->reset_strategy(SUBS_SEMI_JOIN);       // for subsequent executions
   /*TODO: also reset the 'with_subselect' there. */
 
   /* n. Adjust the parent_join->table_count counter */
@@ -1202,6 +1268,8 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
   {
     tl->table->tablenr= table_no;
     tl->table->map= ((table_map)1) << table_no;
+    if (tl->is_jtbm())
+      tl->jtbm_table_no= tl->table->tablenr;
     SELECT_LEX *old_sl= tl->select_lex;
     tl->select_lex= parent_join->select_lex; 
     for (TABLE_LIST *emb= tl->embedding;
@@ -1211,6 +1279,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
     table_no++;
   }
   parent_join->table_count += subq_lex->join->table_count;
+  //parent_join->table_count += subq_lex->leaf_tables.elements;
 
   /* 
     Put the subquery's WHERE into semi-join's sj_on_expr
@@ -1290,6 +1359,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
   {
     emb_tbl_nest->on_expr= and_items(emb_tbl_nest->on_expr, 
                                      sj_nest->sj_on_expr);
+    emb_tbl_nest->on_expr->top_level_item();
     if (!emb_tbl_nest->on_expr->fixed)
       emb_tbl_nest->on_expr->fix_fields(parent_join->thd,
                                         &emb_tbl_nest->on_expr);
@@ -1298,6 +1368,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
   {
     /* Inject into the WHERE */
     parent_join->conds= and_items(parent_join->conds, sj_nest->sj_on_expr);
+    parent_join->conds->top_level_item();
     /*
       fix_fields must update the properties (e.g. st_select_lex::cond_count of
       the correct select_lex.
@@ -1360,8 +1431,9 @@ static bool convert_subq_to_jtbm(JOIN *parent_join,
   double read_time;
   DBUG_ENTER("convert_subq_to_jtbm");
 
-  subq_pred->in_strategy &= ~SUBS_IN_TO_EXISTS;
-  subq_pred->optimize(&rows, &read_time);
+  subq_pred->set_strategy(SUBS_MATERIALIZATION);
+  if (subq_pred->optimize(&rows, &read_time))
+    DBUG_RETURN(TRUE);
 
   subq_pred->jtbm_read_time= read_time;
   subq_pred->jtbm_record_count=rows;
@@ -1479,6 +1551,25 @@ void fix_list_after_tbl_changes(SELECT_LEX *new_parent, List<TABLE_LIST> *tlist)
 }
 
 
+static void set_emb_join_nest(List<TABLE_LIST> *tables, TABLE_LIST *emb_sj_nest)
+{
+  List_iterator<TABLE_LIST> it(*tables);
+  TABLE_LIST *tbl;
+  while ((tbl= it++))
+  {
+    /*
+      Note: check for nested_join first. 
+       derived-merged tables have tbl->table!=NULL &&
+       tbl->table->reginfo==NULL.
+    */
+    if (tbl->nested_join)
+      set_emb_join_nest(&tbl->nested_join->join_list, emb_sj_nest);
+    else if (tbl->table)
+      tbl->table->reginfo.join_tab->emb_sj_nest= emb_sj_nest;
+
+  }
+}
+
 /*
   Pull tables out of semi-join nests, if possible
 
@@ -1534,10 +1625,34 @@ int pull_out_semijoin_tables(JOIN *join)
   /* Try pulling out of the each of the semi-joins */
   while ((sj_nest= sj_list_it++))
   {
-    /* Action #1: Mark the constant tables to be pulled out */
-    table_map pulled_tables= 0;
     List_iterator<TABLE_LIST> child_li(sj_nest->nested_join->join_list);
     TABLE_LIST *tbl;
+
+    /*
+      Don't do table pull-out for nested joins (if we get nested joins here, it
+      means these are outer joins. It is theoretically possible to do pull-out
+      for some of the outer tables but we dont support this currently.
+    */
+    bool have_join_nest_children= FALSE;
+
+    set_emb_join_nest(&sj_nest->nested_join->join_list, sj_nest);
+
+    while ((tbl= child_li++))
+    {
+      if (tbl->nested_join)
+      {
+        have_join_nest_children= TRUE;
+        break;
+      }
+    }
+    
+    
+    table_map pulled_tables= 0;
+    if (have_join_nest_children)
+      goto skip;
+
+    /* Action #1: Mark the constant tables to be pulled out */
+    child_li.rewind();
     while ((tbl= child_li++))
     {
       if (tbl->table)
@@ -1609,6 +1724,7 @@ int pull_out_semijoin_tables(JOIN *join)
     } while (pulled_a_table);
  
     child_li.rewind();
+  skip:
     /*
       Action #3: Move the pulled out TABLE_LIST elements to the parents.
     */
@@ -1688,6 +1804,9 @@ int pull_out_semijoin_tables(JOIN *join)
 
     All obtained information is saved and will be used by the main join
     optimization pass.
+  
+  NOTES 
+    Because of Join::reoptimize(), this function may be called multiple times.
 
   RETURN
     FALSE  Ok 
@@ -1745,9 +1864,12 @@ bool optimize_semijoin_nests(JOIN *join, table_map all_table_map)
 
         sjm->materialization_cost.convert_from_cost(subjoin_read_time);
         sjm->rows= subjoin_out_rows;
-
-        List<Item> &right_expr_list= 
-          sj_nest->sj_subq_pred->unit->first_select()->item_list;
+        
+        // Don't use the following list because it has "stale" items. use
+        // ref_pointer_array instead:
+        //
+        //List<Item> &right_expr_list= 
+        //  sj_nest->sj_subq_pred->unit->first_select()->item_list;
         /*
           Adjust output cardinality estimates. If the subquery has form
 
@@ -1765,17 +1887,20 @@ bool optimize_semijoin_nests(JOIN *join, table_map all_table_map)
           
           See also get_post_group_estimate().
         */
+        SELECT_LEX *subq_select= sj_nest->sj_subq_pred->unit->first_select();
         {
           for (uint i=0 ; i < join->const_tables + sjm->tables ; i++)
           {
             JOIN_TAB *tab= join->best_positions[i].table;
             join->map2table[tab->table->tablenr]= tab;
           }
-          List_iterator<Item> it(right_expr_list);
-          Item *item;
+          //List_iterator<Item> it(right_expr_list);
+          Item **ref_array= subq_select->ref_pointer_array;
+          Item **ref_array_end= ref_array + subq_select->item_list.elements; 
           table_map map= 0;
-          while ((item= it++))
-            map |= item->used_tables();
+          //while ((item= it++))
+          for (;ref_array < ref_array_end; ref_array++)
+            map |= (*ref_array)->used_tables();
           map= map & ~PSEUDO_TABLE_BITS;
           Table_map_iterator tm_it(map);
           int tableno;
@@ -1790,7 +1915,8 @@ bool optimize_semijoin_nests(JOIN *join, table_map all_table_map)
         /*
           Calculate temporary table parameters and usage costs
         */
-        uint rowlen= get_tmp_table_rec_length(right_expr_list);
+        uint rowlen= get_tmp_table_rec_length(subq_select->ref_pointer_array,
+                                              subq_select->item_list.elements);
         double lookup_cost= get_tmp_table_lookup_cost(join->thd,
                                                       subjoin_out_rows, rowlen);
         double write_cost= get_tmp_table_write_cost(join->thd,
@@ -1837,13 +1963,15 @@ bool optimize_semijoin_nests(JOIN *join, table_map all_table_map)
     Length of the temptable record, in bytes
 */
 
-static uint get_tmp_table_rec_length(List<Item> &items)
+static uint get_tmp_table_rec_length(Item **p_items, uint elements)
 {
   uint len= 0;
   Item *item;
-  List_iterator<Item> it(items);
-  while ((item= it++))
+  //List_iterator<Item> it(items);
+  Item **p_item;
+  for (p_item= p_items; p_item < p_items + elements ; p_item++)
   {
+    item = *p_item;
     switch (item->result_type()) {
     case REAL_RESULT:
       len += sizeof(double);
@@ -2053,6 +2181,17 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
   pos->sj_strategy= SJ_OPT_NONE;
   
   pos->prefix_dups_producing_tables= join->cur_dups_producing_tables;
+
+  /* We're performing optimization inside SJ-Materialization nest */
+  if (join->emb_sjm_nest)
+  {
+    pos->invalidate_firstmatch_prefix();
+    pos->first_loosescan_table= MAX_TABLES; 
+    pos->dupsweedout_tables= 0;
+    pos->sjm_scan_need_tables= 0;
+    return;
+  }
+
   /* Initialize the state or copy it from prev. tables */
   if (idx == join->const_tables)
   {
@@ -2091,7 +2230,8 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
   table_map handled_by_fm_or_ls= 0;
   /* FirstMatch Strategy */
   if (new_join_tab->emb_sj_nest &&
-      optimizer_flag(join->thd, OPTIMIZER_SWITCH_FIRSTMATCH))
+      optimizer_flag(join->thd, OPTIMIZER_SWITCH_FIRSTMATCH) &&
+      !join->outer_join)
   {
     const table_map outer_corr_tables=
       new_join_tab->emb_sj_nest->nested_join->sj_corr_tables |
@@ -2186,7 +2326,7 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
       If we got an option to use LooseScan for the current table, start
       considering using LooseScan strategy
     */
-    if (loose_scan_pos->read_time != DBL_MAX)
+    if (loose_scan_pos->read_time != DBL_MAX && !join->outer_join)
     {
       pos->first_loosescan_table= idx;
       pos->loosescan_need_tables=
@@ -2394,6 +2534,15 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
                                  nest->nested_join->sj_depends_on |
                                  nest->nested_join->sj_corr_tables;
     }
+    
+    if (pos->dupsweedout_tables)
+    {
+      /* we're in the process of constructing a DuplicateWeedout range */
+      TABLE_LIST *emb= new_join_tab->table->pos_in_table_list->embedding;
+      /* and we've entered an inner side of an outer join*/
+      if (emb && emb->on_expr)
+        pos->dupsweedout_tables |= emb->nested_join->used_tables;
+    }
 
     if (pos->dupsweedout_tables && 
         !(remaining_tables &
@@ -2549,6 +2698,7 @@ ulonglong get_bound_sj_equalities(TABLE_LIST *sj_nest,
     {
       res |= 1ULL << i;
     }
+    i++;
   }
   return res;
 }
@@ -2840,27 +2990,36 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
     TRUE   Error
 */
 
-bool setup_sj_materialization(JOIN_TAB *sjm_tab)
+bool setup_sj_materialization_part1(JOIN_TAB *sjm_tab)
 {
-  uint i;
   DBUG_ENTER("setup_sj_materialization");
   JOIN_TAB *tab= sjm_tab->bush_children->start;
   TABLE_LIST *emb_sj_nest= tab->table->pos_in_table_list->embedding;
+  
+  /* Walk out of outer join nests until we reach the semi-join nest we're in */
+  while (!emb_sj_nest->sj_mat_info)
+    emb_sj_nest= emb_sj_nest->embedding;
+
   SJ_MATERIALIZATION_INFO *sjm= emb_sj_nest->sj_mat_info;
   THD *thd= tab->join->thd;
   /* First the calls come to the materialization function */
-  List<Item> &item_list= emb_sj_nest->sj_subq_pred->unit->first_select()->item_list;
-
+  //List<Item> &item_list= emb_sj_nest->sj_subq_pred->unit->first_select()->item_list;
+  
+  DBUG_ASSERT(sjm->is_used);
   /* 
     Set up the table to write to, do as select_union::create_result_table does
   */
   sjm->sjm_table_param.init();
-  sjm->sjm_table_param.field_count= item_list.elements;
   sjm->sjm_table_param.bit_fields_as_long= TRUE;
-  List_iterator<Item> it(item_list);
-  Item *right_expr;
-  while((right_expr= it++))
-    sjm->sjm_table_cols.push_back(right_expr);
+  //List_iterator<Item> it(item_list);
+  SELECT_LEX *subq_select= emb_sj_nest->sj_subq_pred->unit->first_select();
+  Item **p_item= subq_select->ref_pointer_array;
+  Item **p_end= p_item + subq_select->item_list.elements;
+  //while((right_expr= it++))
+  for(;p_item != p_end; p_item++)
+    sjm->sjm_table_cols.push_back(*p_item);
+
+  sjm->sjm_table_param.field_count= subq_select->item_list.elements;
 
   if (!(sjm->table= create_tmp_table(thd, &sjm->sjm_table_param, 
                                      sjm->sjm_table_cols, (ORDER*) 0, 
@@ -2878,6 +3037,25 @@ bool setup_sj_materialization(JOIN_TAB *sjm_tab)
   
   sjm->materialized= FALSE;
   sjm_tab->table= sjm->table;
+  sjm->table->pos_in_table_list= emb_sj_nest;
+ 
+  DBUG_RETURN(FALSE);
+}
+
+
+bool setup_sj_materialization_part2(JOIN_TAB *sjm_tab)
+{
+  DBUG_ENTER("setup_sj_materialization_part2");
+  JOIN_TAB *tab= sjm_tab->bush_children->start;
+  TABLE_LIST *emb_sj_nest= tab->table->pos_in_table_list->embedding;
+  /* Walk out of outer join nests until we reach the semi-join nest we're in */
+  while (!emb_sj_nest->sj_mat_info)
+    emb_sj_nest= emb_sj_nest->embedding;
+  SJ_MATERIALIZATION_INFO *sjm= emb_sj_nest->sj_mat_info;
+  THD *thd= tab->join->thd;
+  uint i;
+  //List<Item> &item_list= emb_sj_nest->sj_subq_pred->unit->first_select()->item_list;
+  //List_iterator<Item> it(item_list);
 
   if (!sjm->is_sj_scan)
   {
@@ -2992,12 +3170,14 @@ bool setup_sj_materialization(JOIN_TAB *sjm_tab)
       in the record buffers for the source tables. 
     */
     sjm->copy_field= new Copy_field[sjm->sjm_table_cols.elements];
-    it.rewind();
+    //it.rewind();
+    Item **p_item= emb_sj_nest->sj_subq_pred->unit->first_select()->ref_pointer_array;
     for (uint i=0; i < sjm->sjm_table_cols.elements; i++)
     {
       bool dummy;
       Item_equal *item_eq;
-      Item *item= (it++)->real_item();
+      //Item *item= (it++)->real_item();
+      Item *item= (*(p_item++))->real_item();
       DBUG_ASSERT(item->type() == Item::FIELD_ITEM);
       Field *copy_to= ((Item_field*)item)->field;
       /*
@@ -3032,11 +3212,15 @@ bool setup_sj_materialization(JOIN_TAB *sjm_tab)
       if (item_eq)
       {
         List_iterator<Item> it(item_eq->equal_items);
+        /* We're interested in field items only */
+        if (item_eq->get_const())
+          it++;
         Item *item;
         while ((item= it++))
         {
           if (!(item->used_tables() & ~emb_sj_nest->sj_inner_tables))
           {
+            DBUG_ASSERT(item->real_item()->type() == Item::FIELD_ITEM);
             copy_to= ((Item_field *) (item->real_item()))->field;
             break;
           }
@@ -3203,7 +3387,7 @@ TABLE *create_duplicate_weedout_tmp_table(THD *thd,
   bool using_unique_constraint=FALSE;
   bool use_packed_rows= FALSE;
   Field *field, *key_field;
-  uint blob_count, null_pack_length, null_count;
+  uint null_pack_length, null_count;
   uchar *null_flags;
   uchar *pos;
   DBUG_ENTER("create_duplicate_weedout_tmp_table");
@@ -3284,8 +3468,6 @@ TABLE *create_duplicate_weedout_tmp_table(THD *thd,
   share->keys_for_keyread.init();
   share->keys_in_use.init();
 
-  blob_count= 0;
-
   /* Create the field */
   {
     /*
@@ -3702,6 +3884,8 @@ int setup_semijoin_dups_elimination(JOIN *join, ulonglong options,
       {
         /* We jump from the last table to the first one */
         tab->loosescan_match_tab= tab + pos->n_sj_tables - 1;
+        for (uint j= i; j < i + pos->n_sj_tables; j++)
+          join->join_tab[j].inside_loosescan_range= TRUE;
 
         /* Calculate key length */
         keylen= 0;
@@ -4216,6 +4400,8 @@ bool JOIN::optimize_unflattened_subqueries()
     Currently there are only two possible startup functions, so we have them
     both here inside if (...) branches. In future we could switch to function
     pointers.
+
+  TODO: consider moving this together with JOIN_TAB::preread_init
   
   RETURN 
     NESTED_LOOP_OK - OK
@@ -4325,8 +4511,8 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
   }
   else
     return false;
-
-  DBUG_ASSERT(in_subs->in_strategy); /* A strategy must be chosen earlier. */
+  /* A strategy must be chosen earlier. */
+  DBUG_ASSERT(in_subs->has_strategy());
   DBUG_ASSERT(in_to_exists_where || in_to_exists_having);
   DBUG_ASSERT(!in_to_exists_where || in_to_exists_where->fixed);
   DBUG_ASSERT(!in_to_exists_having || in_to_exists_having->fixed);
@@ -4336,13 +4522,11 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
     strategies are possible and allowed by the user (checked during the prepare
     phase.
   */
-  if (in_subs->in_strategy & SUBS_MATERIALIZATION &&
-      in_subs->in_strategy & SUBS_IN_TO_EXISTS)
+  if (in_subs->test_strategy(SUBS_MATERIALIZATION) &&
+      in_subs->test_strategy(SUBS_IN_TO_EXISTS))
   {
     JOIN *outer_join;
     JOIN *inner_join= this;
-    /* Number of (partial) rows of the outer JOIN filtered by the IN predicate. */
-    double outer_record_count;
     /* Number of unique value combinations filtered by the IN predicate. */
     double outer_lookup_keys;
     /* Cost and row count of the unmodified subquery. */
@@ -4360,40 +4544,24 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
       by the IN predicate.
     */
     outer_join= unit->outer_select() ? unit->outer_select()->join : NULL;
-    if (outer_join)
+    if (outer_join && outer_join->table_count > 0)
     {
-      uint outer_partial_plan_len;
       /*
-        Make_cond_for_table is called for predicates only in the WHERE/ON
-        clauses. In all other cases, predicates are not pushed to any
-        JOIN_TAB, and their joi_tab_idx remains MAX_TABLES. Such predicates
-        are evaluated for each complete row of the outer join.
+        TODO:
+        Currently outer_lookup_keys is computed as the number of rows in
+        the partial join including the JOIN_TAB where the IN predicate is
+        pushed to. In the general case this is a gross overestimate because
+        due to caching we are interested only in the number of unique keys.
+        The search key may be formed by columns from much fewer than all
+        tables in the partial join. Example:
+        select * from t1, t2 where t1.c1 = t2.key AND t2.c2 IN (select ...);
+        If the join order: t1, t2, the number of unique lookup keys is ~ to
+        the number of unique values t2.c2 in the partial join t1 join t2.
       */
-      outer_partial_plan_len= (in_subs->get_join_tab_idx() == MAX_TABLES) ?
-                               outer_join->table_count :
-                               in_subs->get_join_tab_idx() + 1;
-      outer_join->get_partial_cost_and_fanout(outer_partial_plan_len,
+      outer_join->get_partial_cost_and_fanout(in_subs->get_join_tab_idx(),
                                               table_map(-1),
                                               &dummy,
-                                              &outer_record_count);
-
-      if (outer_join->table_count > outer_join->const_tables)
-      {
-        outer_join->get_partial_cost_and_fanout(outer_partial_plan_len,
-                                                in_subs->used_tables(),
-                                                &dummy,
-                                                &outer_lookup_keys);
-        /*
-        outer_lookup_keys= prev_record_reads(outer_join->best_positions,
-                                             outer_partial_plan_len,
-                                             in_subs->used_tables());
-        */
-      }
-      else
-      {
-        /* If all tables are constant, positions is undefined. */
-        outer_lookup_keys= 1;
-      }
+                                              &outer_lookup_keys);
     }
     else
     {
@@ -4401,17 +4569,8 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
         TODO: outer_join can be NULL for DELETE statements.
         How to compute its cost?
       */
-      outer_record_count= 1;
-      outer_lookup_keys=1;
+      outer_lookup_keys= 1;
     }
-    /*
-      There cannot be more lookup keys than the total number of records.
-      TODO: this a temporary solution until we find a better way to compute
-      get_partial_join_cost() and prev_record_reads() in a consitent manner,
-      where it is guaranteed that (outer_lookup_keys <= outer_record_count).
-    */
-    if (outer_lookup_keys > outer_record_count)
-      outer_lookup_keys= outer_record_count;
 
     /*
       B. Estimate the cost and number of records of the subquery both
@@ -4444,7 +4603,9 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
       C. Compute execution costs.
     */
     /* C.1 Compute the cost of the materialization strategy. */
-    uint rowlen= get_tmp_table_rec_length(unit->first_select()->item_list);
+    //uint rowlen= get_tmp_table_rec_length(unit->first_select()->item_list);
+    uint rowlen= get_tmp_table_rec_length(ref_pointer_array, 
+                                          select_lex->item_list.elements);
     /* The cost of writing one row into the temporary table. */
     double write_cost= get_tmp_table_write_cost(thd, inner_record_count_1,
                                                 rowlen);
@@ -4459,16 +4620,24 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
                                  write_cost * inner_record_count_1;
 
     materialize_strategy_cost= materialization_cost +
-                               outer_record_count * lookup_cost;
+                               outer_lookup_keys * lookup_cost;
 
     /* C.2 Compute the cost of the IN=>EXISTS strategy. */
     in_exists_strategy_cost= outer_lookup_keys * inner_read_time_2;
 
     /* C.3 Compare the costs and choose the cheaper strategy. */
     if (materialize_strategy_cost >= in_exists_strategy_cost)
-      in_subs->in_strategy&= ~SUBS_MATERIALIZATION;
+      in_subs->set_strategy(SUBS_IN_TO_EXISTS);
     else
-      in_subs->in_strategy&= ~SUBS_IN_TO_EXISTS;
+      in_subs->set_strategy(SUBS_MATERIALIZATION);
+
+    DBUG_PRINT("info",
+               ("mat_strategy_cost: %.2f, mat_cost: %.2f, write_cost: %.2f, lookup_cost: %.2f",
+                materialize_strategy_cost, materialization_cost, write_cost, lookup_cost));
+    DBUG_PRINT("info",
+               ("inx_strategy_cost: %.2f, inner_read_time_2: %.2f",
+                in_exists_strategy_cost, inner_read_time_2));
+    DBUG_PRINT("info",("outer_lookup_keys: %.2f", outer_lookup_keys));
   }
 
   /*
@@ -4481,7 +4650,7 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
     otherwise
       use materialization.
   */
-  if (in_subs->in_strategy & SUBS_MATERIALIZATION &&
+  if (in_subs->test_strategy(SUBS_MATERIALIZATION) &&
       in_subs->setup_mat_engine())
   {
     /*
@@ -4489,11 +4658,10 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
       but it is not possible to execute it due to limitations in the
       implementation, fall back to IN-TO-EXISTS.
     */
-    in_subs->in_strategy&= ~SUBS_MATERIALIZATION;
-    in_subs->in_strategy|= SUBS_IN_TO_EXISTS;
+    in_subs->set_strategy(SUBS_IN_TO_EXISTS);
   }
 
-  if (in_subs->in_strategy & SUBS_MATERIALIZATION)
+  if (in_subs->test_strategy(SUBS_MATERIALIZATION))
   {
     /* Restore the original query plan used for materialization. */
     if (reopt_result == REOPT_NEW_PLAN)
@@ -4518,23 +4686,18 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
     */
     select_limit= in_subs->unit->select_limit_cnt;
   }
-  else if (in_subs->in_strategy & SUBS_IN_TO_EXISTS)
+  else if (in_subs->test_strategy(SUBS_IN_TO_EXISTS))
   {
     if (reopt_result == REOPT_NONE && in_to_exists_where &&
         const_tables != table_count)
     {
       /*
-        The subquery was not reoptimized either because the user allowed only the
-        IN-EXISTS strategy, or because materialization was not possible based on
-        semantic analysis. Clenup the original plan and reoptimize.
+        The subquery was not reoptimized with the newly injected IN-EXISTS
+        conditions either because the user allowed only the IN-EXISTS strategy,
+        or because materialization was not possible based on semantic analysis.
       */
-      for (uint i= 0; i < table_count; i++)
-      {
-        join_tab[i].keyuse= NULL;
-        join_tab[i].checked_keys.clear_all();
-      }
-      if ((reopt_result= reoptimize(in_to_exists_where, join_tables, NULL)) ==
-          REOPT_ERROR)
+      reopt_result= reoptimize(in_to_exists_where, join_tables, NULL);
+      if (reopt_result == REOPT_ERROR)
         return TRUE;
     }
 
@@ -4604,7 +4767,7 @@ bool JOIN::choose_tableless_subquery_plan()
     {
       Item_in_subselect *in_subs;
       in_subs= (Item_in_subselect*) subs_predicate;
-      in_subs->in_strategy= SUBS_IN_TO_EXISTS;
+      in_subs->set_strategy(SUBS_IN_TO_EXISTS);
       if (in_subs->create_in_to_exists_cond(this) ||
           in_subs->inject_in_to_exists_cond(this))
         return TRUE;
diff --git a/sql/opt_subselect.h b/sql/opt_subselect.h
index 4d89609e1a8..571fcbaa935 100644
--- a/sql/opt_subselect.h
+++ b/sql/opt_subselect.h
@@ -285,7 +285,9 @@ void restore_prev_sj_state(const table_map remaining_tables,
                                   const JOIN_TAB *tab, uint idx);
 
 void fix_semijoin_strategies_for_picked_join_order(JOIN *join);
-bool setup_sj_materialization(JOIN_TAB *tab);
+
+bool setup_sj_materialization_part1(JOIN_TAB *sjm_tab);
+bool setup_sj_materialization_part2(JOIN_TAB *sjm_tab);
 
 TABLE *create_duplicate_weedout_tmp_table(THD *thd, uint uniq_tuple_length_arg,
                                           SJ_TMP_TABLE *sjtbl);
diff --git a/sql/opt_sum.cc b/sql/opt_sum.cc
index 965eb853471..7b4c48497fa 100644
--- a/sql/opt_sum.cc
+++ b/sql/opt_sum.cc
@@ -494,28 +494,30 @@ bool simple_pred(Item_func *func_item, Item **args, bool *inv_order)
     /* MULT_EQUAL_FUNC */
     {
       Item_equal *item_equal= (Item_equal *) func_item;
+      if (!(args[1]= item_equal->get_const()))
+        return 0;
       Item_equal_fields_iterator it(*item_equal);
-      args[0]= it++;
-      if (it++)
+      if (!(item= it++))
         return 0;
-      if (!(args[1]= item_equal->get_const()))
+      args[0]= item->real_item();
+      if (it++)
         return 0;
     }
     break;
   case 1:
     /* field IS NULL */
-    item= func_item->arguments()[0];
+    item= func_item->arguments()[0]->real_item();
     if (item->type() != Item::FIELD_ITEM)
       return 0;
     args[0]= item;
     break;
   case 2:
     /* 'field op const' or 'const op field' */
-    item= func_item->arguments()[0];
+    item= func_item->arguments()[0]->real_item();
     if (item->type() == Item::FIELD_ITEM)
     {
       args[0]= item;
-      item= func_item->arguments()[1];
+      item= func_item->arguments()[1]->real_item();
       if (!item->const_item())
         return 0;
       args[1]= item;
@@ -523,7 +525,7 @@ bool simple_pred(Item_func *func_item, Item **args, bool *inv_order)
     else if (item->const_item())
     {
       args[1]= item;
-      item= func_item->arguments()[1];
+      item= func_item->arguments()[1]->real_item();
       if (item->type() != Item::FIELD_ITEM)
         return 0;
       args[0]= item;
@@ -534,13 +536,13 @@ bool simple_pred(Item_func *func_item, Item **args, bool *inv_order)
     break;
   case 3:
     /* field BETWEEN const AND const */
-    item= func_item->arguments()[0];
+    item= func_item->arguments()[0]->real_item();
     if (item->type() == Item::FIELD_ITEM)
     {
       args[0]= item;
       for (int i= 1 ; i <= 2; i++)
       {
-        item= func_item->arguments()[i];
+        item= func_item->arguments()[i]->real_item();
         if (!item->const_item())
           return 0;
         args[i]= item;
@@ -621,8 +623,10 @@ static bool matching_cond(bool max_fl, TABLE_REF *ref, KEY *keyinfo,
   if (!(cond->used_tables() & field->table->map))
   {
     /* Condition doesn't restrict the used table */
-    DBUG_RETURN(TRUE);
+    DBUG_RETURN(!cond->const_item());
   }
+  else if (cond->is_expensive())
+    DBUG_RETURN(FALSE);
   if (cond->type() == Item::COND_ITEM)
   {
     if (((Item_cond*) cond)->functype() == Item_func::COND_OR_FUNC)
@@ -669,6 +673,8 @@ static bool matching_cond(bool max_fl, TABLE_REF *ref, KEY *keyinfo,
   case Item_func::GE_FUNC:
     break;
   case Item_func::BETWEEN:
+    if (((Item_func_between*) cond)->negated)
+      DBUG_RETURN(FALSE);
     between= 1;
     break;
   case Item_func::MULT_EQUAL_FUNC:
diff --git a/sql/opt_table_elimination.cc b/sql/opt_table_elimination.cc
index c8c61720068..56396181619 100644
--- a/sql/opt_table_elimination.cc
+++ b/sql/opt_table_elimination.cc
@@ -588,10 +588,8 @@ void eliminate_tables(JOIN *join)
   if (!join->outer_join)
     DBUG_VOID_RETURN;
 
-#ifndef DBUG_OFF
   if (!optimizer_flag(thd, OPTIMIZER_SWITCH_TABLE_ELIMINATION))
     DBUG_VOID_RETURN; /* purecov: inspected */
-#endif
 
   /* Find the tables that are referred to from WHERE/HAVING */
   used_tables= (join->conds?  join->conds->used_tables() : 0) | 
diff --git a/sql/parse_file.cc b/sql/parse_file.cc
index 5bc16e55ec0..3ccbfba5aff 100644
--- a/sql/parse_file.cc
+++ b/sql/parse_file.cc
@@ -381,7 +381,7 @@ sql_parse_prepare(const LEX_STRING *file_name, MEM_ROOT *mem_root,
     DBUG_RETURN(0);
   }
 
-  if (!(parser->buff= (char*) alloc_root(mem_root, stat_info.st_size+1)))
+  if (!(parser->buff= (char*) alloc_root(mem_root, (size_t)(stat_info.st_size+1))))
   {
     DBUG_RETURN(0);
   }
@@ -392,7 +392,7 @@ sql_parse_prepare(const LEX_STRING *file_name, MEM_ROOT *mem_root,
   }
   
   if ((len= my_read(file, (uchar *)parser->buff,
-                    stat_info.st_size, MYF(MY_WME))) ==
+                    (size_t) stat_info.st_size, MYF(MY_WME))) ==
       MY_FILE_ERROR)
   {
     my_close(file, MYF(MY_WME));
diff --git a/sql/protocol.cc b/sql/protocol.cc
index a0e14423b73..af4324b4166 100644
--- a/sql/protocol.cc
+++ b/sql/protocol.cc
@@ -515,6 +515,56 @@ void net_end_statement(THD *thd)
 }
 
 
+/**
+   Send a progress report to the client
+
+   What we send is:
+   header (255,255,255,1)
+   stage, max_stage as on byte integers
+   percentage withing the stage as percentage*1000
+   (that is, ratio*100000) as a 3 byte integer
+   proc_info as a string
+*/
+
+const uchar progress_header[2]= {(uchar) 255, (uchar) 255 };
+
+void net_send_progress_packet(THD *thd)
+{
+  uchar buff[200], *pos;
+  const char *proc_info= thd->proc_info ? thd->proc_info : "";
+  uint length= strlen(proc_info);
+  ulonglong progress;
+  DBUG_ENTER("net_send_progress_packet");
+
+  if (unlikely(!thd->net.vio))
+    DBUG_VOID_RETURN;                           // Socket is closed
+
+  pos= buff;
+  /*
+    Store number of strings first. This allows us to later expand the
+    progress indicator if needed.
+  */
+  *pos++= (uchar) 1;                            // Number of strings
+  *pos++= (uchar) thd->progress.stage + 1;
+  /*
+    We have the max() here to avoid problems if max_stage is not set,
+    which may happen during automatic repair of table
+  */
+  *pos++= (uchar) max(thd->progress.max_stage, thd->progress.stage + 1);
+  progress= 0;
+  if (thd->progress.max_counter)
+    progress= 100000ULL * thd->progress.counter / thd->progress.max_counter;
+  int3store(pos, progress);                          // Between 0 & 100000
+  pos+= 3;
+  pos= net_store_data(pos, (const uchar*) proc_info,
+                      min(length, sizeof(buff)-7));
+  net_write_command(&thd->net, (uchar) 255, progress_header,
+                    sizeof(progress_header), (uchar*) buff,
+                    (uint) (pos - buff));
+  DBUG_VOID_RETURN;
+}
+
+  
 /****************************************************************************
   Functions used by the protocol functions (like net_send_ok) to store
   strings and numbers in the header result packet.
diff --git a/sql/protocol.h b/sql/protocol.h
index 4ab5c169928..e07af5208db 100644
--- a/sql/protocol.h
+++ b/sql/protocol.h
@@ -177,6 +177,7 @@ public:
 void send_warning(THD *thd, uint sql_errno, const char *err=0);
 bool net_send_error(THD *thd, uint sql_errno=0, const char *err=0);
 void net_end_statement(THD *thd);
+void net_send_progress_packet(THD *thd);
 uchar *net_store_data(uchar *to,const uchar *from, size_t length);
 uchar *net_store_data(uchar *to,int32 from);
 uchar *net_store_data(uchar *to,longlong from);
diff --git a/sql/records.cc b/sql/records.cc
index 5ce7d8660e9..b125c201621 100644
--- a/sql/records.cc
+++ b/sql/records.cc
@@ -59,7 +59,6 @@ void init_read_record_idx(READ_RECORD *info, THD *thd, TABLE *table,
   bzero((char*) info,sizeof(*info));
   info->thd= thd;
   info->table= table;
-  info->file=  table->file;
   info->record= table->record[0];
   info->print_error= print_error;
   info->unlock_row= rr_unlock_row;
@@ -169,7 +168,6 @@ bool init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
   bzero((char*) info,sizeof(*info));
   info->thd=thd;
   info->table=table;
-  info->file= table->file;
   info->forms= &info->table;		/* Only one table */
   
   if (table->s->tmp_table == NON_TRANSACTIONAL_TMP_TABLE &&
@@ -195,15 +193,6 @@ bool init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
 
   if (select && my_b_inited(&select->file))
     tempfile= &select->file;
-  else if (select && select->quick && select->quick->clustered_pk_range())
-  {
-    /*
-      In case of QUICK_INDEX_MERGE_SELECT with clustered pk range we have to
-      use its own access method(i.e QUICK_INDEX_MERGE_SELECT::get_next()) as
-      sort file does not contain rowids which satisfy clustered pk range.
-    */
-    tempfile= 0;
-  }
   else
     tempfile= table->sort.io_cache;
   if (tempfile && my_b_inited(tempfile) &&
@@ -300,9 +289,9 @@ void end_read_record(READ_RECORD *info)
   {
     filesort_free_buffers(info->table,0);
     if (info->table->created)
-      (void) info->file->extra(HA_EXTRA_NO_CACHE);
+      (void) info->table->file->extra(HA_EXTRA_NO_CACHE);
     if (info->read_record != rr_quick) // otherwise quick_range does it
-      (void) info->file->ha_index_or_rnd_end();
+      (void) info->table->file->ha_index_or_rnd_end();
     info->table=0;
   }
 }
@@ -361,7 +350,7 @@ static int rr_quick(READ_RECORD *info)
 
 static int rr_index_first(READ_RECORD *info)
 {
-  int tmp= info->file->ha_index_first(info->record);
+  int tmp= info->table->file->ha_index_first(info->record);
   info->read_record= rr_index;
   if (tmp)
     tmp= rr_handle_error(info, tmp);
@@ -387,7 +376,7 @@ static int rr_index_first(READ_RECORD *info)
 
 static int rr_index(READ_RECORD *info)
 {
-  int tmp= info->file->ha_index_next(info->record);
+  int tmp= info->table->file->ha_index_next(info->record);
   if (tmp)
     tmp= rr_handle_error(info, tmp);
   return tmp;
@@ -397,7 +386,7 @@ static int rr_index(READ_RECORD *info)
 int rr_sequential(READ_RECORD *info)
 {
   int tmp;
-  while ((tmp= info->file->ha_rnd_next(info->record)))
+  while ((tmp= info->table->file->ha_rnd_next(info->record)))
   {
     /*
       rnd_next can return RECORD_DELETED for MyISAM when one thread is
@@ -422,7 +411,7 @@ static int rr_from_tempfile(READ_RECORD *info)
   {
     if (my_b_read(info->io_cache,info->ref_pos,info->ref_length))
       return -1;					/* End of file */
-    if (!(tmp= info->file->ha_rnd_pos(info->record,info->ref_pos)))
+    if (!(tmp= info->table->file->ha_rnd_pos(info->record,info->ref_pos)))
       break;
     /* The following is extremely unlikely to happen */
     if (tmp == HA_ERR_RECORD_DELETED ||
@@ -473,7 +462,7 @@ static int rr_from_pointers(READ_RECORD *info)
     cache_pos= info->cache_pos;
     info->cache_pos+= info->ref_length;
 
-    if (!(tmp= info->file->ha_rnd_pos(info->record,cache_pos)))
+    if (!(tmp= info->table->file->ha_rnd_pos(info->record,cache_pos)))
       break;
 
     /* The following is extremely unlikely to happen */
@@ -606,7 +595,7 @@ static int rr_from_cache(READ_RECORD *info)
       record=uint3korr(position);
       position+=3;
       record_pos=info->cache+record*info->reclength;
-      if ((error=(int16) info->file->ha_rnd_pos(record_pos,info->ref_pos)))
+      if ((error=(int16) info->table->file->ha_rnd_pos(record_pos,info->ref_pos)))
       {
 	record_pos[info->error_offset]=1;
 	shortstore(record_pos,error);
diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h
index 0ea3c23bfd8..a143f5251af 100644
--- a/sql/rpl_rli.h
+++ b/sql/rpl_rli.h
@@ -425,14 +425,14 @@ public:
 
   /**
     Save pointer to Annotate_rows event and switch on the
-    binlog_annotate_rows_events for this sql thread.
+    binlog_annotate_row_events for this sql thread.
     To be called when sql thread recieves an Annotate_rows event.
   */
   inline void set_annotate_event(Annotate_rows_log_event *event)
   {
     free_annotate_event();
     m_annotate_event= event;
-    sql_thd->variables.binlog_annotate_rows_events= 1;
+    sql_thd->variables.binlog_annotate_row_events= 1;
   }
 
   /**
@@ -446,7 +446,7 @@ public:
 
   /**
     Delete saved Annotate_rows event (if any) and switch off the
-    binlog_annotate_rows_events for this sql thread.
+    binlog_annotate_row_events for this sql thread.
     To be called when sql thread has applied the last (i.e. with
     STMT_END_F flag) rbr event.
   */
@@ -454,7 +454,7 @@ public:
   {
     if (m_annotate_event)
     {
-      sql_thd->variables.binlog_annotate_rows_events= 0;
+      sql_thd->variables.binlog_annotate_row_events= 0;
       delete m_annotate_event;
       m_annotate_event= 0;
     }
diff --git a/sql/scheduler.cc b/sql/scheduler.cc
index 5b8f834aecc..5b83bb4753e 100644
--- a/sql/scheduler.cc
+++ b/sql/scheduler.cc
@@ -376,7 +376,7 @@ void libevent_kill_thd_callback(int Fd, short, void*)
   {
     THD *thd= (THD*)list->data;
     list= list_rest(list);
-    if (thd->killed == THD::KILL_CONNECTION)
+    if ((int) thd->killed >= (int) KILL_CONNECTION)
     {
       /*
         Delete from libevent and add to the processing queue.
@@ -510,7 +510,7 @@ static void libevent_connection_close(THD *thd)
   DBUG_ENTER("libevent_connection_close");
   DBUG_PRINT("enter", ("thd: %p", thd));
 
-  thd->killed= THD::KILL_CONNECTION;          // Avoid error messages
+  thd->killed= KILL_CONNECTION;                // Avoid error messages
 
   if (thd->net.vio->sd >= 0)                  // not already closed
   {
@@ -531,9 +531,9 @@ static void libevent_connection_close(THD *thd)
 
 static bool libevent_should_close_connection(THD* thd)
 {
-  return thd->net.error ||
-         thd->net.vio == 0 ||
-         thd->killed == THD::KILL_CONNECTION;
+  return (thd->net.error ||
+          thd->net.vio == 0 ||
+          (int) thd->killed >= (int) KILL_CONNECTION);
 }
 
 
diff --git a/sql/set_var.cc b/sql/set_var.cc
index 85722f3d64a..ed6bc60e3df 100644
--- a/sql/set_var.cc
+++ b/sql/set_var.cc
@@ -156,6 +156,7 @@ static bool sys_update_slow_log_path(THD *thd, set_var * var);
 static void sys_default_slow_log_path(THD *thd, enum_var_type type);
 static void fix_sys_log_slow_filter(THD *thd, enum_var_type);
 static uchar *get_myisam_mmap_size(THD *thd);
+static uchar *in_transaction(THD *thd);
 static int check_max_allowed_packet(THD *thd,  set_var *var);
 static int check_net_buffer_length(THD *thd,  set_var *var);
 
@@ -188,8 +189,8 @@ static sys_var_const            sys_back_log(&vars, "back_log",
                                              (uchar*) &back_log);
 static sys_var_const_os_str       sys_basedir(&vars, "basedir", mysql_home);
 static sys_var_thd_bool
-sys_binlog_annotate_rows_events(&vars, "binlog_annotate_rows_events",
-                            &SV::binlog_annotate_rows_events);
+sys_binlog_annotate_row_events(&vars, "binlog_annotate_row_events",
+                            &SV::binlog_annotate_row_events);
 static sys_var_long_ptr	sys_binlog_cache_size(&vars, "binlog_cache_size",
 					      &binlog_cache_size);
 static sys_var_thd_binlog_format sys_binlog_format(&vars, "binlog_format",
@@ -264,6 +265,9 @@ static sys_var_thd_ulong sys_deadlock_timeout_long(&vars,
                                  &SV::wt_timeout_long);
 #ifndef DBUG_OFF
 static sys_var_thd_dbug        sys_dbug(&vars, "debug");
+static sys_var_long_ptr        sys_var_debug_binlog_fsync_sleep(&vars,
+                                        "debug_binlog_fsync_sleep",
+                                        &opt_binlog_dbug_fsync_sleep);
 #endif
 static sys_var_enum		sys_delay_key_write(&vars, "delay_key_write",
 					    &delay_key_write_options,
@@ -527,6 +531,10 @@ static sys_var_thd_ulong        sys_optimizer_search_depth(&vars, "optimizer_sea
 static sys_var_thd_optimizer_switch   sys_optimizer_switch(&vars, "optimizer_switch",
                                      &SV::optimizer_switch);
 
+static sys_var_thd_ulong        sys_progress_report_time(&vars,
+                                                         "progress_report_time",
+                                                         &SV::progress_report_time);
+
 static sys_var_const            sys_pid_file(&vars, "pid_file",
                                              OPT_GLOBAL, SHOW_CHAR,
                                              (uchar*) pidfile_name);
@@ -997,6 +1005,12 @@ static sys_var_enum_const     sys_plugin_maturity(&vars, "plugin_maturity",
                                                   &plugin_maturity,
                                                   &plugin_maturity_values);
 
+static sys_var_readonly       sys_in_transaction(&vars, "in_transaction",
+                                                 OPT_SESSION, SHOW_BOOL,
+                                                 in_transaction);
+
+
+
 bool sys_var::check(THD *thd, set_var *var)
 {
   var->save_result.ulonglong_value= var->value->val_int();
@@ -3071,7 +3085,19 @@ void sys_var_thd_time_zone::set_default(THD *thd, enum_var_type type)
 bool sys_var_max_user_conn::check(THD *thd, set_var *var)
 {
   if (var->type == OPT_GLOBAL)
+  {
+    if (! max_user_connections_checking)
+    {
+      /*
+        We can't change the value of max_user_connections from 0 as then
+        connect counting would be wrong.
+      */
+      my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0),
+               "--max-user-connections=0");
+      return TRUE;
+    }
     return sys_var_thd::check(thd, var);
+  }
   else
   {
     /*
@@ -3087,7 +3113,7 @@ bool sys_var_max_user_conn::update(THD *thd, set_var *var)
 {
   DBUG_ASSERT(var->type == OPT_GLOBAL);
   pthread_mutex_lock(&LOCK_global_system_variables);
-  max_user_connections= (uint)var->save_result.ulonglong_value;
+  max_user_connections= (int) var->save_result.ulonglong_value;
   pthread_mutex_unlock(&LOCK_global_system_variables);
   return 0;
 }
@@ -3269,35 +3295,26 @@ static bool set_option_log_bin_bit(THD *thd, set_var *var)
 
 static bool set_option_autocommit(THD *thd, set_var *var)
 {
-  /* The test is negative as the flag we use is NOT autocommit */
-
-  ulonglong org_options= thd->options;
+  ulonglong new_options= thd->options;
 
-  if (var->save_result.ulong_value != 0)
-    thd->options&= ~((sys_var_thd_bit*) var->var)->bit_flag;
+  /* The test is negative as the flag we use is NOT autocommit */
+  if (var->save_result.ulong_value)
+    new_options&= ~OPTION_NOT_AUTOCOMMIT;
   else
-    thd->options|= ((sys_var_thd_bit*) var->var)->bit_flag;
+    new_options|= OPTION_NOT_AUTOCOMMIT;
 
-  if ((org_options ^ thd->options) & OPTION_NOT_AUTOCOMMIT)
+  if ((new_options ^ thd->options) & OPTION_NOT_AUTOCOMMIT)
   {
-    if ((org_options & OPTION_NOT_AUTOCOMMIT))
+    if ((thd->options & OPTION_NOT_AUTOCOMMIT))
     {
-      /* We changed to auto_commit mode */
-      if (thd->transaction.xid_state.xa_state != XA_NOTR)
-      {
-        thd->options= org_options;
-        my_error(ER_XAER_RMFAIL, MYF(0),
-                 xa_state_names[thd->transaction.xid_state.xa_state]);
+      if (end_active_trans(thd))
         return 1;
-      }
-      thd->options&= ~(ulonglong) (OPTION_BEGIN | OPTION_KEEP_LOG);
-      thd->transaction.all.modified_non_trans_table= FALSE;
       thd->server_status|= SERVER_STATUS_AUTOCOMMIT;
-      if (ha_commit(thd))
-	return 1;
+      thd->options= new_options;
     }
     else
     {
+      thd->options= new_options;
       thd->transaction.all.modified_non_trans_table= FALSE;
       thd->server_status&= ~SERVER_STATUS_AUTOCOMMIT;
     }
@@ -3399,6 +3416,12 @@ static uchar *get_myisam_mmap_size(THD *thd)
   return (uchar *)&myisam_mmap_size;
 }
 
+static uchar *in_transaction(THD *thd)
+{
+  thd->sys_var_tmp.my_bool_value=
+    test(thd->server_status & SERVER_STATUS_IN_TRANS);
+  return (uchar*) &thd->sys_var_tmp.my_bool_value;
+}
 
 /****************************************************************************
   Main handling of variables:
diff --git a/sql/share/errmsg.txt b/sql/share/errmsg.txt
index eb5d7ed935d..ac0c2653ee0 100644
--- a/sql/share/errmsg.txt
+++ b/sql/share/errmsg.txt
@@ -6290,3 +6290,11 @@ ER_QUERY_CACHE_IS_GLOBALY_DISABLED
         eng "Query cache is globally disabled and you can't enable it only for this session"
 ER_SPATIAL_MUST_HAVE_GEOM_COL 42000
   eng "A SPATIAL index may only contain a geometrical type column"
+ER_VIEW_ORDERBY_IGNORED
+        eng "View '%-.192s'.'%-.192s' ORDER BY clause ignored because there is other ORDER BY clause already."
+ER_CONNECTION_KILLED 70100 
+        eng "Connection was killed"
+ER_INTERNAL_ERROR
+        eng "Internal error: '%-.192s'"
+
+
diff --git a/sql/slave.cc b/sql/slave.cc
index cfa6c9fdc56..fa7fabb5d99 100644
--- a/sql/slave.cc
+++ b/sql/slave.cc
@@ -524,7 +524,7 @@ terminate_slave_thread(THD *thd,
     IF_DBUG(int err= ) pthread_kill(thd->real_id, thr_client_alarm);
     DBUG_ASSERT(err != EINVAL);
 #endif
-    thd->awake(THD::NOT_KILLED);
+    thd->awake(KILL_CONNECTION);
     pthread_mutex_unlock(&thd->LOCK_thd_data);
 
     /*
@@ -837,6 +837,8 @@ bool is_network_error(uint errorno)
       errorno == CR_SERVER_GONE_ERROR ||
       errorno == CR_SERVER_LOST ||
       errorno == ER_CON_COUNT_ERROR ||
+      errorno == ER_CONNECTION_KILLED ||
+      errorno == ER_NEW_ABORTING_CONNECTION ||
       errorno == ER_SERVER_SHUTDOWN)
     return TRUE;
 
@@ -2012,7 +2014,7 @@ static int request_dump(MYSQL* mysql, Master_info* mi,
   
   *suppress_warnings= FALSE;
 
-  if (opt_log_slave_updates && opt_replicate_annotate_rows_events)
+  if (opt_log_slave_updates && opt_replicate_annotate_row_events)
     binlog_flags|= BINLOG_SEND_ANNOTATE_ROWS_EVENT;
 
   // TODO if big log files: Change next to int8store()
@@ -3076,11 +3078,11 @@ pthread_handler_t handle_slave_sql(void *arg)
   thd->temporary_tables = rli->save_temporary_tables; // restore temp tables
   set_thd_in_use_temporary_tables(rli);   // (re)set sql_thd in use for saved temp tables
   /*
-    binlog_annotate_rows_events must be TRUE only after an Annotate_rows event
+    binlog_annotate_row_events must be TRUE only after an Annotate_rows event
     has been recieved and only till the last corresponding rbr event has been
     applied. In all other cases it must be FALSE.
   */
-  thd->variables.binlog_annotate_rows_events= 0;
+  thd->variables.binlog_annotate_row_events= 0;
   pthread_mutex_lock(&LOCK_thread_count);
   threads.append(thd);
   pthread_mutex_unlock(&LOCK_thread_count);
diff --git a/sql/slave.h b/sql/slave.h
index 3333e583559..bd4302ff3a1 100644
--- a/sql/slave.h
+++ b/sql/slave.h
@@ -106,7 +106,7 @@ extern MYSQL_PLUGIN_IMPORT char *relay_log_info_file;
 extern char *opt_relay_logname, *opt_relaylog_index_name;
 extern my_bool opt_skip_slave_start, opt_reckless_slave;
 extern my_bool opt_log_slave_updates;
-extern my_bool opt_replicate_annotate_rows_events;
+extern my_bool opt_replicate_annotate_row_events;
 extern ulonglong relay_log_space_limit;
 
 /*
diff --git a/sql/sp.cc b/sql/sp.cc
index d2c732c2100..57dbc04be4e 100644
--- a/sql/sp.cc
+++ b/sql/sp.cc
@@ -14,8 +14,8 @@
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
 
 #include "mysql_priv.h"
-#include "sp.h"
 #include "sp_head.h"
+#include "sp.h"
 #include "sp_cache.h"
 #include "sql_trigger.h"
 
@@ -23,7 +23,7 @@
 
 static bool
 create_string(THD *thd, String *buf,
-	      int sp_type,
+	      stored_procedure_type sp_type,
 	      const char *db, ulong dblen,
 	      const char *name, ulong namelen,
 	      const char *params, ulong paramslen,
@@ -33,7 +33,8 @@ create_string(THD *thd, String *buf,
               const LEX_STRING *definer_user,
               const LEX_STRING *definer_host);
 static int
-db_load_routine(THD *thd, int type, sp_name *name, sp_head **sphp,
+db_load_routine(THD *thd, stored_procedure_type type, sp_name *name,
+                sp_head **sphp,
                 ulong sql_mode, const char *params, const char *returns,
                 const char *body, st_sp_chistics &chistics,
                 const char *definer, longlong created, longlong modified,
@@ -490,7 +491,8 @@ static TABLE *open_proc_table_for_update(THD *thd)
 */
 
 static int
-db_find_routine_aux(THD *thd, int type, sp_name *name, TABLE *table)
+db_find_routine_aux(THD *thd, stored_procedure_type type, sp_name *name,
+                    TABLE *table)
 {
   uchar key[MAX_KEY_LENGTH];	// db, name, optional key length type
   DBUG_ENTER("db_find_routine_aux");
@@ -543,7 +545,8 @@ db_find_routine_aux(THD *thd, int type, sp_name *name, TABLE *table)
 */
 
 static int
-db_find_routine(THD *thd, int type, sp_name *name, sp_head **sphp)
+db_find_routine(THD *thd, stored_procedure_type type, sp_name *name,
+                sp_head **sphp)
 {
   TABLE *table;
   const char *params, *returns, *body;
@@ -711,7 +714,8 @@ Silence_deprecated_warning::handle_error(uint sql_errno, const char *message,
 
 
 static int
-db_load_routine(THD *thd, int type, sp_name *name, sp_head **sphp,
+db_load_routine(THD *thd, stored_procedure_type type,
+                sp_name *name, sp_head **sphp,
                 ulong sql_mode, const char *params, const char *returns,
                 const char *body, st_sp_chistics &chistics,
                 const char *definer, longlong created, longlong modified,
@@ -890,7 +894,7 @@ sp_returns_type(THD *thd, String &result, sp_head *sp)
 */
 
 int
-sp_create_routine(THD *thd, int type, sp_head *sp)
+sp_create_routine(THD *thd, stored_procedure_type type, sp_head *sp)
 {
   int ret;
   TABLE *table;
@@ -906,7 +910,8 @@ sp_create_routine(THD *thd, int type, sp_head *sp)
   bool save_binlog_row_based;
 
   DBUG_ENTER("sp_create_routine");
-  DBUG_PRINT("enter", ("type: %d  name: %.*s",type, (int) sp->m_name.length,
+  DBUG_PRINT("enter", ("type: %d  name: %.*s", (int) type,
+                       (int) sp->m_name.length,
                        sp->m_name.str));
   String retstr(64);
   retstr.set_charset(system_charset_info);
@@ -1151,7 +1156,7 @@ done:
 */
 
 int
-sp_drop_routine(THD *thd, int type, sp_name *name)
+sp_drop_routine(THD *thd, stored_procedure_type type, sp_name *name)
 {
   TABLE *table;
   int ret;
@@ -1211,14 +1216,16 @@ sp_drop_routine(THD *thd, int type, sp_name *name)
 */
 
 int
-sp_update_routine(THD *thd, int type, sp_name *name, st_sp_chistics *chistics)
+sp_update_routine(THD *thd, stored_procedure_type type, sp_name *name,
+                  st_sp_chistics *chistics)
 {
   TABLE *table;
   int ret;
   bool save_binlog_row_based;
   DBUG_ENTER("sp_update_routine");
   DBUG_PRINT("enter", ("type: %d  name: %.*s",
-		       type, (int) name->m_name.length, name->m_name.str));
+		       (int) type,
+                       (int) name->m_name.length, name->m_name.str));
 
   DBUG_ASSERT(type == TYPE_ENUM_PROCEDURE ||
               type == TYPE_ENUM_FUNCTION);
@@ -1346,7 +1353,7 @@ err:
 */
 
 bool
-sp_show_create_routine(THD *thd, int type, sp_name *name)
+sp_show_create_routine(THD *thd, stored_procedure_type type, sp_name *name)
 {
   bool err_status= TRUE;
   sp_head *sp;
@@ -1404,8 +1411,8 @@ sp_show_create_routine(THD *thd, int type, sp_name *name)
 */
 
 sp_head *
-sp_find_routine(THD *thd, int type, sp_name *name, sp_cache **cp,
-                bool cache_only)
+sp_find_routine(THD *thd, stored_procedure_type type, sp_name *name,
+                sp_cache **cp, bool cache_only)
 {
   sp_head *sp;
   ulong depth= (type == TYPE_ENUM_PROCEDURE ?
@@ -1562,7 +1569,7 @@ sp_exist_routines(THD *thd, TABLE_LIST *routines, bool any)
 */
 
 int
-sp_routine_exists_in_table(THD *thd, int type, sp_name *name)
+sp_routine_exists_in_table(THD *thd, stored_procedure_type type, sp_name *name)
 {
   TABLE *table;
   int ret;
@@ -1729,7 +1736,7 @@ static bool add_used_routine(LEX *lex, Query_arena *arena,
 */
 
 void sp_add_used_routine(LEX *lex, Query_arena *arena,
-                         sp_name *rt, char rt_type)
+                         sp_name *rt, enum stored_procedure_type rt_type)
 {
   rt->set_routine_type(rt_type);
   (void)add_used_routine(lex, arena, &rt->m_sroutines_key, 0);
@@ -1885,9 +1892,11 @@ sp_cache_routines_and_add_tables_aux(THD *thd, LEX *lex,
   for (Sroutine_hash_entry *rt= start; rt; rt= rt->next)
   {
     sp_name name(thd, rt->key.str, rt->key.length);
-    int type= rt->key.str[0];
+    stored_procedure_type type= (stored_procedure_type) rt->key.str[0];
     sp_head *sp;
 
+    if (type == TYPE_ENUM_TRIGGER)
+      continue;
     if (!(sp= sp_cache_lookup((type == TYPE_ENUM_FUNCTION ?
                               &thd->sp_func_cache : &thd->sp_proc_cache),
                               &name)))
@@ -2076,7 +2085,7 @@ sp_cache_routines_and_add_tables_for_triggers(THD *thd, LEX *lex,
 */
 static bool
 create_string(THD *thd, String *buf,
-              int type,
+              stored_procedure_type type,
               const char *db, ulong dblen,
               const char *name, ulong namelen,
               const char *params, ulong paramslen,
diff --git a/sql/sp.h b/sql/sp.h
index 876287d9704..4e3252392f7 100644
--- a/sql/sp.h
+++ b/sql/sp.h
@@ -39,26 +39,28 @@ int
 sp_drop_db_routines(THD *thd, char *db);
 
 sp_head *
-sp_find_routine(THD *thd, int type, sp_name *name,
+sp_find_routine(THD *thd, stored_procedure_type type, sp_name *name,
                 sp_cache **cp, bool cache_only);
 
 bool
 sp_exist_routines(THD *thd, TABLE_LIST *procs, bool any);
 
 int
-sp_routine_exists_in_table(THD *thd, int type, sp_name *name);
+sp_routine_exists_in_table(THD *thd, stored_procedure_type type,
+                           sp_name *name);
 
 bool
-sp_show_create_routine(THD *thd, int type, sp_name *name);
+sp_show_create_routine(THD *thd, stored_procedure_type type, sp_name *name);
 
 int
-sp_create_routine(THD *thd, int type, sp_head *sp);
+sp_create_routine(THD *thd, stored_procedure_type type, sp_head *sp);
 
 int
-sp_update_routine(THD *thd, int type, sp_name *name, st_sp_chistics *chistics);
+sp_update_routine(THD *thd, stored_procedure_type type, sp_name *name,
+                  st_sp_chistics *chistics);
 
 int
-sp_drop_routine(THD *thd, int type, sp_name *name);
+sp_drop_routine(THD *thd, stored_procedure_type type, sp_name *name);
 
 /*
   Procedures for pre-caching of stored routines and building table list
@@ -67,7 +69,7 @@ sp_drop_routine(THD *thd, int type, sp_name *name);
 void sp_get_prelocking_info(THD *thd, bool *need_prelocking, 
                             bool *first_no_prelocking);
 void sp_add_used_routine(LEX *lex, Query_arena *arena,
-                         sp_name *rt, char rt_type);
+                         sp_name *rt, stored_procedure_type rt_type);
 void sp_remove_not_own_routines(LEX *lex);
 bool sp_update_sp_used_routines(HASH *dst, HASH *src);
 int sp_cache_routines_and_add_tables(THD *thd, LEX *lex,
diff --git a/sql/sp_head.cc b/sql/sp_head.cc
index bd2dcfd8653..fda922f91ae 100644
--- a/sql/sp_head.cc
+++ b/sql/sp_head.cc
@@ -374,8 +374,8 @@ sp_eval_expr(THD *thd, Field *result_field, Item **expr_item_ptr)
   
   thd->count_cuted_fields= CHECK_FIELD_ERROR_FOR_NULL;
   thd->abort_on_warning=
-    thd->variables.sql_mode &
-    (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES);
+    test(thd->variables.sql_mode &
+         (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES));
   thd->transaction.stmt.modified_non_trans_table= FALSE;
 
   /* Save the value in the field. Convert the value if needed. */
@@ -1312,7 +1312,7 @@ sp_head::execute(THD *thd)
 	ctx->enter_handler(hip);
         thd->clear_error();
         thd->is_fatal_error= 0;
-	thd->killed= THD::NOT_KILLED;
+	thd->killed= NOT_KILLED;
         thd->mysys_var->abort= 0;
 	continue;
       }
@@ -1362,7 +1362,7 @@ sp_head::execute(THD *thd)
     If the DB has changed, the pointer has changed too, but the
     original thd->db will then have been freed
   */
-  if (cur_db_changed && thd->killed != THD::KILL_CONNECTION)
+  if (cur_db_changed && thd->killed < KILL_CONNECTION)
   {
     /*
       Force switching back to the saved current database, because it may be
@@ -1796,7 +1796,7 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount,
     thd->options= binlog_save_options;
     if (thd->binlog_evt_union.unioned_events)
     {
-      int errcode = query_error_code(thd, thd->killed == THD::NOT_KILLED);
+      int errcode = query_error_code(thd, thd->killed == NOT_KILLED);
       Query_log_event qinfo(thd, binlog_buf.ptr(), binlog_buf.length(),
                             thd->binlog_evt_union.unioned_events_trans, FALSE, errcode);
       if (mysql_bin_log.write(&qinfo) &&
diff --git a/sql/sp_head.h b/sql/sp_head.h
index d422adc8927..dc237163716 100644
--- a/sql/sp_head.h
+++ b/sql/sp_head.h
@@ -28,11 +28,16 @@
   @ingroup Runtime_Environment
   @{
 */
-// Values for the type enum. This reflects the order of the enum declaration
-// in the CREATE TABLE command.
-#define TYPE_ENUM_FUNCTION  1
-#define TYPE_ENUM_PROCEDURE 2
-#define TYPE_ENUM_TRIGGER   3
+/*
+  Values for the type enum. This reflects the order of the enum declaration
+  in the CREATE TABLE command.
+*/
+enum stored_procedure_type
+{
+  TYPE_ENUM_FUNCTION=1,
+  TYPE_ENUM_PROCEDURE=2,
+  TYPE_ENUM_TRIGGER=3
+};
 
 Item_result
 sp_map_result_type(enum enum_field_types type);
@@ -134,9 +139,9 @@ public:
   // Init. the qualified name from the db and name.
   void init_qname(THD *thd);	// thd for memroot allocation
 
-  void set_routine_type(char type)
+  void set_routine_type(stored_procedure_type type)
   {
-    m_sroutines_key.str[0]= type;
+    m_sroutines_key.str[0]= (char) type;
   }
 
   ~sp_name()
@@ -170,8 +175,7 @@ public:
     HAS_SQLCOM_FLUSH= 4096
   };
 
-  /** TYPE_ENUM_FUNCTION, TYPE_ENUM_PROCEDURE or TYPE_ENUM_TRIGGER */
-  int m_type;
+  stored_procedure_type m_type;
   uint m_flags;                 // Boolean attributes of a stored routine
 
   Create_field m_return_field_def; /**< This is used for FUNCTIONs only. */
diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc
index b133e1e4d68..ae286878cea 100644
--- a/sql/sql_acl.cc
+++ b/sql/sql_acl.cc
@@ -2089,7 +2089,7 @@ static int replace_user_table(THD *thd, TABLE *table, const LEX_USER &combo,
       table->field[next_field+2]->store((longlong) mqh.conn_per_hour, TRUE);
     if (table->s->fields >= 36 &&
         (mqh.specified_limits & USER_RESOURCES::USER_CONNECTIONS))
-      table->field[next_field+3]->store((longlong) mqh.user_conn, TRUE);
+      table->field[next_field+3]->store((longlong) mqh.user_conn, FALSE);
     mqh_used= mqh_used || mqh.questions || mqh.updates || mqh.conn_per_hour;
 
     next_field+=4;
@@ -4578,7 +4578,8 @@ ulong get_column_grant(THD *thd, GRANT_INFO *grant,
 
 /* Help function for mysql_show_grants */
 
-static void add_user_option(String *grant, ulong value, const char *name)
+static void add_user_option(String *grant, long value, const char *name,
+                            my_bool is_signed)
 {
   if (value)
   {
@@ -4586,7 +4587,7 @@ static void add_user_option(String *grant, ulong value, const char *name)
     grant->append(' ');
     grant->append(name, strlen(name));
     grant->append(' ');
-    p=int10_to_str(value, buff, 10);
+    p=int10_to_str(value, buff, is_signed ? -10 : 10);
     grant->append(buff,p-buff);
   }
 }
@@ -4768,13 +4769,13 @@ bool mysql_show_grants(THD *thd,LEX_USER *lex_user)
       if (want_access & GRANT_ACL)
 	global.append(STRING_WITH_LEN(" GRANT OPTION"));
       add_user_option(&global, acl_user->user_resource.questions,
-		      "MAX_QUERIES_PER_HOUR");
+		      "MAX_QUERIES_PER_HOUR", 0);
       add_user_option(&global, acl_user->user_resource.updates,
-		      "MAX_UPDATES_PER_HOUR");
+		      "MAX_UPDATES_PER_HOUR", 0);
       add_user_option(&global, acl_user->user_resource.conn_per_hour,
-		      "MAX_CONNECTIONS_PER_HOUR");
+		      "MAX_CONNECTIONS_PER_HOUR", 0);
       add_user_option(&global, acl_user->user_resource.user_conn,
-		      "MAX_USER_CONNECTIONS");
+		      "MAX_USER_CONNECTIONS", 1);
     }
     protocol->prepare_for_resend();
     protocol->store(global.ptr(),global.length(),global.charset());
@@ -6978,7 +6979,6 @@ bool check_routine_level_acl(THD *thd, const char *db, const char *name,
 #undef HAVE_OPENSSL
 #ifdef NO_EMBEDDED_ACCESS_CHECKS
 #define initialized 0
-#define decrease_user_connections(X)        /* nothing */
 #define check_for_max_user_connections(X,Y)   0
 #define get_or_create_user_conn(A,B,C,D) 0
 #endif
@@ -7478,7 +7478,6 @@ static ulong parse_client_handshake_packet(MPVIO_EXT *mpvio,
   THD *thd= mpvio->thd;
   NET *net= &thd->net;
   char *end;
-
   DBUG_ASSERT(mpvio->status == MPVIO_EXT::FAILURE);
 
   if (pkt_len < MIN_HANDSHAKE_SIZE)
@@ -7490,7 +7489,7 @@ static ulong parse_client_handshake_packet(MPVIO_EXT *mpvio,
   ulong client_capabilities= uint2korr(net->read_pos);
   if (client_capabilities & CLIENT_PROTOCOL_41)
   {
-    client_capabilities|= ((ulong) uint2korr(net->read_pos+2)) << 16;
+    client_capabilities|= ((ulonglong) uint2korr(net->read_pos+2)) << 16;
     thd->max_client_packet_length= uint4korr(net->read_pos+4);
     DBUG_PRINT("info", ("client_character_set: %d", (uint) net->read_pos[8]));
     if (thd_init_client_charset(thd, (uint) net->read_pos[8]))
@@ -7566,21 +7565,15 @@ static ulong parse_client_handshake_packet(MPVIO_EXT *mpvio,
   uint passwd_len= thd->client_capabilities & CLIENT_SECURE_CONNECTION ?
                    (uchar)(*passwd++) : strlen(passwd);
 
-  if (thd->client_capabilities & CLIENT_CONNECT_WITH_DB)
-  {
-    db= db + passwd_len + 1;
-    /* strlen() can't be easily deleted without changing protocol */
-    db_len= strlen(db);
-  }
-  else
-  {
-    db= 0;
-    db_len= 0;
-  }
+  db= thd->client_capabilities & CLIENT_CONNECT_WITH_DB ?
+    db + passwd_len + 1 : 0;
 
-  if (passwd + passwd_len + db_len > (char *)net->read_pos + pkt_len)
+  if (passwd + passwd_len + test(db) > (char *)net->read_pos + pkt_len)
     return packet_error;
 
+  /* strlen() can't be easily deleted without changing protocol */
+  db_len= db ? strlen(db) : 0;
+
   char *client_plugin= passwd + passwd_len + (db ? db_len + 1 : 0);
 
   /* Since 4.1 all database names are stored in utf8 */
@@ -7647,8 +7640,7 @@ static ulong parse_client_handshake_packet(MPVIO_EXT *mpvio,
 
   if (thd->client_capabilities & CLIENT_PLUGIN_AUTH)
   {
-    if ((client_plugin + strlen(client_plugin)) > 
-          (char *)net->read_pos + pkt_len)
+    if (client_plugin >= (char *)net->read_pos + pkt_len)
       return packet_error;
     client_plugin= fix_plugin_ptr(client_plugin);
   }
@@ -8156,10 +8148,16 @@ bool acl_authenticate(THD *thd, uint connect_errors,
       DBUG_RETURN(1);
     }
 
-    /* Don't allow the user to connect if he has done too many queries */
-    if ((acl_user->user_resource.questions || acl_user->user_resource.updates ||
+    /*
+      Don't allow the user to connect if he has done too many queries.
+      As we are testing max_user_connections == 0 here, it means that we
+      can't let the user change max_user_connections from 0 in the server
+      without a restart as it would lead to wrong connect counting.
+    */
+    if ((acl_user->user_resource.questions ||
+         acl_user->user_resource.updates ||
          acl_user->user_resource.conn_per_hour ||
-         acl_user->user_resource.user_conn || max_user_connections) &&
+         acl_user->user_resource.user_conn || max_user_connections_checking) &&
         get_or_create_user_conn(thd,
           (opt_old_style_user_limits ? sctx->user : sctx->priv_user),
           (opt_old_style_user_limits ? sctx->host_or_ip : sctx->priv_host),
@@ -8172,9 +8170,11 @@ bool acl_authenticate(THD *thd, uint connect_errors,
   if (thd->user_connect &&
       (thd->user_connect->user_resources.conn_per_hour ||
        thd->user_connect->user_resources.user_conn ||
-       max_user_connections) &&
+       max_user_connections_checking) &&
       check_for_max_user_connections(thd, thd->user_connect))
   {
+    /* Ensure we don't decrement thd->user_connections->connections twice */
+    thd->user_connect= 0;
     status_var_increment(denied_connections);
     DBUG_RETURN(1); // The error is set in check_for_max_user_connections()
   }
@@ -8215,12 +8215,7 @@ bool acl_authenticate(THD *thd, uint connect_errors,
     if (mysql_change_db(thd, &mpvio.db, FALSE))
     {
       /* mysql_change_db() has pushed the error message. */
-      if (thd->user_connect)
-      {
-        status_var_increment(thd->status_var.access_denied_errors);
-        decrease_user_connections(thd->user_connect);
-        thd->user_connect= 0;
-      }
+      status_var_increment(thd->status_var.access_denied_errors);
       DBUG_RETURN(1);
     }
   }
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index c42551100ae..3e27355cc26 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -62,7 +62,7 @@ private:
 bool
 Prelock_error_handler::handle_error(uint sql_errno,
                                     const char * /* message */,
-                                    MYSQL_ERROR::enum_warning_level /* level */,
+                                    MYSQL_ERROR::enum_warning_level level,
                                     THD * /* thd */)
 {
   if (sql_errno == ER_NO_SUCH_TABLE)
@@ -71,7 +71,8 @@ Prelock_error_handler::handle_error(uint sql_errno,
     return TRUE;
   }
 
-  m_unhandled_errors++;
+  if (level == MYSQL_ERROR::WARN_LEVEL_ERROR)
+    m_unhandled_errors++;
   return FALSE;
 }
 
@@ -675,7 +676,7 @@ void close_handle_and_leave_table_as_lock(TABLE *table)
   */
   if (table->child_l || table->parent)
     detach_merge_children(table, FALSE);
-  table->file->close();
+  table->file->ha_close();
   table->db_stat= 0;                            // Mark file closed
   release_table_share(table->s, RELEASE_NORMAL);
   table->s= share;
@@ -3708,7 +3709,7 @@ TABLE *drop_locked_tables(THD *thd,const char *db, const char *table_name)
         if (table->db_stat)
         {
           table->db_stat= 0;
-          table->file->close();
+          table->file->ha_close();
         }
       }
       else
@@ -5861,10 +5862,15 @@ find_field_in_view(THD *thd, TABLE_LIST *table_list,
       */
       if (*ref && !(*ref)->is_autogenerated_name)
       {
+        if (register_tree_change &&
+            thd->stmt_arena->is_stmt_prepare_or_first_stmt_execute())
+          arena= thd->activate_stmt_arena_if_needed(&backup);
         item->set_name((*ref)->name, (*ref)->name_length,
                        system_charset_info);
         item->real_item()->set_name((*ref)->name, (*ref)->name_length,
                        system_charset_info);
+        if (arena)
+          thd->restore_active_arena(arena, &backup);
       }
       if (register_tree_change)
         thd->change_item_tree(ref, item);
@@ -6441,19 +6447,28 @@ find_field_in_tables(THD *thd, Item_ident *item,
       {
         SELECT_LEX *current_sel= thd->lex->current_select;
         SELECT_LEX *last_select= table_ref->select_lex;
+        bool all_merged= TRUE;
+        for (SELECT_LEX *sl= current_sel; sl && sl!=last_select;
+             sl=sl->outer_select())
+        {
+          Item *subs= sl->master_unit()->item;
+          if (subs->type() == Item::SUBSELECT_ITEM && 
+              ((Item_subselect*)subs)->substype() == Item_subselect::IN_SUBS &&
+              ((Item_in_subselect*)subs)->test_strategy(SUBS_SEMI_JOIN))
+          {
+            continue;
+          }
+          all_merged= FALSE;
+          break;
+        }
         /*
           If the field was an outer referencee, mark all selects using this
           sub query as dependent on the outer query
         */
-        if (current_sel != last_select)
+        if (!all_merged && current_sel != last_select)
         {
           mark_select_range_as_dependent(thd, last_select, current_sel,
                                          found, *ref, item);
-          if (item->can_be_depended)
-          {
-            DBUG_ASSERT((*ref) == (Item*)item);
-            current_sel->register_dependency_item(last_select, ref);
-          }
         }
       }
       return found;
@@ -7789,9 +7804,18 @@ bool setup_tables(THD *thd, Name_resolution_context *context,
   if (select_lex->first_cond_optimization)
   {
     leaves.empty();
-    select_lex->leaf_tables_exec.empty();
-    make_leaves_list(leaves, tables, full_table_list, first_select_table);
- 
+    if (!select_lex->is_prep_leaf_list_saved)
+    {
+      make_leaves_list(leaves, tables, full_table_list, first_select_table);
+      select_lex->leaf_tables_exec.empty();
+    }
+    else
+    {
+      List_iterator_fast <TABLE_LIST> ti(select_lex->leaf_tables_prep);
+      while ((table_list= ti++))
+        leaves.push_back(table_list);
+    }
+      
     while ((table_list= ti++))
     {
       TABLE *table= table_list->table;
@@ -7830,9 +7854,17 @@ bool setup_tables(THD *thd, Name_resolution_context *context,
     select_lex->leaf_tables.empty();
     while ((table_list= ti++))
     {
-      table_list->table->tablenr= table_list->tablenr_exec;
-      table_list->table->map= table_list->map_exec;
-      table_list->table->pos_in_table_list= table_list;
+      if(table_list->jtbm_subselect)
+      {
+        table_list->jtbm_table_no= table_list->tablenr_exec;
+      }
+      else
+      {
+        table_list->table->tablenr= table_list->tablenr_exec;
+        table_list->table->map= table_list->map_exec;
+        table_list->table->maybe_null= table_list->maybe_null_exec;
+        table_list->table->pos_in_table_list= table_list;
+      }
       select_lex->leaf_tables.push_back(table_list);
     }
   }    
@@ -8259,7 +8291,6 @@ int setup_conds(THD *thd, TABLE_LIST *tables, List<TABLE_LIST> &leaves,
   SELECT_LEX *select_lex= thd->lex->current_select;
   Query_arena *arena= thd->stmt_arena, backup;
   TABLE_LIST *table= NULL;	// For HP compilers
-  TABLE_LIST *save_emb_on_expr_nest= thd->thd_marker.emb_on_expr_nest;
   List_iterator<TABLE_LIST> ti(leaves);
   /*
     it_is_update set to TRUE when tables of primary SELECT_LEX (SELECT_LEX
@@ -8300,7 +8331,6 @@ int setup_conds(THD *thd, TABLE_LIST *tables, List<TABLE_LIST> &leaves,
       goto err_no_arena;
   }
 
-  thd->thd_marker.emb_on_expr_nest= NO_JOIN_NEST;
   if (*conds)
   {
     thd->where="where clause";
@@ -8314,11 +8344,11 @@ int setup_conds(THD *thd, TABLE_LIST *tables, List<TABLE_LIST> &leaves,
     */
     if ((*conds)->type() == Item::FIELD_ITEM && !derived)
       wrap_ident(thd, conds);
+    (*conds)->mark_as_condition_AND_part(NO_JOIN_NEST);
     if ((!(*conds)->fixed && (*conds)->fix_fields(thd, conds)) ||
 	(*conds)->check_cols(1))
       goto err_no_arena;
   }
-  thd->thd_marker.emb_on_expr_nest= save_emb_on_expr_nest;
 
   /*
     Apply fix_fields() to all ON clauses at all levels of nesting,
@@ -8334,8 +8364,8 @@ int setup_conds(THD *thd, TABLE_LIST *tables, List<TABLE_LIST> &leaves,
       if (embedded->on_expr)
       {
         /* Make a join an a expression */
-        thd->thd_marker.emb_on_expr_nest= embedded;
         thd->where="on clause";
+        embedded->on_expr->mark_as_condition_AND_part(embedded);
         if ((!embedded->on_expr->fixed &&
              embedded->on_expr->fix_fields(thd, &embedded->on_expr)) ||
 	    embedded->on_expr->check_cols(1))
@@ -8359,7 +8389,6 @@ int setup_conds(THD *thd, TABLE_LIST *tables, List<TABLE_LIST> &leaves,
       }
     }
   }
-  thd->thd_marker.emb_on_expr_nest= save_emb_on_expr_nest;
 
   if (!thd->stmt_arena->is_conventional())
   {
@@ -8881,7 +8910,7 @@ bool remove_table_from_cache(THD *thd, const char *db, const char *table_name,
         {
           if (!in_use->killed)
           {
-            in_use->killed= THD::KILL_CONNECTION;
+            in_use->killed= KILL_SYSTEM_THREAD;
             pthread_mutex_lock(&in_use->mysys_var->mutex);
             if (in_use->mysys_var->current_cond)
             {
@@ -9215,7 +9244,7 @@ void mysql_wait_completed_table(ALTER_PARTITION_PARAM_TYPE *lpt, TABLE *my_table
       if ((in_use->system_thread & SYSTEM_THREAD_DELAYED_INSERT) &&
           ! in_use->killed)
       {
-        in_use->killed= THD::KILL_CONNECTION;
+        in_use->killed= KILL_SYSTEM_THREAD;
         pthread_mutex_lock(&in_use->mysys_var->mutex);
         if (in_use->mysys_var->current_cond)
         {
diff --git a/sql/sql_cache.cc b/sql/sql_cache.cc
index d7040a825fb..8631a13eae7 100644
--- a/sql/sql_cache.cc
+++ b/sql/sql_cache.cc
@@ -391,7 +391,7 @@ static void debug_wait_for_kill(const char *info)
   sql_print_information("%s", info);
   while(!thd->killed)
     my_sleep(1000);
-  thd->killed= THD::NOT_KILLED;
+  thd->killed= NOT_KILLED;
   /*
     Remove the set debug variable, to ensure we don't get stuck on it again
     This is needed as for MyISAM, invalidate_table() may be called twice
@@ -491,11 +491,12 @@ static void make_base_query(String *new_query,
       continue;                                 // Continue with next symbol
     case '/':                                   // Start of comment ?
       /*
-        Comment of format /#!number #/, must be skipped.
+        Comment of format /#!number #/ or /#M!number #/, must be skipped.
         These may include '"' and other comments, but it should
         be safe to parse the content as a normal string.
       */
-      if (query[0] != '*' || query[1] == '!')
+      if (query[0] != '*' || query[1] == '!' ||
+          (query[1] == 'M' && query[2] == '!'))
         break;
 
       query++;                               // skip "/"
@@ -4438,7 +4439,7 @@ void Query_cache::wreck(uint line, const char *message)
   DBUG_PRINT("warning", ("%5d QUERY CACHE WRECK => DISABLED",line));
   DBUG_PRINT("warning", ("=================================="));
   if (thd)
-    thd->killed= THD::KILL_CONNECTION;
+    thd->killed= KILL_CONNECTION;
   cache_dump();
   /* check_integrity(0); */ /* Can't call it here because of locks */
   bins_dump();
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index 97a33b4fe1a..39edcdca3de 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -664,7 +664,7 @@ THD::THD()
    Open_tables_state(refresh_version), rli_fake(0),
    lock_id(&main_lock_id),
    in_sub_stmt(0),
-   sql_log_bin_toplevel(false),
+   sql_log_bin_toplevel(false), log_all_errors(0),
    binlog_table_maps(0), binlog_flags(0UL),
    table_map_for_update(0),
    arg_of_last_insert_id_function(FALSE),
@@ -725,6 +725,8 @@ THD::THD()
   user_time.val= start_time= start_time_sec_part= 0;
   start_utime= prior_thr_create_utime= 0L;
   utime_after_lock= 0L;
+  progress.report_to_client= 0;
+  progress.max_counter= 0;
   current_linfo =  0;
   slave_thread = 0;
   bzero(&variables, sizeof(variables));
@@ -814,6 +816,8 @@ THD::THD()
   memset(&invoker_user, 0, sizeof(invoker_user));
   memset(&invoker_host, 0, sizeof(invoker_host));
   prepare_derived_at_open= FALSE;
+  create_tmp_table_for_derived= FALSE;
+  save_prep_leaf_list= FALSE;
 }
 
 
@@ -990,6 +994,11 @@ void THD::update_all_stats()
   ulonglong end_cpu_time, end_utime;
   double busy_time, cpu_time;
 
+  /* Reset status variables used by information_schema.processlist */
+  progress.max_counter= 0;
+  progress.max_stage= 0;
+  progress.report= 0;
+
   /* This is set at start of query if opt_userstat_running was set */
   if (!userstat_running)
     return;
@@ -1086,6 +1095,11 @@ void THD::cleanup(void)
     lock=locked_tables; locked_tables=0;
     close_thread_tables(this);
   }
+  if (user_connect)
+  {
+    decrease_user_connections(user_connect);
+    user_connect= 0;                            // Safety
+  }
   wt_thd_destroy(&transaction.wt);
 
 #if defined(ENABLED_DEBUG_SYNC)
@@ -1200,6 +1214,7 @@ void add_to_status(STATUS_VAR *to_var, STATUS_VAR *from_var)
   to_var->bytes_sent+=          from_var->bytes_sent;
   to_var->rows_read+=           from_var->rows_read;
   to_var->rows_sent+=           from_var->rows_sent;
+  to_var->rows_tmp_read+=       from_var->rows_tmp_read;
   to_var->binlog_bytes_written+= from_var->binlog_bytes_written;
   to_var->cpu_time+=            from_var->cpu_time;
   to_var->busy_time+=           from_var->busy_time;
@@ -1235,6 +1250,7 @@ void add_diff_to_status(STATUS_VAR *to_var, STATUS_VAR *from_var,
   to_var->bytes_sent+=           from_var->bytes_sent - dec_var->bytes_sent;
   to_var->rows_read+=            from_var->rows_read - dec_var->rows_read;
   to_var->rows_sent+=            from_var->rows_sent - dec_var->rows_sent;
+  to_var->rows_tmp_read+=        from_var->rows_tmp_read - dec_var->rows_tmp_read;
   to_var->binlog_bytes_written+= from_var->binlog_bytes_written -
                                  dec_var->binlog_bytes_written;
   to_var->cpu_time+=             from_var->cpu_time - dec_var->cpu_time;
@@ -1250,15 +1266,24 @@ void add_diff_to_status(STATUS_VAR *to_var, STATUS_VAR *from_var,
 #endif
 
 
-void THD::awake(THD::killed_state state_to_set)
+void THD::awake(killed_state state_to_set)
 {
   DBUG_ENTER("THD::awake");
   DBUG_PRINT("enter", ("this: 0x%lx", (long) this));
   THD_CHECK_SENTRY(this);
   safe_mutex_assert_owner(&LOCK_thd_data);
 
+  if (global_system_variables.log_warnings > 3)
+  {
+    Security_context *sctx= security_ctx;
+    sql_print_warning(ER(ER_NEW_ABORTING_CONNECTION),
+                      thread_id,(db ? db : "unconnected"),
+                      sctx->user ? sctx->user : "unauthenticated",
+                      sctx->host_or_ip,
+                      "KILLED");
+  }
   killed= state_to_set;
-  if (state_to_set != THD::KILL_QUERY)
+  if (state_to_set >= KILL_CONNECTION)
   {
     thr_alarm_kill(thread_id);
     if (!slave_thread)
@@ -1338,6 +1363,38 @@ void THD::awake(THD::killed_state state_to_set)
   DBUG_VOID_RETURN;
 }
 
+
+/*
+  Get error number for killed state
+  Note that the error message can't have any parameters.
+  See thd::kill_message()
+*/
+
+int killed_errno(killed_state killed)
+{
+  switch (killed) {
+  case NOT_KILLED:
+  case KILL_HARD_BIT:
+    return 0;                                   // Probably wrong usage
+  case KILL_BAD_DATA:
+  case KILL_BAD_DATA_HARD:
+    return 0;                                   // Not a real error
+  case KILL_CONNECTION:
+  case KILL_CONNECTION_HARD:
+  case KILL_SYSTEM_THREAD:
+  case KILL_SYSTEM_THREAD_HARD:
+    return ER_CONNECTION_KILLED;
+  case KILL_QUERY:
+  case KILL_QUERY_HARD:
+    return ER_QUERY_INTERRUPTED;
+  case KILL_SERVER:
+  case KILL_SERVER_HARD:
+    return ER_SERVER_SHUTDOWN;
+  }
+  return 0;                                     // Keep compiler happy
+}
+
+
 /*
   Remember the location of thread info, the structure needed for
   sql_alloc() and the structure for the net buffer
@@ -2583,26 +2640,32 @@ bool select_max_min_finder_subselect::cmp_real()
 {
   Item *maxmin= ((Item_singlerow_subselect *)item)->element_index(0);
   double val1= cache->val_real(), val2= maxmin->val_real();
+
+  /* Ignore NULLs for ANY and keep them for ALL subqueries */
+  if (cache->null_value)
+    return (is_all && !maxmin->null_value) || (!is_all && maxmin->null_value);
+  if (maxmin->null_value)
+    return !is_all;
+
   if (fmax)
-    return (cache->null_value && !maxmin->null_value) ||
-      (!cache->null_value && !maxmin->null_value &&
-       val1 > val2);
-  return (maxmin->null_value && !cache->null_value) ||
-    (!cache->null_value && !maxmin->null_value &&
-     val1 < val2);
+    return(val1 > val2);
+  return (val1 < val2);
 }
 
 bool select_max_min_finder_subselect::cmp_int()
 {
   Item *maxmin= ((Item_singlerow_subselect *)item)->element_index(0);
   longlong val1= cache->val_int(), val2= maxmin->val_int();
+
+  /* Ignore NULLs for ANY and keep them for ALL subqueries */
+  if (cache->null_value)
+    return (is_all && !maxmin->null_value) || (!is_all && maxmin->null_value);
+  if (maxmin->null_value)
+    return !is_all;
+
   if (fmax)
-    return (cache->null_value && !maxmin->null_value) ||
-      (!cache->null_value && !maxmin->null_value &&
-       val1 > val2);
-  return (maxmin->null_value && !cache->null_value) ||
-    (!cache->null_value && !maxmin->null_value &&
-     val1 < val2);
+    return(val1 > val2);
+  return (val1 < val2);
 }
 
 bool select_max_min_finder_subselect::cmp_decimal()
@@ -2610,13 +2673,16 @@ bool select_max_min_finder_subselect::cmp_decimal()
   Item *maxmin= ((Item_singlerow_subselect *)item)->element_index(0);
   my_decimal cval, *cvalue= cache->val_decimal(&cval);
   my_decimal mval, *mvalue= maxmin->val_decimal(&mval);
+
+  /* Ignore NULLs for ANY and keep them for ALL subqueries */
+  if (cache->null_value)
+    return (is_all && !maxmin->null_value) || (!is_all && maxmin->null_value);
+  if (maxmin->null_value)
+    return !is_all;
+
   if (fmax)
-    return (cache->null_value && !maxmin->null_value) ||
-      (!cache->null_value && !maxmin->null_value &&
-       my_decimal_cmp(cvalue, mvalue) > 0) ;
-  return (maxmin->null_value && !cache->null_value) ||
-    (!cache->null_value && !maxmin->null_value &&
-     my_decimal_cmp(cvalue,mvalue) < 0);
+    return (my_decimal_cmp(cvalue, mvalue) > 0) ;
+  return (my_decimal_cmp(cvalue,mvalue) < 0);
 }
 
 bool select_max_min_finder_subselect::cmp_str()
@@ -2629,13 +2695,16 @@ bool select_max_min_finder_subselect::cmp_str()
   */
   val1= cache->val_str(&buf1);
   val2= maxmin->val_str(&buf1);
+
+  /* Ignore NULLs for ANY and keep them for ALL subqueries */
+  if (cache->null_value)
+    return (is_all && !maxmin->null_value) || (!is_all && maxmin->null_value);
+  if (maxmin->null_value)
+    return !is_all;
+
   if (fmax)
-    return (cache->null_value && !maxmin->null_value) ||
-      (!cache->null_value && !maxmin->null_value &&
-       sortcmp(val1, val2, cache->collation.collation) > 0) ;
-  return (maxmin->null_value && !cache->null_value) ||
-    (!cache->null_value && !maxmin->null_value &&
-     sortcmp(val1, val2, cache->collation.collation) < 0);
+    return (sortcmp(val1, val2, cache->collation.collation) > 0) ;
+  return (sortcmp(val1, val2, cache->collation.collation) < 0);
 }
 
 int select_exists_subselect::send_data(List<Item> &items)
@@ -3360,12 +3429,128 @@ void THD::restore_backup_open_tables_state(Open_tables_state *backup)
   @param thd  user thread
   @retval 0 the user thread is active
   @retval 1 the user thread has been killed
+
+  This is used to signal a storage engine if it should be killed.
 */
+
 extern "C" int thd_killed(const MYSQL_THD thd)
 {
-  return(thd->killed);
+  if (!(thd->killed & KILL_HARD_BIT))
+    return 0;
+  return thd->killed;
+}
+
+
+/**
+   Send an out-of-band progress report to the client
+
+   The report is sent every 'thd->...progress_report_time' second,
+   however not more often than global.progress_report_time.
+   If global.progress_report_time is 0, then don't send progress reports, but
+   check every second if the value has changed
+*/
+
+static void thd_send_progress(THD *thd)
+{
+  /* Check if we should send the client a progress report */
+  ulonglong report_time= my_interval_timer();
+  if (report_time > thd->progress.next_report_time)
+  {
+    uint seconds_to_next= max(thd->variables.progress_report_time,
+                              global_system_variables.progress_report_time);
+    if (seconds_to_next == 0)             // Turned off
+      seconds_to_next= 1;                 // Check again after 1 second
+
+    thd->progress.next_report_time= (report_time +
+                                     seconds_to_next * 1000000000ULL);
+    if (global_system_variables.progress_report_time &&
+        thd->variables.progress_report_time)
+      net_send_progress_packet(thd);
+  }
 }
 
+
+/** Initialize progress report handling **/
+
+extern "C" void thd_progress_init(MYSQL_THD thd, uint max_stage)
+{
+  /*
+    Send progress reports to clients that supports it, if the command
+    is a high level command (like ALTER TABLE) and we are not in a
+    stored procedure
+  */
+  thd->progress.report= ((thd->client_capabilities & CLIENT_PROGRESS) &&
+                         thd->progress.report_to_client &&
+                         !thd->in_sub_stmt);
+  thd->progress.next_report_time= 0;
+  thd->progress.stage= 0;
+  thd->progress.counter= thd->progress.max_counter= 0;
+  thd->progress.max_stage= max_stage;
+}
+
+
+/* Inform processlist and the client that some progress has been made */
+
+extern "C" void thd_progress_report(MYSQL_THD thd,
+                                    ulonglong progress, ulonglong max_progress)
+{
+  if (thd->progress.max_counter != max_progress)        // Simple optimization
+  {
+    pthread_mutex_lock(&thd->LOCK_thd_data);
+    thd->progress.counter= progress;
+    thd->progress.max_counter= max_progress;
+    pthread_mutex_unlock(&thd->LOCK_thd_data);
+  }
+  else
+    thd->progress.counter= progress;
+
+  if (thd->progress.report)
+    thd_send_progress(thd);
+}
+
+/**
+  Move to next stage in process list handling
+
+  This will reset the timer to ensure the progress is sent to the client
+  if client progress reports are activated.
+*/
+
+extern "C" void thd_progress_next_stage(MYSQL_THD thd)
+{
+  pthread_mutex_lock(&thd->LOCK_thd_data);
+  thd->progress.stage++;
+  thd->progress.counter= 0;
+  DBUG_ASSERT(thd->progress.stage < thd->progress.max_stage);
+  pthread_mutex_unlock(&thd->LOCK_thd_data);
+  if (thd->progress.report)
+  {
+    thd->progress.next_report_time= 0;          // Send new stage info
+    thd_send_progress(thd);
+  }
+}
+
+/**
+  Disable reporting of progress in process list.
+
+  @note
+  This function is safe to call even if one has not called thd_progress_init.
+
+  This function should be called by all parts that does progress
+  reporting to ensure that progress list doesn't contain 100 % done
+  forever.
+*/
+
+
+extern "C" void thd_progress_end(MYSQL_THD thd)
+{
+  /*
+    It's enough to reset max_counter to set disable progress indicator
+    in processlist.
+  */
+  thd->progress.max_counter= 0;
+}
+
+
 /**
   Return the thread id of a user thread
   @param thd user thread
diff --git a/sql/sql_class.h b/sql/sql_class.h
index fe9288c0e52..0085e982abb 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -74,6 +74,22 @@ public:
 
 
 /**
+  Item iterator over List_iterator_fast for Items
+*/
+
+class Item_iterator_list: public Item_iterator
+{
+  List_iterator<Item> list;
+public:
+  Item_iterator_list(List_iterator<Item> &arg_list):
+    list(arg_list) {}
+  void open() { list.rewind(); }
+  Item *next() { return (list++); }
+  void close() {}
+};
+
+
+/**
   Item iterator over Item interface for rows
 */
 
@@ -346,6 +362,38 @@ public:
   LEX_COLUMN (const String& x,const  uint& y ): column (x),rights (y) {}
 };
 
+
+/* Note: these states are actually bit coded with HARD */
+enum killed_state
+{
+  NOT_KILLED= 0,
+  KILL_HARD_BIT= 1,                             /* Bit for HARD KILL */
+  KILL_BAD_DATA= 2,
+  KILL_BAD_DATA_HARD= 3,
+  KILL_QUERY= 4,
+  KILL_QUERY_HARD= 5,
+  /*
+    All of the following killed states will kill the connection
+    KILL_CONNECTION must be the first of these!
+  */
+  KILL_CONNECTION= 6,
+  KILL_CONNECTION_HARD= 7,
+  KILL_SYSTEM_THREAD= 8,
+  KILL_SYSTEM_THREAD_HARD= 9,
+  KILL_SERVER= 10,
+  KILL_SERVER_HARD= 11
+};
+
+extern int killed_errno(killed_state killed);
+#define killed_mask_hard(killed) ((killed_state) ((killed) & ~KILL_HARD_BIT))
+
+enum killed_type
+{
+  KILL_TYPE_ID,
+  KILL_TYPE_USER
+};
+
+
 #include "sql_lex.h"				/* Must be here */
 
 class Delayed_insert;
@@ -443,7 +491,8 @@ struct system_variables
   ulong ndb_index_stat_cache_entries;
   ulong ndb_index_stat_update_freq;
   ulong binlog_format; // binlog format for this thd (see enum_binlog_format)
-  my_bool binlog_annotate_rows_events;
+  ulong progress_report_time;
+  my_bool binlog_annotate_row_events;
   my_bool binlog_direct_non_trans_update;
   /*
     In slave thread we need to know in behalf of which
@@ -530,6 +579,9 @@ typedef struct system_status_var
   ulong ha_rollback_count;
   ulong ha_update_count;
   ulong ha_write_count;
+  /* The following are for internal temporary tables */
+  ulong ha_tmp_update_count;
+  ulong ha_tmp_write_count;
   ulong ha_prepare_count;
   ulong ha_discover_count;
   ulong ha_savepoint_count;
@@ -582,6 +634,7 @@ typedef struct system_status_var
   ulonglong bytes_sent;
   ulonglong rows_read;
   ulonglong rows_sent;
+  ulonglong rows_tmp_read;
   ulonglong binlog_bytes_written;
   double last_query_cost;
   double cpu_time, busy_time;
@@ -1547,6 +1600,25 @@ public:
   ulonglong  prior_thr_create_utime, thr_create_utime;
   ulonglong  start_utime, utime_after_lock;
 
+  // Process indicator
+  struct {
+    /*
+      true, if the currently running command can send progress report
+      packets to a client. Set by mysql_execute_command() for safe commands
+      See CF_REPORT_PROGRESS
+    */
+    bool       report_to_client;
+    /*
+      true, if we will send progress report packets to a client
+      (client has requested them, see CLIENT_PROGRESS; report_to_client
+      is true; not in sub-statement)
+    */
+    bool       report;
+    uint       stage, max_stage;
+    ulonglong  counter, max_counter;
+    ulonglong  next_report_time;
+  } progress;
+
   thr_lock_type update_lock_default;
   Delayed_insert *di;
 
@@ -1556,21 +1628,22 @@ public:
   bool sql_log_bin_toplevel;
   /* True when opt_userstat_running is set at start of query */
   bool userstat_running;
+  /* True if we want to log all errors */
+  bool log_all_errors;
 
   /* container for handler's private per-connection data */
   Ha_data ha_data[MAX_HA];
 
-  /* Place to store various things */
-  union 
-  { 
-    /*
-      Used by subquery optimizations, see Item_in_subselect::emb_on_expr_nest.
-    */
-    TABLE_LIST *emb_on_expr_nest;
-  } thd_marker;
-
   bool prepare_derived_at_open;
 
+  /* 
+    To signal that the tmp table to be created is created for materialized
+    derived table or a view.
+  */ 
+  bool create_tmp_table_for_derived;
+
+  bool save_prep_leaf_list;
+
 #ifndef MYSQL_CLIENT
   int binlog_setup_trx_data();
 
@@ -1932,14 +2005,6 @@ public:
   DYNAMIC_ARRAY user_var_events;        /* For user variables replication */
   MEM_ROOT      *user_var_events_alloc; /* Allocate above array elements here */
 
-  enum killed_state
-  {
-    NOT_KILLED=0,
-    KILL_BAD_DATA=1,
-    KILL_CONNECTION=ER_SERVER_SHUTDOWN,
-    KILL_QUERY=ER_QUERY_INTERRUPTED,
-    KILLED_NO_VALUE      /* means neither of the states */
-  };
   killed_state volatile killed;
 
   /* scramble - random string sent to client on handshake */
@@ -2122,7 +2187,7 @@ public:
   }
   void close_active_vio();
 #endif
-  void awake(THD::killed_state state_to_set);
+  void awake(killed_state state_to_set);
 
 #ifndef MYSQL_CLIENT
   enum enum_binlog_query_type {
@@ -2356,18 +2421,13 @@ public:
   void end_statement();
   inline int killed_errno() const
   {
-    killed_state killed_val; /* to cache the volatile 'killed' */
-    return (killed_val= killed) != KILL_BAD_DATA ? killed_val : 0;
+    return ::killed_errno(killed);
   }
   inline void send_kill_message() const
   {
     int err= killed_errno();
     if (err)
-    {
-      if ((err == KILL_CONNECTION) && !shutdown_in_progress)
-        err = KILL_QUERY;
       my_message(err, ER(err), MYF(0));
-    }
   }
   /* return TRUE if we will abort query if we make a warning now */
   inline bool really_abort_on_warning()
@@ -3179,9 +3239,11 @@ class select_max_min_finder_subselect :public select_subselect
   Item_cache *cache;
   bool (select_max_min_finder_subselect::*op)();
   bool fmax;
+  bool is_all;
 public:
-  select_max_min_finder_subselect(Item_subselect *item_arg, bool mx)
-    :select_subselect(item_arg), cache(0), fmax(mx)
+  select_max_min_finder_subselect(Item_subselect *item_arg, bool mx,
+                                  bool all)
+    :select_subselect(item_arg), cache(0), fmax(mx), is_all(all)
   {}
   void cleanup();
   int send_data(List<Item> &items);
@@ -3546,6 +3608,7 @@ public:
 #define CF_STATUS_COMMAND	4
 #define CF_SHOW_TABLE_COMMAND	8
 #define CF_WRITE_LOGS_COMMAND  16
+
 /**
   Must be set for SQL statements that may contain
   Item expressions and/or use joins and tables.
@@ -3560,6 +3623,7 @@ public:
   joins are currently prohibited in these statements.
 */
 #define CF_REEXECUTION_FRAGILE 32
+#define CF_REPORT_PROGRESS     64
 
 /* Functions in sql_class.cc */
 
@@ -3595,16 +3659,24 @@ inline int handler::ha_index_read_map(uchar * buf, const uchar * key,
   return error;
 }
 
+
+/*
+  @note: Other index lookup/navigation functions require prior
+  handler->index_init() call. This function is different, it requires
+  that the scan is not initialized, and accepts "uint index" as an argument.
+*/
+
 inline int handler::ha_index_read_idx_map(uchar * buf, uint index,
                                           const uchar * key,
                                           key_part_map keypart_map,
                                           enum ha_rkey_function find_flag)
 {
+  DBUG_ASSERT(inited==NONE);
   increment_statistics(&SSV::ha_read_key_count);
   int error= index_read_idx_map(buf, index, key, keypart_map, find_flag);
   if (!error)
   {
-    rows_read++;
+    update_rows_read();
     index_rows_read[index]++;
   }
   table->status=error ? STATUS_NOT_FOUND: 0;
@@ -3671,7 +3743,8 @@ inline int handler::ha_ft_read(uchar *buf)
 {
   int error= ft_read(buf);
   if (!error)
-    rows_read++;
+    update_rows_read();
+
   table->status=error ? STATUS_NOT_FOUND: 0;
   return error;
 }
@@ -3681,7 +3754,7 @@ inline int handler::ha_rnd_next(uchar *buf)
   increment_statistics(&SSV::ha_read_rnd_next_count);
   int error= rnd_next(buf);
   if (!error)
-    rows_read++;
+    update_rows_read();
   table->status=error ? STATUS_NOT_FOUND: 0;
   return error;
 }
@@ -3691,7 +3764,7 @@ inline int handler::ha_rnd_pos(uchar *buf, uchar *pos)
   increment_statistics(&SSV::ha_read_rnd_count);
   int error= rnd_pos(buf, pos);
   if (!error)
-    rows_read++;
+    update_rows_read();
   table->status=error ? STATUS_NOT_FOUND: 0;
   return error;
 }
@@ -3700,7 +3773,7 @@ inline int handler::ha_rnd_pos_by_record(uchar *buf)
 {
   int error= rnd_pos_by_record(buf);
   if (!error)
-    rows_read++;
+    update_rows_read();
   table->status=error ? STATUS_NOT_FOUND: 0;
   return error;
 }
@@ -3709,15 +3782,21 @@ inline int handler::ha_read_first_row(uchar *buf, uint primary_key)
 {
   int error= read_first_row(buf, primary_key);
   if (!error)
-    rows_read++;
+    update_rows_read();
   table->status=error ? STATUS_NOT_FOUND: 0;
   return error;
 }
 
 inline int handler::ha_write_tmp_row(uchar *buf)
 {
-  increment_statistics(&SSV::ha_write_count);
+  increment_statistics(&SSV::ha_tmp_write_count);
   return write_row(buf);
 }
 
+inline int handler::ha_update_tmp_row(const uchar *old_data, uchar *new_data)
+{
+  increment_statistics(&SSV::ha_tmp_update_count);
+  return update_row(old_data, new_data);
+}
+
 #endif /* MYSQL_SERVER */
diff --git a/sql/sql_connect.cc b/sql/sql_connect.cc
index 05e33826da9..cd51fd25558 100644
--- a/sql/sql_connect.cc
+++ b/sql/sql_connect.cc
@@ -49,6 +49,7 @@ int get_or_create_user_conn(THD *thd, const char *user,
 
   DBUG_ASSERT(user != 0);
   DBUG_ASSERT(host != 0);
+  DBUG_ASSERT(thd->user_connect == 0);
 
   user_len= strlen(user);
   temp_len= (strmov(strmov(temp_user, user)+1, host) - temp_user)+1;
@@ -108,15 +109,17 @@ end:
 
 int check_for_max_user_connections(THD *thd, USER_CONN *uc)
 {
-  int error=0;
+  int error= 1;
   DBUG_ENTER("check_for_max_user_connections");
 
   (void) pthread_mutex_lock(&LOCK_user_conn);
+
+  /* Root is not affected by the value of max_user_connections */
   if (max_user_connections && !uc->user_resources.user_conn &&
-      max_user_connections < (uint) uc->connections)
+      max_user_connections < uc->connections &&
+      !(thd->security_ctx->master_access & SUPER_ACL))
   {
     my_error(ER_TOO_MANY_USER_CONNECTIONS, MYF(0), uc->user);
-    error=1;
     goto end;
   }
   time_out_user_resource_limits(thd, uc);
@@ -126,7 +129,6 @@ int check_for_max_user_connections(THD *thd, USER_CONN *uc)
     my_error(ER_USER_LIMIT_REACHED, MYF(0), uc->user,
              "max_user_connections",
              (long) uc->user_resources.user_conn);
-    error= 1;
     goto end;
   }
   if (uc->user_resources.conn_per_hour &&
@@ -135,10 +137,10 @@ int check_for_max_user_connections(THD *thd, USER_CONN *uc)
     my_error(ER_USER_LIMIT_REACHED, MYF(0), uc->user,
              "max_connections_per_hour",
              (long) uc->user_resources.conn_per_hour);
-    error=1;
     goto end;
   }
   uc->conn_per_hour++;
+  error= 0;
 
 end:
   if (error)
@@ -203,7 +205,7 @@ void time_out_user_resource_limits(THD *thd, USER_CONN *uc)
   /* If more than a hour since last check, reset resource checking */
   if (check_time  - uc->reset_utime >= LL(3600000000))
   {
-    uc->questions=1;
+    uc->questions=0;
     uc->updates=0;
     uc->conn_per_hour=0;
     uc->reset_utime= check_time;
@@ -232,7 +234,7 @@ bool check_mqh(THD *thd, uint check_command)
   if (uc->user_resources.questions &&
       uc->questions++ >= uc->user_resources.questions)
   {
-    my_error(ER_USER_LIMIT_REACHED, MYF(0), uc->user, "max_questions",
+    my_error(ER_USER_LIMIT_REACHED, MYF(0), uc->user, "max_queries_per_hour",
              (long) uc->user_resources.questions);
     error=1;
     goto end;
@@ -244,7 +246,7 @@ bool check_mqh(THD *thd, uint check_command)
         (sql_command_flags[check_command] & CF_CHANGES_DATA) &&
 	uc->updates++ >= uc->user_resources.updates)
     {
-      my_error(ER_USER_LIMIT_REACHED, MYF(0), uc->user, "max_updates",
+      my_error(ER_USER_LIMIT_REACHED, MYF(0), uc->user, "max_updates_per_hour",
                (long) uc->user_resources.updates);
       error=1;
       goto end;
@@ -690,6 +692,7 @@ static void update_global_user_stats_with_user(THD *thd,
   user_stats->binlog_bytes_written+=
     (thd->status_var.binlog_bytes_written -
      thd->org_status_var.binlog_bytes_written);
+  /* We are not counting rows in internal temporary tables here ! */
   user_stats->rows_read+=      (thd->status_var.rows_read -
                                 thd->org_status_var.rows_read);
   user_stats->rows_sent+=      (thd->status_var.rows_sent -
@@ -1027,8 +1030,17 @@ void end_connection(THD *thd)
 {
   NET *net= &thd->net;
   plugin_thdvar_cleanup(thd);
+
   if (thd->user_connect)
+  {
+    /*
+      We decrease this variable early to make it easy to log again quickly.
+      This code is not critical as we will in any case do this test
+      again in thd->cleanup()
+    */
     decrease_user_connections(thd->user_connect);
+    thd->user_connect= 0;
+  }
 
   if (thd->killed || (net->error && net->vio != 0))
   {
@@ -1086,7 +1098,7 @@ void prepare_new_connection_state(THD* thd)
     execute_init_command(thd, &sys_init_connect, &LOCK_sys_init_connect);
     if (thd->is_error())
     {
-      thd->killed= THD::KILL_CONNECTION;
+      thd->killed= KILL_CONNECTION;
       sql_print_warning(ER(ER_NEW_ABORTING_CONNECTION),
                         thd->thread_id,(thd->db ? thd->db : "unconnected"),
                         sctx->user ? sctx->user : "unauthenticated",
@@ -1122,6 +1134,8 @@ pthread_handler_t handle_one_connection(void *arg)
   THD *thd= (THD*) arg;
 
   thd->thr_create_utime= microsecond_interval_timer();
+  /* We need to set this because of time_out_user_resource_limits */
+  thd->start_utime= thd->thr_create_utime;
 
   if (thread_scheduler.init_new_connection_thread())
   {
@@ -1172,7 +1186,7 @@ pthread_handler_t handle_one_connection(void *arg)
     prepare_new_connection_state(thd);
 
     while (!net->error && net->vio != 0 &&
-           !(thd->killed == THD::KILL_CONNECTION))
+           thd->killed < KILL_CONNECTION)
     {
       if (do_command(thd))
 	break;
diff --git a/sql/sql_db.cc b/sql/sql_db.cc
index c788d324743..d5579827d1e 100644
--- a/sql/sql_db.cc
+++ b/sql/sql_db.cc
@@ -18,6 +18,7 @@
 
 #include "mysql_priv.h"
 #include <mysys_err.h>
+#include "sp_head.h"
 #include "sp.h"
 #include "events.h"
 #include "sql_handler.h"
diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc
index 6ace7ba8086..98d034a4d5c 100644
--- a/sql/sql_delete.cc
+++ b/sql/sql_delete.cc
@@ -49,7 +49,7 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds,
   bool          triggers_applicable;
   uint usable_index= MAX_KEY;
   SELECT_LEX   *select_lex= &thd->lex->select_lex;
-  THD::killed_state killed_status= THD::NOT_KILLED;
+  killed_state killed_status= NOT_KILLED;
   DBUG_ENTER("mysql_delete");
   bool save_binlog_row_based;
 
@@ -61,8 +61,9 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds,
   if (open_and_lock_tables(thd, table_list))
     DBUG_RETURN(TRUE);
 
-  if (mysql_handle_list_of_derived(thd->lex, table_list, DT_MERGE_FOR_INSERT) ||
-      mysql_handle_list_of_derived(thd->lex, table_list, DT_PREPARE))
+  if (mysql_handle_list_of_derived(thd->lex, table_list, DT_MERGE_FOR_INSERT))
+    DBUG_RETURN(TRUE);
+  if (mysql_handle_list_of_derived(thd->lex, table_list, DT_PREPARE))
     DBUG_RETURN(TRUE);
 
   if (!table_list->updatable)
@@ -382,7 +383,7 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds,
       table->file->unlock_row();  // Row failed selection, release lock on it
   }
   killed_status= thd->killed;
-  if (killed_status != THD::NOT_KILLED || thd->is_error())
+  if (killed_status != NOT_KILLED || thd->is_error())
     error= 1;					// Aborted
   if (will_batch && (loc_error= table->file->end_bulk_delete()))
   {
@@ -449,7 +450,7 @@ cleanup:
       if (error < 0)
         thd->clear_error();
       else
-        errcode= query_error_code(thd, killed_status == THD::NOT_KILLED);
+        errcode= query_error_code(thd, killed_status == NOT_KILLED);
       
       /*
         [binlog]: If 'handler::delete_all_rows()' was called and the
@@ -550,7 +551,7 @@ int mysql_prepare_delete(THD *thd, TABLE_LIST *table_list, Item **conds)
     fix_inner_refs(thd, all_fields, select_lex, select_lex->ref_pointer_array))
     DBUG_RETURN(TRUE);
 
-  select_lex->fix_prepare_information(thd, conds, &fake_conds);
+  select_lex->fix_prepare_information(thd, conds, &fake_conds); 
   DBUG_RETURN(FALSE);
 }
 
@@ -586,10 +587,11 @@ int mysql_multi_delete_prepare(THD *thd)
   TABLE_LIST *target_tbl;
   DBUG_ENTER("mysql_multi_delete_prepare");
 
-  TABLE_LIST *tables= lex->query_tables;
-  if (mysql_handle_derived(lex, DT_INIT) ||
-      mysql_handle_list_of_derived(lex, tables, DT_MERGE_FOR_INSERT) ||
-      mysql_handle_list_of_derived(lex, tables, DT_PREPARE))
+  if (mysql_handle_derived(lex, DT_INIT))
+    DBUG_RETURN(TRUE);
+  if (mysql_handle_derived(lex, DT_MERGE_FOR_INSERT))
+    DBUG_RETURN(TRUE);
+  if (mysql_handle_derived(lex, DT_PREPARE))
     DBUG_RETURN(TRUE);
   /*
     setup_tables() need for VIEWs. JOIN::prepare() will not do it second
@@ -601,9 +603,11 @@ int mysql_multi_delete_prepare(THD *thd)
                                     &thd->lex->select_lex.top_join_list,
                                     lex->query_tables,
                                     lex->select_lex.leaf_tables, FALSE, 
-                                    DELETE_ACL, SELECT_ACL, TRUE))
+                                    DELETE_ACL, SELECT_ACL, FALSE))
     DBUG_RETURN(TRUE);
 
+  if (lex->select_lex.handle_derived(thd->lex, DT_MERGE))  
+    DBUG_RETURN(TRUE);
 
   /*
     Multi-delete can't be constructed over-union => we always have
@@ -616,7 +620,8 @@ int mysql_multi_delete_prepare(THD *thd)
        target_tbl= target_tbl->next_local)
   {
 
-    if (!(target_tbl->table= target_tbl->correspondent_table->table))
+    target_tbl->table= target_tbl->correspondent_table->table;
+    if (target_tbl->correspondent_table->is_multitable())
     {
        my_error(ER_VIEW_DELETE_MERGE_VIEW, MYF(0),
                 target_tbl->correspondent_table->view_db.str,
@@ -651,6 +656,10 @@ int mysql_multi_delete_prepare(THD *thd)
     with further calls to unique_table
   */
   lex->select_lex.exclude_from_table_unique_test= FALSE;
+  
+  if (lex->select_lex.save_prep_leaf_tables(thd))
+    DBUG_RETURN(TRUE);
+  
   DBUG_RETURN(FALSE);
 }
 
@@ -907,7 +916,7 @@ void multi_delete::abort()
     */
     if (mysql_bin_log.is_open())
     {
-      int errcode= query_error_code(thd, thd->killed == THD::NOT_KILLED);
+      int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
       /* possible error of writing binary log is ignored deliberately */
       (void) thd->binlog_query(THD::ROW_QUERY_TYPE,
                               thd->query(), thd->query_length(),
@@ -1057,7 +1066,7 @@ int multi_delete::do_table_deletes(TABLE *table, bool ignore)
 
 bool multi_delete::send_eof()
 {
-  THD::killed_state killed_status= THD::NOT_KILLED;
+  killed_state killed_status= NOT_KILLED;
   thd_proc_info(thd, "deleting from reference tables");
 
   /* Does deletes for the last n - 1 tables, returns 0 if ok */
@@ -1065,7 +1074,7 @@ bool multi_delete::send_eof()
 
   /* compute a total error to know if something failed */
   local_error= local_error || error;
-  killed_status= (local_error == 0)? THD::NOT_KILLED : thd->killed;
+  killed_status= (local_error == 0)? NOT_KILLED : thd->killed;
   /* reset used flags */
   thd_proc_info(thd, "end");
 
@@ -1085,7 +1094,7 @@ bool multi_delete::send_eof()
       if (local_error == 0)
         thd->clear_error();
       else
-        errcode= query_error_code(thd, killed_status == THD::NOT_KILLED);
+        errcode= query_error_code(thd, killed_status == NOT_KILLED);
       if (thd->binlog_query(THD::ROW_QUERY_TYPE,
                             thd->query(), thd->query_length(),
                             transactional_tables, FALSE, errcode) &&
diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc
index 9bfa49c5f2d..e012c161945 100644
--- a/sql/sql_derived.cc
+++ b/sql/sql_derived.cc
@@ -85,15 +85,18 @@ mysql_handle_derived(LEX *lex, uint phases)
 	   cursor && !res;
 	   cursor= cursor->next_local)
       {
+        if (!cursor->is_view_or_derived() && phases == DT_MERGE_FOR_INSERT)
+          continue;
         uint8 allowed_phases= (cursor->is_merged_derived() ? DT_PHASES_MERGE :
-                               DT_PHASES_MATERIALIZE);
+                               DT_PHASES_MATERIALIZE | DT_MERGE_FOR_INSERT);
         /*
           Skip derived tables to which the phase isn't applicable.
           TODO: mark derived at the parse time, later set it's type
           (merged or materialized)
         */
         if ((phase_flag != DT_PREPARE && !(allowed_phases & phase_flag)) ||
-            (cursor->merged_for_insert && phase_flag != DT_REINIT))
+            (cursor->merged_for_insert && phase_flag != DT_REINIT &&
+             phase_flag != DT_PREPARE))
           continue;
 	res= (*processors[phase])(lex->thd, lex, cursor);
       }
@@ -343,43 +346,52 @@ bool mysql_derived_merge(THD *thd, LEX *lex, TABLE_LIST *derived)
   if (derived->merged)
     return FALSE;
 
+ if (thd->lex->sql_command == SQLCOM_UPDATE_MULTI ||
+     thd->lex->sql_command == SQLCOM_DELETE_MULTI)
+   thd->save_prep_leaf_list= TRUE;
+
   arena= thd->activate_stmt_arena_if_needed(&backup);  // For easier test
   derived->merged= TRUE;
-  /*
-    Check whether there is enough free bits in table map to merge subquery.
-    If not - materialize it. This check isn't cached so when there is a big
-    and small subqueries, and the bigger one can't be merged it wouldn't
-    block the smaller one.
-  */
-  if (parent_lex->get_free_table_map(&map, &tablenr))
-  {
-    /* There is no enough table bits, fall back to materialization. */
-    derived->change_refs_to_fields();
-    derived->set_materialized_derived();
-    goto exit_merge;
-  }
 
-  if (dt_select->leaf_tables.elements + tablenr > MAX_TABLES)
+  if (!derived->merged_for_insert || 
+      (derived->is_multitable() && 
+       (thd->lex->sql_command == SQLCOM_UPDATE_MULTI ||
+        thd->lex->sql_command == SQLCOM_DELETE_MULTI)))
   {
-    /* There is no enough table bits, fall back to materialization. */
-    derived->change_refs_to_fields();
-    derived->set_materialized_derived();
-    goto exit_merge;
-  }
+    /*
+      Check whether there is enough free bits in table map to merge subquery.
+      If not - materialize it. This check isn't cached so when there is a big
+      and small subqueries, and the bigger one can't be merged it wouldn't
+      block the smaller one.
+    */
+    if (parent_lex->get_free_table_map(&map, &tablenr))
+    {
+      /* There is no enough table bits, fall back to materialization. */
+      derived->change_refs_to_fields();
+      derived->set_materialized_derived();
+      goto exit_merge;
+    }
+
+    if (dt_select->leaf_tables.elements + tablenr > MAX_TABLES)
+    {
+      /* There is no enough table bits, fall back to materialization. */
+      derived->change_refs_to_fields();
+      derived->set_materialized_derived();
+      goto exit_merge;
+    }
 
-  if (dt_select->options & OPTION_SCHEMA_TABLE)
-    parent_lex->options |= OPTION_SCHEMA_TABLE;
+    if (dt_select->options & OPTION_SCHEMA_TABLE)
+      parent_lex->options |= OPTION_SCHEMA_TABLE;
 
-  parent_lex->cond_count+= dt_select->cond_count;
+    parent_lex->cond_count+= dt_select->cond_count;
 
-  if (!derived->get_unit()->prepared)
-  {
-    dt_select->leaf_tables.empty();
-    make_leaves_list(dt_select->leaf_tables, derived, TRUE, 0);
-  } 
+    if (!derived->get_unit()->prepared)
+    {
+      dt_select->leaf_tables.empty();
+      make_leaves_list(dt_select->leaf_tables, derived, TRUE, 0);
+    } 
 
-  if (!derived->merged_for_insert)
-  {  derived->nested_join= (NESTED_JOIN*) thd->calloc(sizeof(NESTED_JOIN));
+    derived->nested_join= (NESTED_JOIN*) thd->calloc(sizeof(NESTED_JOIN));
     if (!derived->nested_join)
     {
       res= TRUE;
@@ -387,7 +399,7 @@ bool mysql_derived_merge(THD *thd, LEX *lex, TABLE_LIST *derived)
     }
 
     /* Merge derived table's subquery in the parent select. */
-    if (parent_lex->merge_subquery(derived, dt_select, tablenr, map))
+    if (parent_lex->merge_subquery(thd, derived, dt_select, tablenr, map))
     {
       res= TRUE;
       goto exit_merge;
@@ -463,52 +475,21 @@ exit_merge:
 
 bool mysql_derived_merge_for_insert(THD *thd, LEX *lex, TABLE_LIST *derived)
 {
-  SELECT_LEX *dt_select= derived->get_single_select();
-
   if (derived->merged_for_insert)
     return FALSE;
-  /* It's a target view for an INSERT, create field translation only. */
-  if (!derived->updatable || derived->is_materialized_derived())
-  {
-    bool res= derived->create_field_translation(thd);
-    return res;
-  }
+  if (derived->is_materialized_derived())
+    return mysql_derived_prepare(thd, lex, derived);
   if (!derived->is_multitable())
   {
-    TABLE_LIST *tl=((TABLE_LIST*)dt_select->table_list.first);
-    TABLE *table= tl->table;
-    /* preserve old map & tablenr. */
-    if (!derived->merged_for_insert && derived->table)
-      table->set_table_map(derived->table->map, derived->table->tablenr);
-
-    derived->table= table;
-    derived->schema_table=
-      ((TABLE_LIST*)dt_select->table_list.first)->schema_table;
-    if (!derived->merged)
+    if (!derived->updatable)
+      return derived->create_field_translation(thd);
+    if (derived->merge_underlying_list)
     {
-      Query_arena *arena, backup;
-      arena= thd->activate_stmt_arena_if_needed(&backup);  // For easier test
-      derived->select_lex->leaf_tables.push_back(tl);
-      derived->nested_join= (NESTED_JOIN*) thd->calloc(sizeof(NESTED_JOIN));
-      if (derived->nested_join)
-      {
-        derived->wrap_into_nested_join(tl->select_lex->top_join_list);
-        derived->get_unit()->exclude_level(); 
-      }
-      if (arena)
-        thd->restore_active_arena(arena, &backup);
-      derived->merged= TRUE;
-      if (!derived->nested_join)
-        return TRUE;
-    }      
-  }
-  else
-  {
-    if (!derived->merged_for_insert && mysql_derived_merge(thd, lex, derived))
-      return TRUE;
-  }
-  derived->merged_for_insert= TRUE;
-
+      derived->table= derived->merge_underlying_list->table;
+      derived->schema_table= derived->merge_underlying_list->schema_table;
+      derived->merged_for_insert= TRUE;
+    }
+  }  
   return FALSE;
 }
 
@@ -609,7 +590,11 @@ bool mysql_derived_prepare(THD *thd, LEX *lex, TABLE_LIST *derived)
   bool res= FALSE;
 
   // Skip already prepared views/DT
-  if (!unit || unit->prepared || derived->merged_for_insert)
+  if (!unit || unit->prepared ||
+      (derived->merged_for_insert && 
+       !(derived->is_multitable() &&
+         (thd->lex->sql_command == SQLCOM_UPDATE_MULTI ||
+          thd->lex->sql_command == SQLCOM_DELETE_MULTI))))
     DBUG_RETURN(FALSE);
 
   Query_arena *arena= thd->stmt_arena, backup;
@@ -659,13 +644,18 @@ bool mysql_derived_prepare(THD *thd, LEX *lex, TABLE_LIST *derived)
     !unit->union_distinct->next_select() (i.e. it is union and last distinct
     SELECT is last SELECT of UNION).
   */
+  thd->create_tmp_table_for_derived= TRUE;
   if (derived->derived_result->create_result_table(thd, &unit->types, FALSE,
                                                 (first_select->options |
                                                  thd->options |
                                                  TMP_TABLE_ALL_COLUMNS),
                                                 derived->alias,
                                                 FALSE, FALSE))
+  { 
+    thd->create_tmp_table_for_derived= FALSE;
     goto exit;
+  }
+  thd->create_tmp_table_for_derived= FALSE;
 
   derived->table= derived->derived_result->table;
   if (derived->is_derived() && derived->is_merged_derived())
@@ -747,7 +737,7 @@ bool mysql_derived_optimize(THD *thd, LEX *lex, TABLE_LIST *derived)
 
   bool res= FALSE;
 
-  if (unit->optimized && !unit->uncacheable && !unit->describe)
+  if (unit->optimized)
     return FALSE;
   lex->current_select= first_select;
 
diff --git a/sql/sql_error.cc b/sql/sql_error.cc
index 835e60cd6ba..1db4a110d01 100644
--- a/sql/sql_error.cc
+++ b/sql/sql_error.cc
@@ -131,7 +131,7 @@ MYSQL_ERROR *push_warning(THD *thd, MYSQL_ERROR::enum_warning_level level,
     thd->no_warnings_for_error= 1;
     thd->spcont= NULL;
 
-    thd->killed= THD::KILL_BAD_DATA;
+    thd->killed= KILL_BAD_DATA;
     my_message(code, msg, MYF(0));
 
     thd->spcont= spcont;
diff --git a/sql/sql_expression_cache.cc b/sql/sql_expression_cache.cc
index c3450884610..4174f72f080 100644
--- a/sql/sql_expression_cache.cc
+++ b/sql/sql_expression_cache.cc
@@ -16,6 +16,22 @@
 #include "mysql_priv.h"
 #include "sql_select.h"
 
+/**
+  Minimum hit ration to proceed on disk if in memory table overflowed.
+  hit_rate = hit / (miss + hit);
+*/
+#define EXPCACHE_MIN_HIT_RATE_FOR_DISK_TABLE 0.7
+/**
+  Minimum hit ratio to keep in memory table (do not switch cache off)
+  hit_rate = hit / (miss + hit);
+*/
+#define EXPCACHE_MIN_HIT_RATE_FOR_MEM_TABLE  0.2
+/**
+  Number of cache miss to check hit ratio (maximum cache performance
+  impact in the case when the cache is not applicable)
+*/
+#define EXPCACHE_CHECK_HIT_RATIO_AFTER 200
+
 /*
   Expression cache is used only for caching subqueries now, so its statistic
   variables we call subquery_cache*.
@@ -23,10 +39,10 @@
 ulong subquery_cache_miss, subquery_cache_hit;
 
 Expression_cache_tmptable::Expression_cache_tmptable(THD *thd,
-                                                 List<Item*> &dependants,
-                                                 Item *value)
-  :cache_table(NULL), table_thd(thd), list(&dependants), val(value),
-   inited (0)
+                                                     List<Item> &dependants,
+                                                     Item *value)
+  :cache_table(NULL), table_thd(thd), items(dependants), val(value),
+   hit(0), miss(0), inited (0)
 {
   DBUG_ENTER("Expression_cache_tmptable::Expression_cache_tmptable");
   DBUG_VOID_RETURN;
@@ -34,6 +50,17 @@ Expression_cache_tmptable::Expression_cache_tmptable(THD *thd,
 
 
 /**
+  Disable cache
+*/
+
+void Expression_cache_tmptable::disable_cache()
+{
+  free_tmp_table(table_thd, cache_table);
+  cache_table= NULL;
+}
+
+
+/**
   Field enumerator for TABLE::add_tmp_key
 
   @param arg             reference variable with current field number
@@ -60,50 +87,32 @@ static uint field_enumerator(uchar *arg)
 
 void Expression_cache_tmptable::init()
 {
-  List_iterator<Item*> li(*list);
-  Item_iterator_ref_list it(li);
-  Item **item;
+  List_iterator<Item> li(items);
+  Item_iterator_list it(li);
   uint field_counter;
   DBUG_ENTER("Expression_cache_tmptable::init");
   DBUG_ASSERT(!inited);
   inited= TRUE;
   cache_table= NULL;
 
-  while ((item= li++))
-  {
-    DBUG_ASSERT(item);
-    if (*item)
-    {
-      DBUG_ASSERT((*item)->fixed);
-      items.push_back((*item));
-    }
-    else
-    {
-      /*
-        This is possible when optimizer already executed this subquery and
-        optimized out the condition predicate.
-      */
-      li.remove();
-    }
-  }
-
-  if (list->elements == 0)
+  if (items.elements == 0)
   {
     DBUG_PRINT("info", ("All parameters were removed by optimizer."));
     DBUG_VOID_RETURN;
   }
 
+  /* add result field */
+  items.push_front(val);
+
   cache_table_param.init();
   /* dependent items and result */
-  cache_table_param.field_count= list->elements + 1;
+  cache_table_param.field_count= items.elements;
   /* postpone table creation to index description */
   cache_table_param.skip_create_table= 1;
 
-  items.push_front(val);
-
   if (!(cache_table= create_tmp_table(table_thd, &cache_table_param,
                                       items, (ORDER*) NULL,
-                                      FALSE, FALSE,
+                                      FALSE, TRUE,
                                       ((table_thd->options |
                                         TMP_TABLE_ALL_COLUMNS) &
                                        ~(OPTION_BIG_TABLES |
@@ -122,16 +131,13 @@ void Expression_cache_tmptable::init()
     goto error;
   }
 
-  /* This list do not contain result field */
-  it.open();
-
-  field_counter=1;
+  field_counter= 1;
 
   if (cache_table->alloc_keys(1) ||
       cache_table->add_tmp_key(0, items.elements - 1, &field_enumerator,
                                 (uchar*)&field_counter, TRUE) ||
       ref.tmp_table_index_lookup_init(table_thd, cache_table->key_info, it,
-                                      TRUE))
+                                      TRUE, 1 /* skip result field*/))
   {
     DBUG_PRINT("error", ("creating index failed"));
     goto error;
@@ -158,17 +164,19 @@ void Expression_cache_tmptable::init()
   DBUG_VOID_RETURN;
 
 error:
-  /* switch off cache */
-  free_tmp_table(table_thd, cache_table);
-  cache_table= NULL;
+  disable_cache();
   DBUG_VOID_RETURN;
 }
 
 
 Expression_cache_tmptable::~Expression_cache_tmptable()
 {
+  /* Add accumulated statistics */
+  statistic_add(subquery_cache_miss, miss, &LOCK_status);
+  statistic_add(subquery_cache_hit, hit, &LOCK_status);
+
   if (cache_table)
-    free_tmp_table(table_thd, cache_table);
+    disable_cache();
 }
 
 
@@ -192,26 +200,28 @@ Expression_cache::result Expression_cache_tmptable::check_value(Item **value)
   int res;
   DBUG_ENTER("Expression_cache_tmptable::check_value");
 
-  /*
-    We defer cache initialization to get item references that are
-    used at the execution phase.
-  */
-  if (!inited)
-    init();
-
   if (cache_table)
   {
     DBUG_PRINT("info", ("status: %u  has_record %u",
                         (uint)cache_table->status, (uint)ref.has_record));
     if ((res= join_read_key2(table_thd, NULL, cache_table, &ref)) == 1)
       DBUG_RETURN(ERROR);
+
     if (res)
     {
-      subquery_cache_miss++;
+      if (((++miss) == EXPCACHE_CHECK_HIT_RATIO_AFTER) &&
+          ((double)hit / ((double)hit + miss)) <
+          EXPCACHE_MIN_HIT_RATE_FOR_MEM_TABLE)
+      {
+        DBUG_PRINT("info",
+                   ("Early check: hit rate is not so good to keep the cache"));
+        disable_cache();
+      }
+
       DBUG_RETURN(MISS);
     }
 
-    subquery_cache_hit++;
+    hit++;
     *value= cached_result;
     DBUG_RETURN(Expression_cache::HIT);
   }
@@ -249,15 +259,37 @@ my_bool Expression_cache_tmptable::put_value(Item *value)
   if (table_thd->is_error())
     goto err;;
 
-  if ((error= cache_table->file->ha_write_row(cache_table->record[0])))
+  if ((error= cache_table->file->ha_write_tmp_row(cache_table->record[0])))
   {
     /* create_myisam_from_heap will generate error if needed */
-    if (cache_table->file->is_fatal_error(error, HA_CHECK_DUP) &&
-        create_internal_tmp_table_from_heap(table_thd, cache_table,
-                                            cache_table_param.start_recinfo,
-                                            &cache_table_param.recinfo,
-                                            error, 1))
+    if (cache_table->file->is_fatal_error(error, HA_CHECK_DUP))
       goto err;
+    else
+    {
+      double hit_rate= ((double)hit / ((double)hit + miss));
+      DBUG_ASSERT(miss > 0);
+      if (hit_rate < EXPCACHE_MIN_HIT_RATE_FOR_MEM_TABLE)
+      {
+        DBUG_PRINT("info", ("hit rate is not so good to keep the cache"));
+        disable_cache();
+        DBUG_RETURN(FALSE);
+      }
+      else if (hit_rate < EXPCACHE_MIN_HIT_RATE_FOR_DISK_TABLE)
+      {
+        DBUG_PRINT("info", ("hit rate is not so good to go to disk"));
+        if (cache_table->file->ha_delete_all_rows() ||
+            cache_table->file->ha_write_tmp_row(cache_table->record[0]))
+          goto err;
+      }
+      else
+      {
+        if (create_internal_tmp_table_from_heap(table_thd, cache_table,
+                                                cache_table_param.start_recinfo,
+                                                &cache_table_param.recinfo,
+                                                error, 1))
+          goto err;
+      }
+    }
   }
   cache_table->status= 0; /* cache_table->record contains an existed record */
   ref.has_record= TRUE; /* the same as above */
@@ -266,24 +298,24 @@ my_bool Expression_cache_tmptable::put_value(Item *value)
   DBUG_RETURN(FALSE);
 
 err:
-  free_tmp_table(table_thd, cache_table);
-  cache_table= NULL;
+  disable_cache();
   DBUG_RETURN(TRUE);
 }
 
 
 void Expression_cache_tmptable::print(String *str, enum_query_type query_type)
 {
-  List_iterator<Item*> li(*list);
-  Item **item;
+  List_iterator<Item> li(items);
+  Item *item;
   bool is_first= TRUE;
 
   str->append('<');
+  li++;  // skip result field
   while ((item= li++))
   {
     if (!is_first)
       str->append(',');
-    (*item)->print(str, query_type);
+    item->print(str, query_type);
     is_first= FALSE;
   }
   str->append('>');
diff --git a/sql/sql_expression_cache.h b/sql/sql_expression_cache.h
index 88f71e0cf32..32aecc61dc9 100644
--- a/sql/sql_expression_cache.h
+++ b/sql/sql_expression_cache.h
@@ -37,6 +37,15 @@ public:
     Print cache parameters
   */
   virtual void print(String *str, enum_query_type query_type)= 0;
+
+  /**
+    Is this cache initialized
+  */
+  virtual bool is_inited()= 0;
+  /**
+    Initialize this cache
+  */
+  virtual void init()= 0;
 };
 
 struct st_table_ref;
@@ -51,15 +60,17 @@ class Item_field;
 class Expression_cache_tmptable :public Expression_cache
 {
 public:
-  Expression_cache_tmptable(THD *thd, List<Item*> &dependants, Item *value);
+  Expression_cache_tmptable(THD *thd, List<Item> &dependants, Item *value);
   virtual ~Expression_cache_tmptable();
   virtual result check_value(Item **value);
   virtual my_bool put_value(Item *value);
 
   void print(String *str, enum_query_type query_type);
+  bool is_inited() { return inited; };
+  void init();
 
 private:
-  void init();
+  void disable_cache();
 
   /* tmp table parameters */
   TMP_TABLE_PARAM cache_table_param;
@@ -71,12 +82,12 @@ private:
   struct st_table_ref ref;
   /* Cached result */
   Item_field *cached_result;
-  /* List of references to the parameters of the expression */
-  List<Item*> *list;
-  /* List of items */
-  List<Item> items;
+  /* List of parameter items */
+  List<Item> &items;
   /* Value Item example */
   Item *val;
+  /* hit/miss counters */
+  uint hit, miss;
   /* Set on if the object has been succesfully initialized with init() */
   bool inited;
 };
diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc
index 72514d72dc1..cf199c27518 100644
--- a/sql/sql_insert.cc
+++ b/sql/sql_insert.cc
@@ -135,7 +135,7 @@ bool check_view_single_update(List<Item> &fields, List<Item> *values,
     A buffer for the insert values was allocated for the merged view.
     Use it.
   */
-  //tbl->table->insert_values= view->table->insert_values;
+  tbl->table->insert_values= view->table->insert_values;
   view->table= tbl->table;
   *map= tables;
 
@@ -943,7 +943,7 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list,
 	  thd->clear_error();
 	}
         else
-          errcode= query_error_code(thd, thd->killed == THD::NOT_KILLED);
+          errcode= query_error_code(thd, thd->killed == NOT_KILLED);
         
 	/* bug#22725:
 
@@ -957,7 +957,7 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list,
 	routines did not result in any error due to the KILLED.  In
 	such case the flag is ignored for constructing binlog event.
 	*/
-	DBUG_ASSERT(thd->killed != THD::KILL_BAD_DATA || error > 0);
+	DBUG_ASSERT(thd->killed != KILL_BAD_DATA || error > 0);
 	if (thd->binlog_query(THD::ROW_QUERY_TYPE,
                               thd->query(), thd->query_length(),
 			      transactional_table, FALSE,
@@ -2010,7 +2010,7 @@ bool delayed_get_table(THD *thd, TABLE_LIST *table_list)
       /*
         Annotating delayed inserts is not supported.
       */
-      di->thd.variables.binlog_annotate_rows_events= 0;
+      di->thd.variables.binlog_annotate_row_events= 0;
 
       di->thd.set_db(table_list->db, (uint) strlen(table_list->db));
       di->thd.set_query(my_strdup(table_list->table_name, MYF(MY_WME)), 0);
@@ -2362,7 +2362,7 @@ void kill_delayed_threads(void)
   Delayed_insert *di;
   while ((di= it++))
   {
-    di->thd.killed= THD::KILL_CONNECTION;
+    di->thd.killed= KILL_SYSTEM_THREAD;
     pthread_mutex_lock(&di->thd.LOCK_thd_data);
     if (di->thd.mysys_var)
     {
@@ -2447,7 +2447,7 @@ static void handle_delayed_insert_impl(THD *thd, Delayed_insert *di)
 
   for (;;)
   {
-    if (thd->killed == THD::KILL_CONNECTION)
+    if (thd->killed >= KILL_CONNECTION)
     {
       uint lock_count;
       /*
@@ -2495,7 +2495,7 @@ static void handle_delayed_insert_impl(THD *thd, Delayed_insert *di)
 	  break;
 	if (error == ETIMEDOUT || error == ETIME)
 	{
-	  thd->killed= THD::KILL_CONNECTION;
+	  thd->killed= KILL_SYSTEM_THREAD;
 	  break;
 	}
       }
@@ -2528,7 +2528,7 @@ static void handle_delayed_insert_impl(THD *thd, Delayed_insert *di)
       {
 	/* Fatal error */
 	di->dead= 1;
-	thd->killed= THD::KILL_CONNECTION;
+	thd->killed= KILL_SYSTEM_THREAD;
       }
       pthread_cond_broadcast(&di->cond_client);
     }
@@ -2538,7 +2538,7 @@ static void handle_delayed_insert_impl(THD *thd, Delayed_insert *di)
       {
 	/* Some fatal error */
 	di->dead= 1;
-	thd->killed= THD::KILL_CONNECTION;
+	thd->killed= KILL_SYSTEM_THREAD;
       }
     }
     di->status=0;
@@ -2598,7 +2598,7 @@ pthread_handler_t handle_delayed_insert(void *arg)
   thd->thread_id= thd->variables.pseudo_thread_id= thread_id++;
   thd->set_current_time();
   threads.append(thd);
-  thd->killed=abort_loop ? THD::KILL_CONNECTION : THD::NOT_KILLED;
+  thd->killed=abort_loop ? KILL_SYSTEM_THREAD : NOT_KILLED;
   pthread_mutex_unlock(&LOCK_thread_count);
 
   /*
@@ -2632,7 +2632,7 @@ end:
 
   di->table=0;
   di->dead= 1;                                  // If error
-  thd->killed= THD::KILL_CONNECTION;	        // If error
+  thd->killed= KILL_SYSTEM_THREAD;	        // If error
   pthread_mutex_unlock(&di->mutex);
 
   close_thread_tables(thd);			// Free the table
@@ -2711,7 +2711,7 @@ bool Delayed_insert::handle_inserts(void)
   max_rows= delayed_insert_limit;
   if (thd.killed || table->needs_reopen_or_name_lock())
   {
-    thd.killed= THD::KILL_CONNECTION;
+    thd.killed= KILL_SYSTEM_THREAD;
     max_rows= ULONG_MAX;                     // Do as much as possible
   }
 
@@ -2824,7 +2824,7 @@ bool Delayed_insert::handle_inserts(void)
       /* if the delayed insert was killed, the killed status is
          ignored while binlogging */
       int errcode= 0;
-      if (thd.killed == THD::NOT_KILLED)
+      if (thd.killed == NOT_KILLED)
         errcode= query_error_code(&thd, TRUE);
       
       /*
@@ -3015,6 +3015,7 @@ bool mysql_insert_select_prepare(THD *thd)
       select_lex->leaf_tables_exec.push_back(table);
       table->tablenr_exec= table->table->tablenr;
       table->map_exec= table->table->map;
+      table->maybe_null_exec= table->table->maybe_null;
     }
     if (arena)
       thd->restore_active_arena(arena, &backup);
@@ -3352,7 +3353,7 @@ bool select_insert::send_eof()
   bool const trans_table= table->file->has_transactions();
   ulonglong id;
   bool changed;
-  THD::killed_state killed_status= thd->killed;
+  killed_state killed_status= thd->killed;
   DBUG_ENTER("select_insert::send_eof");
   DBUG_PRINT("enter", ("trans_table=%d, table_type='%s'",
                        trans_table, table->file->table_type()));
@@ -3387,7 +3388,7 @@ bool select_insert::send_eof()
     if (!error)
       thd->clear_error();
     else
-      errcode= query_error_code(thd, killed_status == THD::NOT_KILLED);
+      errcode= query_error_code(thd, killed_status == NOT_KILLED);
 
     if (write_to_binlog(trans_table, errcode))
     {
@@ -3461,7 +3462,7 @@ void select_insert::abort() {
     {
         if (mysql_bin_log.is_open())
         {
-          int errcode= query_error_code(thd, thd->killed == THD::NOT_KILLED);
+          int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
           /* error of writing binary log is ignored */
           write_to_binlog(transactional_table, errcode);
         }
@@ -3822,7 +3823,7 @@ select_create::prepare(List<Item> &values, SELECT_LEX_UNIT *u)
           !table->s->tmp_table &&
           !ptr->get_create_info()->table_existed)
       {
-        int errcode= query_error_code(thd, thd->killed == THD::NOT_KILLED);
+        int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
         if (int error= ptr->binlog_show_create_table(tables, count, errcode))
           return error;
       }
@@ -3865,7 +3866,7 @@ select_create::prepare(List<Item> &values, SELECT_LEX_UNIT *u)
                           create_table->table_name);
       if (thd->current_stmt_binlog_row_based)
       {
-        int errcode= query_error_code(thd, thd->killed == THD::NOT_KILLED);
+        int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
         binlog_show_create_table(&(create_table->table), 1, errcode);
       }
       table= create_table->table;
diff --git a/sql/sql_join_cache.cc b/sql/sql_join_cache.cc
index 64b82efd8ff..f53bf738b86 100644
--- a/sql/sql_join_cache.cc
+++ b/sql/sql_join_cache.cc
@@ -225,8 +225,6 @@ void JOIN_CACHE::calc_record_fields()
     flag_fields+= test(tab->table->maybe_null);
     fields+= tab->used_fields;
     blobs+= tab->used_blobs;
-
-    fields+= tab->check_rowid_field();
   }
   if ((with_match_flag= join_tab->use_match_flag()))
     flag_fields++;
@@ -609,11 +607,23 @@ void JOIN_CACHE::create_remaining_fields()
     if (tab->keep_current_rowid)
     {
       copy->str= table->file->ref;
-      copy->length= table->file->ref_length;
-      copy->type= 0;
+      if (copy->str)
+        copy->length= table->file->ref_length;
+      else
+      {
+        /* This may happen only for materialized derived tables and views */
+        copy->length= 0;
+        copy->str= (uchar *) table;
+      } 
+      copy->type= CACHE_ROWID;
       copy->field= 0;
       copy->referenced_field_no= 0;
-      length+= copy->length;
+      /* 
+        Note: this may seem odd, but at this point we have
+        table->file->ref==NULL while table->file->ref_length is already set 
+        to correct value.
+      */
+      length += table->file->ref_length;
       data_field_count++;
       copy++;
     }
@@ -1290,6 +1300,7 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
   if (with_length)
   {
     rec_len_ptr= cp;   
+    DBUG_ASSERT(cp + size_of_rec_len <= buff + buff_size);
     cp+= size_of_rec_len;
   }
 
@@ -1299,6 +1310,7 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
   */
   if (prev_cache)
   {
+    DBUG_ASSERT(cp + prev_cache->get_size_of_rec_offset() <= buff + buff_size);
     cp+= prev_cache->get_size_of_rec_offset();
     prev_cache->store_rec_ref(cp, link);
   } 
@@ -1315,6 +1327,7 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
   flags_pos= cp;
   for ( ; copy < copy_end; copy++)
   {
+    DBUG_ASSERT(cp + copy->length <= buff + buff_size);
     memcpy(cp, copy->str, copy->length);
     cp+= copy->length;
   } 
@@ -1325,8 +1338,7 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
   {
     Field *field= copy->field;
     if (field && field->maybe_null() && field->is_null())
-    {
-      /* Do not copy a field if its value is null */
+    {    
       if (copy->referenced_field_no)
         copy->offset= 0;
       continue;              
@@ -1342,6 +1354,7 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
       {
         last_rec_blob_data_is_in_rec_buff= 1;
         /* Put down the length of the blob and the pointer to the data */  
+        DBUG_ASSERT(cp + copy->length + sizeof(char*) <= buff + buff_size);
 	blob_field->get_image(cp, copy->length+sizeof(char*),
                               blob_field->charset());
 	cp+= copy->length+sizeof(char*);
@@ -1351,6 +1364,7 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
         /* First put down the length of the blob and then copy the data */ 
 	blob_field->get_image(cp, copy->length, 
 			      blob_field->charset());
+        DBUG_ASSERT(cp + copy->length + copy->blob_length <= buff + buff_size);
 	memcpy(cp+copy->length, copy->str, copy->blob_length);               
 	cp+= copy->length+copy->blob_length;
       }
@@ -1361,12 +1375,14 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
       case CACHE_VARSTR1:
         /* Copy the significant part of the short varstring field */ 
         len= (uint) copy->str[0] + 1;
+        DBUG_ASSERT(cp + len <= buff + buff_size);
         memcpy(cp, copy->str, len);
         cp+= len;
         break;
       case CACHE_VARSTR2:
         /* Copy the significant part of the long varstring field */
         len= uint2korr(copy->str) + 2;
+        DBUG_ASSERT(cp + len <= buff + buff_size);
         memcpy(cp, copy->str, len);
         cp+= len;
         break;
@@ -1381,14 +1397,38 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
 	     end > str && end[-1] == ' ';
 	     end--) ;
 	len=(uint) (end-str);
+        DBUG_ASSERT(cp + len + 2 <= buff + buff_size);
         int2store(cp, len);
 	memcpy(cp+2, str, len);
 	cp+= len+2;
         break;
       }
+      case CACHE_ROWID:
+        if (!copy->length)
+	{
+          /*
+            This may happen only for ROWID fields of materialized
+            derived tables and views.
+	  */
+	  TABLE *table= (TABLE *) copy->str;
+          copy->str= table->file->ref;
+          copy->length= table->file->ref_length;
+          if (!copy->str)
+	  {
+            /* 
+              If table is an empty inner table of an outer join and it is
+              a materialized derived table then table->file->ref == NULL.
+	    */
+	    cp+= copy->length;
+            break;
+          }
+        }
+        /* fall through */
       default:      
         /* Copy the entire image of the field from the record buffer */
-	memcpy(cp, copy->str, copy->length);
+        DBUG_ASSERT(cp + copy->length <= buff + buff_size);
+        if (copy->str)
+	  memcpy(cp, copy->str, copy->length);
 	cp+= copy->length;
       }
     }
@@ -1407,6 +1447,7 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
         cnt++;
       }
     }
+    DBUG_ASSERT(cp + size_of_fld_ofs*cnt <= buff + buff_size);
     cp+= size_of_fld_ofs*cnt;
   }
 
@@ -1455,7 +1496,6 @@ uint JOIN_CACHE::write_record_data(uchar * link, bool *is_full)
   RETURN VALUE
     none
 */
-
 void JOIN_CACHE::reset(bool for_writing)
 {
   pos= buff;
@@ -1781,6 +1821,13 @@ uint JOIN_CACHE::read_record_field(CACHE_FIELD *copy, bool blob_in_rec_buff)
       memset(copy->str+len, ' ', copy->length-len);
       len+= 2;
       break;
+    case CACHE_ROWID:
+      if (!copy->str)
+      {
+        len= copy->length;
+        break;
+      }
+      /* fall through */ 
     default:
       /* Copy the entire image of the field from the record buffer */
       len= copy->length;
@@ -2004,6 +2051,7 @@ enum_nested_loop_state JOIN_CACHE::join_records(bool skip_last)
   JOIN_TAB *tab;
   enum_nested_loop_state rc= NESTED_LOOP_OK;
   bool outer_join_first_inner= join_tab->is_first_inner_for_outer_join();
+  DBUG_ENTER("JOIN_CACHE::join_records");
 
   if (outer_join_first_inner && !join_tab->first_unmatched)
     join_tab->not_null_compl= TRUE;   
@@ -2085,7 +2133,8 @@ enum_nested_loop_state JOIN_CACHE::join_records(bool skip_last)
 finish:
   restore_last_record();
   reset(TRUE);
-  return rc;
+  DBUG_PRINT("exit", ("rc: %d", rc));
+  DBUG_RETURN(rc);
 }
 
 
@@ -2147,10 +2196,11 @@ enum_nested_loop_state JOIN_CACHE::join_matching_records(bool skip_last)
   join_tab->table->null_row= 0;
   bool check_only_first_match= join_tab->check_only_first_match();
   bool outer_join_first_inner= join_tab->is_first_inner_for_outer_join();
+  DBUG_ENTER("JOIN_CACHE::join_matching_records");
 
   /* Return at once if there are no records in the join buffer */
   if (!records)     
-    return NESTED_LOOP_OK;   
+    DBUG_RETURN(NESTED_LOOP_OK);
  
   /* 
     When joining we read records from the join buffer back into record buffers.
@@ -2224,7 +2274,7 @@ finish:
     rc= error < 0 ? NESTED_LOOP_NO_MORE_ROWS: NESTED_LOOP_ERROR;
 finish2:    
   join_tab_scan->close();
-  return rc;
+  DBUG_RETURN(rc);
 }
 
 
@@ -2306,6 +2356,7 @@ bool JOIN_CACHE::set_match_flag_if_none(JOIN_TAB *first_inner,
 enum_nested_loop_state JOIN_CACHE::generate_full_extensions(uchar *rec_ptr)
 {
   enum_nested_loop_state rc= NESTED_LOOP_OK;
+  DBUG_ENTER("JOIN_CACHE::generate_full_extensions");
   
   /*
     Check whether the extended partial join record meets
@@ -2323,16 +2374,18 @@ enum_nested_loop_state JOIN_CACHE::generate_full_extensions(uchar *rec_ptr)
       if (rc != NESTED_LOOP_OK && rc != NESTED_LOOP_NO_MORE_ROWS)
       {
         reset(TRUE);
-        return rc;
+        DBUG_RETURN(rc);
       }
     }
     if (res == -1)
     {
       rc= NESTED_LOOP_ERROR;
-      return rc;
+      DBUG_RETURN(rc);
     }
   }
-  return rc;
+  else if (join->thd->is_error())
+    rc= NESTED_LOOP_ERROR;
+  DBUG_RETURN(rc);
 }
 
 
@@ -2357,16 +2410,20 @@ enum_nested_loop_state JOIN_CACHE::generate_full_extensions(uchar *rec_ptr)
   RETURN VALUE
     TRUE   there is a match
     FALSE  there is no match
+           In this case the caller must also check thd->is_error() to see
+           if there was a fatal error for the query.
 */ 
 
 inline bool JOIN_CACHE::check_match(uchar *rec_ptr)
 {
   /* Check whether pushdown conditions are satisfied */
+  DBUG_ENTER("JOIN_CACHE:check_match");
+
   if (join_tab->select && join_tab->select->skip_record(join->thd) <= 0)
-    return FALSE;
+    DBUG_RETURN(FALSE);
 
   if (!join_tab->is_last_inner_table())
-    return TRUE;
+    DBUG_RETURN(TRUE);
 
   /* 
      This is the last inner table of an outer join,
@@ -2379,7 +2436,7 @@ inline bool JOIN_CACHE::check_match(uchar *rec_ptr)
     set_match_flag_if_none(first_inner, rec_ptr);
     if (first_inner->check_only_first_match() &&
         !join_tab->first_inner)
-      return TRUE;
+      DBUG_RETURN(TRUE);
     /* 
       This is the first match for the outer table row.
       The function set_match_flag_if_none has turned the flag
@@ -2393,13 +2450,12 @@ inline bool JOIN_CACHE::check_match(uchar *rec_ptr)
     for (JOIN_TAB *tab= first_inner; tab <= join_tab; tab++)
     {
       if (tab->select && tab->select->skip_record(join->thd) <= 0)
-        return FALSE;
+        DBUG_RETURN(FALSE);
     }
   }
   while ((first_inner= first_inner->first_upper) &&
          first_inner->last_inner == join_tab);
-  
-  return TRUE;
+    DBUG_RETURN(TRUE);
 } 
 
 
@@ -2434,10 +2490,11 @@ enum_nested_loop_state JOIN_CACHE::join_null_complements(bool skip_last)
   ulonglong cnt; 
   enum_nested_loop_state rc= NESTED_LOOP_OK;
   bool is_first_inner= join_tab == join_tab->first_unmatched;
+  DBUG_ENTER("JOIN_CACHE::join_null_complements");
  
   /* Return at once if there are no records in the join buffer */
   if (!records)
-    return NESTED_LOOP_OK;
+    DBUG_RETURN(NESTED_LOOP_OK);
   
   cnt= records - (is_key_access() ? 0 : test(skip_last));
 
@@ -2467,7 +2524,7 @@ enum_nested_loop_state JOIN_CACHE::join_null_complements(bool skip_last)
   }
 
 finish:
-  return rc;
+  DBUG_RETURN(rc);
 }
 
 
diff --git a/sql/sql_join_cache.h b/sql/sql_join_cache.h
index f87cdcec649..f5d64d5530a 100644
--- a/sql/sql_join_cache.h
+++ b/sql/sql_join_cache.h
@@ -16,6 +16,7 @@
 #define CACHE_STRIPPED  2        /* field stripped of trailing spaces */
 #define CACHE_VARSTR1   3        /* short string value (length takes 1 byte) */ 
 #define CACHE_VARSTR2   4        /* long string value (length takes 2 bytes) */
+#define CACHE_ROWID     5        /* ROWID field */
 
 /*
   The CACHE_FIELD structure used to describe fields of records that
diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc
index d3613d0d660..117a866555c 100644
--- a/sql/sql_lex.cc
+++ b/sql/sql_lex.cc
@@ -21,8 +21,8 @@
 #include "item_create.h"
 #include <m_ctype.h>
 #include <hash.h>
-#include "sp.h"
 #include "sp_head.h"
+#include "sp.h"
 #include "sql_select.h"
 
 /*
@@ -339,6 +339,7 @@ void lex_start(THD *thd)
   lex->event_parse_data= NULL;
   lex->profile_options= PROFILE_NONE;
   lex->nest_level=0 ;
+  lex->select_lex.nest_level_base= &lex->unit;
   lex->allow_sum_func= 0;
   lex->in_sum_func= NULL;
   lex->protect_against_global_read_lock= FALSE;
@@ -1272,39 +1273,39 @@ int MYSQLlex(void *arg, void *yythd)
 
       lip->save_in_comment_state();
 
+      if (lip->yyPeekn(2) == 'M' && lip->yyPeekn(3) == '!')
+      {
+        /* Skip MariaDB unique marker */
+        lip->set_echo(FALSE);
+        lip->yySkip();
+        /* The following if will be true */
+      }
       if (lip->yyPeekn(2) == '!')
       {
         lip->in_comment= DISCARD_COMMENT;
         /* Accept '/' '*' '!', but do not keep this marker. */
         lip->set_echo(FALSE);
-        lip->yySkip();
-        lip->yySkip();
-        lip->yySkip();
+        lip->yySkipn(3);
 
         /*
           The special comment format is very strict:
-          '/' '*' '!', followed by exactly
+          '/' '*' '!', followed by an optional 'M' and exactly
           1 digit (major), 2 digits (minor), then 2 digits (dot).
           32302 -> 3.23.02
           50032 -> 5.0.32
           50114 -> 5.1.14
         */
-        char version_str[6];
-        version_str[0]= lip->yyPeekn(0);
-        version_str[1]= lip->yyPeekn(1);
-        version_str[2]= lip->yyPeekn(2);
-        version_str[3]= lip->yyPeekn(3);
-        version_str[4]= lip->yyPeekn(4);
-        version_str[5]= 0;
-        if (  my_isdigit(cs, version_str[0])
-           && my_isdigit(cs, version_str[1])
-           && my_isdigit(cs, version_str[2])
-           && my_isdigit(cs, version_str[3])
-           && my_isdigit(cs, version_str[4])
+        if (  my_isdigit(cs, lip->yyPeekn(0))
+           && my_isdigit(cs, lip->yyPeekn(1))
+           && my_isdigit(cs, lip->yyPeekn(2))
+           && my_isdigit(cs, lip->yyPeekn(3))
+           && my_isdigit(cs, lip->yyPeekn(4))
            )
         {
           ulong version;
-          version=strtol(version_str, NULL, 10);
+          char *end_ptr= (char*) lip->get_ptr()+5;
+          int error;
+          version= (ulong) my_strtoll10(lip->get_ptr(), &end_ptr, &error);
 
           if (version <= MYSQL_VERSION_ID)
           {
@@ -1604,6 +1605,7 @@ void st_select_lex::init_query()
   top_join_list.empty();
   join_list= &top_join_list;
   embedding= 0;
+  leaf_tables_prep.empty();
   leaf_tables.empty();
   item_list.empty();
   join= 0;
@@ -1638,6 +1640,7 @@ void st_select_lex::init_query()
   nest_level= 0;
   link_next= 0;
   lock_option= TL_READ_DEFAULT;
+  is_prep_leaf_list_saved= FALSE;
 
   bzero((char*) expr_cache_may_be_used, sizeof(expr_cache_may_be_used));
 }
@@ -1646,6 +1649,7 @@ void st_select_lex::init_select()
 {
   st_select_lex_node::init_select();
   sj_nests.empty();
+  sj_subselects.empty();
   group_list.empty();
   type= db= 0;
   having= 0;
@@ -1876,55 +1880,6 @@ void st_select_lex_unit::exclude_tree()
 }
 
 
-/**
-  Register reference to an item which the subqueries depends on
-
-  @param def_sel         select against which the item is resolved
-  @param dependency      reference to the item
-
-  @details
-  This function puts the reference dependency to an item that is either an
-  outer field or an aggregate function resolved against an outer select into
-  the list 'depends_on'. It adds it to the 'depends_on' lists for each
-  subquery between this one and 'def_sel' - the subquery against which the
-  item is resolved.
-*/
-
-void st_select_lex::register_dependency_item(st_select_lex *def_sel,
-                                             Item **dependency)
-{
-  SELECT_LEX *s= this;
-  DBUG_ENTER("st_select_lex::register_dependency_item");
-  DBUG_ASSERT(this != def_sel);
-  DBUG_ASSERT(*dependency);
-  do
-  {
-    /* check duplicates */
-    List_iterator_fast<Item*> li(s->master_unit()->item->depends_on);
-    Item **dep;
-    while ((dep= li++))
-    {
-      if ((*dep)->eq(*dependency, FALSE))
-      {
-         DBUG_PRINT("info", ("dependency %s already present",
-                             ((*dependency)->name ?
-                              (*dependency)->name :
-                              "<no name>")));
-         DBUG_VOID_RETURN;
-      }
-    }
-
-    s->master_unit()->item->depends_on.push_back(dependency);
-    DBUG_PRINT("info", ("depends_on: Select: %d  added: %s",
-                        s->select_number,
-                        ((*dependency)->name ?
-                         (*dependency)->name :
-                         "<no name>")));
-  } while ((s= s->outer_select()) != def_sel);
-  DBUG_VOID_RETURN;
-}
-
-
 /*
   st_select_lex_node::mark_as_dependent mark all st_select_lex struct from 
   this to 'last' as dependent
@@ -3229,12 +3184,6 @@ bool st_select_lex::get_free_table_map(table_map *map, uint *tablenr)
   *map= 0;
   *tablenr= 0;
   TABLE_LIST *tl;
-  if (!join)
-  {
-    (*map)= 1<<1;
-    (*tablenr)++;
-    return FALSE;
-  }
   List_iterator<TABLE_LIST> ti(leaf_tables);
   while ((tl= ti++))
   {
@@ -3272,18 +3221,19 @@ void st_select_lex::append_table_to_list(TABLE_LIST *TABLE_LIST::*link,
   tl->*link= table;
 }
 
+
 /*
   @brief
-  Remove given table from the leaf_tables list.
+  Replace given table from the leaf_tables list for a list of tables 
 
-  @param link  Offset to which list in table structure to use
-  @param table Table to remove
+  @param table Table to replace
+  @param list  List to substititute the table for
 
   @details
-  Remove 'table' from the leaf_tables list using the 'link' offset.
+  Replace 'table' from the leaf_tables list for a list of tables 'tbl_list'.
 */
 
-void st_select_lex::remove_table_from_list(TABLE_LIST *table)
+void st_select_lex::replace_leaf_table(TABLE_LIST *table, List<TABLE_LIST> &tbl_list)
 {
   TABLE_LIST *tl;
   List_iterator<TABLE_LIST> ti(leaf_tables);
@@ -3291,7 +3241,7 @@ void st_select_lex::remove_table_from_list(TABLE_LIST *table)
   {
     if (tl == table)
     {
-      ti.remove();
+      ti.replace(tbl_list);
       break;
     }
   }
@@ -3391,39 +3341,34 @@ void st_select_lex::remap_tables(TABLE_LIST *derived, table_map map,
   @return FALSE ok
 */
 
-bool SELECT_LEX::merge_subquery(TABLE_LIST *derived, SELECT_LEX *subq_select,
+bool SELECT_LEX::merge_subquery(THD *thd, TABLE_LIST *derived,
+                                SELECT_LEX *subq_select,
                                 uint table_no, table_map map)
 {
   derived->wrap_into_nested_join(subq_select->top_join_list);
-  /* Reconnect the next_leaf chain. */
-  leaf_tables.concat(&subq_select->leaf_tables);
 
   ftfunc_list->concat(subq_select->ftfunc_list);
-  if (join)
-  {
-    Item_in_subselect **in_subq;
-    Item_in_subselect **in_subq_end;
-    for (in_subq= subq_select->join->sj_subselects.front(), 
-         in_subq_end= subq_select->join->sj_subselects.back(); 
-         in_subq != in_subq_end; 
-         in_subq++)
+  if (join ||
+      thd->lex->sql_command == SQLCOM_UPDATE_MULTI ||
+      thd->lex->sql_command == SQLCOM_DELETE_MULTI)
+  {
+    List_iterator_fast<Item_in_subselect> li(subq_select->sj_subselects);
+    Item_in_subselect *in_subq;
+    while ((in_subq= li++))
     {
-      join->sj_subselects.append(join->thd->mem_root, *in_subq);
-      (*in_subq)->emb_on_expr_nest= derived;
+      sj_subselects.push_back(in_subq);
+      if (in_subq->emb_on_expr_nest == NO_JOIN_NEST)
+         in_subq->emb_on_expr_nest= derived;
     }
   }
-  /*
-    Remove merged table from chain.
-    When merge_subquery is called at a subquery-to-semijoin transformation
-    the derived isn't in the leaf_tables list, so in this case the call of
-    remove_table_from_list does not cause any actions.
-  */
-  remove_table_from_list(derived);
 
   /* Walk through child's tables and adjust table map, tablenr,
    * parent_lex */
   subq_select->remap_tables(derived, map, table_no, this);
   subq_select->merged_into= this;
+
+  replace_leaf_table(derived, subq_select->leaf_tables);
+
   return FALSE;
 }
 
@@ -3464,14 +3409,49 @@ void SELECT_LEX::update_used_tables()
 {
   TABLE_LIST *tl;
   List_iterator<TABLE_LIST> ti(leaf_tables);
+
   while ((tl= ti++))
   {
+    if (tl->table && !tl->is_view_or_derived())
+    {
+      TABLE_LIST *embedding= tl->embedding;
+      for (embedding= tl->embedding; embedding; embedding=embedding->embedding)
+      {
+        if (embedding->is_view_or_derived())
+	{
+          DBUG_ASSERT(embedding->is_merged_derived());
+          TABLE *tab= tl->table;
+          tab->covering_keys= tab->s->keys_for_keyread;
+          tab->covering_keys.intersect(tab->keys_in_use_for_query);
+          tab->merge_keys.clear_all();
+          bitmap_clear_all(tab->read_set);
+          bitmap_clear_all(tab->vcol_set);
+          break;
+        }
+      }
+    }
+  }
+
+  ti.rewind();
+  while ((tl= ti++))
+  {
+    TABLE_LIST *embedding= tl;
+    do
+    {
+      bool maybe_null;
+      if ((maybe_null= test(embedding->outer_join)))
+      {
+	tl->table->maybe_null= maybe_null;
+        break;
+      }
+    }
+    while ((embedding= embedding->embedding));
     if (tl->on_expr)
     {
       tl->on_expr->update_used_tables();
       tl->on_expr->walk(&Item::eval_not_null_tables, 0, NULL);
     }
-    TABLE_LIST *embedding= tl->embedding;
+    embedding= tl->embedding;
     while (embedding)
     {
       if (embedding->on_expr && 
@@ -3484,11 +3464,37 @@ void SELECT_LEX::update_used_tables()
       embedding= tl->embedding;
     }
   }
+
   if (join->conds)
   {
     join->conds->update_used_tables();
     join->conds->walk(&Item::eval_not_null_tables, 0, NULL);
   }
+  if (join->having)
+  {
+    join->having->update_used_tables();
+  }
+
+  Item *item;
+  List_iterator_fast<Item> it(join->fields_list);
+  while ((item= it++))
+  {
+    item->update_used_tables();
+  }
+  Item_outer_ref *ref;
+  List_iterator_fast<Item_outer_ref> ref_it(inner_refs_list);
+  while ((ref= ref_it++))
+  {
+    item= ref->outer_ref;
+    item->update_used_tables();
+  }
+  for (ORDER *order= group_list.first; order; order= order->next)
+    (*order->item)->update_used_tables();
+  if (!master_unit()->is_union())
+  {
+    for (ORDER *order= order_list.first; order; order= order->next)
+      (*order->item)->update_used_tables();
+  }      
 }
 
 
@@ -3585,6 +3591,7 @@ void SELECT_LEX::mark_const_derived(bool empty)
   }
 }
 
+
 bool st_select_lex::save_leaf_tables(THD *thd)
 {
   Query_arena *arena= thd->stmt_arena, backup;
@@ -3601,7 +3608,38 @@ bool st_select_lex::save_leaf_tables(THD *thd)
       return 1;
     table->tablenr_exec= table->table->tablenr;
     table->map_exec= table->table->map;
+    if (join && (join->select_options & SELECT_DESCRIBE))
+      table->maybe_null_exec= 0;
+    else
+      table->maybe_null_exec= table->table->maybe_null;
+  }
+  if (arena)
+    thd->restore_active_arena(arena, &backup);
+
+  return 0;
+}
+
+
+bool st_select_lex::save_prep_leaf_tables(THD *thd)
+{
+  if (!thd->save_prep_leaf_list)
+    return 0;
+
+  Query_arena *arena= thd->stmt_arena, backup;
+  if (arena->is_conventional())
+    arena= 0;                                  
+  else
+    thd->set_n_backup_active_arena(arena, &backup);
+
+  List_iterator_fast<TABLE_LIST> li(leaf_tables);
+  TABLE_LIST *table;
+  while ((table= li++))
+  {
+    if (leaf_tables_prep.push_back(table))
+      return 1;
   }
+  thd->lex->select_lex.is_prep_leaf_list_saved= TRUE; 
+  thd->save_prep_leaf_list= FALSE;
   if (arena)
     thd->restore_active_arena(arena, &backup);
 
@@ -3609,6 +3647,47 @@ bool st_select_lex::save_leaf_tables(THD *thd)
 }
 
 
+/*
+  Return true if this select_lex has been converted into a semi-join nest
+  within 'ancestor'.
+
+  We need a loop to check this because there could be several nested
+  subselects, like
+
+    SELECT ... FROM grand_parent 
+      WHERE expr1 IN (SELECT ... FROM parent 
+                        WHERE expr2 IN ( SELECT ... FROM child)
+
+  which were converted into:
+  
+    SELECT ... 
+    FROM grand_parent SEMI_JOIN (parent JOIN child) 
+    WHERE 
+      expr1 AND expr2
+
+  In this case, both parent and child selects were merged into the parent.
+*/
+
+bool st_select_lex::is_merged_child_of(st_select_lex *ancestor)
+{
+  bool all_merged= TRUE;
+  for (SELECT_LEX *sl= this; sl && sl!=ancestor;
+       sl=sl->outer_select())
+  {
+    Item *subs= sl->master_unit()->item;
+    if (subs && subs->type() == Item::SUBSELECT_ITEM && 
+        ((Item_subselect*)subs)->substype() == Item_subselect::IN_SUBS &&
+        ((Item_in_subselect*)subs)->test_strategy(SUBS_SEMI_JOIN))
+    {
+      continue;
+    }
+    all_merged= FALSE;
+    break;
+  }
+  return all_merged;
+}
+
+
 /**
   A routine used by the parser to decide whether we are specifying a full
   partitioning or if only partitions to add or to split.
diff --git a/sql/sql_lex.h b/sql/sql_lex.h
index 5a8b5a5f361..4caf2a1dbb3 100644
--- a/sql/sql_lex.h
+++ b/sql/sql_lex.h
@@ -637,8 +637,16 @@ public:
     tables. Unlike 'next_local', this in this list views are *not*
     leaves. Created in setup_tables() -> make_leaves_list().
   */
+  /* 
+    Subqueries that will need to be converted to semi-join nests, including
+    those converted to jtbm nests. The list is emptied when conversion is done.
+  */
+  List<Item_in_subselect> sj_subselects;
+
   List<TABLE_LIST> leaf_tables;
   List<TABLE_LIST> leaf_tables_exec;
+  List<TABLE_LIST> leaf_tables_prep;
+  bool is_prep_leaf_list_saved;
   uint insert_tables;
   st_select_lex *merged_into; /* select which this select is merged into */
                               /* (not 0 only for views/derived tables)   */
@@ -676,6 +684,13 @@ public:
   ulong table_join_options;
   uint in_sum_expr;
   uint select_number; /* number of select (used for EXPLAIN) */
+
+  /*
+    nest_levels are local to the query or VIEW,
+    and that view merge procedure does not re-calculate them.
+    So we also have to remember unit against which we count levels.
+  */
+  SELECT_LEX_UNIT *nest_level_base;
   int nest_level;     /* nesting level of select */
   Item_sum *inner_sum_func_list; /* list of sum func in nested selects */ 
   uint with_wild; /* item list contain '*' */
@@ -785,7 +800,6 @@ public:
   inline bool is_subquery_function() { return master_unit()->item != 0; }
 
   bool mark_as_dependent(THD *thd, st_select_lex *last, Item *dependency);
-  void register_dependency_item(st_select_lex *last, Item **dependency);
 
   bool set_braces(bool value);
   bool inc_in_sum_expr();
@@ -881,10 +895,10 @@ public:
   bool handle_derived(struct st_lex *lex, uint phases);
   void append_table_to_list(TABLE_LIST *TABLE_LIST::*link, TABLE_LIST *table);
   bool get_free_table_map(table_map *map, uint *tablenr);
-  void remove_table_from_list(TABLE_LIST *table);
+  void replace_leaf_table(TABLE_LIST *table, List<TABLE_LIST> &tbl_list);
   void remap_tables(TABLE_LIST *derived, table_map map,
                     uint tablenr, st_select_lex *parent_lex);
-  bool merge_subquery(TABLE_LIST *derived, st_select_lex *subq_lex,
+  bool merge_subquery(THD *thd, TABLE_LIST *derived, st_select_lex *subq_lex,
                       uint tablenr, table_map map);
   inline bool is_mergeable()
   {
@@ -899,7 +913,8 @@ public:
   void mark_const_derived(bool empty);
 
   bool save_leaf_tables(THD *thd);
-
+  bool save_prep_leaf_tables(THD *thd);
+  bool is_merged_child_of(st_select_lex *ancestor);
 private:  
   /* current index hint kind. used in filling up index_hints */
   enum index_hint_type current_index_hint_type;
@@ -1727,6 +1742,9 @@ typedef struct st_lex : public Query_tables_list
   LEX_SERVER_OPTIONS server_options;
   USER_RESOURCES mqh;
   ulong type;
+  /* The following is used by KILL */
+  killed_state kill_signal;
+  killed_type  kill_type;
   /*
     This variable is used in post-parse stage to declare that sum-functions,
     or functions which have sense only if GROUP BY is present, are allowed.
diff --git a/sql/sql_list.h b/sql/sql_list.h
index 38235a10c18..4655b4e3577 100644
--- a/sql/sql_list.h
+++ b/sql/sql_list.h
@@ -152,6 +152,7 @@ struct list_node :public Sql_alloc
   }
 };
 
+typedef bool List_eq(void *a, void *b);
 
 extern MYSQL_PLUGIN_IMPORT list_node end_of_list;
 
@@ -295,10 +296,40 @@ public:
   inline void **head_ref() { return first != &end_of_list ? &first->info : 0; }
   inline bool is_empty() { return first == &end_of_list ; }
   inline list_node *last_ref() { return &end_of_list; }
+  inline bool add_unique(void *info, List_eq *eq)
+  {
+    list_node *node= first;
+    for (;
+         node != &end_of_list && (!(*eq)(node->info, info));
+         node= node->next) ;
+    if (node == &end_of_list)
+      return push_back(info);
+    return 1;
+  }
   friend class base_list_iterator;
   friend class error_list;
   friend class error_list_iterator;
 
+  /*
+    Debugging help: return N-th element in the list, or NULL if the list has
+    less than N elements.
+  */
+  inline void *nth_element(int n)
+  {
+    list_node *node= first;
+    void *data= NULL;
+    for (int i=0; i <= n; i++)
+    {
+      if (node == &end_of_list)
+      {
+        data= NULL;
+        break;
+      }
+      data= node->info;
+      node= node->next;
+    }
+    return data;
+  }
 #ifdef LIST_EXTRA_DEBUG
   /*
     Check list invariants and print results into trace. Invariants are:
@@ -465,6 +496,8 @@ public:
   inline void concat(List<T> *list) { base_list::concat(list); }
   inline void disjoin(List<T> *list) { base_list::disjoin(list); }
   inline void prepand(List<T> *list) { base_list::prepand(list); }
+  inline bool add_unique(T *a, bool (*eq)(T *a, T *b))
+  { return base_list::add_unique(a, (List_eq *)eq); }
   void delete_elements(void)
   {
     list_node *element,*next;
@@ -475,6 +508,7 @@ public:
     }
     empty();
   }
+  inline T *nth_element(int n) { return (T*)base_list::nth_element(n); }
 };
 
 
diff --git a/sql/sql_load.cc b/sql/sql_load.cc
index 089c148778e..3df6305f620 100644
--- a/sql/sql_load.cc
+++ b/sql/sql_load.cc
@@ -63,6 +63,8 @@ public:
     ::end_io_cache(&cache);
     need_end_io_cache = 0;
   }
+  my_off_t file_length() { return cache.end_of_file; }
+  my_off_t position()    { return my_b_tell(&cache); }
 
   /*
     Either this method, or we need to make cache public
@@ -128,7 +130,7 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list,
   bool is_fifo=0;
 #ifndef EMBEDDED_LIBRARY
   LOAD_FILE_INFO lf_info;
-  THD::killed_state killed_status;
+  killed_state killed_status;
 #endif
   char *db = table_list->db;			// This is never null
   /*
@@ -416,9 +418,9 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list,
     }
   }
 
+  thd_proc_info(thd, "reading file");
   if (!(error=test(read_info.error)))
   {
-
     table->next_number_field=table->found_next_number_field;
     if (ignore ||
 	handle_duplicates == DUP_REPLACE)
@@ -436,6 +438,7 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list,
                              (MODE_STRICT_TRANS_TABLES |
                               MODE_STRICT_ALL_TABLES)));
 
+    thd_progress_init(thd, 2);
     if (!field_term->length() && !enclosed->length())
       error= read_fixed_length(thd, info, table_list, fields_vars,
                                set_fields, set_values, read_info,
@@ -444,6 +447,9 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list,
       error= read_sep_field(thd, info, table_list, fields_vars,
                             set_fields, set_values, read_info,
 			    *enclosed, skip_lines, ignore);
+    
+    thd_proc_info(thd, "End bulk insert");
+    thd_progress_next_stage(thd);
     if (!thd->prelocked_mode && table->file->ha_end_bulk_insert() && !error)
     {
       table->file->print_error(my_errno, MYF(0));
@@ -465,11 +471,11 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list,
   DBUG_EXECUTE_IF("simulate_kill_bug27571",
                   {
                     error=1;
-                    thd->killed= THD::KILL_QUERY;
+                    thd->killed= KILL_QUERY;
                   };);
 
 #ifndef EMBEDDED_LIBRARY
-  killed_status= (error == 0) ? THD::NOT_KILLED : thd->killed;
+  killed_status= (error == 0) ? NOT_KILLED : thd->killed;
 #endif
 
   /*
@@ -513,7 +519,7 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list,
 	/* If the file was not empty, wrote_create_file is true */
 	if (lf_info.wrote_create_file)
 	{
-          int errcode= query_error_code(thd, killed_status == THD::NOT_KILLED);
+          int errcode= query_error_code(thd, killed_status == NOT_KILLED);
           
           /* since there is already an error, the possible error of
              writing binary log will be ignored */
@@ -564,7 +570,7 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list,
       read_info.end_io_cache();
       if (lf_info.wrote_create_file)
       {
-        int errcode= query_error_code(thd, killed_status == THD::NOT_KILLED);
+        int errcode= query_error_code(thd, killed_status == NOT_KILLED);
         error= write_execute_load_query_log_event(thd, ex,
                                                   table_list->db, table_list->table_name,
                                                   handle_duplicates, ignore,
@@ -736,9 +742,16 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
   List_iterator_fast<Item> it(fields_vars);
   Item_field *sql_field;
   TABLE *table= table_list->table;
-  bool err;
+  bool err, progress_reports;
+  ulonglong counter, time_to_report_progress;
   DBUG_ENTER("read_fixed_length");
 
+  counter= 0;
+  time_to_report_progress= MY_HOW_OFTEN_TO_WRITE/10;
+  progress_reports= 1;
+  if ((thd->progress.max_counter= read_info.file_length()) == ~(my_off_t) 0)
+    progress_reports= 0;
+
   while (!read_info.read_fixed_length())
   {
     if (thd->killed)
@@ -746,6 +759,16 @@ read_fixed_length(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
       thd->send_kill_message();
       DBUG_RETURN(1);
     }
+    if (progress_reports)
+    {
+      thd->progress.counter= read_info.position();
+      if (++counter >= time_to_report_progress)
+      {
+        time_to_report_progress+= MY_HOW_OFTEN_TO_WRITE/10;
+        thd_progress_report(thd, thd->progress.counter,
+                            thd->progress.max_counter);
+      }
+    }
     if (skip_lines)
     {
       /*
@@ -864,11 +887,18 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
   Item *item;
   TABLE *table= table_list->table;
   uint enclosed_length;
-  bool err;
+  bool err, progress_reports;
+  ulonglong counter, time_to_report_progress;
   DBUG_ENTER("read_sep_field");
 
   enclosed_length=enclosed.length();
 
+  counter= 0;
+  time_to_report_progress= MY_HOW_OFTEN_TO_WRITE/10;
+  progress_reports= 1;
+  if ((thd->progress.max_counter= read_info.file_length()) == ~(my_off_t) 0)
+    progress_reports= 0;
+
   for (;;it.rewind())
   {
     if (thd->killed)
@@ -877,6 +907,16 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
       DBUG_RETURN(1);
     }
 
+    if (progress_reports)
+    {
+      thd->progress.counter= read_info.position();
+      if (++counter >= time_to_report_progress)
+      {
+        time_to_report_progress+= MY_HOW_OFTEN_TO_WRITE/10;
+        thd_progress_report(thd, thd->progress.counter,
+                            thd->progress.max_counter);
+      }
+    }
     restore_record(table, s->default_values);
 
     while ((item= it++))
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index f219c8d6876..a1c373ce426 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -265,17 +265,17 @@ void init_update_queries(void)
   bzero((uchar*) &sql_command_flags, sizeof(sql_command_flags));
 
   sql_command_flags[SQLCOM_CREATE_TABLE]=   CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE;
-  sql_command_flags[SQLCOM_CREATE_INDEX]=   CF_CHANGES_DATA;
-  sql_command_flags[SQLCOM_ALTER_TABLE]=    CF_CHANGES_DATA | CF_WRITE_LOGS_COMMAND;
+  sql_command_flags[SQLCOM_CREATE_INDEX]=   CF_CHANGES_DATA | CF_REPORT_PROGRESS;
+  sql_command_flags[SQLCOM_ALTER_TABLE]=    CF_CHANGES_DATA | CF_WRITE_LOGS_COMMAND | CF_REPORT_PROGRESS;
   sql_command_flags[SQLCOM_TRUNCATE]=       CF_CHANGES_DATA | CF_WRITE_LOGS_COMMAND;
   sql_command_flags[SQLCOM_DROP_TABLE]=     CF_CHANGES_DATA;
-  sql_command_flags[SQLCOM_LOAD]=           CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_LOAD]=           CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE | CF_REPORT_PROGRESS;
   sql_command_flags[SQLCOM_CREATE_DB]=      CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_DROP_DB]=        CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_RENAME_TABLE]=   CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_BACKUP_TABLE]=   CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_RESTORE_TABLE]=  CF_CHANGES_DATA;
-  sql_command_flags[SQLCOM_DROP_INDEX]=     CF_CHANGES_DATA;
+  sql_command_flags[SQLCOM_DROP_INDEX]=     CF_CHANGES_DATA | CF_REPORT_PROGRESS;
   sql_command_flags[SQLCOM_CREATE_VIEW]=    CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE;
   sql_command_flags[SQLCOM_DROP_VIEW]=      CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_CREATE_EVENT]=   CF_CHANGES_DATA;
@@ -368,9 +368,11 @@ void init_update_queries(void)
     The following admin table operations are allowed
     on log tables.
   */
-  sql_command_flags[SQLCOM_REPAIR]=           CF_WRITE_LOGS_COMMAND;
-  sql_command_flags[SQLCOM_OPTIMIZE]=         CF_WRITE_LOGS_COMMAND;
-  sql_command_flags[SQLCOM_ANALYZE]=          CF_WRITE_LOGS_COMMAND;
+  sql_command_flags[SQLCOM_REPAIR]=           CF_WRITE_LOGS_COMMAND | CF_REPORT_PROGRESS;
+  sql_command_flags[SQLCOM_OPTIMIZE]=         CF_WRITE_LOGS_COMMAND | CF_REPORT_PROGRESS;
+  sql_command_flags[SQLCOM_ANALYZE]=          CF_WRITE_LOGS_COMMAND | CF_REPORT_PROGRESS;
+  sql_command_flags[SQLCOM_CHECK]=            CF_REPORT_PROGRESS;
+  sql_command_flags[SQLCOM_CHECKSUM]=         CF_REPORT_PROGRESS;
 }
 
 
@@ -790,7 +792,17 @@ int end_trans(THD *thd, enum enum_mysql_completiontype completion)
   if (res < 0)
     my_error(thd->killed_errno(), MYF(0));
   else if ((res == 0) && do_release)
-    thd->killed= THD::KILL_CONNECTION;
+  {
+    thd->killed= KILL_CONNECTION;
+    if (global_system_variables.log_warnings > 3)
+    {
+      Security_context *sctx= &thd->main_security_ctx;
+      sql_print_warning(ER(ER_NEW_ABORTING_CONNECTION),
+                        thd->thread_id,(thd->db ? thd->db : "unconnected"),
+                        sctx->user ? sctx->user : "unauthenticated",
+                        sctx->host_or_ip, "RELEASE");
+    }
+  }
 
   DBUG_RETURN(res);
 }
@@ -1130,12 +1142,15 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
 
     /* Ensure we don't free security_ctx->user in case we have to revert */
     thd->security_ctx->user= 0;
+    thd->user_connect= 0;
 
     if (acl_authenticate(thd, 0, packet_length))
     {
       /* Free user if allocated by acl_authenticate */
       x_free(thd->security_ctx->user);
       *thd->security_ctx= save_security_ctx;
+      if (thd->user_connect)
+	decrease_user_connections(thd->user_connect);
       thd->user_connect= save_user_connect;
       thd->reset_db(save_db, save_db_length);
       thd->variables.character_set_client= save_character_set_client;
@@ -1482,7 +1497,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
     {
       char *end= buff + length;
       length+= my_snprintf(end, buff_len - length - 1,
-                           end,"  Memory in use: %ldK  Max memory used: %ldK",
+                           "  Memory in use: %ldK  Max memory used: %ldK",
                            (sf_malloc_cur_memory+1023L)/1024L,
                            (sf_malloc_max_memory+1023L)/1024L);
     }
@@ -1515,7 +1530,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
   {
     status_var_increment(thd->status_var.com_stat[SQLCOM_KILL]);
     ulong id=(ulong) uint4korr(packet);
-    sql_kill(thd,id,false);
+    sql_kill(thd,id, KILL_CONNECTION_HARD);
     break;
   }
   case COM_SET_OPTION:
@@ -1557,15 +1572,18 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
   }
 
   /* report error issued during command execution */
-  if (thd->killed_errno())
-  {
-    if (! thd->main_da.is_set())
-      thd->send_kill_message();
-  }
-  if (thd->killed == THD::KILL_QUERY || thd->killed == THD::KILL_BAD_DATA)
+  if (thd->killed)
   {
-    thd->killed= THD::NOT_KILLED;
-    thd->mysys_var->abort= 0;
+    if  (thd->killed_errno())
+    {
+      if (! thd->main_da.is_set())
+        thd->send_kill_message();
+    }
+    if (thd->killed < KILL_CONNECTION)
+    {
+      thd->killed= NOT_KILLED;
+      thd->mysys_var->abort= 0;
+    }
   }
 
   /* If commit fails, we should be able to reset the OK status. */
@@ -1609,6 +1627,9 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
   thd_proc_info(thd, 0);
   thd->packet.shrink(thd->variables.net_buffer_length);	// Reclaim some memory
   free_root(thd->mem_root,MYF(MY_KEEP_PREALLOC));
+
+  /* Check that some variables are reset properly */
+  DBUG_ASSERT(thd->abort_on_warning == 0);
   DBUG_RETURN(error);
 }
 
@@ -2166,6 +2187,8 @@ mysql_execute_command(THD *thd)
   } /* endif unlikely slave */
 #endif
   status_var_increment(thd->status_var.com_stat[lex->sql_command]);
+  thd->progress.report_to_client= test(sql_command_flags[lex->sql_command] &
+                                       CF_REPORT_PROGRESS);
 
   DBUG_ASSERT(thd->transaction.stmt.modified_non_trans_table == FALSE);
   
@@ -2995,6 +3018,7 @@ end_with_restore_list:
     thd->enable_slow_log= opt_log_slow_admin_statements;
     thd->query_plan_flags|= QPLAN_ADMIN;
     res= mysql_analyze_table(thd, first_table, &lex->check_opt);
+
     /* ! we write after unlocking the table */
     if (!res && !lex->no_write_to_binlog)
     {
@@ -4017,8 +4041,6 @@ end_with_restore_list:
   }
   case SQLCOM_KILL:
   {
-    Item *it= (Item *)lex->value_list.head();
-
     if (lex->table_or_sp_used())
     {
       my_error(ER_NOT_SUPPORTED_YET, MYF(0), "Usage of subqueries or stored "
@@ -4026,13 +4048,20 @@ end_with_restore_list:
       break;
     }
 
-    if ((!it->fixed && it->fix_fields(lex->thd, &it)) || it->check_cols(1))
+    if (lex->kill_type == KILL_TYPE_ID)
     {
-      my_message(ER_SET_CONSTANTS_ONLY, ER(ER_SET_CONSTANTS_ONLY),
-		 MYF(0));
-      goto error;
+      Item *it= (Item *)lex->value_list.head();
+      if ((!it->fixed && it->fix_fields(lex->thd, &it)) || it->check_cols(1))
+      {
+        my_message(ER_SET_CONSTANTS_ONLY, ER(ER_SET_CONSTANTS_ONLY),
+                   MYF(0));
+        goto error;
+      }
+      sql_kill(thd, (ulong) it->val_int(), lex->kill_signal);
     }
-    sql_kill(thd, (ulong)it->val_int(), lex->type & ONLY_KILL_QUERY);
+    else
+      sql_kill_user(thd, get_current_user(thd, lex->users_list.head()),
+                    lex->kill_signal);
     break;
   }
 #ifndef NO_EMBEDDED_ACCESS_CHECKS
@@ -4486,9 +4515,10 @@ create_sp_error:
           */
           /* Conditionally writes to binlog */
 
-          int type= lex->sql_command == SQLCOM_ALTER_PROCEDURE ?
-                    TYPE_ENUM_PROCEDURE :
-                    TYPE_ENUM_FUNCTION;
+          stored_procedure_type type;
+          type= (lex->sql_command == SQLCOM_ALTER_PROCEDURE ?
+                 TYPE_ENUM_PROCEDURE :
+                 TYPE_ENUM_FUNCTION);
 
           sp_result= sp_update_routine(thd,
                                        type,
@@ -4516,8 +4546,8 @@ create_sp_error:
   case SQLCOM_DROP_FUNCTION:
     {
       int sp_result;
-      int type= (lex->sql_command == SQLCOM_DROP_PROCEDURE ?
-                 TYPE_ENUM_PROCEDURE : TYPE_ENUM_FUNCTION);
+      stored_procedure_type type= (lex->sql_command == SQLCOM_DROP_PROCEDURE ?
+                                   TYPE_ENUM_PROCEDURE : TYPE_ENUM_FUNCTION);
 
       sp_result= sp_routine_exists_in_table(thd, type, lex->spname);
       mysql_reset_errors(thd, 0);
@@ -4544,9 +4574,10 @@ create_sp_error:
 #endif
         /* Conditionally writes to binlog */
 
-        int type= lex->sql_command == SQLCOM_DROP_PROCEDURE ?
-                  TYPE_ENUM_PROCEDURE :
-                  TYPE_ENUM_FUNCTION;
+        stored_procedure_type type;
+        type= (lex->sql_command == SQLCOM_DROP_PROCEDURE ?
+               TYPE_ENUM_PROCEDURE :
+               TYPE_ENUM_FUNCTION);
 
         sp_result= sp_drop_routine(thd, type, lex->spname);
       }
@@ -5044,7 +5075,6 @@ static bool execute_sqlcom_select(THD *thd, TABLE_LIST *all_tables)
       param->select_limit=
         new Item_int((ulonglong) thd->variables.select_limit);
   }
-  thd->thd_marker.emb_on_expr_nest= NULL;
   if (!(res= open_and_lock_tables(thd, all_tables)))
   {
     if (lex->describe)
@@ -5818,7 +5848,6 @@ void mysql_reset_thd_for_next_command(THD *thd, my_bool calculate_userstat)
 
   thd->query_plan_flags= QPLAN_INIT;
   thd->query_plan_fsort_passes= 0;
-  thd->thd_marker.emb_on_expr_nest= NULL;
 
   /*
     Because we come here only for start of top-statements, binlog format is
@@ -5888,6 +5917,7 @@ mysql_new_select(LEX *lex, bool move_down)
     DBUG_RETURN(1);
   }
   select_lex->nest_level= lex->nest_level;
+  select_lex->nest_level_base= &thd->lex->unit;
   if (move_down)
   {
     SELECT_LEX_UNIT *unit;
@@ -6790,6 +6820,28 @@ push_new_name_resolution_context(THD *thd,
 
 
 /**
+  Fix condition which contains only field (f turns to  f <> 0 )
+
+  @param cond            The condition to fix
+
+  @return fixed condition
+*/
+
+Item *normalize_cond(Item *cond)
+{
+  if (cond)
+  {
+    Item::Type type= cond->type();
+    if (type == Item::FIELD_ITEM || type == Item::REF_ITEM)
+    {
+      cond= new Item_func_ne(cond, new Item_int(0));
+    }
+  }
+  return cond;
+}
+
+
+/**
   Add an ON condition to the second operand of a JOIN ... ON.
 
     Add an ON condition to the right operand of a JOIN ... ON clause.
@@ -6807,6 +6859,7 @@ void add_join_on(TABLE_LIST *b, Item *expr)
 {
   if (expr)
   {
+    expr= normalize_cond(expr);
     if (!b->on_expr)
       b->on_expr= expr;
     else
@@ -7132,12 +7185,13 @@ bool reload_acl_and_cache(THD *thd, ulong options, TABLE_LIST *tables,
     This is written such that we have a short lock on LOCK_thread_count
 */
 
-uint kill_one_thread(THD *thd, ulong id, bool only_kill_query)
+uint kill_one_thread(THD *thd, ulong id, killed_state kill_signal)
 {
   THD *tmp;
   uint error=ER_NO_SUCH_THREAD;
   DBUG_ENTER("kill_one_thread");
-  DBUG_PRINT("enter", ("id=%lu only_kill=%d", id, only_kill_query));
+  DBUG_PRINT("enter", ("id: %lu  signal: %u", id, (uint) kill_signal));
+
   VOID(pthread_mutex_lock(&LOCK_thread_count)); // For unlink from list
   I_List_iterator<THD> it(threads);
   while ((tmp=it++))
@@ -7168,12 +7222,16 @@ uint kill_one_thread(THD *thd, ulong id, bool only_kill_query)
 
       If user of both killer and killee are non-NULL, proceed with
       slayage if both are string-equal.
+
+      It's ok to also kill DELAYED threads with KILL_CONNECTION instead of
+      KILL_SYSTEM_THREAD; The difference is that KILL_CONNECTION may be
+      faster and do a harder kill than KILL_SYSTEM_THREAD;
     */
 
     if ((thd->security_ctx->master_access & SUPER_ACL) ||
         thd->security_ctx->user_matches(tmp->security_ctx))
     {
-      tmp->awake(only_kill_query ? THD::KILL_QUERY : THD::KILL_CONNECTION);
+      tmp->awake(kill_signal);
       error=0;
     }
     else
@@ -7185,6 +7243,76 @@ uint kill_one_thread(THD *thd, ulong id, bool only_kill_query)
 }
 
 
+/**
+  kill all threads from one user
+
+  @param thd			Thread class
+  @param user_name		User name for threads we should kill
+  @param only_kill_query        Should it kill the query or the connection
+
+  @note
+    This is written such that we have a short lock on LOCK_thread_count
+
+    If we can't kill all threads because of security issues, no threads
+    are killed.
+*/
+
+static uint kill_threads_for_user(THD *thd, LEX_USER *user,
+                                  killed_state kill_signal, ha_rows *rows)
+{
+  THD *tmp;
+  List<THD> threads_to_kill;
+  DBUG_ENTER("kill_threads_for_user");
+
+  *rows= 0;
+
+  if (thd->is_fatal_error)                       // If we run out of memory
+    DBUG_RETURN(ER_OUT_OF_RESOURCES);
+
+  DBUG_PRINT("enter", ("user: %s  signal: %u", user->user.str,
+                       (uint) kill_signal));
+
+  VOID(pthread_mutex_lock(&LOCK_thread_count)); // For unlink from list
+  I_List_iterator<THD> it(threads);
+  while ((tmp=it++))
+  {
+    if (tmp->command == COM_DAEMON)
+      continue;
+    /*
+      Check that hostname (if given) and user name matches.
+
+      host.str[0] == '%' means that host name was not given. See sql_yacc.yy
+    */
+    if (((user->host.str[0] == '%' && !user->host.str[1]) ||
+         !strcmp(tmp->security_ctx->host, user->host.str)) &&
+        !strcmp(tmp->security_ctx->user, user->user.str))
+    {
+      if (!(thd->security_ctx->master_access & SUPER_ACL) &&
+          !thd->security_ctx->user_matches(tmp->security_ctx))
+      {
+        VOID(pthread_mutex_unlock(&LOCK_thread_count));
+        DBUG_RETURN(ER_KILL_DENIED_ERROR);
+      }
+      if (!threads_to_kill.push_back(tmp, tmp->mem_root))
+        pthread_mutex_lock(&tmp->LOCK_thd_data); // Lock from delete
+    }
+  }
+  VOID(pthread_mutex_unlock(&LOCK_thread_count));
+  if (!threads_to_kill.is_empty())
+  {
+    List_iterator_fast<THD> it(threads_to_kill);
+    THD *ptr;
+    while ((ptr= it++))
+    {
+      ptr->awake(kill_signal);
+      pthread_mutex_unlock(&ptr->LOCK_thd_data);
+      (*rows)++;
+    }
+  }
+  DBUG_RETURN(0);
+}
+
+
 /*
   kills a thread and sends response
 
@@ -7195,16 +7323,33 @@ uint kill_one_thread(THD *thd, ulong id, bool only_kill_query)
     only_kill_query     Should it kill the query or the connection
 */
 
-void sql_kill(THD *thd, ulong id, bool only_kill_query)
+void sql_kill(THD *thd, ulong id, killed_state state)
 {
   uint error;
-  if (!(error= kill_one_thread(thd, id, only_kill_query)))
+  if (!(error= kill_one_thread(thd, id, state)))
     my_ok(thd);
   else
     my_error(error, MYF(0), id);
 }
 
 
+void sql_kill_user(THD *thd, LEX_USER *user, killed_state state)
+{
+  uint error;
+  ha_rows rows;
+  if (!(error= kill_threads_for_user(thd, user, state, &rows)))
+    my_ok(thd, rows);
+  else
+  {
+    /*
+      This is probably ER_OUT_OF_RESOURCES, but in the future we may
+      want to write the name of the user we tried to kill
+    */
+    my_error(error, MYF(0), user->host.str, user->user.str);
+  }
+}
+
+
 /** If pointer is not a null pointer, append filename to it. */
 
 bool append_file_to_dir(THD *thd, const char **filename_ptr,
diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc
index a83990d25f4..8e6f065c1c4 100644
--- a/sql/sql_plugin.cc
+++ b/sql/sql_plugin.cc
@@ -1432,15 +1432,23 @@ int plugin_init(int *argc, char **argv, int flags)
       if (register_builtin(plugin, &tmp, &plugin_ptr))
         goto err_unlock;
 
-      /* only initialize MyISAM and CSV at this stage */
-      if (!(is_myisam=
-            !my_strcasecmp(&my_charset_latin1, plugin->name, "MyISAM")) &&
-          my_strcasecmp(&my_charset_latin1, plugin->name, "CSV"))
-        continue;
+      is_myisam= !my_strcasecmp(&my_charset_latin1, plugin->name, "MyISAM");
 
-      if (plugin_ptr->state == PLUGIN_IS_UNINITIALIZED &&
-          plugin_initialize(plugin_ptr))
-        goto err_unlock;
+      /*
+        strictly speaking, we should to initialize all plugins,
+        even for mysqld --help, because important subsystems
+        may be disabled otherwise, and the help will be incomplete.
+        For example, if the mysql.plugin table is not MyISAM.
+        But for now it's an unlikely corner case, and to optimize
+        mysqld --help for all other users, we will only initialize
+        MyISAM here.
+      */
+      if (!(flags & PLUGIN_INIT_SKIP_INITIALIZATION) || is_myisam)
+      {
+        if (plugin_ptr->state == PLUGIN_IS_UNINITIALIZED &&
+            plugin_initialize(plugin_ptr))
+          goto err_unlock;
+      }
 
       /*
         initialize the global default storage engine so that it may
@@ -1653,13 +1661,6 @@ static void plugin_load(MEM_ROOT *tmp_root, int *argc, char **argv)
     goto end;
   }
   table->use_all_columns();
-  /*
-    there're no other threads running yet, so we don't need a mutex.
-    but plugin_add() before is designed to work in multi-threaded
-    environment, and it uses safe_mutex_assert_owner(), so we lock
-    the mutex here to satisfy the assert
-  */
-  pthread_mutex_lock(&LOCK_plugin);
   while (!(error= read_record_info.read_record(&read_record_info)))
   {
     DBUG_PRINT("info", ("init plugin record"));
@@ -1670,12 +1671,19 @@ static void plugin_load(MEM_ROOT *tmp_root, int *argc, char **argv)
     LEX_STRING name= {(char *)str_name.ptr(), str_name.length()};
     LEX_STRING dl= {(char *)str_dl.ptr(), str_dl.length()};
 
+    /*
+      there're no other threads running yet, so we don't need a mutex.
+      but plugin_add() before is designed to work in multi-threaded
+      environment, and it uses safe_mutex_assert_owner(), so we lock
+      the mutex here to satisfy the assert
+    */
+    pthread_mutex_lock(&LOCK_plugin);
     if (plugin_add(tmp_root, &name, &dl, argc, argv, REPORT_TO_LOG))
       sql_print_warning("Couldn't load plugin named '%s' with soname '%s'.",
                         str_name.c_ptr(), str_dl.c_ptr());
     free_root(tmp_root, MYF(MY_MARK_BLOCKS_FREE));
+    pthread_mutex_unlock(&LOCK_plugin);
   }
-  pthread_mutex_unlock(&LOCK_plugin);
   if (error > 0)
     sql_print_error(ER(ER_GET_ERRNO), my_errno);
   end_read_record(&read_record_info);
@@ -3376,6 +3384,19 @@ static int construct_options(MEM_ROOT *mem_root, struct st_plugin_int *tmp,
                             opt->name, plugin_name);
         }
       }
+      /*
+        PLUGIN_VAR_STR command-line options without PLUGIN_VAR_MEMALLOC, point
+        directly to values in the argv[] array. For plugins started at the
+        server startup, argv[] array is allocated with load_defaults(), and
+        freed when the server is shut down.  But for plugins loaded with
+        INSTALL PLUGIN, the memory allocated with load_defaults() is freed with
+        freed() at the end of mysql_install_plugin(). Which means we cannot
+        allow any pointers into that area.
+        Thus, for all plugins loaded after the server was started,
+        we force all command-line options to be PLUGIN_VAR_MEMALLOC
+      */
+      if (mysqld_server_started && !(opt->flags & PLUGIN_VAR_NOCMDOPT))
+        opt->flags|= PLUGIN_VAR_MEMALLOC;
       break;
     case PLUGIN_VAR_ENUM:
       if (!opt->check)
@@ -3532,6 +3553,10 @@ static int test_plugin_options(MEM_ROOT *tmp_root, struct st_plugin_int *tmp,
   if (!my_strcasecmp(&my_charset_latin1, tmp->name.str, "ndbcluster"))
     plugin_load_policy= PLUGIN_OFF;
 #endif
+#ifdef WITH_FEEDBACK_PLUGIN
+  if (!my_strcasecmp(&my_charset_latin1, tmp->name.str, "feedback"))
+    plugin_load_policy= PLUGIN_OFF;
+#endif
 
   for (opt= tmp->plugin->system_vars; opt && *opt; opt++)
     count+= 2; /* --{plugin}-{optname} and --plugin-{plugin}-{optname} */
diff --git a/sql/sql_plugin_services.h b/sql/sql_plugin_services.h
index 8d4055dd764..497e2c8d6bc 100644
--- a/sql/sql_plugin_services.h
+++ b/sql/sql_plugin_services.h
@@ -38,9 +38,18 @@ static struct thd_alloc_service_st thd_alloc_handler= {
   thd_make_lex_string
 };
 
+static struct progress_report_service_st progress_report_handler= {
+  thd_progress_init,
+  thd_progress_report,
+  thd_progress_next_stage,
+  thd_progress_end,
+  set_thd_proc_info
+};
+
 static struct st_service_ref list_of_services[] __attribute__((unused)) =
 {
   { "my_snprintf_service", VERSION_my_snprintf, &my_snprintf_handler },
-  { "thd_alloc_service",   VERSION_thd_alloc,   &thd_alloc_handler }
+  { "thd_alloc_service",   VERSION_thd_alloc,   &thd_alloc_handler },
+  { "progress_report_service", VERSION_progress_report, &progress_report_handler }
 };
 #endif
diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc
index ece4b8a1014..74ff1f0ebe1 100644
--- a/sql/sql_prepare.cc
+++ b/sql/sql_prepare.cc
@@ -1269,8 +1269,9 @@ static int mysql_test_update(Prepared_statement *stmt,
     thd->fill_derived_tables() is false here for sure (because it is
     preparation of PS, so we even do not check it).
   */
-  if (table_list->handle_derived(thd->lex, DT_MERGE_FOR_INSERT) ||
-      table_list->handle_derived(thd->lex, DT_PREPARE))
+  if (table_list->handle_derived(thd->lex, DT_MERGE_FOR_INSERT))
+    goto error;
+  if (table_list->handle_derived(thd->lex, DT_PREPARE))
     goto error;
 
   if (!table_list->updatable)
@@ -1340,9 +1341,11 @@ static bool mysql_test_delete(Prepared_statement *stmt,
       open_tables(thd, &table_list, &table_count, 0))
     goto error;
 
-  if (mysql_handle_derived(thd->lex, DT_INIT) ||
-      mysql_handle_list_of_derived(thd->lex, table_list, DT_MERGE_FOR_INSERT) ||
-      mysql_handle_list_of_derived(thd->lex, table_list, DT_PREPARE))
+  if (mysql_handle_derived(thd->lex, DT_INIT))
+    goto error;
+  if (mysql_handle_derived(thd->lex, DT_MERGE_FOR_INSERT))
+    goto error;
+  if (mysql_handle_derived(thd->lex, DT_PREPARE))
     goto error;
 
   if (!table_list->updatable)
@@ -1410,7 +1413,6 @@ static int mysql_test_select(Prepared_statement *stmt,
     goto error;
 
   thd->used_tables= 0;                        // Updated by setup_fields
-  thd->thd_marker.emb_on_expr_nest= 0;
 
   /*
     JOIN::prepare calls
diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc
index fc472c17e57..e7363bffa9a 100644
--- a/sql/sql_repl.cc
+++ b/sql/sql_repl.cc
@@ -1277,9 +1277,9 @@ err:
   idle, then this could last long, and if the slave reconnects, we could have 2
   Binlog_dump threads in SHOW PROCESSLIST, until a query is written to the
   binlog. To avoid this, when the slave reconnects and sends COM_BINLOG_DUMP,
-  the master kills any existing thread with the slave's server id (if this id is
-  not zero; it will be true for real slaves, but false for mysqlbinlog when it
-  sends COM_BINLOG_DUMP to get a remote binlog dump).
+  the master kills any existing thread with the slave's server id (if this id
+  is not zero; it will be true for real slaves, but false for mysqlbinlog when
+  it sends COM_BINLOG_DUMP to get a remote binlog dump).
 
   SYNOPSIS
     kill_zombie_dump_threads()
@@ -1311,7 +1311,7 @@ void kill_zombie_dump_threads(uint32 slave_server_id)
       it will be slow because it will iterate through the list
       again. We just to do kill the thread ourselves.
     */
-    tmp->awake(THD::KILL_QUERY);
+    tmp->awake(KILL_QUERY);
     pthread_mutex_unlock(&tmp->LOCK_thd_data);
   }
 }
@@ -1978,10 +1978,10 @@ static sys_var_const    sys_log_slave_updates(&vars, "log_slave_updates",
                                               OPT_GLOBAL, SHOW_MY_BOOL,
                                               (uchar*) &opt_log_slave_updates);
 static sys_var_const
-sys_replicate_annotate_rows_events(&vars,
-                                "replicate_annotate_rows_events",
+sys_replicate_annotate_row_events(&vars,
+                                "replicate_annotate_row_events",
                                 OPT_GLOBAL, SHOW_MY_BOOL,
-                                (uchar*) &opt_replicate_annotate_rows_events);
+                                (uchar*) &opt_replicate_annotate_row_events);
 static sys_var_const    sys_relay_log(&vars, "relay_log",
                                       OPT_GLOBAL, SHOW_CHAR_PTR,
                                       (uchar*) &opt_relay_logname);
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 6212f841793..8d83b4589d5 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -90,7 +90,7 @@ static store_key *get_store_key(THD *thd,
 				KEYUSE *keyuse, table_map used_tables,
 				KEY_PART_INFO *key_part, uchar *key_buff,
 				uint maybe_null);
-static void make_outerjoin_info(JOIN *join);
+static bool make_outerjoin_info(JOIN *join);
 static Item*
 make_cond_after_sjm(Item *root_cond, Item *cond, table_map tables, table_map sjm_tables);
 static bool make_join_select(JOIN *join,SQL_SELECT *select,COND *item);
@@ -175,14 +175,14 @@ int join_read_always_key_or_null(JOIN_TAB *tab);
 int join_read_next_same_or_null(READ_RECORD *info);
 static COND *make_cond_for_table(THD *thd, Item *cond,table_map table,
                                  table_map used_table,
-                                 uint join_tab_idx_arg,
+                                 int join_tab_idx_arg,
                                  bool exclude_expensive_cond,
                                  bool retain_ref_cond);
 static COND *make_cond_for_table_from_pred(THD *thd, Item *root_cond,
                                            Item *cond,
                                            table_map tables,
                                            table_map used_table,
-                                           uint join_tab_idx_arg,
+                                           int join_tab_idx_arg,
                                            bool exclude_expensive_cond,
                                            bool retain_ref_cond);
 
@@ -558,7 +558,7 @@ JOIN::prepare(Item ***rref_pointer_array,
   List_iterator_fast<TABLE_LIST> li(select_lex->leaf_tables);
   while ((tbl= li++))
   {
-    table_count++; /* Count the number of tables in the join. */
+    //table_count++; /* Count the number of tables in the join. */
     /*
       If the query uses implicit grouping where the select list contains both
       aggregate functions and non-aggregate fields, any non-aggregated field
@@ -689,6 +689,10 @@ JOIN::prepare(Item ***rref_pointer_array,
     aggregate functions with implicit grouping (there is no GROUP BY).
   */
   if (thd->variables.sql_mode & MODE_ONLY_FULL_GROUP_BY && !group_list &&
+      !(select_lex->master_unit()->item &&
+        select_lex->master_unit()->item->is_in_predicate() &&
+        ((Item_in_subselect*)select_lex->master_unit()->item)->
+        test_set_strategy(SUBS_MAXMIN_INJECTED)) &&
       select_lex->full_group_by_flag == (NON_AGG_FIELD_USED | SUM_FUNC_USED))
   {
     my_message(ER_MIX_OF_GROUP_FUNC_AND_FIELDS,
@@ -813,7 +817,7 @@ inject_jtbm_conds(JOIN *join, List<TABLE_LIST> *join_list, Item **join_where)
       double rows;
       double read_time;
 
-      subq_pred->in_strategy &= ~SUBS_IN_TO_EXISTS;
+      DBUG_ASSERT(subq_pred->test_set_strategy(SUBS_MATERIALIZATION));
       subq_pred->optimize(&rows, &read_time);
 
       subq_pred->jtbm_read_time= read_time;
@@ -826,6 +830,7 @@ inject_jtbm_conds(JOIN *join, List<TABLE_LIST> *join_list, Item **join_where)
       
       //repeat of convert_subq_to_jtbm:
       table->table= hash_sj_engine->tmp_table;
+      table->table->pos_in_table_list= table;
 
       setup_table_map(table->table, table, table->jtbm_table_no);
 
@@ -864,6 +869,7 @@ JOIN::optimize()
   uint no_jbuf_after;
 
   DBUG_ENTER("JOIN::optimize");
+  do_send_rows = (unit->select_limit_cnt) ? 1 : 0;
   // to prevent double initialization on EXPLAIN
   if (optimized)
     DBUG_RETURN(0);
@@ -897,11 +903,10 @@ JOIN::optimize()
     /* dump_TABLE_LIST_graph(select_lex, select_lex->leaf_tables); */
     select_lex->update_used_tables();
 
-    /* Save this info for the next executions */
-    if (select_lex->save_leaf_tables(thd))
-      DBUG_RETURN(1);
   }
   
+  eval_select_list_used_tables();
+  
   table_count= select_lex->leaf_tables.elements;
 
   if (setup_ftfuncs(select_lex)) /* should be after having->fix_fields */
@@ -913,7 +918,6 @@ JOIN::optimize()
   select_limit= unit->select_limit_cnt;
   if (having || (select_options & OPTION_FOUND_ROWS))
     select_limit= HA_POS_ERROR;
-  do_send_rows = (unit->select_limit_cnt) ? 1 : 0;
   // Ignore errors of execution if option IGNORE present
   if (thd->lex->ignore)
     thd->lex->current_select->no_error= 1;
@@ -958,17 +962,22 @@ JOIN::optimize()
 
     /* Convert all outer joins to inner joins if possible */
     conds= simplify_joins(this, join_list, conds, TRUE, FALSE);
+    if (select_lex->save_leaf_tables(thd))
+      DBUG_RETURN(1);
     build_bitmap_for_nested_joins(join_list, 0);
 
     sel->prep_where= conds ? conds->copy_andor_structure(thd) : 0;
 
+    sel->where= conds;
+
     if (arena)
       thd->restore_active_arena(arena, &backup);
   }
   
   inject_jtbm_conds(this, join_list, &conds);
 
-  conds= optimize_cond(this, conds, join_list, &cond_value, &cond_equal);   
+  conds= optimize_cond(this, conds, join_list, &cond_value, &cond_equal);
+     
   if (thd->is_error())
   {
     error= 1;
@@ -985,10 +994,17 @@ JOIN::optimize()
       DBUG_RETURN(1);
     }
     if (select_lex->where)
+    {
       select_lex->cond_value= cond_value;
+      if (sel->where != conds && cond_value == Item::COND_OK)
+        thd->change_item_tree(&sel->where, conds);
+    }  
     if (select_lex->having)
+    {
       select_lex->having_value= having_value;
-
+      if (sel->having != having && having_value == Item::COND_OK)
+        thd->change_item_tree(&sel->having, having);    
+    }
     if (cond_value == Item::COND_FALSE || having_value == Item::COND_FALSE || 
         (!unit->select_limit_cnt && !(select_options & OPTION_FOUND_ROWS)))
     {						/* Impossible cond */
@@ -1076,7 +1092,7 @@ JOIN::optimize()
       if (conds && !(thd->lex->describe & DESCRIBE_EXTENDED))
       {
         COND *table_independent_conds=
-          make_cond_for_table(thd, conds, PSEUDO_TABLE_BITS, 0, MAX_TABLES,
+          make_cond_for_table(thd, conds, PSEUDO_TABLE_BITS, 0, -1,
                               FALSE, FALSE);
         DBUG_EXECUTE("where",
                      print_where(table_independent_conds,
@@ -1164,7 +1180,10 @@ JOIN::optimize()
   }
   
   reset_nj_counters(this, join_list);
-  make_outerjoin_info(this);
+  if (make_outerjoin_info(this))
+  {
+    DBUG_RETURN(1);
+  }
 
   /*
     Among the equal fields belonging to the same multiple equality
@@ -1719,7 +1738,7 @@ int JOIN::init_execution()
 
     if (exec_tmp_table1->distinct)
     {
-      table_map used_tables= thd->used_tables;
+      table_map used_tables= select_list_used_tables;
       JOIN_TAB *last_join_tab= join_tab + top_join_tab_count - 1;
       do
       {
@@ -2369,7 +2388,6 @@ JOIN::exec()
       
       thd_proc_info(thd, "Copying to group table");
       DBUG_PRINT("info", ("%s", thd->proc_info));
-      tmp_error= -1;
       if (curr_join != this)
       {
 	if (sum_funcs2)
@@ -2394,6 +2412,7 @@ JOIN::exec()
         JOIN_TAB *first_tab= curr_join->join_tab + curr_join->const_tables;
         first_tab->sorted= test(first_tab->loosescan_match_tab);
       }
+      tmp_error= -1;
       if (setup_sum_funcs(curr_join->thd, curr_join->sum_funcs) ||
 	  (tmp_error= do_select(curr_join, (List<Item> *) 0, curr_tmp_table,
 				0)))
@@ -2520,7 +2539,7 @@ JOIN::exec()
 
       Item* sort_table_cond= make_cond_for_table(thd, curr_join->tmp_having,
 						 used_tables,
-						 (table_map)0, MAX_TABLES,
+						 (table_map)0, -1,
 						 FALSE, FALSE);
       if (sort_table_cond)
       {
@@ -2550,7 +2569,9 @@ JOIN::exec()
         if (curr_table->pre_idx_push_select_cond &&
             !curr_table->pre_idx_push_select_cond->fixed)
           curr_table->pre_idx_push_select_cond->fix_fields(thd, 0);
-          
+
+        curr_table->select->pre_idx_push_select_cond=
+          curr_table->pre_idx_push_select_cond;
         curr_table->set_select_cond(curr_table->select->cond, __LINE__);
 	curr_table->select_cond->top_level_item();
 	DBUG_EXECUTE("where",print_where(curr_table->select->cond,
@@ -2558,7 +2579,7 @@ JOIN::exec()
                                          QT_ORDINARY););
 	curr_join->tmp_having= make_cond_for_table(thd, curr_join->tmp_having,
 						   ~ (table_map) 0,
-						   ~used_tables, MAX_TABLES,
+						   ~used_tables, -1,
 						   FALSE, FALSE);
 	DBUG_EXECUTE("where",print_where(curr_join->tmp_having,
                                          "having after sort",
@@ -2977,6 +2998,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
   uint i,table_count,const_count,key;
   table_map found_const_table_map, all_table_map, found_ref, refs;
   key_map const_ref, eq_part;
+  bool has_expensive_keyparts;
   TABLE **table_vector;
   JOIN_TAB *stat,*stat_end,*s,**stat_ref;
   KEYUSE *keyuse,*start_keyuse;
@@ -3056,12 +3078,23 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
         s->embedding_map|= embedding->nested_join->nj_map;
       continue;
     }
-    if (embedding && !(embedding->sj_on_expr && ! embedding->embedding))
+    if (embedding)
     {
       /* s belongs to a nested join, maybe to several embedded joins */
       s->embedding_map= 0;
+      bool inside_an_outer_join= FALSE;
       do
       {
+        /* 
+          If this is a semi-join nest, skip it, and proceed upwards. Maybe
+          we're in some outer join nest
+        */
+        if (embedding->sj_on_expr)
+        {
+          embedding= embedding->embedding;
+          continue;
+        }
+        inside_an_outer_join= TRUE;
         NESTED_JOIN *nested_join= embedding->nested_join;
         s->embedding_map|=nested_join->nj_map;
         s->dependent|= embedding->dep_tables;
@@ -3069,7 +3102,8 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
         outer_join|= nested_join->used_tables;
       }
       while (embedding);
-      continue;
+      if (inside_an_outer_join)
+        continue;
     }
 #ifdef WITH_PARTITION_STORAGE_ENGINE
     const bool no_partitions_used= table->no_partitions_used;
@@ -3151,7 +3185,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
     */
     bool skip_unprefixed_keyparts=
       !(join->is_in_subquery() &&
-        ((Item_in_subselect*)join->unit->item)->in_strategy & SUBS_IN_TO_EXISTS);
+        ((Item_in_subselect*)join->unit->item)->test_strategy(SUBS_IN_TO_EXISTS));
 
     if (keyuse_array->elements &&
         sort_and_filter_keyuse(join->thd, keyuse_array,
@@ -3250,7 +3284,9 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
 	  continue;
 	if (table->file->stats.records <= 1L &&
 	    (table->file->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT) &&
-            !table->pos_in_table_list->embedding)
+            !table->pos_in_table_list->embedding &&
+	      !((outer_join & table->map) && 
+		(*s->on_expr_ref)->is_expensive()))
 	{					// system table
 	  int tmp= 0;
 	  s->type=JT_SYSTEM;
@@ -3284,12 +3320,17 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
 	  refs=0;
           const_ref.clear_all();
 	  eq_part.clear_all();
+          has_expensive_keyparts= false;
 	  do
 	  {
 	    if (keyuse->val->type() != Item::NULL_ITEM && !keyuse->optimize)
 	    {
 	      if (!((~found_const_table_map) & keyuse->used_tables))
+              {
 		const_ref.set_bit(keyuse->keypart);
+                if (keyuse->val->is_expensive())
+                  has_expensive_keyparts= true;
+              }
 	      else
 		refs|=keyuse->used_tables;
 	      eq_part.set_bit(keyuse->keypart);
@@ -3310,7 +3351,10 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
 	  {
             if (table->key_info[key].flags & HA_NOSAME)
             {
-	      if (const_ref == eq_part)
+	      if (const_ref == eq_part &&
+                  !has_expensive_keyparts &&
+                  !((outer_join & table->map) &&
+                    (*s->on_expr_ref)->is_expensive()))
 	      {					// Found everything for ref.
 	        int tmp;
 	        ref_changed = 1;
@@ -3789,7 +3833,9 @@ add_key_field(JOIN *join,
   uint optimize= 0;  
   if (eq_func &&
       ((join->is_allowed_hash_join_access() &&
-        field->hash_join_is_possible()) ||
+        field->hash_join_is_possible() && 
+        !(field->table->pos_in_table_list->is_materialized_derived() &&
+          field->table->created)) ||
        (field->table->pos_in_table_list->is_materialized_derived() &&
         !field->table->created)))
   {
@@ -4745,12 +4791,15 @@ static void optimize_keyuse(JOIN *join, DYNAMIC_ARRAY *keyuse_array)
 	(map= (keyuse->used_tables & ~join->const_table_map &
 	       ~OUTER_REF_TABLE_BIT)))
     {
-      uint tablenr;
-      tablenr= my_count_bits(map);
-      if (map == 1)			// Only one table
+      uint n_tables= my_count_bits(map);
+      if (n_tables == 1)			// Only one table
       {
+        Table_map_iterator it(map);
+        int tablenr= it.next_bit();
+        DBUG_ASSERT(tablenr != Table_map_iterator::BITMAP_END);
 	TABLE *tmp_table=join->table[tablenr];
-	keyuse->ref_table_rows= max(tmp_table->file->stats.records, 100);
+        if (tmp_table) // already created
+          keyuse->ref_table_rows= max(tmp_table->file->stats.records, 100);
       }
     }
     /*
@@ -5136,7 +5185,7 @@ best_access_path(JOIN      *join,
               tmp= table->file->keyread_time(key, 1, (ha_rows) tmp);
             else
               tmp= table->file->read_time(key, 1,
-                                          (ha_rows) min(tmp,s->worst_seeks)-1);
+                                          (ha_rows) min(tmp,s->worst_seeks));
             tmp*= record_count;
           }
         }
@@ -5300,13 +5349,14 @@ best_access_path(JOIN      *join,
               tmp= table->file->keyread_time(key, 1, (ha_rows) tmp);
             else
               tmp= table->file->read_time(key, 1,
-                                          (ha_rows) min(tmp,s->worst_seeks)-1);
+                                          (ha_rows) min(tmp,s->worst_seeks));
             tmp*= record_count;
           }
           else
             tmp= best_time;                    // Do nothing
         }
 
+        DBUG_ASSERT(tmp > 0 || record_count == 0);
         tmp += s->startup_cost;
         loose_scan_opt.check_ref_access_part2(key, start_key, records, tmp);
       } /* not ft_key */
@@ -5342,7 +5392,7 @@ best_access_path(JOIN      *join,
     /* Estimate the cost of  the hash join access to the table */
     ha_rows rnd_records= matching_candidates_in_table(s, found_constraint);
 
-    tmp= s->quick ? s->quick->read_time : s->table->file->scan_time();
+    tmp= s->quick ? s->quick->read_time : s->scan_time();
     tmp+= (s->records - rnd_records)/(double) TIME_FOR_COMPARE;
 
     /* We read the table as many times as join buffer becomes full. */
@@ -6006,7 +6056,7 @@ greedy_search(JOIN      *join,
     read_time_arg and record_count_arg contain the computed cost and fanout
 */
 
-void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
+void JOIN::get_partial_cost_and_fanout(int end_tab_idx,
                                        table_map filter_map,
                                        double *read_time_arg, 
                                        double *record_count_arg)
@@ -6016,14 +6066,14 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
   double sj_inner_fanout= 1.0;
   JOIN_TAB *end_tab= NULL;
   JOIN_TAB *tab;
-  uint i;
-  uint last_sj_table= MAX_TABLES;
+  int i;
+  int last_sj_table= MAX_TABLES;
 
   /* 
     Handle a special case where the join is degenerate, and produces no
     records
   */
-  if (table_count == 0)
+  if (table_count == const_tables)
   {
     *read_time_arg= 0.0;
     /*
@@ -6033,6 +6083,7 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
          calculations.
     */
     *record_count_arg=1.0;
+    return;
   }
 
   for (tab= first_depth_first_tab(this), i= const_tables;
@@ -6045,19 +6096,17 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
   }
 
   for (tab= first_depth_first_tab(this), i= const_tables;
-       tab;
+       ;
        tab= next_depth_first_tab(this, tab), i++)
   {
-    /* 
-      We've entered the SJM nest that contains the end_tab. The caller is
-      actually 
-      - interested in fanout inside the nest (because that's how many times 
-        we'll invoke the attached WHERE conditions)
-      - not interested in cost
-    */
     if (end_tab->bush_root_tab && end_tab->bush_root_tab == tab)
     {
-      /* Ok, end_tab is inside SJM nest and we're entering that nest now */
+      /* 
+        We've entered the SJM nest that contains the end_tab. The caller is
+        - interested in fanout inside the nest (because that's how many times 
+          we'll invoke the attached WHERE conditions)
+        - not interested in cost
+      */
       record_count= 1.0;
       read_time= 0.0;
     }
@@ -6071,8 +6120,18 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
       sj_inner_fanout= 1.0;
       last_sj_table= i + tab->n_sj_tables;
     }
-
-    if (tab->records_read && (tab->table->map & filter_map))
+    
+    table_map cur_table_map;
+    if (tab->table)
+      cur_table_map= tab->table->map;
+    else
+    {
+      /* This is a SJ-Materialization nest. Check all of its tables */
+      TABLE *first_child= tab->bush_children->start->table;
+      TABLE_LIST *sjm_nest= first_child->pos_in_table_list->embedding;
+      cur_table_map= sjm_nest->nested_join->used_tables;
+    }
+    if (tab->records_read && (cur_table_map & filter_map))
     {
       record_count *= tab->records_read;
       read_time += tab->read_time;
@@ -6086,6 +6145,9 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
       sj_inner_fanout= 1.0;
       last_sj_table= MAX_TABLES;
     }
+
+    if (tab == end_tab)
+      break;
   }
   *read_time_arg= read_time;// + record_count / TIME_FOR_COMPARE;
   *record_count_arg= record_count;
@@ -6491,7 +6553,8 @@ find_best(JOIN *join,table_map rest_tables,uint idx,double record_count,
 
 void JOIN_TAB::calc_used_field_length(bool max_fl)
 {
-  uint null_fields,blobs,fields,rec_length;
+  uint null_fields,blobs,fields;
+  ulong rec_length;
   Field **f_ptr,*field;
   uint uneven_bit_fields;
   MY_BITMAP *read_set= table->read_set;
@@ -6517,24 +6580,32 @@ void JOIN_TAB::calc_used_field_length(bool max_fl)
     rec_length+=(table->s->null_fields+7)/8;
   if (table->maybe_null)
     rec_length+=sizeof(my_bool);
+
+  /* Take into account that DuplicateElimination may need to store rowid */
+  uint rowid_add_size= 0;
+  if (keep_current_rowid)
+  {
+    rowid_add_size= table->file->ref_length; 
+    rec_length += rowid_add_size;
+    fields++;
+  }
+
   if (max_fl)
   {
     // TODO: to improve this estimate for max expected length 
     if (blobs)
     {
-      uint blob_length=(uint) (table->file->stats.mean_rec_length-
-			      (table->s->reclength-rec_length));
-      rec_length+=(uint) max(sizeof(void*) * blobs, blob_length);
+      ulong blob_length= table->file->stats.mean_rec_length;
+      if (ULONG_MAX - rec_length > blob_length)
+        rec_length+=  blob_length;
+      else
+        rec_length= ULONG_MAX;
     }
     max_used_fieldlength= rec_length;
   } 
-  else if (table->file->stats.mean_rec_length)           
-    set_if_smaller(rec_length, table->file->stats.mean_rec_length);
+  else if (table->file->stats.mean_rec_length)
+    set_if_smaller(rec_length, table->file->stats.mean_rec_length + rowid_add_size);
       
-  /*
-    TODO: why we don't count here rowid that we might need to store when 
-    using DuplicateElimination?
-  */
   used_fields=fields;
   used_fieldlength=rec_length;
   used_blobs=blobs;
@@ -6571,7 +6642,7 @@ int JOIN_TAB::make_scan_filter()
   if (cond &&
       (tmp= make_cond_for_table(join->thd, cond,
                                join->const_table_map | table->map,
-			       table->map, MAX_TABLES, FALSE, TRUE)))
+			       table->map, -1, FALSE, TRUE)))
   {
      DBUG_EXECUTE("where",print_where(tmp,"cache", QT_ORDINARY););
      if (!(cache_select=
@@ -7043,6 +7114,7 @@ get_best_combination(JOIN *join)
       goto loop_end;					// Handled in make_join_stat..
 
     j->loosescan_match_tab= NULL;  //non-nulls will be set later
+    j->inside_loosescan_range= FALSE;
     j->ref.key = -1;
     j->ref.key_parts=0;
 
@@ -7150,7 +7222,8 @@ static bool create_hj_key_for_table(JOIN *join, JOIN_TAB *join_tab,
         (first_keyuse || keyuse->keypart != (keyuse-1)->keypart))
     {
       Field *field= table->field[keyuse->keypart];
-      table->create_key_part_by_field(keyinfo, key_part_info, field);
+      uint fieldnr= keyuse->keypart+1;
+      table->create_key_part_by_field(keyinfo, key_part_info, field, fieldnr);
       first_keyuse= FALSE;
       key_part_info++;
     }
@@ -7162,6 +7235,33 @@ static bool create_hj_key_for_table(JOIN *join, JOIN_TAB *join_tab,
   DBUG_RETURN(FALSE);
 }
 
+/* 
+  Check if a set of tables specified by used_tables can be accessed when
+  we're doing scan on join_tab jtab.
+*/
+static bool are_tables_local(JOIN_TAB *jtab, table_map used_tables)
+{
+  if (jtab->bush_root_tab)
+  {
+    /*
+      jtab is inside execution join nest. We may not refer to outside tables,
+      except the const tables.
+    */
+    table_map local_tables= jtab->emb_sj_nest->nested_join->used_tables |
+                            jtab->join->const_table_map |
+                            OUTER_REF_TABLE_BIT;
+    return !test(used_tables & ~local_tables);
+  }
+
+  /* 
+    If we got here then jtab is at top level. 
+     - all other tables at top level are accessible,
+     - tables in join nests are accessible too, because all their columns that 
+       are needed at top level will be unpacked when scanning the
+       materialization table.
+  */
+  return TRUE;
+}
 
 static bool create_ref_for_key(JOIN *join, JOIN_TAB *j,
                                KEYUSE *org_keyuse, table_map used_tables)
@@ -7207,14 +7307,17 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j,
     {
       if (!(~used_tables & keyuse->used_tables))
       {
-        if ((is_hash_join_key_no(key) && 
-            (keyparts == 0 || keyuse->keypart != (keyuse-1)->keypart)) ||
-            (!is_hash_join_key_no(key) && keyparts == keyuse->keypart &&
-             !(found_part_ref_or_null & keyuse->optimize)))
+        if  (are_tables_local(j, keyuse->val->used_tables()))
         {
-           length+= keyinfo->key_part[keyparts].store_length;
-           keyparts++;
-           found_part_ref_or_null|= keyuse->optimize & ~KEY_OPTIMIZE_EQ;
+          if ((is_hash_join_key_no(key) && 
+              (keyparts == 0 || keyuse->keypart != (keyuse-1)->keypart)) ||
+              (!is_hash_join_key_no(key) && keyparts == keyuse->keypart &&
+               !(found_part_ref_or_null & keyuse->optimize)))
+          {
+             length+= keyinfo->key_part[keyparts].store_length;
+             keyparts++;
+             found_part_ref_or_null|= keyuse->optimize & ~KEY_OPTIMIZE_EQ;
+          }
         }
       }
       keyuse++;
@@ -7263,7 +7366,8 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j,
       while (((~used_tables) & keyuse->used_tables) || 
 	     (keyuse->keypart != 
               (is_hash_join_key_no(key) ?
-                 keyinfo->key_part[i].field->field_index : i))) 
+                 keyinfo->key_part[i].field->field_index : i)) || 
+             !are_tables_local(j, keyuse->val->used_tables())) 
 	 keyuse++;                              	/* Skip other parts */ 
 
       uint maybe_null= test(keyinfo->key_part[i].null_bit);
@@ -7360,7 +7464,7 @@ get_store_key(THD *thd, KEYUSE *keyuse, table_map used_tables,
 			       maybe_null ? key_buff : 0,
 			       key_part->length,
 			       ((Item_field*) keyuse->val->real_item())->field,
-			       keyuse->val->full_name());
+			       keyuse->val->real_item()->full_name());
   return new store_key_item(thd,
 			    key_part->field,
 			    key_buff + maybe_null,
@@ -7620,6 +7724,12 @@ static void add_not_null_conds(JOIN *join)
   nested outer join and so on until it reaches root_tab
   (root_tab can be 0).
 
+  In other words:
+  add_found_match_trig_cond(tab->first_inner_tab, y, 0) is the way one should 
+  wrap parts of WHERE.  The idea is that the part of WHERE should be only
+  evaluated after we've finished figuring out whether outer joins.
+  ^^^ is the above correct?
+
   @param tab       the first inner table for most nested outer join
   @param cond      the predicate to be guarded (must be set)
   @param root_tab  the first inner table to stop
@@ -7647,6 +7757,12 @@ add_found_match_trig_cond(JOIN_TAB *tab, COND *cond, JOIN_TAB *root_tab)
 }
 
 
+bool TABLE_LIST::is_active_sjm()
+{ 
+  return sj_mat_info && sj_mat_info->is_used;
+}
+
+
 /**
   Fill in outer join related info for the execution plan structure.
 
@@ -7664,6 +7780,12 @@ add_found_match_trig_cond(JOIN_TAB *tab, COND *cond, JOIN_TAB *root_tab)
     corresponding first inner table through the field t0->on_expr_ref.
     Here ti are structures of the JOIN_TAB type.
 
+    In other words, for each join tab, set
+     - first_inner
+     - last_inner
+     - first_upper
+     - on_expr_ref, cond_equal
+
   EXAMPLE. For the query: 
   @code
         SELECT * FROM t1
@@ -7689,20 +7811,33 @@ add_found_match_trig_cond(JOIN_TAB *tab, COND *cond, JOIN_TAB *root_tab)
     has been chosen.
 */
 
-static void
+static bool
 make_outerjoin_info(JOIN *join)
 {
   DBUG_ENTER("make_outerjoin_info");
+  
+  /*
+    Create temp. tables for merged SJ-Materialization nests. We need to do
+    this now, because further code relies on tab->table and
+    tab->table->pos_in_table_list being set.
+  */
+  JOIN_TAB *tab;
+  for (tab= first_linear_tab(join, WITHOUT_CONST_TABLES); 
+       tab; 
+       tab= next_linear_tab(join, tab, WITH_BUSH_ROOTS))
+  {
+    if (tab->bush_children)
+    {
+      if (setup_sj_materialization_part1(tab))
+        DBUG_RETURN(TRUE);
+      tab->table->reginfo.join_tab= tab;
+    }
+  }
+
   for (JOIN_TAB *tab= first_linear_tab(join, WITHOUT_CONST_TABLES); tab; 
-       tab= next_linear_tab(join, tab, WITHOUT_BUSH_ROOTS))
+       tab= next_linear_tab(join, tab, WITH_BUSH_ROOTS))
   {
-    TABLE *table=tab->table;
-    /* 
-      psergey: The following is probably incorrect, fix it when we get 
-      semi+outer joins processing to work: 
-    */
-    if (!table)
-      continue;
+    TABLE *table= tab->table;
     TABLE_LIST *tbl= table->pos_in_table_list;
     TABLE_LIST *embedding= tbl->embedding;
 
@@ -7716,11 +7851,16 @@ make_outerjoin_info(JOIN *join)
       tab->last_inner= tab->first_inner= tab;
       tab->on_expr_ref= &tbl->on_expr;
       tab->cond_equal= tbl->cond_equal;
-      if (embedding)
+      if (embedding && !embedding->is_active_sjm())
         tab->first_upper= embedding->nested_join->first_nested;
     }    
     for ( ; embedding ; embedding= embedding->embedding)
     {
+      if (embedding->is_active_sjm())
+      {
+        /* We're trying to walk out of an SJ-Materialization nest. Don't do this.  */
+        break;
+      }
       /* Ignore sj-nests: */
       if (!(embedding->on_expr && embedding->outer_join))
         continue;
@@ -7752,7 +7892,7 @@ make_outerjoin_info(JOIN *join)
       } 
     }
   }
-  DBUG_VOID_RETURN;
+  DBUG_RETURN(FALSE);
 }
 
 
@@ -7791,7 +7931,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
         join->exec_const_cond=
 	  make_cond_for_table(thd, cond,
                               join->const_table_map,
-                              (table_map) 0, MAX_TABLES, FALSE, FALSE);
+                              (table_map) 0, -1, FALSE, FALSE);
         /* Add conditions added by add_not_null_conds(). */
         for (uint i= 0 ; i < join->const_tables ; i++)
           add_cond_and_fix(thd, &join->exec_const_cond,
@@ -7810,7 +7950,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
         COND *outer_ref_cond= make_cond_for_table(thd, cond, 
                                                   OUTER_REF_TABLE_BIT,
                                                   OUTER_REF_TABLE_BIT,
-                                                  MAX_TABLES, FALSE, FALSE);
+                                                  -1, FALSE, FALSE);
         if (outer_ref_cond)
 	{
           add_cond_and_fix(thd, &outer_ref_cond, join->outer_ref_cond);
@@ -7838,7 +7978,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
       */
       JOIN_TAB *first_inner_tab= tab->first_inner;
 
-      if (tab->table)
+      if (!tab->bush_children)
         current_map= tab->table->map;
       else
         current_map= tab->bush_children->start->emb_sj_nest->sj_inner_tables;
@@ -7980,7 +8120,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
             {
               COND *push_cond= 
               make_cond_for_table(thd, tmp, current_map, current_map,
-                                  MAX_TABLES, FALSE, FALSE);
+                                  -1, FALSE, FALSE);
               if (push_cond)
               {
                 /* Push condition to handler */
@@ -8127,17 +8267,32 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
         the null complemented row.
       */ 
 
-      /* First push down constant conditions from on expressions */
-      for (JOIN_TAB *join_tab= first_linear_tab(join, WITHOUT_CONST_TABLES); 
-           join_tab; 
-           join_tab= next_linear_tab(join, join_tab, WITH_BUSH_ROOTS))
+      /* 
+        First push down constant conditions from ON expressions. 
+         - Each pushed-down condition is wrapped into trigger which is 
+           enabled only for non-NULL-complemented record
+         - The condition is attached to the first_inner_table.
+        
+        With regards to join nests:
+         - if we start at top level, don't walk into nests
+         - if we start inside a nest, stay within that nest.
+      */
+      JOIN_TAB *start_from= tab->bush_root_tab? 
+                               tab->bush_root_tab->bush_children->start : 
+                               join->join_tab + join->const_tables;
+      JOIN_TAB *end_with= tab->bush_root_tab? 
+                               tab->bush_root_tab->bush_children->end : 
+                               join->join_tab + join->top_join_tab_count;
+      for (JOIN_TAB *join_tab= start_from;
+           join_tab != end_with;
+           join_tab++)
       {
         if (*join_tab->on_expr_ref)
         {
           JOIN_TAB *cond_tab= join_tab->first_inner;
           COND *tmp= make_cond_for_table(thd, *join_tab->on_expr_ref,
                                          join->const_table_map,
-                                         (table_map) 0, MAX_TABLES, FALSE, FALSE);
+                                         (table_map) 0, -1, FALSE, FALSE);
           if (!tmp)
             continue;
           tmp= new Item_func_trig_cond(tmp, &cond_tab->not_null_compl);
@@ -8155,9 +8310,14 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
         }       
       }
 
+
       /* Push down non-constant conditions from ON expressions */
-      //JOIN_TAB *first_tab= join->join_tab+join->const_tables;
       JOIN_TAB *last_tab= tab;
+
+      /*
+        while we're inside of an outer join and last_tab is 
+        the last of its tables ... 
+      */
       while (first_inner_tab && first_inner_tab->last_inner == last_tab)
       { 
         /* 
@@ -8168,26 +8328,20 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
 
         table_map used_tables2= (join->const_table_map |
                                  OUTER_REF_TABLE_BIT | RAND_TABLE_BIT);
-        for (JOIN_TAB *tab= first_linear_tab(join, WITHOUT_CONST_TABLES); 
-             tab; 
-             tab= (tab == last_tab)? NULL: next_linear_tab(join, tab, 
-                                                           WITH_BUSH_ROOTS))
+
+        start_from= tab->bush_root_tab? 
+                      tab->bush_root_tab->bush_children->start : 
+                      join->join_tab + join->const_tables;
+        for (JOIN_TAB *tab= start_from; tab <= last_tab; tab++)
         {
-          if (!tab->table)
-          {
-            /* 
-              psergey-todo: this is probably incorrect, fix this when we get
-              correct processing for outer joins + semi joins 
-            */
-            continue;
-          }
+          DBUG_ASSERT(tab->table);
           current_map= tab->table->map;
           used_tables2|= current_map;
           /*
-            psergey: have put the MAX_TABLES below. It's bad, will need to fix it.
+            psergey: have put the -1 below. It's bad, will need to fix it.
           */
           COND *tmp_cond= make_cond_for_table(thd, on_expr, used_tables2,
-                                              current_map, /*(tab - first_tab)*/ MAX_TABLES,
+                                              current_map, /*(tab - first_tab)*/ -1,
 					      FALSE, FALSE);
           if (tab == first_inner_tab && tab->on_precond)
             add_cond_and_fix(thd, &tmp_cond, tab->on_precond);
@@ -8257,6 +8411,7 @@ uint get_next_field_for_derived_key(uchar *arg)
     keyuse->keypart= keypart;
   if (keyuse->key != key)
     keyuse= 0;
+  *((KEYUSE **) arg)= keyuse;
   return fldno;
 }
 
@@ -8267,34 +8422,43 @@ bool generate_derived_keys_for_table(KEYUSE *keyuse, uint count, uint keys)
   TABLE *table= keyuse->table;
   if (table->alloc_keys(keys))
     return TRUE;
-  uint keyno= 0;
+  uint key_count= 0;
   KEYUSE *first_keyuse= keyuse;
-  uint prev_part= (uint) (-1);
+  uint prev_part= keyuse->keypart;
   uint parts= 0;
   uint i= 0;
-  do
+
+  for ( ; i < count && key_count < keys; )
   {
-    keyuse->key= keyno;
-    keyuse->keypart_map= (key_part_map) (1 << parts);
-    keyuse++;
-    if (++i == count || keyuse->used_tables != first_keyuse->used_tables)
+    do
     {
-      if (table->add_tmp_key(keyno, ++parts, 
+      keyuse->key= table->s->keys;
+      keyuse->keypart_map= (key_part_map) (1 << parts);     
+      keyuse++;
+      i++;
+    } 
+    while (i < count && keyuse->used_tables == first_keyuse->used_tables &&
+           keyuse->keypart == prev_part);
+    parts++;
+    if (i < count && keyuse->used_tables == first_keyuse->used_tables)
+    {
+      prev_part= keyuse->keypart;
+    }
+    else
+    {
+      if (table->add_tmp_key(table->s->keys, parts, 
                              get_next_field_for_derived_key, 
                              (uchar *) &first_keyuse,
                              FALSE))
         return TRUE;
-      table->reginfo.join_tab->keys.set_bit(keyno);
+      table->reginfo.join_tab->keys.set_bit(table->s->keys);
       first_keyuse= keyuse;
-      keyno++;
+      key_count++;
       parts= 0;
-    }
-    else if (keyuse->keypart != prev_part)
-    {
-      parts++;
       prev_part= keyuse->keypart;
     }
-  } while (keyno < keys);
+  }             
+
   return FALSE;
 }
    
@@ -8316,12 +8480,23 @@ bool generate_derived_keys(DYNAMIC_ARRAY *keyuse_array)
     TABLE_LIST *derived= NULL;
     if (keyuse->table != prev_table)
       derived= keyuse->table->pos_in_table_list;
-    while (derived && derived->is_materialized_derived() && 
-           keyuse->key == MAX_KEY)
+    while (derived && derived->is_materialized_derived())
     {
       if (keyuse->table != prev_table)
       {
         prev_table= keyuse->table;
+        while (keyuse->table == prev_table && keyuse->key != MAX_KEY)
+	{
+          keyuse++;
+          i++;
+        }
+        if (keyuse->table != prev_table)
+	{
+          keyuse--;
+          i--;
+          derived= NULL;
+          continue;
+        }
         first_table_keyuse= keyuse;
         last_used_tables= keyuse->used_tables;
         count= 0;
@@ -8334,11 +8509,13 @@ bool generate_derived_keys(DYNAMIC_ARRAY *keyuse_array)
       }
       count++;
       keyuse++;
+      i++;
       if (keyuse->table != prev_table)
       {
         if (generate_derived_keys_for_table(first_table_keyuse, count, ++keys))
           return TRUE;
         keyuse--;
+        i--;
 	derived= NULL;
       }
     }
@@ -8369,14 +8546,40 @@ void JOIN::drop_unused_derived_keys()
     TABLE *table=tab->table;
     if (!table)
       continue;
-    if (!table->pos_in_table_list->is_materialized_derived() ||
-        table->max_keys <= 1)
+    if (!table->pos_in_table_list->is_materialized_derived())
       continue;
-    table->use_index(tab->ref.key);
-    tab->ref.key= 0;
+    if (table->max_keys > 1)
+      table->use_index(tab->ref.key);
+    if (table->s->keys && tab->ref.key >= 0)
+      tab->ref.key= 0;
+    tab->keys= (key_map) (table->s->keys ? 1 : 0);
   }
 }
 
+
+/*
+  Evaluate the bitmap of used tables for items from the select list
+*/
+
+inline void JOIN::eval_select_list_used_tables()
+{
+  select_list_used_tables= 0;
+  Item *item;
+  List_iterator_fast<Item> it(fields_list);
+  while ((item= it++))
+  {
+    select_list_used_tables|= item->used_tables();
+  }
+  Item_outer_ref *ref;
+  List_iterator_fast<Item_outer_ref> ref_it(select_lex->inner_refs_list);
+  while ((ref= ref_it++))
+  {
+    item= ref->outer_ref;
+    select_list_used_tables|= item->used_tables();
+  }
+}
+
+
 /*
   Determine {after which table we'll produce ordered set} 
 
@@ -8478,7 +8681,7 @@ void set_join_cache_denial(JOIN_TAB *join_tab)
 void rr_unlock_row(st_join_table *tab)
 {
   READ_RECORD *info= &tab->read_record;
-  info->file->unlock_row();
+  info->table->file->unlock_row();
 }
 
 
@@ -8812,6 +9015,15 @@ uint check_join_cache_usage(JOIN_TAB *tab,
 
   if (tab->use_quick == 2)
     goto no_join_cache;
+  
+  /*
+    Don't use join cache if we're inside a join tab range covered by LooseScan
+    strategy (TODO: LooseScan is very similar to FirstMatch so theoretically it 
+    should be possible to use join buffering in the same way we're using it for
+    multi-table firstmatch ranges).
+  */
+  if (tab->inside_loosescan_range)
+    goto no_join_cache;
 
   if (tab->is_inner_table_of_semi_join_with_first_match() &&
       !join->allowed_semijoin_with_cache)
@@ -8893,6 +9105,9 @@ uint check_join_cache_usage(JOIN_TAB *tab,
   case JT_EQ_REF:
     if (cache_level <=2 || (no_hashed_cache && no_bka_cache))
       goto no_join_cache;
+    if (tab->ref.is_access_triggered())
+      goto no_join_cache;
+      
     if (!tab->is_ref_for_hash_join())
     {
       flags= HA_MRR_NO_NULL_ENDPOINTS | HA_MRR_SINGLE_POINT;
@@ -9131,14 +9346,13 @@ make_join_readinfo(JOIN *join, ulonglong options, uint no_jbuf_after)
   {
     if (tab->bush_children)
     {
-      if (setup_sj_materialization(tab))
+      if (setup_sj_materialization_part2(tab))
         return TRUE;
     }
 
     TABLE *table=tab->table;
     uint jcl= tab->used_join_cache_level;
     tab->read_record.table= table;
-    tab->read_record.file=table->file;
     tab->read_record.unlock_row= rr_unlock_row;
     tab->sorted= sorted;
     sorted= 0;                                  // only first must be sorted
@@ -9184,9 +9398,9 @@ make_join_readinfo(JOIN *join, ulonglong options, uint no_jbuf_after)
         table->key_read=1;
         table->file->extra(HA_EXTRA_KEYREAD);
       }
-      else if (!jcl || jcl > 4) 
+      else if ((!jcl || jcl > 4) && !tab->ref.is_access_triggered())
         push_index_cond(tab, tab->ref.key);
-        break;
+      break;
     case JT_EQ_REF:
       tab->read_record.unlock_row= join_read_key_unlock_row;
       /* fall through */
@@ -9196,7 +9410,7 @@ make_join_readinfo(JOIN *join, ulonglong options, uint no_jbuf_after)
 	table->key_read=1;
 	table->file->extra(HA_EXTRA_KEYREAD);
       }
-      else if (!jcl || jcl > 4) 
+      else if ((!jcl || jcl > 4) && !tab->ref.is_access_triggered())
         push_index_cond(tab, tab->ref.key);
       break;
     case JT_REF_OR_NULL:
@@ -9211,7 +9425,7 @@ make_join_readinfo(JOIN *join, ulonglong options, uint no_jbuf_after)
       if (table->covering_keys.is_set(tab->ref.key) &&
 	  !table->no_keyread)
         table->enable_keyread();
-      else if (!jcl || jcl > 4)
+      else if ((!jcl || jcl > 4) && !tab->ref.is_access_triggered())
         push_index_cond(tab, tab->ref.key);
       break;
     case JT_ALL:
@@ -9420,7 +9634,6 @@ void JOIN_TAB::cleanup()
         table->pos_in_table_list->jtbm_subselect)
     {
       end_read_record(&read_record);
-      //psergey-merge:
       table->pos_in_table_list->jtbm_subselect->cleanup();
       DBUG_VOID_RETURN;
     }
@@ -9465,7 +9678,7 @@ double JOIN_TAB::scan_time()
   else
   {
     found_records= records=table->file->stats.records;
-    read_time= found_records ? found_records: 10;// TODO:fix this stub
+    read_time= found_records ? (double)found_records: 10.0;// TODO:fix this stub
     res= read_time;
   }
   return res;
@@ -9474,6 +9687,8 @@ double JOIN_TAB::scan_time()
 /**
   Initialize the join_tab before reading.
   Currently only derived table/view materialization is done here.
+
+  TODO: consider moving this together with join_tab_execution_startup
 */
 bool JOIN_TAB::preread_init()
 {
@@ -9490,6 +9705,8 @@ bool JOIN_TAB::preread_init()
                                     derived, DT_CREATE | DT_FILL))
       return TRUE;
   preread_init_done= TRUE;
+  if (select && select->quick)
+    select->quick->replace_handler(table->file);
   return FALSE;
 }
 
@@ -9501,6 +9718,7 @@ bool JOIN_TAB::preread_init()
   @param thd             Thread handle
   @param tmp_key         The temporary table key
   @param it              The iterator of items for lookup in the key
+  @param skip            Number of fields from the beginning to skip
 
   @details
   Build TABLE_REF object for lookup in the key 'tmp_key' using items
@@ -9513,9 +9731,11 @@ bool JOIN_TAB::preread_init()
 bool TABLE_REF::tmp_table_index_lookup_init(THD *thd,
                                             KEY *tmp_key,
                                             Item_iterator &it,
-                                            bool value)
+                                            bool value,
+                                            uint skip)
 {
   uint tmp_key_parts= tmp_key->key_parts;
+  uint i;
   DBUG_ENTER("TABLE_REF::tmp_table_index_lookup_init");
 
   key= 0; /* The only temp table index. */
@@ -9536,7 +9756,8 @@ bool TABLE_REF::tmp_table_index_lookup_init(THD *thd,
   uchar *cur_ref_buff= key_buff;
 
   it.open();
-  for (uint i= 0; i < tmp_key_parts; i++, cur_key_part++, ref_key++)
+  for (i= 0; i < skip; i++) it.next();
+  for (i= 0; i < tmp_key_parts; i++, cur_key_part++, ref_key++)
   {
     Item *item= it.next();
     DBUG_ASSERT(item);
@@ -9561,6 +9782,22 @@ bool TABLE_REF::tmp_table_index_lookup_init(THD *thd,
 }
 
 
+/*
+  Check if ref access uses "Full scan on NULL key" (i.e. it actually alternates
+  between ref access and full table scan)
+*/
+
+bool TABLE_REF::is_access_triggered()
+{
+  for (uint i = 0; i < key_parts; i++)
+  {
+    if (cond_guards[i])
+      return TRUE;
+  }
+  return FALSE;
+}
+
+
 /**
   Partially cleanup JOIN after it has executed: close index or rnd read
   (table cursors), free quick selects.
@@ -9612,8 +9849,7 @@ void JOIN::join_free()
     Optimization: if not EXPLAIN and we are done with the JOIN,
     free all tables.
   */
-  bool full= (!(select_lex->uncacheable) &&
-              !thd->lex->describe);
+  bool full= !(select_lex->uncacheable);
   bool can_unlock= full;
   DBUG_ENTER("JOIN::join_free");
 
@@ -10031,7 +10267,9 @@ return_zero_rows(JOIN *join, select_result *result, List<TABLE_LIST> &tables,
     TABLE_LIST *table;
     while ((table= ti++))
       mark_as_null_row(table->table);		// All fields are NULL
-    if (having && having->val_int() == 0)
+    if (having &&
+        !having->walk(&Item::clear_sum_processor, FALSE, NULL) &&
+        having->val_int() == 0)
       send_row=0;
   }
   if (!(result->send_fields(fields,
@@ -10334,7 +10572,7 @@ static bool check_simple_equality(Item *left_item, Item *right_item,
         !((Item_field*)left_item)->get_depended_from() &&
         right_item->const_item() && !right_item->is_expensive())
     {
-      orig_field_item= left_item;
+      orig_field_item= orig_left_item;
       field_item= (Item_field *) left_item;
       const_item= right_item;
     }
@@ -10342,7 +10580,7 @@ static bool check_simple_equality(Item *left_item, Item *right_item,
              !((Item_field*)right_item)->get_depended_from() &&
              left_item->const_item() && !left_item->is_expensive())
     {
-      orig_field_item= right_item;
+      orig_field_item= orig_right_item;
       field_item= (Item_field *) right_item;
       const_item= left_item;
     }
@@ -11464,10 +11702,10 @@ propagate_cond_constants(THD *thd, I_List<COND_CMP> *save_list,
     {
       Item_func_eq *func=(Item_func_eq*) cond;
       Item **args= func->arguments();
-      bool left_const= args[0]->const_item();
-      bool right_const= args[1]->const_item();
+      bool left_const= args[0]->const_item() && !args[0]->is_expensive();
+      bool right_const= args[1]->const_item() && !args[1]->is_expensive();
       if (!(left_const && right_const) &&
-          args[0]->result_type() == args[1]->result_type())
+          args[0]->cmp_type() == args[1]->cmp_type())
       {
 	if (right_const)
 	{
@@ -11778,6 +12016,18 @@ simplify_joins(JOIN *join, List<TABLE_LIST> *join_list, COND *conds, bool top,
          leave it intact (otherwise it is flattened)
        */
       join->select_lex->sj_nests.push_back(table);
+
+      /* 
+        Also, walk through semi-join children and mark those that are now
+        top-level
+      */
+      TABLE_LIST *tbl;
+      List_iterator<TABLE_LIST> it(nested_join->join_list);
+      while ((tbl= it++))
+      {
+        if (!tbl->on_expr && tbl->table)
+          tbl->table->maybe_null= FALSE;
+      }
     }
     else if (nested_join && !table->on_expr)
     {
@@ -11888,8 +12138,8 @@ static uint reset_nj_counters(JOIN *join, List<TABLE_LIST> *join_list)
       if (!nested_join->n_tables)
         is_eliminated_nest= TRUE;
     }
-    if ((!table->table && !is_eliminated_nest) || 
-        (table->table && (table->table->map & ~join->eliminated_tables)))
+    if ((table->nested_join && !is_eliminated_nest) || 
+        (!table->nested_join && (table->table->map & ~join->eliminated_tables)))
       n++;
   }
   DBUG_RETURN(n);
@@ -12771,8 +13021,9 @@ Field *create_tmp_field(THD *thd, TABLE *table,Item *item, Item::Type type,
       If item have to be able to store NULLs but underlaid field can't do it,
       create_tmp_field_from_field() can't be used for tmp field creation.
     */
-    if (((field->maybe_null && field->in_rollup) ||
-         (orig_item && orig_item->maybe_null)) && /* for outer joined views/dt*/        
+    if (((field->maybe_null && field->in_rollup) ||      
+	(thd->create_tmp_table_for_derived  &&    /* for mat. view/dt */
+	 orig_item && orig_item->maybe_null)) &&         
         !field->field->maybe_null())
     {
       bool save_maybe_null= FALSE;
@@ -12944,7 +13195,7 @@ void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps)
 */
 
 TABLE *
-create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
+create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields,
 		 ORDER *group, bool distinct, bool save_sum_fields,
 		 ulonglong select_options, ha_rows rows_limit,
                  char *table_alias, bool do_not_open)
@@ -12962,7 +13213,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
   bool  using_unique_constraint= 0;
   bool  use_packed_rows= 0;
   bool  not_all_columns= !(select_options & TMP_TABLE_ALL_COLUMNS);
-  char  *tmpname,path[FN_REFLEN], tmp_table_name[50];
+  char  *tmpname,path[FN_REFLEN];
   uchar	*pos, *group_buff, *bitmaps;
   uchar *null_flags;
   Field **reg_field, **from_field, **default_field;
@@ -12993,12 +13244,12 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
     temp_pool_slot = bitmap_lock_set_next(&temp_pool);
 
   if (temp_pool_slot != MY_BIT_NONE) // we got a slot
-    sprintf(tmp_table_name, "%s_%lx_%i", tmp_file_prefix,
+    sprintf(path, "%s_%lx_%i", tmp_file_prefix,
             current_pid, temp_pool_slot);
   else
   {
     /* if we run out of slots or we are not using tempool */
-    sprintf(tmp_table_name, "%s%lx_%lx_%x", tmp_file_prefix,current_pid,
+    sprintf(path, "%s%lx_%lx_%x", tmp_file_prefix,current_pid,
             thd->thread_id, thd->tmp_table++);
   }
 
@@ -13006,7 +13257,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
     No need to change table name to lower case as we are only creating
     MyISAM, Aria or HEAP tables here
   */
-  fn_format(path, tmp_table_name, mysql_tmpdir, "",
+  fn_format(path, path, mysql_tmpdir, "",
             MY_REPLACE_EXT|MY_UNPACK_FILENAME);
 
   if (group)
@@ -13059,7 +13310,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
                         sizeof(*key_part_info)*(param->group_parts+1),
                         &param->start_recinfo,
                         sizeof(*param->recinfo)*(field_count*2+4),
-                        &tmpname, (uint) strlen(tmp_table_name)+1,
+                        &tmpname, (uint) strlen(path)+1,
                         &group_buff, (group && ! using_unique_constraint ?
                                       param->group_length : 0),
                         &bitmaps, bitmap_buffer_size(field_count)*4,
@@ -13078,7 +13329,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
     DBUG_RETURN(NULL);				/* purecov: inspected */
   }
   param->items_to_copy= copy_func;
-  strmov(tmpname, tmp_table_name);
+  strmov(tmpname, path);
   /* make table according to fields */
 
   bzero((char*) table,sizeof(*table));
@@ -13310,6 +13561,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
   *reg_field= 0;
   *blob_field= 0;				// End marker
   share->fields= field_count;
+  share->column_bitmap_size= bitmap_buffer_size(share->fields);
 
   /* If result table is small; use a heap */
   /* future: storage engine selection can be made dynamic? */
@@ -13535,6 +13787,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
       bool maybe_null=(*cur_group->item)->maybe_null;
       key_part_info->null_bit=0;
       key_part_info->field=  field;
+      key_part_info->fieldnr= field->field_index + 1;
       if (cur_group == group)
         field->key_start.set_bit(0);
       key_part_info->offset= field->offset(table->record[0]);
@@ -13545,15 +13798,30 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
 	 (ha_base_keytype) key_part_info->type == HA_KEYTYPE_VARTEXT1 ||
 	 (ha_base_keytype) key_part_info->type == HA_KEYTYPE_VARTEXT2) ?
 	0 : FIELDFLAG_BINARY;
+
       if (!using_unique_constraint)
       {
 	cur_group->buff=(char*) group_buff;
+
+        if (maybe_null && !field->null_bit)
+        {
+          /*
+            This can only happen in the unusual case where an outer join
+            table was found to be not-nullable by the optimizer and we
+            the item can't really be null.
+            We solve this by marking the item as !maybe_null to ensure
+            that the key,field and item definition match.
+          */
+          (*cur_group->item)->maybe_null= maybe_null= 0;
+        }
+
 	if (!(cur_group->field= field->new_key_field(thd->mem_root,table,
                                                      group_buff +
                                                      test(maybe_null),
                                                      field->null_ptr,
                                                      field->null_bit)))
 	  goto err; /* purecov: inspected */
+
 	if (maybe_null)
 	{
 	  /*
@@ -13575,6 +13843,12 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
       }
       keyinfo->key_length+=  key_part_info->length;
     }
+    /*
+      Ensure we didn't overrun the group buffer. The < is only true when
+      some maybe_null fields was changed to be not null fields.
+    */
+    DBUG_ASSERT(using_unique_constraint ||
+                group_buff <= param->group_buff + param->group_length);
   }
 
   if (distinct && field_count != param->hidden_field_count)
@@ -13637,6 +13911,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
       key_part_info->field->init(table);
       key_part_info->key_type=FIELDFLAG_BINARY;
       key_part_info->type=    HA_KEYTYPE_BINARY;
+      key_part_info->fieldnr= key_part_info->field->field_index + 1;
       key_part_info++;
     }
     /* Create a distinct key over the columns we are going to return */
@@ -13654,6 +13929,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
 
       key_part_info->offset=   (*reg_field)->offset(table->record[0]);
       key_part_info->length=   (uint16) (*reg_field)->pack_length();
+      key_part_info->fieldnr= (*reg_field)->field_index + 1;
       /* TODO:
         The below method of computing the key format length of the
         key part is a copy/paste from opt_range.cc, and table.cc.
@@ -13917,6 +14193,13 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo,
 	keyinfo->key_parts > table->file->max_key_parts() ||
 	share->uniques)
     {
+      if (!share->uniques && !(keyinfo->flags & HA_NOSAME))
+      {
+        my_error(ER_INTERNAL_ERROR, MYF(0),
+                 "Using too big key for internal temp tables");
+        DBUG_RETURN(1);
+      }
+
       /* Can't create a key; Make a unique constraint instead of a key */
       share->keys=    0;
       share->uniques= 1;
@@ -13935,9 +14218,9 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo,
     }
     else
     {
-      /* Create an unique key */
+      /* Create a key */
       bzero((char*) &keydef,sizeof(keydef));
-      keydef.flag=HA_NOSAME;
+      keydef.flag= keyinfo->flags & HA_NOSAME;
       keydef.keysegs=  keyinfo->key_parts;
       keydef.seg= seg;
     }
@@ -14113,7 +14396,8 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo,
     {
       /* Create an unique key */
       bzero((char*) &keydef,sizeof(keydef));
-      keydef.flag=HA_NOSAME | HA_BINARY_PACK_KEY | HA_PACK_KEY;
+      keydef.flag= ((keyinfo->flags & HA_NOSAME) | HA_BINARY_PACK_KEY |
+                    HA_PACK_KEY);
       keydef.keysegs=  keyinfo->key_parts;
       keydef.seg= seg;
     }
@@ -14297,7 +14581,7 @@ create_internal_tmp_table_from_heap2(THD *thd, TABLE *table,
 
   /* remove heap table and change to use myisam table */
   (void) table->file->ha_rnd_end();
-  (void) table->file->close();                  // This deletes the table !
+  (void) table->file->ha_close();          // This deletes the table !
   delete table->file;
   table->file=0;
   plugin_unlock(0, table->s->db_plugin);
@@ -14318,7 +14602,7 @@ create_internal_tmp_table_from_heap2(THD *thd, TABLE *table,
   table->file->print_error(write_err, MYF(0));
 err_killed:
   (void) table->file->ha_rnd_end();
-  (void) new_table.file->close();
+  (void) new_table.file->ha_close();
  err1:
   new_table.file->ha_delete_table(new_table.s->table_name.str);
  err2:
@@ -14518,14 +14802,6 @@ do_select(JOIN *join,List<Item> *fields,TABLE *table,Procedure *procedure)
       {
         List<Item> *columns_list= (procedure ? &join->procedure_fields_list :
                                    fields);
-        /*
-          With implicit grouping all fields of special row produced for an
-          empty result are NULL. See return_zero_rows() for the same behavior.
-        */
-        TABLE_LIST *table;
-        List_iterator_fast<TABLE_LIST> li(join->select_lex->leaf_tables);
-        while ((table= li++))
-          mark_as_null_row(table->table);
         rc= join->result->send_data(*columns_list) > 0;
       }
     }
@@ -14642,10 +14918,12 @@ sub_select_cache(JOIN *join, JOIN_TAB *join_tab, bool end_of_records)
 {
   enum_nested_loop_state rc;
   JOIN_CACHE *cache= join_tab->cache;
-
   DBUG_ENTER("sub_select_cache");
 
-  /* This function cannot be called if join_tab has no associated join buffer */
+  /*
+    This function cannot be called if join_tab has no associated join
+    buffer
+  */
   DBUG_ASSERT(cache != NULL);
 
   join_tab->cache->reset_join(join);
@@ -14980,7 +15258,6 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab,
       condition is true => a match is found.
     */
     bool found= 1;
-    bool use_not_exists_opt= 0;
     while (join_tab->first_unmatched && found)
     {
       /*
@@ -14996,8 +15273,6 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab,
       first_unmatched->found= 1;
       for (JOIN_TAB *tab= first_unmatched; tab <= join_tab; tab++)
       {
-        if (tab->table->reginfo.not_exists_optimize)
-          use_not_exists_opt= 1;
         /* Check all predicates that has just been activated. */
         /*
           Actually all predicates non-guarded by first_unmatched->found
@@ -15008,7 +15283,11 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab,
         {
           /* The condition attached to table tab is false */
           if (tab == join_tab)
+          {
             found= 0;
+            if (tab->table->reginfo.not_exists_optimize)
+              DBUG_RETURN(NESTED_LOOP_NO_MORE_ROWS);
+          }            
           else
           {
             /*
@@ -15016,7 +15295,10 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab,
               not to the last table of the current nest level.
             */
             join->return_tab= tab;
-            DBUG_RETURN(NESTED_LOOP_OK);
+            if (tab->table->reginfo.not_exists_optimize)
+              DBUG_RETURN(NESTED_LOOP_NO_MORE_ROWS);
+            else
+              DBUG_RETURN(NESTED_LOOP_OK);
           }
         }
       }
@@ -15030,8 +15312,6 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab,
       join_tab->first_unmatched= first_unmatched;
     }
 
-    if (use_not_exists_opt)
-      DBUG_RETURN(NESTED_LOOP_NO_MORE_ROWS);
     JOIN_TAB *return_tab= join->return_tab;
     join_tab->found_match= TRUE;
 
@@ -15158,6 +15438,26 @@ evaluate_null_complemented_join_record(JOIN *join, JOIN_TAB *join_tab)
   /*
     The row complemented by nulls satisfies all conditions
     attached to inner tables.
+  */
+  if (join_tab->check_weed_out_table)
+  {
+    int res= do_sj_dups_weedout(join->thd, join_tab->check_weed_out_table);
+    if (res == -1)
+      return NESTED_LOOP_ERROR;
+    else if (res == 1)
+      return NESTED_LOOP_OK;
+  }
+  else if (join_tab->do_firstmatch)
+  {
+    /* 
+      We should return to the join_tab->do_firstmatch after we have 
+      enumerated all the suffixes for current prefix row combination
+    */
+    if (join_tab->do_firstmatch < join->return_tab)
+      join->return_tab= join_tab->do_firstmatch;
+  }
+
+  /*
     Send the row complemented by nulls to be joined with the
     remaining tables.
   */
@@ -15440,7 +15740,7 @@ int join_read_key2(THD *thd, JOIN_TAB *tab, TABLE *table, TABLE_REF *table_ref)
     */
     if (tab && tab->ref.has_record && tab->ref.use_count == 0)
     {
-      tab->read_record.file->unlock_row();
+      tab->read_record.table->file->unlock_row();
       table_ref->has_record= FALSE;
     }
     error=table->file->ha_index_read_map(table->record[0],
@@ -15626,7 +15926,7 @@ join_init_quick_read_record(JOIN_TAB *tab)
 int init_read_record_seq(JOIN_TAB *tab)
 {
   tab->read_record.read_record= rr_sequential;
-  if (tab->read_record.file->ha_rnd_init_with_error(1))
+  if (tab->read_record.table->file->ha_rnd_init_with_error(1))
     return 1;
   return (*tab->read_record.read_record)(&tab->read_record);
 }
@@ -15693,7 +15993,6 @@ join_read_first(JOIN_TAB *tab)
   tab->table->status=0;
   tab->read_record.read_record=join_read_next;
   tab->read_record.table=table;
-  tab->read_record.file=table->file;
   tab->read_record.index=tab->index;
   tab->read_record.record=table->record[0];
   if (!table->file->inited)
@@ -15714,7 +16013,7 @@ static int
 join_read_next(READ_RECORD *info)
 {
   int error;
-  if ((error= info->file->ha_index_next(info->record)))
+  if ((error= info->table->file->ha_index_next(info->record)))
     return report_error(info->table, error);
 
   return 0;
@@ -15732,7 +16031,6 @@ join_read_last(JOIN_TAB *tab)
   tab->table->status=0;
   tab->read_record.read_record=join_read_prev;
   tab->read_record.table=table;
-  tab->read_record.file=table->file;
   tab->read_record.index=tab->index;
   tab->read_record.record=table->record[0];
   if (!table->file->inited)
@@ -15750,7 +16048,7 @@ static int
 join_read_prev(READ_RECORD *info)
 {
   int error;
-  if ((error= info->file->ha_index_prev(info->record)))
+  if ((error= info->table->file->ha_index_prev(info->record)))
     return report_error(info->table, error);
   return 0;
 }
@@ -15784,7 +16082,7 @@ static int
 join_ft_read_next(READ_RECORD *info)
 {
   int error;
-  if ((error= info->file->ha_ft_read(info->table->record[0])))
+  if ((error= info->table->file->ha_ft_read(info->table->record[0])))
     return report_error(info->table, error);
   return 0;
 }
@@ -16158,8 +16456,8 @@ end_update(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
   {						/* Update old record */
     restore_record(table,record[1]);
     update_tmptable_sum_func(join->sum_funcs,table);
-    if ((error= table->file->ha_update_row(table->record[1],
-                                           table->record[0])))
+    if ((error= table->file->ha_update_tmp_row(table->record[1],
+                                               table->record[0])))
     {
       table->file->print_error(error,MYF(0));	/* purecov: inspected */
       DBUG_RETURN(NESTED_LOOP_ERROR);            /* purecov: inspected */
@@ -16242,8 +16540,8 @@ end_unique_update(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
     }
     restore_record(table,record[1]);
     update_tmptable_sum_func(join->sum_funcs,table);
-    if ((error= table->file->ha_update_row(table->record[1],
-                                           table->record[0])))
+    if ((error= table->file->ha_update_tmp_row(table->record[1],
+                                               table->record[0])))
     {
       table->file->print_error(error,MYF(0));	/* purecov: inspected */
       DBUG_RETURN(NESTED_LOOP_ERROR);            /* purecov: inspected */
@@ -16376,16 +16674,12 @@ bool test_if_ref(Item *root_cond, Item_field *left_item,Item *right_item)
       between ref access and full table scan), then no equality can be
       guaranteed to be true.
     */
-    for (uint i = 0; i < join_tab->ref.key_parts; i++)
-    {
-      if (join_tab->ref.cond_guards[i])
-      {
-        return FALSE;
-      }
-    }
+    if (join_tab->ref.is_access_triggered())
+      return FALSE;
 
     Item *ref_item=part_of_refkey(field->table,field);
-    if (ref_item && ref_item->eq(right_item,1))
+    if (ref_item && (ref_item->eq(right_item,1) || 
+		     ref_item->real_item()->eq(right_item,1)))
     {
       right_item= right_item->real_item();
       if (right_item->type() == Item::FIELD_ITEM)
@@ -16463,7 +16757,7 @@ bool test_if_ref(Item *root_cond, Item_field *left_item,Item *right_item)
 static Item *
 make_cond_for_table(THD *thd, Item *cond, table_map tables,
                     table_map used_table,
-                    uint join_tab_idx_arg,
+                    int join_tab_idx_arg,
                     bool exclude_expensive_cond __attribute__((unused)),
 		    bool retain_ref_cond)
 {
@@ -16477,7 +16771,7 @@ make_cond_for_table(THD *thd, Item *cond, table_map tables,
 static Item *
 make_cond_for_table_from_pred(THD *thd, Item *root_cond, Item *cond,
                               table_map tables, table_map used_table,
-                              uint join_tab_idx_arg,
+                              int join_tab_idx_arg,
                               bool exclude_expensive_cond __attribute__
                               ((unused)),
                               bool retain_ref_cond)
@@ -17653,8 +17947,10 @@ check_reverse_order:
           condition are not relevant anymore
         */
         if (tab->select && tab->select->pre_idx_push_select_cond)
+	{
           tab->set_cond(tab->select->pre_idx_push_select_cond);
-
+           tab->table->file->cancel_pushed_idx_cond();
+        }
         /*
           TODO: update the number of records in join->best_positions[tablenr]
         */
@@ -17713,14 +18009,11 @@ skipped_filesort:
     delete save_quick;
     save_quick= NULL;
   }
-          /*
-            orig_cond is a part of pre_idx_push_cond,
-            no need to restore it.
-          */
-          orig_cond= 0;
-          orig_cond_saved= false;
   if (orig_cond_saved && !changed_key)
     tab->set_cond(orig_cond);
+  if (!no_changes && changed_key && table->file->pushed_idx_cond)
+    table->file->cancel_pushed_idx_cond();
+
   DBUG_RETURN(1);
 
 use_filesort:
@@ -17732,6 +18025,7 @@ use_filesort:
   }
   if (orig_cond_saved)
     tab->set_cond(orig_cond);
+
   DBUG_RETURN(0);
 }
 
@@ -19659,7 +19953,6 @@ static bool add_ref_to_table_cond(THD *thd, JOIN_TAB *join_tab)
   }
   if (thd->is_fatal_error)
     DBUG_RETURN(TRUE);
-
   if (!cond->fixed)
   {
     Item *tmp_item= (Item*) cond;
@@ -19668,11 +19961,21 @@ static bool add_ref_to_table_cond(THD *thd, JOIN_TAB *join_tab)
   }
   if (join_tab->select)
   {
+    Item *cond_copy;
+    UNINIT_VAR(cond_copy); // used when pre_idx_push_select_cond!=NULL
+    if (join_tab->select->pre_idx_push_select_cond)
+      cond_copy= cond->copy_andor_structure(thd);
     if (join_tab->select->cond)
       error=(int) cond->add(join_tab->select->cond);
     join_tab->select->cond= cond;
     if (join_tab->select->pre_idx_push_select_cond)
-      join_tab->select->pre_idx_push_select_cond= cond;
+    {
+      Item *new_cond= and_conds(cond_copy, join_tab->select->pre_idx_push_select_cond);
+      if (!new_cond->fixed && new_cond->fix_fields(thd, &new_cond))
+        error= 1;
+      join_tab->pre_idx_push_select_cond=
+        join_tab->select->pre_idx_push_select_cond= new_cond;
+    }
     join_tab->set_select_cond(cond, __LINE__);
   }
   else if ((join_tab->select= make_select(join_tab->table, 0, 0, cond, 0,
@@ -20467,7 +20770,7 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order,
                  keylen_str_buf);
         tmp3.append(keylen_str_buf, length, cs);
       }         
-      if (tab->select && tab->select->quick)
+      if (tab->type != JT_CONST && tab->select && tab->select->quick)
         tab->select->quick->add_keys_and_lengths(&tmp2, &tmp3);
       if (key_info || (tab->select && tab->select->quick))
       {
@@ -20538,8 +20841,7 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order,
             examined_rows= tab->limit;
           else
           {
-            if (!tab->table->pos_in_table_list ||
-                tab->table->is_filled_at_execution()) // temporary, is_filled_at_execution
+            if (tab->table->is_filled_at_execution())
             {
               examined_rows= tab->records;
             }
@@ -20714,7 +21016,8 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order,
 	  need_order=0;
 	  extra.append(STRING_WITH_LEN("; Using filesort"));
 	}
-	if (distinct & test_all_bits(used_tables,thd->used_tables))
+	if (distinct & test_all_bits(used_tables,
+                                     join->select_list_used_tables))
 	  extra.append(STRING_WITH_LEN("; Distinct"));
         if (tab->loosescan_match_tab)
         {
@@ -21321,6 +21624,28 @@ void JOIN::save_query_plan(Join_plan_state *save_to)
   memcpy((uchar*) save_to->best_positions, (uchar*) best_positions,
          sizeof(POSITION) * (table_count + 1));
   memset(best_positions, 0, sizeof(POSITION) * (table_count + 1));
+  
+  /* Save SJM nests */
+  List_iterator<TABLE_LIST> it(select_lex->sj_nests);
+  TABLE_LIST *tlist;
+  SJ_MATERIALIZATION_INFO **p_info= save_to->sj_mat_info;
+  while ((tlist= it++))
+  {
+    *(p_info++)= tlist->sj_mat_info;
+  }
+}
+
+
+/**
+  Reset a query execution plan so that it can be reoptimized in-place.
+*/
+void JOIN::reset_query_plan()
+{
+  for (uint i= 0; i < table_count; i++)
+  {
+    join_tab[i].keyuse= NULL;
+    join_tab[i].checked_keys.clear_all();
+  }
 }
 
 
@@ -21348,6 +21673,14 @@ void JOIN::restore_query_plan(Join_plan_state *restore_from)
   }
   memcpy((uchar*) best_positions, (uchar*) restore_from->best_positions,
          sizeof(POSITION) * (table_count + 1));
+  /* Restore SJM nests */
+  List_iterator<TABLE_LIST> it(select_lex->sj_nests);
+  TABLE_LIST *tlist;
+  SJ_MATERIALIZATION_INFO **p_info= restore_from->sj_mat_info;
+  while ((tlist= it++))
+  {
+    tlist->sj_mat_info= *(p_info++);
+  }
 }
 
 
@@ -21357,7 +21690,8 @@ void JOIN::restore_query_plan(Join_plan_state *restore_from)
 
   @param added_where  An extra conjunct to the WHERE clause to reoptimize with
   @param join_tables  The set of tables to reoptimize
-  @param save_to      If != NULL, save here the state of the current query plan
+  @param save_to      If != NULL, save here the state of the current query plan,
+                      otherwise reuse the existing query plan structures.
 
   @notes
   Given a query plan that was already optimized taking into account some WHERE
@@ -21401,6 +21735,8 @@ JOIN::reoptimize(Item *added_where, table_map join_tables,
 
   if (save_to)
     save_query_plan(save_to);
+  else
+    reset_query_plan();
 
   if (!keyuse.buffer &&
       my_init_dynamic_array(&keyuse, sizeof(KEYUSE), 20, 64))
@@ -21434,6 +21770,9 @@ JOIN::reoptimize(Item *added_where, table_map join_tables,
     return REOPT_ERROR;
   optimize_keyuse(this, &keyuse);
 
+  if (optimize_semijoin_nests(this, join_tables))
+    return REOPT_ERROR;
+
   /* Re-run the join optimizer to compute a new query plan. */
   if (choose_plan(this, join_tables))
     return REOPT_ERROR;
diff --git a/sql/sql_select.h b/sql/sql_select.h
index abb9a98030e..5476ef9b46c 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -135,7 +135,8 @@ typedef struct st_table_ref
   bool          disable_cache;
 
   bool tmp_table_index_lookup_init(THD *thd, KEY *tmp_key, Item_iterator &it,
-                                   bool value);
+                                   bool value, uint skip= 0);
+  bool is_access_triggered();
 } TABLE_REF;
 
 
@@ -204,7 +205,13 @@ typedef struct st_join_table {
     NULL means no index condition pushdown was performed.
   */
   Item          *pre_idx_push_select_cond;
-  Item	       **on_expr_ref;   /**< pointer to the associated on expression   */
+  /*
+    Pointer to the associated ON expression. on_expr_ref=!NULL except for
+    degenerate joins. 
+    *on_expr_ref!=NULL for tables that are first inner tables within an outer
+    join.
+  */
+  Item	       **on_expr_ref;
   COND_EQUAL    *cond_equal;    /**< multiple equalities for the on expression */
   st_join_table *first_inner;   /**< first inner table for including outerjoin */
   bool           found;         /**< true after all matches or null complement */
@@ -282,7 +289,6 @@ typedef struct st_join_table {
   ulong         max_used_fieldlength;
   uint          used_blobs;
   uint          used_null_fields;
-  uint          used_rowid_fields;
   uint          used_uneven_bit_fields;
   enum join_type type;
   bool		cached_eq_ref_table,eq_ref_table,not_used_in_distinct;
@@ -341,6 +347,9 @@ typedef struct st_join_table {
      NULL - Not doing a loose scan on this join tab.
   */
   struct st_join_table *loosescan_match_tab;
+  
+  /* TRUE <=> we are inside LooseScan range */
+  bool inside_loosescan_range;
 
   /* Buffer to save index tuple to be able to skip duplicates */
   uchar *loosescan_buf;
@@ -378,15 +387,6 @@ typedef struct st_join_table {
             (select->quick->get_type() ==
              QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX));
   }
-  bool check_rowid_field()
-  {
-    if (keep_current_rowid && !used_rowid_fields)
-    {
-      used_rowid_fields= 1;
-      used_fieldlength+= table->file->ref_length;
-    }
-    return test(used_rowid_fields);
-  }
   bool is_inner_table_of_semi_join_with_first_match()
   {
     return first_sj_inner_tab != NULL;
@@ -478,6 +478,8 @@ typedef struct st_join_table {
   }
   double scan_time();
   bool preread_init();
+
+  bool is_sjm_nest() { return test(bush_children); }
 } JOIN_TAB;
 
 
@@ -665,6 +667,7 @@ protected:
     KEYUSE *join_tab_keyuse[MAX_TABLES];
     /* Copies of JOIN_TAB::checked_keys for each JOIN_TAB. */
     key_map join_tab_checked_keys[MAX_TABLES];
+    SJ_MATERIALIZATION_INFO *sj_mat_info[MAX_TABLES];
   public:
     Join_plan_state()
     {   
@@ -690,6 +693,7 @@ protected:
   enum_reopt_result reoptimize(Item *added_where, table_map join_tables,
                                Join_plan_state *save_to);
   void save_query_plan(Join_plan_state *save_to);
+  void reset_query_plan();
   void restore_query_plan(Join_plan_state *restore_from);
   /* Choose a subquery plan for a table-less subquery. */
   bool choose_tableless_subquery_plan();
@@ -759,6 +763,8 @@ public:
      make_join_statistics)
   */
   table_map outer_join;
+  /* Bitmap of tables used in the select list items */
+  table_map select_list_used_tables;
   ha_rows  send_records,found_records,examined_rows,row_limit, select_limit;
   /**
     Used to fetch no more than given amount of rows per one
@@ -956,11 +962,6 @@ public:
   bool optimized; ///< flag to avoid double optimization in EXPLAIN
   bool initialized; ///< flag to avoid double init_execution calls
 
-  /* 
-    Subqueries that will need to be converted to semi-join nests, including
-    those converted to jtbm nests. The list is emptied when conversion is done.
-  */
-  Array<Item_in_subselect> sj_subselects;
   /*
     Additional WHERE and HAVING predicates to be considered for IN=>EXISTS
     subquery transformation of a JOIN object.
@@ -990,7 +991,7 @@ public:
 
   JOIN(THD *thd_arg, List<Item> &fields_arg, ulonglong select_options_arg,
        select_result *result_arg)
-    :fields_list(fields_arg), sj_subselects(thd_arg->mem_root, 4)
+    :fields_list(fields_arg)
   {
     init(thd_arg, fields_arg, select_options_arg, result_arg);
   }
@@ -1126,6 +1127,7 @@ public:
     return (table_map(1) << table_count) - 1;
   }
   void drop_unused_derived_keys();
+  inline void eval_select_list_used_tables();
   /* 
     Return the table for which an index scan can be used to satisfy 
     the sort order needed by the ORDER BY/(implicit) GROUP BY clause 
@@ -1147,7 +1149,7 @@ public:
            max_allowed_join_cache_level > JOIN_CACHE_HASHED_BIT;
   }
   bool choose_subquery_plan(table_map join_tables);
-  void get_partial_cost_and_fanout(uint end_tab_idx,
+  void get_partial_cost_and_fanout(int end_tab_idx,
                                    table_map filter_map,
                                    double *read_time_arg, 
                                    double *record_count_arg);
diff --git a/sql/sql_show.cc b/sql/sql_show.cc
index 0cf7a9b1dc2..c7cd692000f 100644
--- a/sql/sql_show.cc
+++ b/sql/sql_show.cc
@@ -22,8 +22,8 @@
 #include "create_options.h"
 #include "sql_show.h"
 #include "repl_failsafe.h"
-#include "sp.h"
 #include "sp_head.h"
+#include "sp.h"
 #include "sql_trigger.h"
 #include "authors.h"
 #include "contributors.h"
@@ -1274,7 +1274,7 @@ int store_create_info(THD *thd, TABLE_LIST *table_list, String *packet,
   handler *file= table->file;
   TABLE_SHARE *share= table->s;
   HA_CREATE_INFO create_info;
-  bool show_table_options= FALSE;
+  bool show_table_options __attribute__ ((unused))= FALSE;
   bool foreign_db_mode=  (thd->variables.sql_mode & (MODE_POSTGRESQL |
                                                      MODE_ORACLE |
                                                      MODE_MSSQL |
@@ -1507,7 +1507,9 @@ int store_create_info(THD *thd, TABLE_LIST *table_list, String *packet,
   packet->append(STRING_WITH_LEN("\n)"));
   if (!(thd->variables.sql_mode & MODE_NO_TABLE_OPTIONS) && !foreign_db_mode)
   {
+#ifdef WITH_PARTITION_STORAGE_ENGINE
     show_table_options= TRUE;
+#endif
     /*
       Get possible table space definitions and append them
       to the CREATE TABLE statement
@@ -1870,6 +1872,7 @@ public:
   uint   command;
   const char *user,*host,*db,*proc_info,*state_info;
   char *query;
+  double progress;
 };
 
 #ifdef HAVE_EXPLICIT_TEMPLATE_INSTANTIATION
@@ -1898,6 +1901,11 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose)
   field->maybe_null=1;
   field_list.push_back(field=new Item_empty_string("Info",max_query_length));
   field->maybe_null=1;
+  if (!thd->variables.old_mode)
+  {
+    field_list.push_back(field= new Item_float("Progress", 0.0, 3, 7));
+    field->maybe_null= 0;
+  }
   if (protocol->send_fields(&field_list,
                             Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF))
     DBUG_VOID_RETURN;
@@ -1937,7 +1945,8 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose)
         pthread_mutex_lock(&tmp->LOCK_thd_data);
         if ((mysys_var= tmp->mysys_var))
           pthread_mutex_lock(&mysys_var->mutex);
-        thd_info->proc_info= (char*) (tmp->killed == THD::KILL_CONNECTION? "Killed" : 0);
+        thd_info->proc_info= (char*) (tmp->killed >= KILL_QUERY ?
+                                      "Killed" : 0);
 #ifndef EMBEDDED_LIBRARY
         thd_info->state_info= (char*) (tmp->net.reading_or_writing ?
                                        (tmp->net.reading_or_writing == 2 ?
@@ -1957,6 +1966,8 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose)
 
         thd_info->start_time= tmp->start_time;
         thd_info->query=0;
+        thd_info->progress= 0.0;
+
         /* Lock THD mutex that protects its data when looking at it. */
         pthread_mutex_lock(&tmp->LOCK_thd_data);
         if (tmp->query())
@@ -1964,6 +1975,20 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose)
           uint length= min(max_query_length, tmp->query_length());
           thd_info->query= (char*) thd->strmake(tmp->query(),length);
         }
+
+        /*
+          Progress report. We need to do this under a lock to ensure that all
+          is from the same stage.
+        */
+        if (tmp->progress.max_counter)
+        {
+          uint max_stage= max(tmp->progress.max_stage, 1);
+          thd_info->progress= (((tmp->progress.stage / (double) max_stage) +
+                                ((tmp->progress.counter /
+                                  (double) tmp->progress.max_counter) /
+                                 (double) max_stage)) *
+                               100.0);
+        }
         pthread_mutex_unlock(&tmp->LOCK_thd_data);
         thread_infos.append(thd_info);
       }
@@ -1973,6 +1998,9 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose)
 
   thread_info *thd_info;
   time_t now= my_time(0);
+  char buff[20];                                // For progress
+  String store_buffer(buff, sizeof(buff), system_charset_info);
+
   while ((thd_info=thread_infos.get()))
   {
     protocol->prepare_for_resend();
@@ -1990,6 +2018,8 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose)
       protocol->store_null();
     protocol->store(thd_info->state_info, system_charset_info);
     protocol->store(thd_info->query, system_charset_info);
+    if (!thd->variables.old_mode)
+      protocol->store(thd_info->progress, 3, &store_buffer);
     if (protocol->write())
       break; /* purecov: inspected */
   }
@@ -2020,6 +2050,7 @@ int fill_schema_processlist(THD* thd, TABLE_LIST* tables, COND* cond)
       Security_context *tmp_sctx= tmp->security_ctx;
       struct st_my_thread_var *mysys_var;
       const char *val;
+      ulonglong max_counter;
 
       if ((!tmp->vio_ok() && !tmp->system_thread) ||
           (user && (!tmp_sctx->user || strcmp(tmp_sctx->user, user))))
@@ -2054,7 +2085,8 @@ int fill_schema_processlist(THD* thd, TABLE_LIST* tables, COND* cond)
       if ((mysys_var= tmp->mysys_var))
         pthread_mutex_lock(&mysys_var->mutex);
       /* COMMAND */
-      if ((val= (char *) (tmp->killed == THD::KILL_CONNECTION? "Killed" : 0)))
+      if ((val= (char *) ((tmp->killed >= KILL_QUERY ?
+                           "Killed" : 0))))
         table->field[4]->store(val, strlen(val), cs);
       else
         table->field[4]->store(command_name[tmp->command].str,
@@ -2087,6 +2119,9 @@ int fill_schema_processlist(THD* thd, TABLE_LIST* tables, COND* cond)
       if (mysys_var)
         pthread_mutex_unlock(&mysys_var->mutex);
 
+      /* TIME_MS */
+      table->field[8]->store((double)(utime / (HRTIME_RESOLUTION / 1000.0)));
+
       /* INFO */
       /* Lock THD mutex that protects its data when looking at it. */
       pthread_mutex_lock(&tmp->LOCK_thd_data);
@@ -2097,10 +2132,19 @@ int fill_schema_processlist(THD* thd, TABLE_LIST* tables, COND* cond)
                                    tmp->query_length()), cs);
         table->field[7]->set_notnull();
       }
-      pthread_mutex_unlock(&tmp->LOCK_thd_data);
 
-      /* TIME_MS */
-      table->field[8]->store((double)(utime / (HRTIME_RESOLUTION / 1000.0)));
+      /*
+        Progress report. We need to do this under a lock to ensure that all
+        is from the same stage.
+      */
+      if ((max_counter= tmp->progress.max_counter))
+      {
+        table->field[9]->store((longlong) tmp->progress.stage + 1, 1);
+        table->field[10]->store((longlong) tmp->progress.max_stage, 1);
+        table->field[11]->store((double) tmp->progress.counter /
+                                (double) max_counter*100.0);
+      }
+      pthread_mutex_unlock(&tmp->LOCK_thd_data);
 
       if (schema_table_store_record(thd, table))
       {
@@ -2397,7 +2441,7 @@ static bool show_status_array(THD *thd, const char *wild,
           end= strmov(buff, *(my_bool*) value ? "ON" : "OFF");
           break;
         case SHOW_INT:
-          end= int10_to_str((long) *(uint32*) value, buff, 10);
+          end= int10_to_str((long) *(int*) value, buff, -10);
           break;
         case SHOW_HAVE:
         {
@@ -2814,7 +2858,7 @@ typedef struct st_lookup_field_values
 bool schema_table_store_record(THD *thd, TABLE *table)
 {
   int error;
-  if ((error= table->file->ha_write_row(table->record[0])))
+  if ((error= table->file->ha_write_tmp_row(table->record[0])))
   {
     TMP_TABLE_PARAM *param= table->pos_in_table_list->schema_table_param;
     if (create_internal_tmp_table_from_heap(thd, table, param->start_recinfo, 
@@ -6703,11 +6747,11 @@ bool get_schema_tables_result(JOIN *join,
       {
         result= 1;
         join->error= 1;
-        tab->read_record.file= table_list->table->file;
+        tab->read_record.table->file= table_list->table->file;
         table_list->schema_table_state= executed_place;
         break;
       }
-      tab->read_record.file= table_list->table->file;
+      tab->read_record.table->file= table_list->table->file;
       table_list->schema_table_state= executed_place;
     }
   }
@@ -7300,6 +7344,10 @@ ST_FIELD_INFO processlist_fields_info[]=
    SKIP_OPEN_TABLE},
   {"TIME_MS", 100 * (MY_INT64_NUM_DECIMAL_DIGITS + 1) + 3, MYSQL_TYPE_DECIMAL,
    0, 0, "Time_ms", SKIP_OPEN_TABLE},
+  {"STAGE", 2, MYSQL_TYPE_TINY,  0, 0, "Stage", SKIP_OPEN_TABLE},
+  {"MAX_STAGE", 2, MYSQL_TYPE_TINY,  0, 0, "Max_stage", SKIP_OPEN_TABLE},
+  {"PROGRESS", 703, MYSQL_TYPE_DECIMAL,  0, 0, "Progress",
+   SKIP_OPEN_TABLE},
   {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE}
 };
 
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 9d71f2482b4..90844824368 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -36,7 +36,8 @@ const char *primary_key_name="PRIMARY";
 
 static bool check_if_keyname_exists(const char *name,KEY *start, KEY *end);
 static char *make_unique_key_name(const char *field_name,KEY *start,KEY *end);
-static int copy_data_between_tables(TABLE *,TABLE *, List<Create_field> &, bool,
+static int copy_data_between_tables(THD *thd, TABLE *,TABLE *,
+                                    List<Create_field> &, bool,
 				    uint, ORDER *, ha_rows *,ha_rows *,
                                     enum enum_enable_or_disable, bool);
 
@@ -58,7 +59,7 @@ static void wait_for_kill_signal(THD *thd)
   while (thd->killed == 0)
     sleep(1);
   // Reset signal and continue as if nothing happend
-  thd->killed= THD::NOT_KILLED;
+  thd->killed= NOT_KILLED;
 }
 #endif
 
@@ -4689,6 +4690,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
   int result_code;
   bool need_repair_or_alter= 0;
   DBUG_ENTER("mysql_admin_table");
+  DBUG_PRINT("enter", ("extra_open_options: %u", extra_open_options));
 
   if (end_active_trans(thd))
     DBUG_RETURN(1);
@@ -4713,9 +4715,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
     bool fatal_error=0;
 
     DBUG_PRINT("admin", ("table: '%s'.'%s'", table->db, table->table_name));
-    DBUG_PRINT("admin", ("extra_open_options: %u", extra_open_options));
     strxmov(table_name, db, ".", table->table_name, NullS);
-    thd->open_options|= extra_open_options;
     table->lock_type= lock_type;
     /* open only one table from local list of command */
     {
@@ -4742,12 +4742,13 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
           lex->sql_command == SQLCOM_ANALYZE ||
           lex->sql_command == SQLCOM_OPTIMIZE)
 	thd->prepare_derived_at_open= TRUE;
+      thd->open_options|= extra_open_options;
       open_and_lock_tables(thd, table);
+      thd->open_options&= ~extra_open_options;
       thd->prepare_derived_at_open= FALSE;
       thd->no_warnings_for_error= 0;
       table->next_global= save_next_global;
       table->next_local= save_next_local;
-      thd->open_options&= ~extra_open_options;
 #ifdef WITH_PARTITION_STORAGE_ENGINE
       if (table->table)
       {
@@ -4931,7 +4932,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
         /* We use extra_open_options to be able to open crashed tables */
         thd->open_options|= extra_open_options;
         result_code= admin_recreate_table(thd, table);
-        thd->open_options= ~extra_open_options;
+        thd->open_options&= ~extra_open_options;
         goto send_result;
       }
       if (check_old_types || check_for_upgrade)
@@ -7505,8 +7506,7 @@ view_err:
     /* We don't want update TIMESTAMP fields during ALTER TABLE. */
     new_table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET;
     new_table->next_number_field=new_table->found_next_number_field;
-    thd_proc_info(thd, "copy to tmp table");
-    error= copy_data_between_tables(table, new_table,
+    error= copy_data_between_tables(thd, table, new_table,
                                     alter_info->create_list, ignore,
                                     order_num, order, &copied, &deleted,
                                     alter_info->keys_onoff,
@@ -7913,7 +7913,7 @@ err_with_placeholders:
 /* Copy all rows from one table to another */
 
 static int
-copy_data_between_tables(TABLE *from,TABLE *to,
+copy_data_between_tables(THD *thd, TABLE *from,TABLE *to,
 			 List<Create_field> &create,
                          bool ignore,
 			 uint order_num, ORDER *order,
@@ -7925,7 +7925,6 @@ copy_data_between_tables(TABLE *from,TABLE *to,
   int error= 1, errpos= 0;
   Copy_field *copy= NULL, *copy_end;
   ha_rows found_count= 0, delete_count= 0;
-  THD *thd= current_thd;
   uint length= 0;
   SORT_FIELD *sortorder;
   READ_RECORD info;
@@ -7935,11 +7934,14 @@ copy_data_between_tables(TABLE *from,TABLE *to,
   ha_rows examined_rows;
   bool auto_increment_field_copied= 0;
   ulong save_sql_mode= thd->variables.sql_mode;
-  ulonglong prev_insert_id;
+  ulonglong prev_insert_id, time_to_report_progress;
   List_iterator<Create_field> it(create);
   Create_field *def;
   DBUG_ENTER("copy_data_between_tables");
 
+  /* Two or 3 stages; Sorting, copying data and update indexes */
+  thd_progress_init(thd, 2 + test(order));
+
   /*
     Turn off recovery logging since rollback of an alter table is to
     delete the new table so there is no need to log the changes to it.
@@ -8013,6 +8015,7 @@ copy_data_between_tables(TABLE *from,TABLE *to,
       tables.alias= tables.table_name= from->s->table_name.str;
       tables.db= from->s->db.str;
 
+      thd_proc_info(thd, "Sorting");
       if (thd->lex->select_lex.setup_ref_array(thd, order_num) ||
           setup_order(thd, thd->lex->select_lex.ref_pointer_array,
                       &tables, fields, all_fields, order) ||
@@ -8023,8 +8026,10 @@ copy_data_between_tables(TABLE *from,TABLE *to,
           HA_POS_ERROR)
         goto err;
     }
-  };
+    thd_progress_next_stage(thd);
+  }
 
+  thd_proc_info(thd, "copy to tmp table");
   /* Tell handler that we have values for all columns in the to table */
   to->use_all_columns();
   to->mark_virtual_columns_for_write(TRUE);
@@ -8035,6 +8040,10 @@ copy_data_between_tables(TABLE *from,TABLE *to,
     to->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
   thd->row_count= 0;
   restore_record(to, s->default_values);        // Create empty record
+
+  thd->progress.max_counter= from->file->records();
+  time_to_report_progress= MY_HOW_OFTEN_TO_WRITE/10;
+
   while (!(error=info.read_record(&info)))
   {
     if (thd->killed)
@@ -8045,6 +8054,13 @@ copy_data_between_tables(TABLE *from,TABLE *to,
     }
     update_virtual_fields(thd, from);
     thd->row_count++;
+    if (++thd->progress.counter >= time_to_report_progress)
+    {
+      time_to_report_progress+= MY_HOW_OFTEN_TO_WRITE/10;
+      thd_progress_report(thd, thd->progress.counter,
+                          thd->progress.max_counter);
+    }
+
     /* Return error if source table isn't empty. */
     if (error_if_not_empty)
     {
@@ -8108,6 +8124,9 @@ err:
   free_io_cache(from);
   delete [] copy;
 
+  thd_proc_info(thd, "Enabling keys");
+  thd_progress_next_stage(thd);
+
   if (error > 0)
     to->file->extra(HA_EXTRA_PREPARE_FOR_DROP);
   if (errpos >= 3 && to->file->ha_end_bulk_insert() && error <= 0)
diff --git a/sql/sql_union.cc b/sql/sql_union.cc
index 50505dd94a9..50de42d153c 100644
--- a/sql/sql_union.cc
+++ b/sql/sql_union.cc
@@ -60,7 +60,7 @@ int select_union::send_data(List<Item> &values)
   if (thd->is_error())
     return 1;
 
-  if ((write_err= table->file->ha_write_row(table->record[0])))
+  if ((write_err= table->file->ha_write_tmp_row(table->record[0])))
   {
     if (write_err == HA_ERR_FOUND_DUPP_KEY)
     {
@@ -371,6 +371,7 @@ bool st_select_lex_unit::prepare(THD *thd_arg, select_result *sel_result,
     ulonglong create_options;
     uint save_tablenr= 0;
     table_map save_map= 0;
+    uint save_maybe_null= 0;
 
     while ((type= tp++))
     {
@@ -429,6 +430,7 @@ bool st_select_lex_unit::prepare(THD *thd_arg, select_result *sel_result,
     {
       save_tablenr= result_table_list.tablenr_exec;
       save_map= result_table_list.map_exec;
+      save_maybe_null= result_table_list.maybe_null_exec;
     }
     bzero((char*) &result_table_list, sizeof(result_table_list));
     result_table_list.db= (char*) "";
@@ -438,6 +440,7 @@ bool st_select_lex_unit::prepare(THD *thd_arg, select_result *sel_result,
     {
       result_table_list.tablenr_exec= save_tablenr;
       result_table_list.map_exec= save_map;
+      result_table_list.maybe_null_exec= save_maybe_null;
     }
 
     thd_arg->lex->current_select= lex_select_save;
@@ -642,6 +645,7 @@ bool st_select_lex_unit::exec()
 	if (!saved_error)
 	{
 	  examined_rows+= thd->examined_row_count;
+          thd->examined_row_count= 0;
 	  if (union_result->flush())
 	  {
 	    thd->lex->current_select= lex_select_save;
diff --git a/sql/sql_update.cc b/sql/sql_update.cc
index 4821fc2bd8f..80eb823c346 100644
--- a/sql/sql_update.cc
+++ b/sql/sql_update.cc
@@ -221,7 +221,7 @@ int mysql_update(THD *thd,
   bool          need_reopen;
   ulonglong     id;
   List<Item> all_fields;
-  THD::killed_state killed_status= THD::NOT_KILLED;
+  killed_state killed_status= NOT_KILLED;
   DBUG_ENTER("mysql_update");
 
   for ( ; ; )
@@ -248,6 +248,7 @@ int mysql_update(THD *thd,
       DBUG_RETURN(1);
     close_tables_for_reopen(thd, &table_list);
   }
+
   if (table_list->handle_derived(thd->lex, DT_MERGE_FOR_INSERT))
     DBUG_RETURN(1);
   if (table_list->handle_derived(thd->lex, DT_PREPARE))
@@ -787,9 +788,9 @@ int mysql_update(THD *thd,
   // simulated killing after the loop must be ineffective for binlogging
   DBUG_EXECUTE_IF("simulate_kill_bug27571",
                   {
-                    thd->killed= THD::KILL_QUERY;
+                    thd->killed= KILL_QUERY;
                   };);
-  error= (killed_status == THD::NOT_KILLED)?  error : 1;
+  error= (killed_status == NOT_KILLED)?  error : 1;
   
   if (error &&
       will_batch &&
@@ -850,7 +851,7 @@ int mysql_update(THD *thd,
       if (error < 0)
         thd->clear_error();
       else
-        errcode= query_error_code(thd, killed_status == THD::NOT_KILLED);
+        errcode= query_error_code(thd, killed_status == NOT_KILLED);
 
       if (thd->binlog_query(THD::ROW_QUERY_TYPE,
                             thd->query(), thd->query_length(),
@@ -1037,9 +1038,10 @@ reopen_tables:
     second time, but this call will do nothing (there are check for second
     call in setup_tables()).
   */
+
   //We need to merge for insert prior to prepare.
-  if (mysql_handle_list_of_derived(lex, table_list, DT_MERGE_FOR_INSERT))
-    DBUG_RETURN(1);
+  if (mysql_handle_derived(lex, DT_MERGE_FOR_INSERT))
+    DBUG_RETURN(TRUE);
   if (mysql_handle_derived(lex, DT_PREPARE))
     DBUG_RETURN(TRUE);
 
@@ -1047,7 +1049,10 @@ reopen_tables:
                                     &lex->select_lex.top_join_list,
                                     table_list,
                                     lex->select_lex.leaf_tables, FALSE,
-                                    UPDATE_ACL, SELECT_ACL, TRUE))
+                                    UPDATE_ACL, SELECT_ACL, FALSE))
+    DBUG_RETURN(TRUE);
+
+  if (lex->select_lex.handle_derived(thd->lex, DT_MERGE))  
     DBUG_RETURN(TRUE);
 
   if (setup_fields_with_no_wrap(thd, 0, *fields, MARK_COLUMNS_WRITE, 0, 0))
@@ -1237,6 +1242,9 @@ reopen_tables:
     further check in multi_update::prepare whether to use record cache.
   */
   lex->select_lex.exclude_from_table_unique_test= FALSE;
+
+  if (lex->select_lex.save_prep_leaf_tables(thd))
+    DBUG_RETURN(TRUE);
  
   DBUG_RETURN (FALSE);
 }
@@ -1330,13 +1338,6 @@ int multi_update::prepare(List<Item> &not_used_values,
   thd->cuted_fields=0L;
   thd_proc_info(thd, "updating main table");
 
-  SELECT_LEX *select_lex= lex_unit->first_select();
-  if (select_lex->first_cond_optimization)
-  {
-    if (select_lex->handle_derived(thd->lex, DT_MERGE))
-      DBUG_RETURN(TRUE);
-  }
-
   tables_to_update= get_table_map(fields);
 
   if (!tables_to_update)
@@ -1868,7 +1869,7 @@ int multi_update::send_data(List<Item> &not_used_values)
                   *values_for_table[offset], TRUE, FALSE);
 
       /* Write row, ignoring duplicated updates to a row */
-      error= tmp_table->file->ha_write_row(tmp_table->record[0]);
+      error= tmp_table->file->ha_write_tmp_row(tmp_table->record[0]);
       if (error != HA_ERR_FOUND_DUPP_KEY && error != HA_ERR_FOUND_DUPP_UNIQUE)
       {
         if (error &&
@@ -1936,7 +1937,7 @@ void multi_update::abort()
         got caught and if happens later the killed error is written
         into repl event.
       */
-      int errcode= query_error_code(thd, thd->killed == THD::NOT_KILLED);
+      int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
       /* the error of binary logging is ignored */
       (void)thd->binlog_query(THD::ROW_QUERY_TYPE,
                               thd->query(), thd->query_length(),
@@ -2150,7 +2151,7 @@ bool multi_update::send_eof()
 {
   char buff[STRING_BUFFER_USUAL_SIZE];
   ulonglong id;
-  THD::killed_state killed_status= THD::NOT_KILLED;
+  killed_state killed_status= NOT_KILLED;
   DBUG_ENTER("multi_update::send_eof");
   thd_proc_info(thd, "updating reference tables");
 
@@ -2163,7 +2164,7 @@ bool multi_update::send_eof()
     if local_error is not set ON until after do_updates() then
     later carried out killing should not affect binlogging.
   */
-  killed_status= (local_error == 0)? THD::NOT_KILLED : thd->killed;
+  killed_status= (local_error == 0) ? NOT_KILLED : thd->killed;
   thd_proc_info(thd, "end");
 
   /* We must invalidate the query cache before binlog writing and
@@ -2192,7 +2193,7 @@ bool multi_update::send_eof()
       if (local_error == 0)
         thd->clear_error();
       else
-        errcode= query_error_code(thd, killed_status == THD::NOT_KILLED);
+        errcode= query_error_code(thd, killed_status == NOT_KILLED);
       if (thd->binlog_query(THD::ROW_QUERY_TYPE,
                             thd->query(), thd->query_length(),
                             transactional_tables, FALSE, errcode))
diff --git a/sql/sql_view.cc b/sql/sql_view.cc
index 89ccfd10a45..9d71a61f411 100644
--- a/sql/sql_view.cc
+++ b/sql/sql_view.cc
@@ -18,8 +18,8 @@
 #include "mysql_priv.h"
 #include "sql_select.h"
 #include "parse_file.h"
-#include "sp.h"
 #include "sp_head.h"
+#include "sp.h"
 #include "sp_cache.h"
 
 #define MD5_BUFF_LENGTH 33
@@ -1483,8 +1483,22 @@ bool mysql_make_view(THD *thd, File_parser *parser, TABLE_LIST *table,
         We can safely ignore the VIEW's ORDER BY if we merge into union 
         branch, as order is not important there.
       */
-      if (!table->select_lex->master_unit()->is_union())
+      if (!table->select_lex->master_unit()->is_union() &&
+          table->select_lex->order_list.elements == 0)
         table->select_lex->order_list.push_back(&lex->select_lex.order_list);
+      else
+      {
+        if (old_lex->sql_command == SQLCOM_SELECT &&
+            (old_lex->describe & DESCRIBE_EXTENDED) &&
+            lex->select_lex.order_list.elements &&
+            !table->select_lex->master_unit()->is_union())
+        {
+          push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_NOTE,
+                              ER_VIEW_ORDERBY_IGNORED,
+                              ER(ER_VIEW_ORDERBY_IGNORED),
+                              table->db, table->table_name);
+        }
+      }
       /*
 	This SELECT_LEX will be linked in global SELECT_LEX list
 	to make it processed by mysql_handle_derived(),
diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy
index 3337b1b487e..0d0e0ef98a4 100644
--- a/sql/sql_yacc.yy
+++ b/sql/sql_yacc.yy
@@ -682,10 +682,10 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 
 %pure_parser                                    /* We have threads */
 /*
-  Currently there are 171 shift/reduce conflicts.
+  Currently there are 174 shift/reduce conflicts.
   We should not introduce new conflicts any more.
 */
-%expect 171
+%expect 174
 
 /*
    Comments for TOKENS.
@@ -901,6 +901,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  GROUP_CONCAT_SYM
 %token  GT_SYM                        /* OPERATOR */
 %token  HANDLER_SYM
+%token  HARD_SYM
 %token  HASH_SYM
 %token  HAVING                        /* SQL-2003-R */
 %token  HELP_SYM
@@ -1169,6 +1170,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  SMALLINT                      /* SQL-2003-R */
 %token  SNAPSHOT_SYM
 %token  SOCKET_SYM
+%token  SOFT_SYM
 %token  SONAME_SYM
 %token  SOUNDS_SYM
 %token  SOURCE_SYM
@@ -1348,7 +1350,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
         opt_ev_status opt_ev_on_completion ev_on_completion opt_ev_comment
         ev_alter_on_schedule_completion opt_ev_rename_to opt_ev_sql_stmt
         optional_flush_tables_arguments opt_dyncol_type dyncol_type
-        opt_time_precision
+        opt_time_precision kill_type kill_option int_num
 
 %type <ulong_num>
         ulong_num real_ulong_num merge_insert_types
@@ -1380,7 +1382,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
         function_call_keyword
         function_call_nonkeyword
         function_call_generic
-        function_call_conflict
+        function_call_conflict kill_expr
 
 %type <item_num>
         NUM_literal
@@ -7172,7 +7174,7 @@ expr:
         | expr XOR expr %prec XOR
           {
             /* XOR is a proprietary extension */
-            $$ = new (YYTHD->mem_root) Item_cond_xor($1, $3);
+            $$ = new (YYTHD->mem_root) Item_func_xor($1, $3);
             if ($$ == NULL)
               MYSQL_YYABORT;
           }
@@ -9426,7 +9428,7 @@ where_clause:
           expr
           {
             SELECT_LEX *select= Select;
-            select->where= $3;
+            select->where= normalize_cond($3);
             select->parsing_place= NO_MATTER;
             if ($3)
               $3->top_level_item();
@@ -9442,7 +9444,7 @@ having_clause:
           expr
           {
             SELECT_LEX *sel= Select;
-            sel->having= $3;
+            sel->having= normalize_cond($3);
             sel->parsing_place= NO_MATTER;
             if ($3)
               $3->top_level_item();
@@ -9674,6 +9676,12 @@ delete_limit_clause:
           }
         ;
 
+int_num:
+          NUM           { int error; $$= (int) my_strtoll10($1.str, (char**) 0, &error); }
+        | '-' NUM       { int error; $$= -(int) my_strtoll10($2.str, (char**) 0, &error); }
+        | '-' LONG_NUM  { int error; $$= -(int) my_strtoll10($2.str, (char**) 0, &error); }
+        ;
+
 ulong_num:
           NUM           { int error; $$= (ulong) my_strtoll10($1.str, (char**) 0, &error); }
         | HEX_NUM       { $$= (ulong) strtol($1.str, (char**) 0, 16); }
@@ -10909,7 +10917,7 @@ wild_and_where:
           }
         | WHERE expr
           {
-            Select->where= $2;
+            Select->where= normalize_cond($2);
             if ($2)
               $2->top_level_item();
           }
@@ -11081,19 +11089,41 @@ purge_option:
 /* kill threads */
 
 kill:
-          KILL_SYM kill_option expr
+          KILL_SYM
           {
             LEX *lex=Lex;
             lex->value_list.empty();
-            lex->value_list.push_front($3);
+            lex->users_list.empty();
             lex->sql_command= SQLCOM_KILL;
           }
+          kill_type kill_option kill_expr
+          {
+            Lex->kill_signal= (killed_state) ($3 | $4);
+          }
         ;
 
+kill_type:
+        /* Empty */    { $$= (int) KILL_HARD_BIT; }
+        | HARD_SYM     { $$= (int) KILL_HARD_BIT; }
+        | SOFT_SYM     { $$= 0; }
+
 kill_option:
-          /* empty */ { Lex->type= 0; }
-        | CONNECTION_SYM { Lex->type= 0; }
-        | QUERY_SYM      { Lex->type= ONLY_KILL_QUERY; }
+          /* empty */    { $$= (int) KILL_CONNECTION; }
+        | CONNECTION_SYM { $$= (int) KILL_CONNECTION; }
+        | QUERY_SYM      { $$= (int) KILL_QUERY; }
+        ;
+
+kill_expr:
+        expr
+        {
+          Lex->value_list.push_front($$);
+          Lex->kill_type= KILL_TYPE_ID;
+         }
+        | USER user
+          {
+            Lex->users_list.push_back($2);
+            Lex->kill_type= KILL_TYPE_USER;
+          }
         ;
 
 /* change database */
@@ -12208,6 +12238,7 @@ keyword_sp:
         | GRANTS                   {}
         | GLOBAL_SYM               {}
         | HASH_SYM                 {}
+        | HARD_SYM                 {}
         | HOSTS_SYM                {}
         | HOUR_SYM                 {}
         | IDENTIFIED_SYM           {}
@@ -12339,6 +12370,7 @@ keyword_sp:
         | SHUTDOWN                 {}
         | SLOW_SYM                 {}
         | SNAPSHOT_SYM             {}
+        | SOFT_SYM                 {}
         | SOUNDS_SYM               {}
         | SOURCE_SYM               {}
         | SQL_CACHE_SYM            {}
@@ -13408,7 +13440,7 @@ grant_option:
             lex->mqh.conn_per_hour= $2;
             lex->mqh.specified_limits|= USER_RESOURCES::CONNECTIONS_PER_HOUR;
           }
-        | MAX_USER_CONNECTIONS_SYM ulong_num
+        | MAX_USER_CONNECTIONS_SYM int_num
           {
             LEX *lex=Lex;
             lex->mqh.user_conn= $2;
diff --git a/sql/structs.h b/sql/structs.h
index 29ccde3fb03..61354cbd01c 100644
--- a/sql/structs.h
+++ b/sql/structs.h
@@ -149,7 +149,7 @@ struct READ_RECORD {			/* Parameter to read_record */
   typedef int (*Read_func)(READ_RECORD*);
   typedef void (*Unlock_row_func)(st_join_table *);
   struct st_table *table;			/* Head-form */
-  handler *file;
+ // handler *file_;
   struct st_table **forms;			/* head and ref forms */
 
   Read_func read_record;
@@ -222,8 +222,11 @@ typedef struct user_resources {
   uint updates;
   /* Maximum number of connections established per hour. */
   uint conn_per_hour;
-  /* Maximum number of concurrent connections. */
-  uint user_conn;
+  /*
+    Maximum number of concurrent connections. If -1 then no new
+    connections allowed
+  */
+  int user_conn;
   /*
      Values of this enum and specified_limits member are used by the
      parser to store which user limits were specified in GRANT statement.
@@ -256,7 +259,7 @@ typedef struct  user_conn {
   /* Total length of the key. */
   uint len;
   /* Current amount of concurrent connections for this account. */
-  uint connections;
+  int connections;
   /*
      Current number of connections per hour, number of updating statements
      per hour and total number of statements per hour for this account.
diff --git a/sql/table.cc b/sql/table.cc
index 8427ca6e112..23c43f91d22 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -989,14 +989,13 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head,
 #endif
       next_chunk+= 5 + partition_info_len;
     }
-    if (share->mysql_version >= 50110)
+    if (share->mysql_version >= 50110 && next_chunk < buff_end)
     {
       /* New auto_partitioned indicator introduced in 5.1.11 */
 #ifdef WITH_PARTITION_STORAGE_ENGINE
       share->auto_partitioned= *next_chunk;
 #endif
       next_chunk++;
-      DBUG_ASSERT(next_chunk <= buff_end);
     }
     keyinfo= share->key_info;
     for (i= 0; i < keys; i++, keyinfo++)
@@ -2460,7 +2459,7 @@ int closefrm(register TABLE *table, bool free_share)
   {
     if (table->s->deleting)
       table->file->extra(HA_EXTRA_PREPARE_FOR_DROP);
-    error=table->file->close();
+    error=table->file->ha_close();
   }
   table->alias.free();
   if (table->expr_arena)
@@ -4537,7 +4536,7 @@ Item *create_view_field(THD *thd, TABLE_LIST *view, Item **field_ref,
     Force creation of nullable item for the result tmp table for outer joined
     views/derived tables.
   */
-  if (view->outer_join)
+  if (view->table && view->table->maybe_null)
     item->maybe_null= TRUE;
   /* Save item in case we will need to fall back to materialization. */
   view->used_items.push_back(item);
@@ -5221,10 +5220,11 @@ void st_table::mark_virtual_columns_for_write(bool insert_fl)
   @brief
   Allocate space for keys
 
-  @param key_count  number of keys to allocate
+  @param key_count  number of keys to allocate additionally
 
   @details
-  The function allocates memory  to fit 'key_count' keys for this table.
+  The function allocates memory  to fit additionally 'key_count' keys 
+  for this table.
 
   @return FALSE   space was successfully allocated
   @return TRUE    an error occur
@@ -5232,22 +5232,25 @@ void st_table::mark_virtual_columns_for_write(bool insert_fl)
 
 bool TABLE::alloc_keys(uint key_count)
 {
-  DBUG_ASSERT(!s->keys);
-  key_info= s->key_info= (KEY*) alloc_root(&mem_root, sizeof(KEY)*key_count);
-  max_keys= key_count;
+  key_info= (KEY*) alloc_root(&mem_root, sizeof(KEY)*(s->keys+key_count));
+  if (s->keys)
+    memmove(key_info, s->key_info, sizeof(KEY)*s->keys);
+  s->key_info= key_info;
+  max_keys= s->keys+key_count;
   return !(key_info);
 }
 
 
 void TABLE::create_key_part_by_field(KEY *keyinfo,
                                      KEY_PART_INFO *key_part_info,
-                                     Field *field)
+                                     Field *field, uint fieldnr)
 {   
   field->flags|= PART_KEY_FLAG;
   key_part_info->null_bit= field->null_bit;
   key_part_info->null_offset= (uint) (field->null_ptr -
                                       (uchar*) record[0]);
   key_part_info->field= field;
+  key_part_info->fieldnr= fieldnr;
   key_part_info->offset= field->offset(record[0]);
   key_part_info->length=   (uint16) field->pack_length();
   keyinfo->key_length+= key_part_info->length;
@@ -5340,11 +5343,12 @@ bool TABLE::add_tmp_key(uint key, uint key_parts,
 
   for (i= 0; i < key_parts; i++)
   {
-    reg_field= field + next_field_no(arg);
+    uint fld_idx= next_field_no(arg); 
+    reg_field= field + fld_idx;
     if (key_start)
       (*reg_field)->key_start.set_bit(key);
     (*reg_field)->part_of_key.set_bit(key);
-    create_key_part_by_field(keyinfo, key_part_info, *reg_field);
+    create_key_part_by_field(keyinfo, key_part_info, *reg_field, fld_idx+1);
     key_start= FALSE;
     key_part_info++;
   }
@@ -5372,12 +5376,12 @@ void TABLE::use_index(int key_to_save)
   DBUG_ASSERT(!created && key_to_save < (int)s->keys);
   if (key_to_save >= 0)
     /* Save the given key. */
-    memcpy(key_info, key_info + key_to_save, sizeof(KEY));
+    memmove(key_info, key_info + key_to_save, sizeof(KEY));
   else
     /* Drop all keys; */
     i= 0;
 
-  s->keys= (key_to_save < 0) ? 0 : 1;
+  s->keys= i;
 }
 
 /**
@@ -5408,9 +5412,11 @@ bool st_table::is_children_attached(void)
 
 bool st_table::is_filled_at_execution()
 { 
-  return test(pos_in_table_list->jtbm_subselect);
+  return test(pos_in_table_list->jtbm_subselect || 
+              pos_in_table_list->is_active_sjm());
 }
 
+
 /*
   Cleanup this table for re-execution.
 
@@ -5686,7 +5692,7 @@ int update_virtual_fields(THD *thd, TABLE *table, bool for_write)
 {
   DBUG_ENTER("update_virtual_fields");
   Field **vfield_ptr, *vfield;
-  int error= 0;
+  int error __attribute__ ((unused))= 0;
   if (!table || !table->vfield)
     DBUG_RETURN(0);
 
diff --git a/sql/table.h b/sql/table.h
index 4205b81dc12..5dd464ac876 100644
--- a/sql/table.h
+++ b/sql/table.h
@@ -959,7 +959,7 @@ struct st_table {
                    uint (*next_field_no) (uchar *), uchar *arg,
                    bool unique);
   void create_key_part_by_field(KEY *keyinfo, KEY_PART_INFO *key_part_info,
-                                Field *field);
+                                Field *field, uint fieldnr);
   void use_index(int key_to_save);
   void set_table_map(table_map map_arg, uint tablenr_arg)
   {
@@ -1320,6 +1320,7 @@ struct TABLE_LIST
 
   /* If this is a jtbm semi-join object: corresponding subselect predicate */
   Item_in_subselect  *jtbm_subselect;
+  /* TODO: check if this can be joined with tablenr_exec */
   uint jtbm_table_no;
 
   SJ_MATERIALIZATION_INFO *sj_mat_info;
@@ -1445,7 +1446,9 @@ struct TABLE_LIST
 
   table_map view_used_tables;
   table_map     map_exec;
+  /* TODO: check if this can be joined with jtbm_table_no */
   uint          tablenr_exec;
+  uint          maybe_null_exec;
 
   /* Ptr to parent MERGE table list item. See top comment in ha_myisammrg.cc */
   TABLE_LIST    *parent_l;
@@ -1765,6 +1768,8 @@ struct TABLE_LIST
      respectively.
    */
   char *get_table_name() { return view != NULL ? view_name.str : table_name; }
+  bool is_active_sjm();
+  bool is_jtbm() { return test(jtbm_subselect!=NULL); }
   st_select_lex_unit *get_unit();
   st_select_lex *get_single_select();
   void wrap_into_nested_join(List<TABLE_LIST> &join_list);