5.3 merge

127 files changed, 2461 insertions, 1248 deletions

diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
index ce6c96e55f2..50ba3712fb8 100644
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@ -1,4 +1,4 @@
-# Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2006, 2011, Oracle and/or its affiliates.
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
diff --git a/sql/authors.h b/sql/authors.h
index 018c8fabc31..84c29f8c127 100644
--- a/sql/authors.h
+++ b/sql/authors.h
@@ -1,7 +1,7 @@
 #ifndef AUTHORS_INCLUDED
 #define AUTHORS_INCLUDED
 
-/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2005, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/debug_sync.cc b/sql/debug_sync.cc
index 95f512983da..a789763dd25 100644
--- a/sql/debug_sync.cc
+++ b/sql/debug_sync.cc
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2009, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/debug_sync.h b/sql/debug_sync.h
index 930e6c35be9..50881f491fe 100644
--- a/sql/debug_sync.h
+++ b/sql/debug_sync.h
@@ -1,7 +1,7 @@
 #ifndef DEBUG_SYNC_INCLUDED
 #define DEBUG_SYNC_INCLUDED
 
-/* Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2009, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/derror.cc b/sql/derror.cc
index 23319ac0c99..baf7163790d 100644
--- a/sql/derror.cc
+++ b/sql/derror.cc
@@ -153,7 +153,6 @@ bool read_texts(const char *file_name, const char *language,
   char lang_path[FN_REFLEN];
   uchar *buff;
   uchar head[32],*pos;
-  const char *errmsg;
   DBUG_ENTER("read_texts");
 
   *point= 0;
@@ -231,20 +230,10 @@ Error message file '%s' had only %d error messages, but it should contain at lea
   DBUG_RETURN(i);
 
 err:
-  switch (funktpos) {
-  case 3:
-    errmsg= "Not enough memory for messagefile '%s'";
-    break;
-  case 2:
-    errmsg= "Incompatible header in messagefile '%s'. Probably from another version of MariaDB";
-  case 1:
-    errmsg= "Can't read from messagefile '%s'";
-    break;
-  default:
-    errmsg= "Can't find messagefile '%s'";
-    break;
-  }
-  sql_print_error(errmsg, name);
+  sql_print_error((funktpos == 3) ? "Not enough memory for messagefile '%s'" :
+                  (funktpos == 2) ? "Incompatible header in messagefile '%s'. Probably from another version of MariaDB" :
+                  ((funktpos == 1) ? "Can't read from messagefile '%s'" :
+                   "Can't find messagefile '%s'"), name);
   if (file != FERR)
     (void) mysql_file_close(file, MYF(MY_WME));
   DBUG_RETURN(1);
diff --git a/sql/event_data_objects.cc b/sql/event_data_objects.cc
index eabe215bc3a..c41194d1a8d 100644
--- a/sql/event_data_objects.cc
+++ b/sql/event_data_objects.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2005, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/event_db_repository.cc b/sql/event_db_repository.cc
index 8099d87b580..0eb65cff91e 100644
--- a/sql/event_db_repository.cc
+++ b/sql/event_db_repository.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2006, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/event_db_repository.h b/sql/event_db_repository.h
index bb708b2b520..a2862790be1 100644
--- a/sql/event_db_repository.h
+++ b/sql/event_db_repository.h
@@ -1,8 +1,6 @@
 #ifndef _EVENT_DB_REPOSITORY_H_
 #define _EVENT_DB_REPOSITORY_H_
-
-/*
-   Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2006, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -78,7 +76,6 @@ public:
   bool
   create_event(THD *thd, Event_parse_data *parse_data, bool create_if_not,
                bool *event_already_exists);
-
   bool
   update_event(THD *thd, Event_parse_data *parse_data, LEX_STRING *new_dbname,
                LEX_STRING *new_name);
diff --git a/sql/event_parse_data.cc b/sql/event_parse_data.cc
index b53e366e27c..ad812a6aa5d 100644
--- a/sql/event_parse_data.cc
+++ b/sql/event_parse_data.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2008, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/event_parse_data.h b/sql/event_parse_data.h
index 1ea22edf488..faf42db623a 100644
--- a/sql/event_parse_data.h
+++ b/sql/event_parse_data.h
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2008, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/events.cc b/sql/events.cc
index 00299463dba..8b4bab9e3a6 100644
--- a/sql/events.cc
+++ b/sql/events.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2005, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -360,8 +360,8 @@ Events::create_event(THD *thd, Event_parse_data *parse_data,
                                                      parse_data->name,
                                                      new_element)))
       {
-        if (!db_repository->drop_event(thd, parse_data->dbname, parse_data->name,
-                                       TRUE))
+        if (!db_repository->drop_event(thd, parse_data->dbname,
+                                       parse_data->name, TRUE))
           dropped= 1;
         delete new_element;
       }
@@ -383,16 +383,19 @@ Events::create_event(THD *thd, Event_parse_data *parse_data,
       String log_query;
       if (create_query_string(thd, &log_query))
       {
-        sql_print_error("Event Error: An error occurred while creating query string, "
-                        "before writing it into binary log.");
+        sql_print_error("Event Error: An error occurred while creating query "
+                        "string, before writing it into binary log.");
         ret= true;
       }
       else
+      {
         /*
-          If the definer is not set or set to CURRENT_USER, the value of CURRENT_USER
-          will be written into the binary log as the definer for the SQL thread.
+          If the definer is not set or set to CURRENT_USER, the value
+          of CURRENT_USER will be written into the binary log as the
+          definer for the SQL thread.
         */
         ret= write_bin_log(thd, TRUE, log_query.ptr(), log_query.length());
+      }
     }
   }
   /* Restore the state of binlog format */
diff --git a/sql/field.cc b/sql/field.cc
index 480d36c2e28..84ee0f8b15a 100644
--- a/sql/field.cc
+++ b/sql/field.cc
@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2009-2011 Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/field.h b/sql/field.h
index 84d60012a85..fee141d30b0 100644
--- a/sql/field.h
+++ b/sql/field.h
@@ -1,7 +1,7 @@
 #ifndef FIELD_INCLUDED
 #define FIELD_INCLUDED
 /* Copyright (c) 2000, 2011 Oracle and/or its affiliates.
-   Copyright (c) 2009-2011 Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -2277,6 +2277,23 @@ public:
   uchar *from_null_ptr,*to_null_ptr;
   bool *null_row;
   uint	from_bit,to_bit;
+  /**
+    Number of bytes in the fields pointed to by 'from_ptr' and
+    'to_ptr'. Usually this is the number of bytes that are copied from
+    'from_ptr' to 'to_ptr'.
+
+    For variable-length fields (VARCHAR), the first byte(s) describe
+    the actual length of the text. For VARCHARs with length 
+       < 256 there is 1 length byte 
+       >= 256 there is 2 length bytes
+    Thus, if from_field is VARCHAR(10), from_length (and in most cases
+    to_length) is 11. For VARCHAR(1024), the length is 1026. @see
+    Field_varstring::length_bytes
+
+    Note that for VARCHARs, do_copy() will be do_varstring*() which
+    only copies the length-bytes (1 or 2) + the actual length of the
+    text instead of from/to_length bytes. @see get_copy_func()
+  */
   uint from_length,to_length;
   Field *from_field,*to_field;
   String tmp;					// For items
diff --git a/sql/field_conv.cc b/sql/field_conv.cc
index 8808d73f622..89ac499b6c0 100644
--- a/sql/field_conv.cc
+++ b/sql/field_conv.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -250,6 +251,25 @@ static void do_outer_field_null(Copy_field *copy)
   }
 }
 
+/*
+  Copy: (not-NULL field in table that can be NULL-complemented) -> (not-NULL
+  field)
+*/
+static void do_copy_nullable_row_to_notnull(Copy_field *copy)
+{
+  if (*copy->null_row ||
+      (copy->from_null_ptr && (*copy->from_null_ptr & copy->from_bit)))
+  {
+    copy->to_field->set_warning(MYSQL_ERROR::WARN_LEVEL_WARN,
+                                WARN_DATA_TRUNCATED, 1);
+    copy->to_field->reset();
+  }
+  else
+  {
+    (copy->do_copy2)(copy);
+  }
+
+}
 
 /* Copy: (NULL-able field) -> (not NULL-able field) */
 static void do_copy_not_null(Copy_field *copy)
@@ -640,7 +660,15 @@ void Copy_field::set(Field *to,Field *from,bool save)
       else if (to_field == to_field->table->next_number_field)
         do_copy= do_copy_next_number;
       else
-        do_copy= do_copy_not_null;
+      {
+        if (!from_null_ptr)
+        {
+          null_row= &from->table->null_row;
+          do_copy= do_copy_nullable_row_to_notnull;
+        }
+        else
+          do_copy= do_copy_not_null;
+      }
     }
   }
   else if (to_field->real_maybe_null())
@@ -731,15 +759,11 @@ Copy_field::get_copy_func(Field *to,Field *from)
         if (((Field_varstring*) to)->length_bytes !=
             ((Field_varstring*) from)->length_bytes)
           return do_field_string;
-        if (to_length != from_length)
-          return (((Field_varstring*) to)->length_bytes == 1 ?
-                  (from->charset()->mbmaxlen == 1 ? do_varstring1 :
-                                                    do_varstring1_mb) :
-                  (from->charset()->mbmaxlen == 1 ? do_varstring2 :
-                                                    do_varstring2_mb));
-        else 
-          return  (((Field_varstring*) from)->length_bytes == 1 ?
-                    do_varstring1 : do_varstring2);
+        return (((Field_varstring*) to)->length_bytes == 1 ?
+                (from->charset()->mbmaxlen == 1 ? do_varstring1 :
+                 do_varstring1_mb) :
+                (from->charset()->mbmaxlen == 1 ? do_varstring2 :
+                 do_varstring2_mb));
       }
       else if (to_length < from_length)
 	return (from->charset()->mbmaxlen == 1 ?
diff --git a/sql/filesort.cc b/sql/filesort.cc
index a9db198e9d3..f9fe0d731cd 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -879,7 +880,7 @@ static void make_sortkey(register SORTPARAM *param,
         if (sort_field->need_strxnfrm)
         {
           char *from=(char*) res->ptr();
-          uint tmp_length;
+          uint tmp_length __attribute__((unused));
           if ((uchar*) from == to)
           {
             set_if_smaller(length,sort_field->length);
@@ -1006,10 +1007,21 @@ static void make_sortkey(register SORTPARAM *param,
       if (addonf->null_bit && field->is_null())
       {
         nulls[addonf->null_offset]|= addonf->null_bit;
+#ifdef HAVE_valgrind
+	bzero(to, addonf->length);
+#endif
       }
       else
       {
+#ifdef HAVE_valgrind
+        uchar *end= field->pack(to, field->ptr);
+	uint length= (uint) ((to + addonf->length) - end);
+	DBUG_ASSERT((int) length >= 0);
+	if (length)
+	  bzero(end, length);
+#else
         (void) field->pack(to, field->ptr);
+#endif
       }
       to+= addonf->length;
     }
diff --git a/sql/gcalc_slicescan.cc b/sql/gcalc_slicescan.cc
index cc2b26a8665..cd3717f186a 100644
--- a/sql/gcalc_slicescan.cc
+++ b/sql/gcalc_slicescan.cc
@@ -408,7 +408,7 @@ static int de_weak_check(long double a, long double b, long double ex)
 
 static int de_check(long double a, long double b)
 {
-  return de_weak_check(a, b, (long double) 1e-10);
+  return de_weak_check(a, b, (long double) 1e-9);
 }
 #endif /*GCALC_CHECK_WITH_FLOAT*/
 
@@ -434,7 +434,7 @@ void gcalc_mul_coord(Gcalc_internal_coord *result, int result_len,
       gcalc_coord2 cur_b= n_b ? b[n_b] : FIRST_DIGIT(b[0]);
       gcalc_coord2 mul= cur_a * cur_b + carry + result[n_a + n_b + 1];
       result[n_a + n_b + 1]= mul % GCALC_DIG_BASE;
-      carry= (gcalc_digit_t) (mul / GCALC_DIG_BASE);
+      carry= (gcalc_digit_t) (mul / (gcalc_coord2) GCALC_DIG_BASE);
     } while (n_b--);
     if (carry)
     {
@@ -794,10 +794,51 @@ static int cmp_intersections(const Gcalc_heap::Info *i1,
 /* Internal coordinates implementation end */
 
 
+#define GCALC_SCALE_1 1e18
+
+static double find_scale(double extent)
+{
+  double scale= 1e-2;
+  while (scale < extent)
+    scale*= (double ) 10;
+  return GCALC_SCALE_1 / scale / 10;
+}
+
+
+void Gcalc_heap::set_extent(double xmin, double xmax, double ymin, double ymax)
+{
+  xmin= fabs(xmin);
+  xmax= fabs(xmax);
+  ymin= fabs(ymin);
+  ymax= fabs(ymax);
+
+  if (xmax < xmin)
+    xmax= xmin;
+  if (ymax < ymin)
+    ymax= ymin;
+
+  coord_extent= xmax > ymax ? xmax : ymax;
+  coord_extent= find_scale(coord_extent);
+#ifdef GCALC_CHECK_WITH_FLOAT
+  gcalc_coord_extent= &coord_extent;
+#endif /*GCALC_CHECK_WITH_FLOAT*/
+}
+
+
+void Gcalc_heap::free_point_info(Gcalc_heap::Info *i,
+                                 Gcalc_dyn_list::Item **i_hook)
+{
+  if (m_hook == &i->next)
+    m_hook= i_hook;
+  *i_hook= i->next;
+  free_item(i);
+  m_n_points--;
+}
+
+
 Gcalc_heap::Info *Gcalc_heap::new_point_info(double x, double y,
                                              gcalc_shape_info shape)
 {
-  double abs= fabs(x);
   Info *result= (Info *)new_item();
   if (!result)
     return NULL;
@@ -808,17 +849,8 @@ Gcalc_heap::Info *Gcalc_heap::new_point_info(double x, double y,
   result->shape= shape;
   result->top_node= 1;
   result->type= nt_shape_node;
-  if (m_n_points)
-  {
-    if (abs > coord_extent)
-      coord_extent= abs;
-  }
-  else
-    coord_extent= abs;
-
-  abs= fabs(y);
-  if (abs > coord_extent)
-    coord_extent= abs;
+  gcalc_set_double(result->ix, x, coord_extent);
+  gcalc_set_double(result->iy, y, coord_extent);
 
   m_n_points++;
   return result;
@@ -929,32 +961,12 @@ static int compare_point_info(const void *e0, const void *e1)
 }
 
 
-#define GCALC_SCALE_1 1e18
-
-static double find_scale(double extent)
-{
-  double scale= 1e-2;
-  while (scale < extent)
-    scale*= (double ) 10;
-  return GCALC_SCALE_1 / scale / 10;
-}
-
-
 void Gcalc_heap::prepare_operation()
 {
   Info *cur;
   GCALC_DBUG_ASSERT(m_hook);
-  coord_extent= find_scale(coord_extent);
-#ifdef GCALC_CHECK_WITH_FLOAT
-  gcalc_coord_extent= &coord_extent;
-#endif /*GCALC_CHECK_WITH_FLOAT*/
   *m_hook= NULL;
   m_hook= NULL; /* just to check it's not called twice */
-  for (cur= get_first(); cur; cur= cur->get_next())
-  {
-    gcalc_set_double(cur->ix, cur->x, coord_extent);
-    gcalc_set_double(cur->iy, cur->y, coord_extent);
-  }
   m_first= sort_list(compare_point_info, m_first, m_n_points);
 
   /* TODO - move this to the 'normal_scan' loop */
@@ -992,18 +1004,28 @@ int Gcalc_shape_transporter::int_add_point(gcalc_shape_info Info,
                                            double x, double y)
 {
   Gcalc_heap::Info *point;
-  GCALC_DBUG_ASSERT(!m_prev || m_prev->x != x || m_prev->y != y);
+  Gcalc_dyn_list::Item **hook;
+
+  hook= m_heap->get_cur_hook();
 
   if (!(point= m_heap->new_point_info(x, y, Info)))
     return 1;
   if (m_first)
   {
+    if (cmp_point_info(m_prev, point) == 0)
+    {
+      /* Coinciding points, do nothing */
+      m_heap->free_point_info(point, hook);
+      return 0;
+    }
+    GCALC_DBUG_ASSERT(!m_prev || m_prev->x != x || m_prev->y != y);
     m_prev->left= point;
     point->right= m_prev;
   }
   else
     m_first= point;
   m_prev= point;
+  m_prev_hook= hook;
   return 0;
 }
 
@@ -1031,10 +1053,20 @@ void Gcalc_shape_transporter::int_complete()
     return;
   }
 
-  GCALC_DBUG_ASSERT(m_prev->x != m_first->x || m_prev->y != m_first->y);
   /* polygon */
-  m_first->right= m_prev;
-  m_prev->left= m_first;
+  if (cmp_point_info(m_first, m_prev) == 0)
+  {
+    /* Coinciding points, remove the last one from the list */
+    m_prev->right->left= m_first;
+    m_first->right= m_prev->right;
+    m_heap->free_point_info(m_prev, m_prev_hook);
+  }
+  else
+  {
+    GCALC_DBUG_ASSERT(m_prev->x != m_first->x || m_prev->y != m_first->y);
+    m_first->right= m_prev;
+    m_prev->left= m_first;
+  }
 }
 
 
diff --git a/sql/gcalc_slicescan.h b/sql/gcalc_slicescan.h
index 9fc4cea5199..55de497f1ee 100644
--- a/sql/gcalc_slicescan.h
+++ b/sql/gcalc_slicescan.h
@@ -229,7 +229,9 @@ public:
     Gcalc_dyn_list(blk_size, sizeof(Info)),
     m_hook(&m_first), m_n_points(0)
   {}
+  void set_extent(double xmin, double xmax, double ymin, double ymax);
   Info *new_point_info(double x, double y, gcalc_shape_info shape);
+  void free_point_info(Info *i, Gcalc_dyn_list::Item **i_hook);
   Info *new_intersection(const Info *p1, const Info *p2,
                          const Info *p3, const Info *p4);
   void prepare_operation();
@@ -242,6 +244,8 @@ public:
   long double get_double(const Gcalc_internal_coord *c) const;
 #endif /*GCALC_CHECK_WITH_FLOAT*/
   double coord_extent;
+  Gcalc_dyn_list::Item **get_cur_hook() { return m_hook; }
+
 private:
   Gcalc_dyn_list::Item *m_first;
   Gcalc_dyn_list::Item **m_hook;
@@ -269,6 +273,7 @@ class Gcalc_shape_transporter
 private:
   Gcalc_heap::Info *m_first;
   Gcalc_heap::Info *m_prev;
+  Gcalc_dyn_list::Item **m_prev_hook;
   int m_shape_started;
   void int_complete();
 protected:
diff --git a/sql/gcalc_tools.cc b/sql/gcalc_tools.cc
index 187acbacc4c..864437401b7 100644
--- a/sql/gcalc_tools.cc
+++ b/sql/gcalc_tools.cc
@@ -650,15 +650,6 @@ Gcalc_operation_reducer(Gcalc_function *fn, modes mode, size_t blk_size) :
 }
 
 
-#ifdef TMP_BLOCK
-void Gcalc_operation_reducer::res_point::set(const Gcalc_scan_iterator *si)
-{
-  if ((intersection_point= si->intersection_step()))
-    ii= si->get_cur_ii();
-  else
-    pi= si->get_cur_pi();
-}
-#endif /*TMP_BLOCK*/
 void Gcalc_operation_reducer::res_point::set(const Gcalc_scan_iterator *si)
 {
   intersection_point= si->intersection_step();
@@ -1362,6 +1353,10 @@ int Gcalc_operation_reducer::get_result(Gcalc_result_receiver *storage)
   GCALC_DBUG_ENTER("Gcalc_operation_reducer::get_result");
   *m_res_hook= NULL;
 
+  /* This is to workaround an old gcc's bug */
+  if (m_res_hook == (Gcalc_dyn_list::Item **) &m_result)
+    goto done;
+
   while (m_result)
   {
     Gcalc_function::shape_type shape= m_result->type;
@@ -1412,6 +1407,7 @@ int Gcalc_operation_reducer::get_result(Gcalc_result_receiver *storage)
     }
   }
   
+done:
   m_res_hook= (Gcalc_dyn_list::Item **)&m_result;
   storage->done();
   GCALC_DBUG_RETURN(0);
diff --git a/sql/gen_lex_hash.cc b/sql/gen_lex_hash.cc
index e2ced8c8326..7b50b26c6ca 100644
--- a/sql/gen_lex_hash.cc
+++ b/sql/gen_lex_hash.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -85,7 +85,6 @@ So, we can read full search-structure as 32-bit word
 #include <stdio.h>
 #include <string.h>
 
-#include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */
 
 struct hash_lex_struct
 {
@@ -377,9 +376,25 @@ int main(int argc,char **argv)
   /* Broken up to indicate that it's not advice to you, gentle reader. */
   printf("/*\n\n  Do " "not " "edit " "this " "file " "directly!\n\n*/\n");
 
-  puts("/*");
-  puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000, 2011"));
-  puts("*/");
+  printf("\
+/* Copyright (C) 2000-2008 MySQL AB, 2008 Sun Microsystems, Inc.\n\
+   Copyright (C) 2008-2011 Oracle\n\
+\n\
+   This program is free software; you can redistribute it and/or modify\n\
+   it under the terms of the GNU General Public License as published by\n\
+   the Free Software Foundation; version 2 of the License.\n\
+\n\
+   This program is distributed in the hope that it will be useful,\n\
+   but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n\
+   GNU General Public License for more details.\n\
+\n\
+   You should have received a copy of the GNU General Public License\n\
+   along with this program; see the file COPYING. If not, write to the\n\
+   Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston\n\
+   MA  02110-1301  USA. */\n\
+\n\
+");
 
   /* Broken up to indicate that it's not advice to you, gentle reader. */
   printf("/* Do " "not " "edit " "this " "file!  This is generated by "
diff --git a/sql/ha_ndbcluster.cc b/sql/ha_ndbcluster.cc
index 7a62efb1917..2878f25ed14 100644
--- a/sql/ha_ndbcluster.cc
+++ b/sql/ha_ndbcluster.cc
@@ -1,4 +1,4 @@
-/* Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2004, 2011, Oracle and/or its affiliates.
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc
index 3d140b92977..f91e19df08e 100644
--- a/sql/ha_ndbcluster_binlog.cc
+++ b/sql/ha_ndbcluster_binlog.cc
@@ -1,5 +1,4 @@
-/*
-  Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2006, 2011, Oracle and/or its affiliates.
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc
index a1e45b36b82..1f6c81cd459 100644
--- a/sql/ha_partition.cc
+++ b/sql/ha_partition.cc
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
+  Copyright (c) 2005, 2011, Oracle and/or its affiliates.
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
diff --git a/sql/ha_partition.h b/sql/ha_partition.h
index 46a8924c883..8723682904a 100644
--- a/sql/ha_partition.h
+++ b/sql/ha_partition.h
@@ -2,7 +2,7 @@
 #define HA_PARTITION_INCLUDED
 
 /*
-   Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2005, 2011, Oracle and/or its affiliates.
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
diff --git a/sql/handler.cc b/sql/handler.cc
index 1d0f676493d..8c2756977cf 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -2879,7 +2879,23 @@ void handler::print_error(int error, myf errflag)
       char key[MAX_KEY_LENGTH];
       String str(key,sizeof(key),system_charset_info);
       /* Table is opened and defined at this point */
-      key_unpack(&str,table,(uint) key_nr);
+
+      /*
+        Use primary_key instead of key_nr because key_nr is a key
+        number in the child FK table, not in our 'table'. See
+        Bug#12661768 UPDATE IGNORE CRASHES SERVER IF TABLE IS INNODB
+        AND IT IS PARENT FOR OTHER ONE This bug gets a better fix in
+        MySQL 5.6, but it is too risky to get that in 5.1 and 5.5
+        (extending the handler interface and adding new error message
+        codes)
+      */
+      if (table->s->primary_key < MAX_KEY)
+        key_unpack(&str,table,table->s->primary_key);
+      else
+      {
+        LEX_CUSTRING tmp= {USTRING_WITH_LEN("Unknown key value")};
+        str.set((const char*) tmp.str, tmp.length, system_charset_info);
+      }
       max_length= (MYSQL_ERRMSG_SIZE-
                    (uint) strlen(ER(ER_FOREIGN_DUPLICATE_KEY)));
       if (str.length() >= max_length)
@@ -2887,15 +2903,15 @@ void handler::print_error(int error, myf errflag)
         str.length(max_length-4);
         str.append(STRING_WITH_LEN("..."));
       }
-      my_error(ER_FOREIGN_DUPLICATE_KEY, MYF(0), table_share->table_name.str,
-        str.c_ptr_safe(), key_nr+1);
+      my_error(ER_FOREIGN_DUPLICATE_KEY, errflag, table_share->table_name.str,
+               str.c_ptr_safe(), key_nr+1);
       DBUG_VOID_RETURN;
     }
     textno= ER_DUP_KEY;
     break;
   }
   case HA_ERR_NULL_IN_SPATIAL:
-    my_error(ER_CANT_CREATE_GEOMETRY_OBJECT, MYF(0));
+    my_error(ER_CANT_CREATE_GEOMETRY_OBJECT, errflag);
     DBUG_VOID_RETURN;
   case HA_ERR_FOUND_DUPP_UNIQUE:
     textno=ER_DUP_UNIQUE;
@@ -2965,21 +2981,21 @@ void handler::print_error(int error, myf errflag)
   {
     String str;
     get_error_message(error, &str);
-    my_error(ER_ROW_IS_REFERENCED_2, MYF(0), str.c_ptr_safe());
+    my_error(ER_ROW_IS_REFERENCED_2, errflag, str.c_ptr_safe());
     DBUG_VOID_RETURN;
   }
   case HA_ERR_NO_REFERENCED_ROW:
   {
     String str;
     get_error_message(error, &str);
-    my_error(ER_NO_REFERENCED_ROW_2, MYF(0), str.c_ptr_safe());
+    my_error(ER_NO_REFERENCED_ROW_2, errflag, str.c_ptr_safe());
     DBUG_VOID_RETURN;
   }
   case HA_ERR_TABLE_DEF_CHANGED:
     textno=ER_TABLE_DEF_CHANGED;
     break;
   case HA_ERR_NO_SUCH_TABLE:
-    my_error(ER_NO_SUCH_TABLE, MYF(0), table_share->db.str,
+    my_error(ER_NO_SUCH_TABLE, errflag, table_share->db.str,
              table_share->table_name.str);
     DBUG_VOID_RETURN;
   case HA_ERR_RBR_LOGGING_FAILED:
@@ -2991,7 +3007,7 @@ void handler::print_error(int error, myf errflag)
     uint key_nr= get_dup_key(error);
     if ((int) key_nr >= 0)
       ptr= table->key_info[key_nr].name;
-    my_error(ER_DROP_INDEX_FK, MYF(0), ptr);
+    my_error(ER_DROP_INDEX_FK, errflag, ptr);
     DBUG_VOID_RETURN;
   }
   case HA_ERR_TABLE_NEEDS_UPGRADE:
@@ -3032,12 +3048,12 @@ void handler::print_error(int error, myf errflag)
       {
 	const char* engine= table_type();
 	if (temporary)
-	  my_error(ER_GET_TEMPORARY_ERRMSG, MYF(0), error, str.c_ptr(),
+	  my_error(ER_GET_TEMPORARY_ERRMSG, errflag, error, str.c_ptr(),
                    engine);
 	else
         {
           SET_FATAL_ERROR;
-	  my_error(ER_GET_ERRMSG, MYF(0), error, str.c_ptr(), engine);
+	  my_error(ER_GET_ERRMSG, errflag, error, str.c_ptr(), engine);
         }
       }
       else
@@ -3045,15 +3061,19 @@ void handler::print_error(int error, myf errflag)
       DBUG_VOID_RETURN;
     }
   }
-  if (fatal_error && (debug_assert_if_crashed_table ||
-                      global_system_variables.log_warnings > 1))
+  if (fatal_error)
   {
-    /*
-      Log error to log before we crash or if extended warnings are requested
-    */
-    errflag|= ME_NOREFRESH;
-  }
-    
+    /* Ensure this becomes a true error */
+    errflag&= ~(ME_JUST_WARNING | ME_JUST_INFO);
+    if ((debug_assert_if_crashed_table ||
+                      global_system_variables.log_warnings > 1))
+    {
+      /*
+        Log error to log before we crash or if extended warnings are requested
+      */
+      errflag|= ME_NOREFRESH;
+    }
+  }    
   my_error(textno, errflag, table_share->table_name.str, error);
   DBUG_VOID_RETURN;
 }
diff --git a/sql/item.cc b/sql/item.cc
index 028cc8c8e30..02cbe9b7f49 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -4227,7 +4227,7 @@ String* Item_ref_null_helper::val_str(String* s)
 
 bool Item_ref_null_helper::get_date(MYSQL_TIME *ltime, ulonglong fuzzydate)
 {  
-  return (owner->was_null|= null_value= (*ref)->get_date(ltime, fuzzydate));
+  return (owner->was_null|= null_value= (*ref)->get_date_result(ltime, fuzzydate));
 }
 
 
@@ -8654,6 +8654,20 @@ void Item_cache::print(String *str, enum_query_type query_type)
   str->append(')');
 }
 
+/**
+  Assign to this cache NULL value if it is possible
+*/
+
+void Item_cache::set_null()
+{
+  if (maybe_null)
+  {
+    null_value= TRUE;
+    value_cached= TRUE;
+  }
+}
+
+
 bool  Item_cache_int::cache_value()
 {
   if (!example)
@@ -9090,6 +9104,20 @@ void Item_cache_row::bring_value()
 }
 
 
+/**
+  Assign to this cache NULL value if it is possible
+*/
+
+void Item_cache_row::set_null()
+{
+  Item_cache::set_null();
+  if (!values)
+    return;
+  for (uint i= 0; i < item_count; i++)
+    values[i]->set_null();
+};
+
+
 Item_type_holder::Item_type_holder(THD *thd, Item *item)
   :Item(thd, item), enum_set_typelib(0), fld_type(get_real_type(item))
 {
diff --git a/sql/item.h b/sql/item.h
index 9186ba084db..8b452c303d6 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -1150,6 +1150,10 @@ public:
   virtual bool eval_not_null_tables(uchar *opt_arg) { return 0; }
   virtual bool clear_sum_processor(uchar *opt_arg) { return 0; }
   virtual bool is_subquery_processor (uchar *opt_arg) { return 0; }
+  virtual bool limit_index_condition_pushdown_processor(uchar *opt_arg)
+  { 
+    return FALSE;
+  }
 
   /* To call bool function for all arguments */
   struct bool_func_call_args
@@ -3885,6 +3889,7 @@ public:
       return false;
     return example->is_expensive_processor(arg);
   }
+  virtual void set_null();
 };
 
 
@@ -4055,6 +4060,7 @@ public:
     DBUG_VOID_RETURN;
   }
   bool cache_value();
+  virtual void set_null();
 };
 
 
diff --git a/sql/item_buff.cc b/sql/item_buff.cc
index e72c63daf7d..86e0fd32774 100644
--- a/sql/item_buff.cc
+++ b/sql/item_buff.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_create.cc b/sql/item_create.cc
index 9c8d6b73e60..96837a8f262 100644
--- a/sql/item_create.cc
+++ b/sql/item_create.cc
@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2011 Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_create.h b/sql/item_create.h
index a8a8b9d97dd..ac6b0f8454f 100644
--- a/sql/item_create.h
+++ b/sql/item_create.h
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2010, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2011, Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_func.cc b/sql/item_func.cc
index 1d3a32ba4f1..034ffbea344 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -1,5 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates.
-   Copyright (c) 2011 Monty Program Ab
+/* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_func.h b/sql/item_func.h
index 8e9fbd9af5c..f5c43360ee8 100644
--- a/sql/item_func.h
+++ b/sql/item_func.h
@@ -1763,6 +1763,7 @@ public:
     table= 0;           // required by Item_func_match::eq()
     DBUG_VOID_RETURN;
   }
+  bool is_expensive_processor(uchar *arg) { return TRUE; }
   enum Functype functype() const { return FT_FUNC; }
   const char *func_name() const { return "match"; }
   void update_used_tables() {}
@@ -1957,6 +1958,10 @@ public:
   {
     return trace_unsupported_by_check_vcol_func_processor(func_name());
   }
+  bool limit_index_condition_pushdown_processor(uchar *opt_arg)
+  {
+    return TRUE;
+  }
 };
 
 
diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc
index 5ccdb198ecb..e1f06824d5b 100644
--- a/sql/item_geofunc.cc
+++ b/sql/item_geofunc.cc
@@ -1,5 +1,6 @@
-/* Copyright (c) 2000, 2010 Oracle and/or its affiliates.
-   Copyright (C) 2011 Monty Program Ab.
+/*
+   Copyright (c) 2003-2007 MySQL AB, 2009, 2010 Sun Microsystems, Inc.
+   Use is subject to license terms.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -659,140 +660,6 @@ static double distance_points(const Gcalc_heap::Info *a,
 }
 
 
-/*
-  Calculates the distance between objects.
-*/
-
-#ifdef TMP_BLOCK
-static int calc_distance(double *result, Gcalc_heap *collector, uint obj2_si,
-                         Gcalc_function *func, Gcalc_scan_iterator *scan_it)
-{
-  bool above_cur_point, cur_point_edge;
-  const Gcalc_scan_iterator::point *evpos;
-  const Gcalc_heap::Info *cur_point= NULL;
-  const Gcalc_heap::Info *dist_point;
-  const Gcalc_scan_iterator::point *ev;
-  double t, distance, cur_distance;
-  double ex, ey, vx, vy, e_sqrlen;
-  int o1, o2;
-
-  DBUG_ENTER("calc_distance");
-
-  above_cur_point= false;
-  distance= DBL_MAX;
-
-  while (scan_it->more_points())
-  {
-    if (scan_it->step())
-      goto mem_error;
-    evpos= scan_it->get_event_position();
-    ev= scan_it->get_events();
-    cur_point= NULL;
-
-    if (ev->simple_event())
-    {
-      cur_point= ev->pi;
-      goto calculate_distance;
-    }
-
-    /*
-       handling intersection we only need to check if it's the intersecion
-       of objects 1 and 2. In this case distance is 0
-    */
-    o1= 0;
-    o2= 0;
-    for (; ev; ev= ev->get_next())
-    {
-      if (ev->event != scev_intersection)
-        cur_point= ev->pi;
-      if (ev->pi->shape >= obj2_si)
-        o2= 1;
-      else
-        o1= 1;
-      if (o1 && o2)
-      {
-        distance= 0;
-        goto exit;
-      }
-    }
-    if (!cur_point)
-      continue;
-
-#ifdef TO_REMOVE
-    goto calculate_distance;
-    /*
-       having these events we need to check for possible intersection
-       of objects
-       scev_thread | scev_two_threads | scev_single_point
-    */
-    DBUG_ASSERT(ev & (scev_thread | scev_two_threads | scev_single_point));
-
-    func->clear_state();
-    for (Gcalc_point_iterator pit(scan_it); pit.point() != evpos; ++pit)
-    {
-      gcalc_shape_info si= pit.point()->get_shape();
-      if ((func->get_shape_kind(si) == Gcalc_function::shape_polygon))
-        func->invert_state(si);
-    }
-    func->invert_state(evpos->get_shape());
-    if (func->count())
-    {
-      /* Point of one object is inside the other - intersection found */
-      distance= 0;
-      goto exit;
-    }
-#endif /*TO_REMOVE*/
-
-calculate_distance:
-    if (cur_point->shape >= obj2_si)
-      continue;
-    cur_point_edge= !cur_point->is_bottom();
-
-    for (dist_point= collector->get_first(); dist_point; dist_point= dist_point->get_next())
-    {
-      /* We only check vertices of object 2 */
-      if (dist_point->shape < obj2_si)
-        continue;
-
-      /* if we have an edge to check */
-      if (dist_point->left)
-      {
-        t= count_edge_t(dist_point, dist_point->left, cur_point,
-                        ex, ey, vx, vy, e_sqrlen);
-        if ((t > 0.0) && (t < 1.0))
-        {
-          cur_distance= distance_to_line(ex, ey, vx, vy, e_sqrlen);
-          if (distance > cur_distance)
-            distance= cur_distance;
-        }
-      }
-      if (cur_point_edge)
-      {
-        t= count_edge_t(cur_point, cur_point->left, dist_point,
-                        ex, ey, vx, vy, e_sqrlen);
-        if ((t > 0.0) && (t < 1.0))
-        {
-          cur_distance= distance_to_line(ex, ey, vx, vy, e_sqrlen);
-          if (distance > cur_distance)
-            distance= cur_distance;
-        }
-      }
-      cur_distance= distance_points(cur_point, dist_point);
-      if (distance > cur_distance)
-        distance= cur_distance;
-    }
-  }
-
-exit:
-  *result= distance;
-  DBUG_RETURN(0);
-
-mem_error:
-  DBUG_RETURN(1);
-}
-#endif /*TMP_BLOCK*/
-
-
 #define GIS_ZERO 0.00000000001
 
 longlong Item_func_spatial_rel::val_int()
@@ -806,6 +673,8 @@ longlong Item_func_spatial_rel::val_int()
   int result= 0;
   int mask= 0;
   uint shape_a, shape_b;
+  MBR umbr, mbr1, mbr2;
+  const char *c_end;
 
   res1= args[0]->val_str(&tmp_value1);
   res2= args[1]->val_str(&tmp_value2);
@@ -820,19 +689,35 @@ longlong Item_func_spatial_rel::val_int()
 	!(g2= Geometry::construct(&buffer2, res2->ptr(), res2->length())))))
     goto exit;
 
+  g1->get_mbr(&mbr1, &c_end);
+  g2->get_mbr(&mbr2, &c_end);
+
+  umbr= mbr1;
+  umbr.add_mbr(&mbr2);
+  collector.set_extent(umbr.xmin, umbr.xmax, umbr.ymin, umbr.ymax);
+
+  mbr1.buffer(1e-5);
+
   switch (spatial_rel) {
     case SP_CONTAINS_FUNC:
+      if (!mbr1.contains(&mbr2))
+        goto exit;
       mask= 1;
       func.add_operation(Gcalc_function::op_difference, 2);
       /* Mind the g2 goes first. */
       null_value= g2->store_shapes(&trn) || g1->store_shapes(&trn);
       break;
     case SP_WITHIN_FUNC:
+      mbr2.buffer(2e-5);
+      if (!mbr1.within(&mbr2))
+        goto exit;
       mask= 1;
       func.add_operation(Gcalc_function::op_difference, 2);
       null_value= g1->store_shapes(&trn) || g2->store_shapes(&trn);
       break;
     case SP_EQUALS_FUNC:
+      if (!mbr1.contains(&mbr2))
+        goto exit;
       mask= 1;
       func.add_operation(Gcalc_function::op_symdifference, 2);
       null_value= g1->store_shapes(&trn) || g2->store_shapes(&trn);
@@ -843,6 +728,8 @@ longlong Item_func_spatial_rel::val_int()
       null_value= g1->store_shapes(&trn) || g2->store_shapes(&trn);
       break;
     case SP_INTERSECTS_FUNC:
+      if (!mbr1.intersects(&mbr2))
+        goto exit;
       func.add_operation(Gcalc_function::op_intersection, 2);
       null_value= g1->store_shapes(&trn) || g2->store_shapes(&trn);
       break;
@@ -900,15 +787,6 @@ longlong Item_func_spatial_rel::val_int()
   scan_it.init(&collector);
   scan_it.killed= (int *) &(current_thd->killed);
 
-#ifdef TMP_BLOCK
-  if (spatial_rel == SP_EQUALS_FUNC)
-  {
-    result= (g1->get_class_info()->m_type_id == g1->get_class_info()->m_type_id) &&
-            func_equals();
-    goto exit;
-  }
-#endif /*TMP_BLOCK*/
-
   if (func.alloc_states())
     goto exit;
 
@@ -937,6 +815,8 @@ String *Item_func_spatial_operation::val_str(String *str_value)
   Geometry *g1, *g2;
   uint32 srid= 0;
   Gcalc_operation_transporter trn(&func, &collector);
+  MBR mbr1, mbr2;
+  const char *c_end;
 
   if (func.reserve_op_buffer(1))
     DBUG_RETURN(0);
@@ -945,14 +825,23 @@ String *Item_func_spatial_operation::val_str(String *str_value)
   if ((null_value=
        (args[0]->null_value || args[1]->null_value ||
 	!(g1= Geometry::construct(&buffer1, res1->ptr(), res1->length())) ||
-	!(g2= Geometry::construct(&buffer2, res2->ptr(), res2->length())) ||
-	g1->store_shapes(&trn) || g2->store_shapes(&trn))))
+	!(g2= Geometry::construct(&buffer2, res2->ptr(), res2->length())))))
   {
     str_value= 0;
     goto exit;
   }
 
+  g1->get_mbr(&mbr1, &c_end);
+  g2->get_mbr(&mbr2, &c_end);
+  mbr1.add_mbr(&mbr2);
+  collector.set_extent(mbr1.xmin, mbr1.xmax, mbr1.ymin, mbr1.ymax);
   
+  if ((null_value= g1->store_shapes(&trn) || g2->store_shapes(&trn)))
+  {
+    str_value= 0;
+    goto exit;
+  }
+
   collector.prepare_operation();
   if (func.alloc_states())
     goto exit;
@@ -1376,12 +1265,18 @@ String *Item_func_buffer::val_str(String *str_value)
   uint32 srid= 0;
   String *str_result= NULL;
   Transporter trn(&func, &collector, dist);
+  MBR mbr;
+  const char *c_end;
 
   null_value= 1;
   if (args[0]->null_value || args[1]->null_value ||
-      !(g= Geometry::construct(&buffer, obj->ptr(), obj->length())))
+      !(g= Geometry::construct(&buffer, obj->ptr(), obj->length())) ||
+      g->get_mbr(&mbr, &c_end))
     goto mem_error;
 
+  if (dist > 0.0)
+    mbr.buffer(dist);
+  collector.set_extent(mbr.xmin, mbr.xmax, mbr.ymin, mbr.ymax);
   /*
     If the distance given is 0, the Buffer function is in fact NOOP,
     so it's natural just to return the argument1.
@@ -1448,6 +1343,8 @@ longlong Item_func_issimple::val_int()
   Geometry *g;
   int result= 1;
   const Gcalc_scan_iterator::event_point *ev;
+  MBR mbr;
+  const char *c_end;
 
   DBUG_ENTER("Item_func_issimple::val_int");
   DBUG_ASSERT(fixed == 1);
@@ -1456,6 +1353,8 @@ longlong Item_func_issimple::val_int()
       !(g= Geometry::construct(&buffer, swkb->ptr(), swkb->length())))
     DBUG_RETURN(0);
 
+  g->get_mbr(&mbr, &c_end);
+  collector.set_extent(mbr.xmin, mbr.xmax, mbr.ymin, mbr.ymax);
 
   if (g->get_class_info()->m_type_id == Geometry::wkb_point)
     DBUG_RETURN(1);
@@ -1684,6 +1583,8 @@ double Item_func_distance::val_real()
   String *res2= args[1]->val_str(&tmp_value2);
   Geometry_buffer buffer1, buffer2;
   Geometry *g1, *g2;
+  MBR mbr1, mbr2;
+  const char *c_end;
 
 
   if ((null_value= (args[0]->null_value || args[1]->null_value ||
@@ -1691,6 +1592,11 @@ double Item_func_distance::val_real()
           !(g2= Geometry::construct(&buffer2, res2->ptr(), res2->length())))))
     goto mem_error;
 
+  g1->get_mbr(&mbr1, &c_end);
+  g2->get_mbr(&mbr2, &c_end);
+  mbr1.add_mbr(&mbr2);
+  collector.set_extent(mbr1.xmin, mbr1.xmax, mbr1.ymin, mbr1.ymax);
+
   if ((g1->get_class_info()->m_type_id == Geometry::wkb_point) &&
       (g2->get_class_info()->m_type_id == Geometry::wkb_point))
   {
diff --git a/sql/item_row.cc b/sql/item_row.cc
index ccd7a37e9b7..530a40c55dc 100644
--- a/sql/item_row.cc
+++ b/sql/item_row.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2002, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_row.h b/sql/item_row.h
index 1d34ecfc310..aa56068f8ba 100644
--- a/sql/item_row.h
+++ b/sql/item_row.h
@@ -1,7 +1,8 @@
 #ifndef ITEM_ROW_INCLUDED
 #define ITEM_ROW_INCLUDED
 
-/* Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2002, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index f570c309d79..48029863878 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h
index 69800e7e674..8c56cea990b 100644
--- a/sql/item_strfunc.h
+++ b/sql/item_strfunc.h
@@ -1,7 +1,8 @@
 #ifndef ITEM_STRFUNC_INCLUDED
 #define ITEM_STRFUNC_INCLUDED
 
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc
index f25722ab30b..5f78cdf167b 100644
--- a/sql/item_subselect.cc
+++ b/sql/item_subselect.cc
@@ -188,7 +188,7 @@ void Item_allany_subselect::cleanup()
   */
   for (SELECT_LEX *sl= unit->first_select();
        sl; sl= sl->next_select())
-    if (test_strategy(SUBS_MAXMIN_INJECTED))
+    if (test_set_strategy(SUBS_MAXMIN_INJECTED))
       sl->with_sum_func= false;
   Item_in_subselect::cleanup();
 }
@@ -221,8 +221,7 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref)
   bool res;
 
   DBUG_ASSERT(fixed == 0);
-  /* There is no reason to get a different THD. */
-  DBUG_ASSERT(thd == thd_param);
+  engine->set_thd((thd= thd_param));
   if (!done_first_fix_fields)
   {
     done_first_fix_fields= TRUE;
@@ -905,7 +904,10 @@ void Item_singlerow_subselect::reset()
 {
   Item_subselect::reset();
   if (value)
-    value->null_value= TRUE;
+  {
+    for(uint i= 0; i < engine->cols(); i++)
+      row[i]->set_null();
+  }
 }
 
 
@@ -1016,6 +1018,11 @@ void Item_singlerow_subselect::fix_length_and_dec()
   */
   if (engine->no_tables())
     maybe_null= engine->may_be_null();
+  else
+  {
+    for (uint i= 0; i < max_columns; i++)
+      row[i]->maybe_null= TRUE;
+  }
 }
 
 
@@ -1201,7 +1208,8 @@ Item_in_subselect::Item_in_subselect(Item * left_exp,
   Item_exists_subselect(), 
   left_expr_cache(0), first_execution(TRUE), in_strategy(SUBS_NOT_TRANSFORMED),
   optimizer(0), pushed_cond_guards(NULL), emb_on_expr_nest(NULL),
-  is_jtbm_merged(FALSE), is_flattenable_semijoin(FALSE),
+  is_jtbm_merged(FALSE), is_jtbm_const_tab(FALSE), 
+  is_flattenable_semijoin(FALSE),
   is_registered_semijoin(FALSE), 
   upper_item(0)
 {
@@ -1674,6 +1682,13 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
       thd->change_item_tree(it.ref(), item);
     }
 
+    DBUG_EXECUTE("where",
+                 print_where(item, "rewrite with MIN/MAX", QT_ORDINARY););
+    if (thd->variables.sql_mode & MODE_ONLY_FULL_GROUP_BY)
+    {
+      select_lex->set_non_agg_field_used(false);
+    }
+
     save_allow_sum_func= thd->lex->allow_sum_func;
     thd->lex->allow_sum_func|= 1 << thd->lex->current_select->nest_level;
     /*
@@ -2906,6 +2921,7 @@ int subselect_single_select_engine::exec()
   char const *save_where= thd->where;
   SELECT_LEX *save_select= thd->lex->current_select;
   thd->lex->current_select= select_lex;
+
   if (!join->optimized)
   {
     SELECT_LEX_UNIT *unit= select_lex->master_unit();
@@ -2914,7 +2930,7 @@ int subselect_single_select_engine::exec()
     if (join->optimize())
     {
       thd->where= save_where;
-      optimize_error= 1;
+      executed= optimize_error= 1;
       thd->lex->current_select= save_select;
       DBUG_RETURN(join->error ? join->error : 1);
     }
@@ -4043,6 +4059,7 @@ ulonglong subselect_hash_sj_engine::rowid_merge_buff_size(
   uint rowid_length= tmp_table->file->ref_length;
   select_materialize_with_stats *result_sink=
     (select_materialize_with_stats *) result;
+  ha_rows max_null_row;
 
   /* Size of the subselect_rowid_merge_engine::row_num_to_rowid buffer. */
   buff_size= row_count * rowid_length * sizeof(uchar);
@@ -4065,7 +4082,18 @@ ulonglong subselect_hash_sj_engine::rowid_merge_buff_size(
       buff_size+= (row_count - result_sink->get_null_count_of_col(i)) *
                          sizeof(rownum_t);
       /* Add the size of Ordered_key::null_key */
-      buff_size+= bitmap_buffer_size(result_sink->get_max_null_of_col(i));
+      max_null_row= result_sink->get_max_null_of_col(i);
+      if (max_null_row >= UINT_MAX)
+      {
+        /*
+          There can be at most UINT_MAX bits in a MY_BITMAP that is used to
+          store NULLs in an Ordered_key. Return a number of bytes bigger than
+          the maximum allowed memory buffer for partial matching to disable
+          the rowid merge strategy.
+        */
+        return ULONGLONG_MAX;
+      }
+      buff_size+= bitmap_buffer_size(max_null_row);
     }
   }
 
@@ -4181,6 +4209,8 @@ bool subselect_hash_sj_engine::init(List<Item> *tmp_columns, uint subquery_id)
   */
   if (tmp_table->s->keys == 0)
   {
+    //fprintf(stderr, "Q: %s\n", current_thd->query());
+    DBUG_ASSERT(0);
     DBUG_ASSERT(
       tmp_table->s->uniques ||
       tmp_table->key_info->key_length >= tmp_table->file->max_key_length() ||
@@ -5564,6 +5594,8 @@ bool subselect_rowid_merge_engine::test_null_row(rownum_t row_num)
 
 /**
   Test if a subset of NULL-able columns contains a row of NULLs.
+  @retval TRUE  if such a row exists
+  @retval FALSE no complementing null row
 */
 
 bool subselect_rowid_merge_engine::
@@ -5571,34 +5603,34 @@ exists_complementing_null_row(MY_BITMAP *keys_to_complement)
 {
   rownum_t highest_min_row= 0;
   rownum_t lowest_max_row= UINT_MAX;
-  uint count_null_keys, i, j;
+  uint count_null_keys, i;
   Ordered_key *cur_key;
 
-  count_null_keys= keys_to_complement->n_bits -
-                   bitmap_bits_set(keys_to_complement);
-  if (count_null_keys == 1)
+  if (!count_columns_with_nulls)
   {
     /*
-      The caller guarantees that the complement to keys_to_complement
-      contains only columns with NULLs. Therefore if there is only one column,
-      it is guaranteed to contain NULLs.
+      If there are both NULLs and non-NUll values in the outer reference, and
+      the subquery contains no NULLs, a complementing NULL row cannot exist.
     */
-    return TRUE;
+    return FALSE;
   }
 
-  for (i= (non_null_key ? 1 : 0), j= 0; i < merge_keys_count; i++)
+  for (i= (non_null_key ? 1 : 0), count_null_keys= 0; i < merge_keys_count; i++)
   {
     cur_key= merge_keys[i];
     if (bitmap_is_set(keys_to_complement, cur_key->get_keyid()))
       continue;
-    DBUG_ASSERT(cur_key->get_null_count());
+    if (!cur_key->get_null_count())
+    {
+      /* If there is column without NULLs, there cannot be a partial match. */
+      return FALSE;
+    }
     if (cur_key->get_min_null_row() > highest_min_row)
       highest_min_row= cur_key->get_min_null_row();
     if (cur_key->get_max_null_row() < lowest_max_row)
       lowest_max_row= cur_key->get_max_null_row();
-    null_bitmaps[j++]= cur_key->get_null_key();
+    null_bitmaps[count_null_keys++]= cur_key->get_null_key();
   }
-  DBUG_ASSERT(count_null_keys == j);
 
   if (lowest_max_row < highest_min_row)
   {
@@ -5608,7 +5640,7 @@ exists_complementing_null_row(MY_BITMAP *keys_to_complement)
 
   return bitmap_exists_intersection((const MY_BITMAP**) null_bitmaps,
                                     count_null_keys,
-                                    highest_min_row, lowest_max_row);
+                                    (uint)highest_min_row, (uint)lowest_max_row);
 }
 
 
diff --git a/sql/item_subselect.h b/sql/item_subselect.h
index 6007812fa7d..57f413f1ccd 100644
--- a/sql/item_subselect.h
+++ b/sql/item_subselect.h
@@ -1,7 +1,7 @@
 #ifndef ITEM_SUBSELECT_INCLUDED
 #define ITEM_SUBSELECT_INCLUDED
 
-/* Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2002, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -241,6 +241,10 @@ public:
   virtual bool expr_cache_is_needed(THD *);
   virtual void get_cache_parameters(List<Item> &parameters);
   virtual bool is_subquery_processor (uchar *opt_arg) { return 1; }
+  bool limit_index_condition_pushdown_processor(uchar *opt_arg) 
+  {
+    return TRUE;
+   }
 
   friend class select_result_interceptor;
   friend class Item_in_optimizer;
@@ -460,10 +464,44 @@ public:
     Same as above, but they also allow to scan the materialized table. 
   */
   bool sjm_scan_allowed;
+
+  /* 
+    JoinTaB Materialization (JTBM) members
+  */
+  
+  /* 
+    TRUE <=> This subselect has been converted into non-mergeable semi-join
+    table.
+  */
+  bool is_jtbm_merged;
+  
+  /* (Applicable if is_jtbm_merged==TRUE) Time required to run the materialized join */
   double jtbm_read_time;
+
+  /* (Applicable if is_jtbm_merged==TRUE) Number of output rows in materialized join */
   double jtbm_record_count;   
-  bool is_jtbm_merged;
+  
+  /*
+    (Applicable if is_jtbm_merged==TRUE) TRUE <=> The materialized subselect is
+    a degenerate subselect which produces 0 or 1 rows, which we know at
+    optimization phase.
+    Examples:
+    1. subquery has "Impossible WHERE": 
 
+      SELECT * FROM ot WHERE ot.column IN (SELECT it.col FROM it WHERE 2 > 3)
+    
+    2. Subquery produces one row which opt_sum.cc is able to get with one lookup:
+
+      SELECT * FROM ot WHERE ot.column IN (SELECT MAX(it.key_col) FROM it)
+  */
+  bool is_jtbm_const_tab;
+  
+  /* 
+    (Applicable if is_jtbm_const_tab==TRUE) Whether the subquery has produced 
+     the row (or not)
+  */
+  bool jtbm_const_row_found;
+  
   /*
     TRUE<=>this is a flattenable semi-join, false overwise.
   */
@@ -504,7 +542,7 @@ public:
     :Item_exists_subselect(), left_expr_cache(0), first_execution(TRUE),
      abort_on_null(0), in_strategy(SUBS_NOT_TRANSFORMED), optimizer(0),
     pushed_cond_guards(NULL), func(NULL), emb_on_expr_nest(NULL), 
-    is_jtbm_merged(FALSE),
+    is_jtbm_merged(FALSE), is_jtbm_const_tab(FALSE),
     upper_item(0)
     {}
   void cleanup();
@@ -725,9 +763,9 @@ protected:
 
 class subselect_single_select_engine: public subselect_engine
 {
-  bool prepared; /* simple subselect is prepared */
-  bool executed; /* simple subselect is executed */
-  bool optimize_error; ///< simple subselect optimization failed
+  bool prepared;       /* simple subselect is prepared */
+  bool executed;       /* simple subselect is executed */
+  bool optimize_error; /* simple subselect optimization failed */
   st_select_lex *select_lex; /* corresponding select_lex */
   JOIN * join; /* corresponding JOIN structure */
 public:
@@ -755,6 +793,9 @@ public:
 
   friend class subselect_hash_sj_engine;
   friend class Item_in_subselect;
+  friend bool setup_jtbm_semi_joins(JOIN *join, List<TABLE_LIST> *join_list,
+                                    Item **join_where);
+
 };
 
 
diff --git a/sql/item_sum.cc b/sql/item_sum.cc
index 9bf786dbbd4..8824ec5b699 100644
--- a/sql/item_sum.cc
+++ b/sql/item_sum.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2011, Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_sum.h b/sql/item_sum.h
index c7a222bcd84..7233fe39ead 100644
--- a/sql/item_sum.h
+++ b/sql/item_sum.h
@@ -1,7 +1,7 @@
 #ifndef ITEM_SUM_INCLUDED
 #define ITEM_SUM_INCLUDED
 /* Copyright (c) 2000, 2011 Oracle and/or its affiliates.
-   Copyright (c) 2010, 2011 Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc
index 0d21d619421..30357a1bbe8 100644
--- a/sql/item_timefunc.cc
+++ b/sql/item_timefunc.cc
@@ -2288,8 +2288,7 @@ String *Item_char_typecast::val_str(String *str)
 			cast_cs == &my_charset_bin ?
                         "cast_as_binary" : func_name(),
                         current_thd->variables.max_allowed_packet);
-    null_value= 1;
-    return 0;
+    cast_length= current_thd->variables.max_allowed_packet;
   }
 
   if (!charset_conversion)
@@ -2360,6 +2359,18 @@ String *Item_char_typecast::val_str(String *str)
     }
   }
   null_value= 0;
+
+  if (res->length() > current_thd->variables.max_allowed_packet)
+  {
+    push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
+			ER_WARN_ALLOWED_PACKET_OVERFLOWED,
+			ER(ER_WARN_ALLOWED_PACKET_OVERFLOWED),
+			cast_cs == &my_charset_bin ?
+                        "cast_as_binary" : func_name(),
+                        current_thd->variables.max_allowed_packet);
+    null_value= 1;
+    return 0;
+  }
   return res;
 }
 
diff --git a/sql/item_xmlfunc.cc b/sql/item_xmlfunc.cc
index b93a83c05a9..150c2c212d7 100644
--- a/sql/item_xmlfunc.cc
+++ b/sql/item_xmlfunc.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2005, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/lock.cc b/sql/lock.cc
index 57a9e753ce7..a7029548493 100644
--- a/sql/lock.cc
+++ b/sql/lock.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -705,8 +706,10 @@ MYSQL_LOCK *get_lock_data(THD *thd, TABLE **table_ptr, uint count, uint flags)
   for (i=tables=lock_count=0 ; i < count ; i++)
   {
     TABLE *t= table_ptr[i];
-
-    if (t->s->tmp_table != NON_TRANSACTIONAL_TMP_TABLE)
+    
+    
+    if (t->s->tmp_table != NON_TRANSACTIONAL_TMP_TABLE && 
+        t->s->tmp_table != INTERNAL_TMP_TABLE)
     {
       tables+= t->file->lock_count();
       lock_count++;
@@ -734,8 +737,9 @@ MYSQL_LOCK *get_lock_data(THD *thd, TABLE **table_ptr, uint count, uint flags)
     TABLE *table;
     enum thr_lock_type lock_type;
     THR_LOCK_DATA **locks_start;
-
-    if ((table=table_ptr[i])->s->tmp_table == NON_TRANSACTIONAL_TMP_TABLE)
+    table= table_ptr[i];
+    if (table->s->tmp_table == NON_TRANSACTIONAL_TMP_TABLE ||
+        table->s->tmp_table == INTERNAL_TMP_TABLE) 
       continue;
     lock_type= table->reginfo.lock_type;
     DBUG_ASSERT(lock_type != TL_WRITE_DEFAULT && lock_type != TL_READ_DEFAULT);
diff --git a/sql/log.cc b/sql/log.cc
index b882aab15e9..5a4d09a2744 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -3464,7 +3464,6 @@ bool MYSQL_BIN_LOG::reset_logs(THD* thd)
   DBUG_ENTER("reset_logs");
 
   ha_reset_logs(thd);
-
   /*
     We need to get both locks to be sure that no one is trying to
     write to the index log file.
diff --git a/sql/log.h b/sql/log.h
index bad86e7615d..1e3d0698eee 100644
--- a/sql/log.h
+++ b/sql/log.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2005, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/log_event.cc b/sql/log_event.cc
index 38af12cab9f..676fa457c15 100644
--- a/sql/log_event.cc
+++ b/sql/log_event.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/log_event.h b/sql/log_event.h
index 0accbd0dd1c..5c0a82b2dac 100644
--- a/sql/log_event.h
+++ b/sql/log_event.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/log_event_old.cc b/sql/log_event_old.cc
index a52c584726a..49d35197d60 100644
--- a/sql/log_event_old.cc
+++ b/sql/log_event_old.cc
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2007, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/message.h b/sql/message.h
index 8ef0032e423..dac0576d0c4 100644
--- a/sql/message.h
+++ b/sql/message.h
@@ -68,8 +68,8 @@
 // MessageText:

 //

 //  %1For more information, see Help and Support Center at http://www.mysql.com.

-//  
-//  
+//  

+//  

 //

 #define MSG_DEFAULT                      0xC0000064L

 

diff --git a/sql/my_decimal.cc b/sql/my_decimal.cc
index 39be927fd27..21611afd87b 100644
--- a/sql/my_decimal.cc
+++ b/sql/my_decimal.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2005, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/my_decimal.h b/sql/my_decimal.h
index e0c47029940..bd03782cb18 100644
--- a/sql/my_decimal.h
+++ b/sql/my_decimal.h
@@ -73,15 +73,30 @@ inline int my_decimal_int_part(uint precision, uint decimals)
 
 class my_decimal :public decimal_t
 {
+  /*
+    Several of the routines in strings/decimal.c have had buffer
+    overrun/underrun problems. These are *not* caught by valgrind.
+    To catch them, we allocate dummy fields around the buffer,
+    and test that their values do not change.
+   */
+#if !defined(DBUG_OFF)
+  int foo1;
+#endif
+
   decimal_digit_t buffer[DECIMAL_BUFF_LENGTH];
 
+#if !defined(DBUG_OFF)
+  int foo2;
+  static const int test_value= 123;
+#endif
+
 public:
 
   my_decimal(const my_decimal &rhs) : decimal_t(rhs)
   {
+    init();
     for (uint i= 0; i < DECIMAL_BUFF_LENGTH; i++)
       buffer[i]= rhs.buffer[i];
-    fix_buffer_pointer();
   }
 
   my_decimal& operator=(const my_decimal &rhs)
@@ -97,14 +112,30 @@ public:
 
   void init()
   {
+#if !defined(DBUG_OFF)
+    foo1= test_value;
+    foo2= test_value;
+#endif
     len= DECIMAL_BUFF_LENGTH;
     buf= buffer;
+    TRASH_ALLOC(buffer, sizeof(buffer));
   }
 
   my_decimal()
   {
     init();
   }
+  ~my_decimal()
+  {
+    sanity_check();
+  }
+
+  void sanity_check()
+  {
+    DBUG_ASSERT(foo1 == test_value);
+    DBUG_ASSERT(foo2 == test_value);
+  }
+
   void fix_buffer_pointer() { buf= buffer; }
 
   bool sign() const { return decimal_t::sign; }
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index d3b7e1e7085..fc58866ac17 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -1,5 +1,5 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
-   Copyright (c) 2009-2011 Monty Program Ab
+/* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -275,8 +275,6 @@ extern "C" sig_handler handle_segfault(int sig);
 
 /* Constants */
 
-#include <welcome_copyright_notice.h> // ORACLE_WELCOME_COPYRIGHT_NOTICE
-
 const char *show_comp_option_name[]= {"YES", "NO", "DISABLED"};
 
 static const char *tc_heuristic_recover_names[]=
@@ -7220,8 +7218,14 @@ static void usage(void)
   if (!default_collation_name)
     default_collation_name= (char*) default_charset_info->name;
   print_version();
-  puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000, 2011"));
-  puts("Starts the MySQL database server.\n");
+  puts("\
+Copyright (C) 2000-2008 MySQL AB, by Monty and others.\n\
+Copyright (C) 2000, 2011 Oracle.\n\
+Copyright (C) 2009-2011 Monty Program Ab.\n\
+This software comes with ABSOLUTELY NO WARRANTY. This is free software,\n\
+and you are welcome to modify and redistribute it under the GPL license\n\n\
+Starts the MariaDB database server.\n");
+
   printf("Usage: %s [OPTIONS]\n", my_progname);
   if (!opt_verbose)
     puts("\nFor more help options (several pages), use mysqld --verbose --help.");
diff --git a/sql/opt_index_cond_pushdown.cc b/sql/opt_index_cond_pushdown.cc
index 12a732176a5..bec3bf6dab6 100644
--- a/sql/opt_index_cond_pushdown.cc
+++ b/sql/opt_index_cond_pushdown.cc
@@ -29,6 +29,11 @@
 bool uses_index_fields_only(Item *item, TABLE *tbl, uint keyno,
                             bool other_tbls_ok)
 {
+  if (item->walk(&Item::limit_index_condition_pushdown_processor, FALSE, NULL))
+  {
+    return FALSE;
+  }
+
   if (item->const_item())
     return TRUE;
 
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index c6e1bf584e3..69273398585 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2011, Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -11708,6 +11708,8 @@ get_best_group_min_max(PARAM *param, SEL_TREE *tree, double read_time)
     DBUG_RETURN(NULL);
   if (table->s->keys == 0)        /* There are no indexes to use. */
     DBUG_RETURN(NULL);
+  if (join->conds && join->conds->used_tables() & OUTER_REF_TABLE_BIT)
+    DBUG_RETURN(NULL); /* Cannot execute with correlated conditions. */
 
   /* Check (SA1,SA4) and store the only MIN/MAX argument - the C attribute.*/
   if (join->make_sum_func_list(join->all_fields, join->fields_list, 1))
@@ -13053,6 +13055,7 @@ int QUICK_GROUP_MIN_MAX_SELECT::reset(void)
   int result;
   DBUG_ENTER("QUICK_GROUP_MIN_MAX_SELECT::reset");
 
+  seen_first_key= FALSE;
   if (!head->key_read)
   {
     doing_key_read= 1;
diff --git a/sql/opt_range.h b/sql/opt_range.h
index cf8da1acb0d..545e9e3c7b8 100644
--- a/sql/opt_range.h
+++ b/sql/opt_range.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index 02a57a9e7df..3b1991d1686 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -10,7 +10,9 @@
 #pragma implementation				// gcc: Class implementation
 #endif
 
+#include "sql_base.h"
 #include "sql_select.h"
+#include "filesort.h"
 #include "opt_subselect.h"
 #include "sql_test.h"
 #include <my_bit.h>
@@ -303,6 +305,14 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
   st_select_lex *select_lex= join->select_lex;
   st_select_lex_unit* parent_unit= select_lex->master_unit();
   DBUG_ENTER("check_and_do_in_subquery_rewrites");
+
+  /*
+    IN/ALL/ANY rewrites are not applicable for so called fake select
+    (this select exists only to filter results of union if it is needed).
+  */
+  if (select_lex == select_lex->master_unit()->fake_select_lex)
+    DBUG_RETURN(0);
+
   /*
     If 
       1) this join is inside a subquery (of any type except FROM-clause 
@@ -375,14 +385,6 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
       if (failure)
         DBUG_RETURN(-1); /* purecov: deadcode */
     }
-    if (select_lex == parent_unit->fake_select_lex)
-    {
-      /*
-        The join and its select_lex object represent the 'fake' select used
-        to compute the result of a UNION.
-      */
-      DBUG_RETURN(0);
-    }
 
     DBUG_PRINT("info", ("Checking if subq can be converted to semi-join"));
     /*
@@ -596,7 +598,7 @@ bool subquery_types_allow_materialization(Item_in_subselect *in_subs)
       break;
     case TIME_RESULT:
       if (mysql_type_to_time_type(outer->field_type()) !=
-          mysql_type_to_time_type(outer->field_type()))
+          mysql_type_to_time_type(inner->field_type()))
         DBUG_RETURN(FALSE);
     default:
       /* suitable for materialization */
@@ -1265,10 +1267,9 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
   List_iterator_fast<TABLE_LIST> si(subq_lex->leaf_tables);
   while ((tl= si++))
   {
-    tl->table->tablenr= table_no;
-    tl->table->map= ((table_map)1) << table_no;
+    tl->set_tablenr(table_no);
     if (tl->is_jtbm())
-      tl->jtbm_table_no= tl->table->tablenr;
+      tl->jtbm_table_no= table_no;
     SELECT_LEX *old_sl= tl->select_lex;
     tl->select_lex= parent_join->select_lex; 
     for (TABLE_LIST *emb= tl->embedding;
@@ -1426,25 +1427,12 @@ static bool convert_subq_to_jtbm(JOIN *parent_join,
   List<TABLE_LIST> *emb_join_list= &parent_lex->top_join_list;
   TABLE_LIST *emb_tbl_nest= NULL; // will change when we learn to handle outer joins
   TABLE_LIST *tl;
-  double rows;
-  double read_time;
   DBUG_ENTER("convert_subq_to_jtbm");
-
+  bool optimization_delayed= TRUE;
   subq_pred->set_strategy(SUBS_MATERIALIZATION);
-  if (subq_pred->optimize(&rows, &read_time))
-    DBUG_RETURN(TRUE);
 
-  subq_pred->jtbm_read_time= read_time;
-  subq_pred->jtbm_record_count=rows;
   subq_pred->is_jtbm_merged= TRUE;
 
-  if (subq_pred->engine->engine_type() != subselect_engine::HASH_SJ_ENGINE)
-  {
-    *remove_item= FALSE;
-    DBUG_RETURN(FALSE);
-  }
-
-
   *remove_item= TRUE;
 
   TABLE_LIST *jtbm;
@@ -1480,7 +1468,18 @@ static bool convert_subq_to_jtbm(JOIN *parent_join,
   tl->next_local= jtbm;
 
   /* A theory: no need to re-connect the next_global chain */
+  if (optimization_delayed)
+  {
+    DBUG_ASSERT(parent_join->table_count < MAX_TABLES);
+
+    jtbm->jtbm_table_no= parent_join->table_count;
 
+    create_subquery_temptable_name(tbl_alias, 
+                                   subq_pred->unit->first_select()->select_number);
+    jtbm->alias= tbl_alias;
+    parent_join->table_count++;
+    DBUG_RETURN(FALSE);
+  }
   subselect_hash_sj_engine *hash_sj_engine=
     ((subselect_hash_sj_engine*)subq_pred->engine);
   jtbm->table= hash_sj_engine->tmp_table;
@@ -2165,227 +2164,172 @@ bool find_eq_ref_candidate(TABLE *table, table_map sj_inner_tables)
     join prefixes each strategy can handle.
 */
 
-void advance_sj_state(JOIN *join, table_map remaining_tables, 
-                      const JOIN_TAB *new_join_tab, uint idx, 
-                      double *current_record_count, double *current_read_time, 
+bool is_multiple_semi_joins(JOIN *join, POSITION *prefix, uint idx, table_map inner_tables)
+{
+  for (int i= (int)idx; i >= 0; i--)
+  {
+    TABLE_LIST *emb_sj_nest;
+    if ((emb_sj_nest= prefix[i].table->emb_sj_nest))
+    {
+      if (inner_tables & emb_sj_nest->sj_inner_tables)
+        return !test(inner_tables == (emb_sj_nest->sj_inner_tables &
+                                      ~join->const_table_map));
+    }
+  }
+  return FALSE;
+}
+
+
+void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, 
+                      double *current_record_count, double *current_read_time,
                       POSITION *loose_scan_pos)
 {
-  TABLE_LIST *emb_sj_nest;
   POSITION *pos= join->positions + idx;
-  remaining_tables &= ~new_join_tab->table->map;
-  bool disable_jbuf= join->thd->variables.join_cache_level == 0;
-
-  pos->prefix_cost.convert_from_cost(*current_read_time);
-  pos->prefix_record_count= *current_record_count;
-  pos->sj_strategy= SJ_OPT_NONE;
-  
-  pos->prefix_dups_producing_tables= join->cur_dups_producing_tables;
+  const JOIN_TAB *new_join_tab= pos->table; 
+  Semi_join_strategy_picker *pickers[]=
+  {
+    &pos->firstmatch_picker,
+    &pos->loosescan_picker,
+    &pos->sjmat_picker,
+    &pos->dups_weedout_picker,
+    NULL,
+  };
 
-  /* We're performing optimization inside SJ-Materialization nest */
   if (join->emb_sjm_nest)
   {
-    pos->invalidate_firstmatch_prefix();
-    pos->first_loosescan_table= MAX_TABLES; 
-    pos->dupsweedout_tables= 0;
-    pos->sjm_scan_need_tables= 0;
+    /* 
+      We're performing optimization inside SJ-Materialization nest:
+       - there are no other semi-joins inside semi-join nests
+       - attempts to build semi-join strategies here will confuse
+         the optimizer, so bail out.
+    */
+    pos->sj_strategy= SJ_OPT_NONE;
     return;
   }
 
-  /* Initialize the state or copy it from prev. tables */
+  /* 
+    Update join->cur_sj_inner_tables (Used by FirstMatch in this function and
+    LooseScan detector in best_access_path)
+  */
+  remaining_tables &= ~new_join_tab->table->map;
+  pos->prefix_dups_producing_tables= join->cur_dups_producing_tables;
+  TABLE_LIST *emb_sj_nest;
+  if ((emb_sj_nest= new_join_tab->emb_sj_nest))
+    join->cur_dups_producing_tables |= emb_sj_nest->sj_inner_tables;
+
+  Semi_join_strategy_picker **strategy;
   if (idx == join->const_tables)
   {
-    pos->invalidate_firstmatch_prefix();
-    pos->first_loosescan_table= MAX_TABLES; 
-    pos->dupsweedout_tables= 0;
-    pos->sjm_scan_need_tables= 0;
-    LINT_INIT(pos->sjm_scan_last_inner);
+    /* First table, initialize pickers */
+    for (strategy= pickers; *strategy != NULL; strategy++)
+      (*strategy)->set_empty();
+    pos->inner_tables_handled_with_other_sjs= 0;
   }
   else
   {
-    // FirstMatch
-    pos->first_firstmatch_table=
-      (pos[-1].sj_strategy == SJ_OPT_FIRST_MATCH) ?
-      MAX_TABLES : pos[-1].first_firstmatch_table;
-    pos->first_firstmatch_rtbl= pos[-1].first_firstmatch_rtbl;
-    pos->firstmatch_need_tables= pos[-1].firstmatch_need_tables;
-
-    // LooseScan
-    pos->first_loosescan_table=
-      (pos[-1].sj_strategy == SJ_OPT_LOOSE_SCAN) ?
-      MAX_TABLES : pos[-1].first_loosescan_table;
-    pos->loosescan_need_tables= pos[-1].loosescan_need_tables;
-
-    // SJ-Materialization Scan
-    pos->sjm_scan_need_tables=
-      (pos[-1].sj_strategy == SJ_OPT_MATERIALIZE_SCAN) ?
-      0 : pos[-1].sjm_scan_need_tables;
-    pos->sjm_scan_last_inner= pos[-1].sjm_scan_last_inner;
-
-    // Duplicate Weedout
-    pos->dupsweedout_tables=      pos[-1].dupsweedout_tables;
-    pos->first_dupsweedout_table= pos[-1].first_dupsweedout_table;
-  }
-  
-  table_map handled_by_fm_or_ls= 0;
-  /* FirstMatch Strategy */
-  if (new_join_tab->emb_sj_nest &&
-      optimizer_flag(join->thd, OPTIMIZER_SWITCH_FIRSTMATCH) &&
-      !join->outer_join)
-  {
-    const table_map outer_corr_tables=
-      new_join_tab->emb_sj_nest->nested_join->sj_corr_tables |
-      new_join_tab->emb_sj_nest->nested_join->sj_depends_on;
-    const table_map sj_inner_tables=
-      new_join_tab->emb_sj_nest->sj_inner_tables & ~join->const_table_map;
-
-    /* 
-      Enter condition:
-       1. The next join tab belongs to semi-join nest
-          (verified for the encompassing code block above).
-       2. We're not in a duplicate producer range yet
-       3. All outer tables that
-           - the subquery is correlated with, or
-           - referred to from the outer_expr 
-          are in the join prefix
-       4. All inner tables are still part of remaining_tables.
-    */
-    if (!join->cur_sj_inner_tables &&              // (2)
-        !(remaining_tables & outer_corr_tables) && // (3)
-        (sj_inner_tables ==                        // (4)
-         ((remaining_tables | new_join_tab->table->map) & sj_inner_tables)))
+    for (strategy= pickers; *strategy != NULL; strategy++)
     {
-      /* Start tracking potential FirstMatch range */
-      pos->first_firstmatch_table= idx;
-      pos->firstmatch_need_tables= sj_inner_tables;
-      pos->first_firstmatch_rtbl= remaining_tables;
+      (*strategy)->set_from_prev(pos - 1);
     }
+    pos->inner_tables_handled_with_other_sjs=
+       pos[-1].inner_tables_handled_with_other_sjs;
+  }
+
+  pos->prefix_cost.convert_from_cost(*current_read_time);
+  pos->prefix_record_count= *current_record_count;
+
+  {
+    pos->sj_strategy= SJ_OPT_NONE;
 
-    if (pos->in_firstmatch_prefix())
+    for (strategy= pickers; *strategy != NULL; strategy++)
     {
-      if (outer_corr_tables & pos->first_firstmatch_rtbl)
+      table_map handled_fanout;
+      sj_strategy_enum sj_strategy;
+      double rec_count= *current_record_count;
+      double read_time= *current_read_time;
+      if ((*strategy)->check_qep(join, idx, remaining_tables, 
+                                 new_join_tab,
+                                 &rec_count,
+                                 &read_time,
+                                 &handled_fanout,
+                                 &sj_strategy,
+                                 loose_scan_pos))
       {
         /*
-          Trying to add an sj-inner table whose sj-nest has an outer correlated 
-          table that was not in the prefix. This means FirstMatch can't be used.
+          It's possible to use the strategy. Use it, if
+           - it removes semi-join fanout that was not removed before
+           - using it is cheaper than using something else,
+               and {if some other strategy has removed fanout
+               that this strategy is trying to remove, then it
+               did remove the fanout only for one semi-join}
+               This is to avoid a situation when
+                1. strategy X removes fanout for semijoin X,Y
+                2. using strategy Z is cheaper, but it only removes
+                   fanout from semijoin X.
+                3. We have no clue what to do about fanount of semi-join Y.
         */
-        pos->invalidate_firstmatch_prefix();
-      }
-      else
-      {
-        /* Record that we need all of this semi-join's inner tables, too */
-        pos->firstmatch_need_tables|= sj_inner_tables;
-      }
-    
-      if (pos->in_firstmatch_prefix() && 
-          !(pos->firstmatch_need_tables & remaining_tables))
-      {
-        /*
-          Got a complete FirstMatch range.
-            Calculate correct costs and fanout
-        */
-        optimize_wo_join_buffering(join, pos->first_firstmatch_table, idx,
-                                   remaining_tables, FALSE, idx,
-                                   current_record_count, 
-                                   current_read_time);
-        /*
-          We don't yet know what are the other strategies, so pick the
-          FirstMatch.
-
-          We ought to save the alternate POSITIONs produced by
-          optimize_wo_join_buffering but the problem is that providing save
-          space uses too much space. Instead, we will re-calculate the
-          alternate POSITIONs after we've picked the best QEP.
-        */
-        pos->sj_strategy= SJ_OPT_FIRST_MATCH;
-        handled_by_fm_or_ls=  pos->firstmatch_need_tables;
+        if ((join->cur_dups_producing_tables & handled_fanout) ||
+            (read_time < *current_read_time && 
+             !(handled_fanout & pos->inner_tables_handled_with_other_sjs)))
+        {
+          /* Mark strategy as used */ 
+          (*strategy)->mark_used();
+          pos->sj_strategy= sj_strategy;
+          *current_read_time= read_time;
+          *current_record_count= rec_count;
+          join->cur_dups_producing_tables &= ~handled_fanout;
+          //TODO: update bitmap of semi-joins that were handled together with
+          // others.
+          if (is_multiple_semi_joins(join, join->positions, idx, handled_fanout))
+            pos->inner_tables_handled_with_other_sjs |= handled_fanout;
+        }
+        else
+        {
+          /* We decided not to apply the strategy. */
+          (*strategy)->set_empty();
+        }
       }
     }
   }
 
-  /* LooseScan Strategy */
-  {
-    POSITION *first=join->positions+pos->first_loosescan_table; 
-    /* 
-      LooseScan strategy can't handle interleaving between tables from the 
-      semi-join that LooseScan is handling and any other tables.
-
-      If we were considering LooseScan for the join prefix (1)
-         and the table we're adding creates an interleaving (2)
-      then 
-         stop considering loose scan
-    */
-    if ((pos->first_loosescan_table != MAX_TABLES) &&   // (1)
-        (first->table->emb_sj_nest->sj_inner_tables & remaining_tables) && //(2)
-        new_join_tab->emb_sj_nest != first->table->emb_sj_nest) //(2)
-    {
-      pos->first_loosescan_table= MAX_TABLES;
-    }
-
-    /*
-      If we got an option to use LooseScan for the current table, start
-      considering using LooseScan strategy
-    */
-    if (loose_scan_pos->read_time != DBL_MAX && !join->outer_join)
-    {
-      pos->first_loosescan_table= idx;
-      pos->loosescan_need_tables=
-        new_join_tab->emb_sj_nest->sj_inner_tables | 
-        new_join_tab->emb_sj_nest->nested_join->sj_depends_on |
-        new_join_tab->emb_sj_nest->nested_join->sj_corr_tables;
-    }
-    
-    if ((pos->first_loosescan_table != MAX_TABLES) && 
-        !(remaining_tables & pos->loosescan_need_tables))
-    {
-      /* 
-        Ok we have LooseScan plan and also have all LooseScan sj-nest's
-        inner tables and outer correlated tables into the prefix.
-      */
-
-      first=join->positions + pos->first_loosescan_table; 
-      uint n_tables= my_count_bits(first->table->emb_sj_nest->sj_inner_tables);
-      /* Got a complete LooseScan range. Calculate its cost */
-      /*
-        The same problem as with FirstMatch - we need to save POSITIONs
-        somewhere but reserving space for all cases would require too
-        much space. We will re-calculate POSITION structures later on. 
-      */
-      optimize_wo_join_buffering(join, pos->first_loosescan_table, idx,
-                                 remaining_tables, 
-                                 TRUE,  //first_alt
-                                 disable_jbuf ? join->table_count :
-                                   pos->first_loosescan_table + n_tables,
-                                 current_record_count,
-                                 current_read_time);
-      /*
-        We don't yet have any other strategies that could handle this
-        semi-join nest (the other options are Duplicate Elimination or
-        Materialization, which need at least the same set of tables in 
-        the join prefix to be considered) so unconditionally pick the 
-        LooseScan.
-      */
-      pos->sj_strategy= SJ_OPT_LOOSE_SCAN;
-      handled_by_fm_or_ls= first->table->emb_sj_nest->sj_inner_tables;
-    }
-  }
-
-  /* 
-    Update join->cur_sj_inner_tables (Used by FirstMatch in this function and
-    LooseScan detector in best_access_path)
-  */
   if ((emb_sj_nest= new_join_tab->emb_sj_nest))
   {
     join->cur_sj_inner_tables |= emb_sj_nest->sj_inner_tables;
-    join->cur_dups_producing_tables |= emb_sj_nest->sj_inner_tables;
 
     /* Remove the sj_nest if all of its SJ-inner tables are in cur_table_map */
     if (!(remaining_tables &
           emb_sj_nest->sj_inner_tables & ~new_join_tab->table->map))
       join->cur_sj_inner_tables &= ~emb_sj_nest->sj_inner_tables;
   }
-  join->cur_dups_producing_tables &= ~handled_by_fm_or_ls;
 
-  /* 4. SJ-Materialization and SJ-Materialization-scan strategy handler */
+  pos->prefix_cost.convert_from_cost(*current_read_time);
+  pos->prefix_record_count= *current_record_count;
+}
+
+
+void Sj_materialization_picker::set_from_prev(struct st_position *prev)
+{
+  if (prev->sjmat_picker.is_used)
+    set_empty();
+  else
+  {
+    sjm_scan_need_tables= prev->sjmat_picker.sjm_scan_need_tables; 
+    sjm_scan_last_inner=  prev->sjmat_picker.sjm_scan_last_inner;
+  }
+  is_used= FALSE;
+}
+
+
+bool Sj_materialization_picker::check_qep(JOIN *join,
+                                          uint idx,
+                                          table_map remaining_tables, 
+                                          const JOIN_TAB *new_join_tab,
+                                          double *record_count,
+                                          double *read_time,
+                                          table_map *handled_fanout,
+                                          sj_strategy_enum *strategy,
+                                          POSITION *loose_scan_pos)
+{
   bool sjm_scan;
   SJ_MATERIALIZATION_INFO *mat_info;
   if ((mat_info= at_sjmat_pos(join, remaining_tables,
@@ -2412,11 +2356,11 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
         The simple way to model this is to remove SJM-SCAN(...) fanout once
         we reach the point #2.
       */
-      pos->sjm_scan_need_tables=
+      sjm_scan_need_tables=
         new_join_tab->emb_sj_nest->sj_inner_tables | 
         new_join_tab->emb_sj_nest->nested_join->sj_depends_on |
         new_join_tab->emb_sj_nest->nested_join->sj_corr_tables;
-      pos->sjm_scan_last_inner= idx;
+      sjm_scan_last_inner= idx;
     }
     else
     {
@@ -2439,34 +2383,31 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
       mat_read_time += mat_info->materialization_cost.total_cost() +
                        prefix_rec_count * mat_info->lookup_cost.total_cost();
 
-      if (mat_read_time < *current_read_time || join->cur_dups_producing_tables)
-      {
-        /*
-          NOTE: When we pick to use SJM[-Scan] we don't memcpy its POSITION
-          elements to join->positions as that makes it hard to return things
-          back when making one step back in join optimization. That's done 
-          after the QEP has been chosen.
-        */
-        pos->sj_strategy= SJ_OPT_MATERIALIZE;
-        *current_read_time=    mat_read_time;
-        *current_record_count= prefix_rec_count;
-        join->cur_dups_producing_tables&=
-          ~new_join_tab->emb_sj_nest->sj_inner_tables;
-      }
+      /*
+        NOTE: When we pick to use SJM[-Scan] we don't memcpy its POSITION
+        elements to join->positions as that makes it hard to return things
+        back when making one step back in join optimization. That's done 
+        after the QEP has been chosen.
+      */
+      *read_time=    mat_read_time;
+      *record_count= prefix_rec_count;
+      *handled_fanout= new_join_tab->emb_sj_nest->sj_inner_tables;
+      *strategy= SJ_OPT_MATERIALIZE;
+      return TRUE;
     }
   }
   
   /* 4.A SJM-Scan second phase check */
-  if (pos->sjm_scan_need_tables && /* Have SJM-Scan prefix */
-      !(pos->sjm_scan_need_tables & remaining_tables))
+  if (sjm_scan_need_tables && /* Have SJM-Scan prefix */
+      !(sjm_scan_need_tables & remaining_tables))
   {
     TABLE_LIST *mat_nest= 
-      join->positions[pos->sjm_scan_last_inner].table->emb_sj_nest;
+      join->positions[sjm_scan_last_inner].table->emb_sj_nest;
     SJ_MATERIALIZATION_INFO *mat_info= mat_nest->sj_mat_info;
 
     double prefix_cost;
     double prefix_rec_count;
-    int first_tab= pos->sjm_scan_last_inner + 1 - mat_info->tables;
+    int first_tab= sjm_scan_last_inner + 1 - mat_info->tables;
     /* Get the prefix cost */
     if (first_tab == (int)join->const_tables)
     {
@@ -2491,6 +2432,7 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
 
     POSITION curpos, dummy;
     /* Need to re-run best-access-path as we prefix_rec_count has changed */
+    bool disable_jbuf= (join->thd->variables.join_cache_level == 0);
     for (i= first_tab + mat_info->tables; i <= idx; i++)
     {
       best_access_path(join, join->positions[i].table, rem_tables, i,
@@ -2499,142 +2441,350 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
       prefix_cost += curpos.read_time;
     }
 
+    *strategy= SJ_OPT_MATERIALIZE_SCAN;
+    *read_time=    prefix_cost;
+    *record_count= prefix_rec_count;
+    *handled_fanout= mat_nest->sj_inner_tables;
+    return TRUE;
+  }
+  return FALSE;
+}
+
+
+void LooseScan_picker::set_from_prev(struct st_position *prev)
+{
+  if (prev->loosescan_picker.is_used)
+    set_empty();
+  else
+  {
+    first_loosescan_table= prev->loosescan_picker.first_loosescan_table;
+    loosescan_need_tables= prev->loosescan_picker.loosescan_need_tables;
+  }
+  is_used= FALSE;
+}
+
+
+bool LooseScan_picker::check_qep(JOIN *join,
+                                 uint idx,
+                                 table_map remaining_tables, 
+                                 const JOIN_TAB *new_join_tab,
+                                 double *record_count, 
+                                 double *read_time,
+                                 table_map *handled_fanout,
+                                 sj_strategy_enum *strategy,
+                                 struct st_position *loose_scan_pos)
+{
+  POSITION *first= join->positions + first_loosescan_table; 
+  /* 
+    LooseScan strategy can't handle interleaving between tables from the 
+    semi-join that LooseScan is handling and any other tables.
+
+    If we were considering LooseScan for the join prefix (1)
+       and the table we're adding creates an interleaving (2)
+    then 
+       stop considering loose scan
+  */
+  if ((first_loosescan_table != MAX_TABLES) &&   // (1)
+      (first->table->emb_sj_nest->sj_inner_tables & remaining_tables) && //(2)
+      new_join_tab->emb_sj_nest != first->table->emb_sj_nest) //(2)
+  {
+    first_loosescan_table= MAX_TABLES;
+  }
+
+  /*
+    If we got an option to use LooseScan for the current table, start
+    considering using LooseScan strategy
+  */
+  if (loose_scan_pos->read_time != DBL_MAX && !join->outer_join)
+  {
+    first_loosescan_table= idx;
+    loosescan_need_tables=
+      new_join_tab->emb_sj_nest->sj_inner_tables | 
+      new_join_tab->emb_sj_nest->nested_join->sj_depends_on |
+      new_join_tab->emb_sj_nest->nested_join->sj_corr_tables;
+  }
+  
+  if ((first_loosescan_table != MAX_TABLES) && 
+      !(remaining_tables & loosescan_need_tables) &&
+      (new_join_tab->table->map & loosescan_need_tables))
+  {
+    /* 
+      Ok we have LooseScan plan and also have all LooseScan sj-nest's
+      inner tables and outer correlated tables into the prefix.
+    */
+
+    first= join->positions + first_loosescan_table; 
+    uint n_tables= my_count_bits(first->table->emb_sj_nest->sj_inner_tables);
+    /* Got a complete LooseScan range. Calculate its cost */
     /*
-      Use the strategy if 
-       * it is cheaper then what we've had, or
-       * we haven't picked any other semi-join strategy yet
-      In the second case, we pick this strategy unconditionally because
-      comparing cost without semi-join duplicate removal with cost with
-      duplicate removal is not an apples-to-apples comparison.
+      The same problem as with FirstMatch - we need to save POSITIONs
+      somewhere but reserving space for all cases would require too
+      much space. We will re-calculate POSITION structures later on. 
     */
-    if (prefix_cost < *current_read_time || join->cur_dups_producing_tables)
-    {
-      pos->sj_strategy= SJ_OPT_MATERIALIZE_SCAN;
-      *current_read_time=    prefix_cost;
-      *current_record_count= prefix_rec_count;
-      join->cur_dups_producing_tables&= ~mat_nest->sj_inner_tables;
+    bool disable_jbuf= (join->thd->variables.join_cache_level == 0);
+    optimize_wo_join_buffering(join, first_loosescan_table, idx,
+                               remaining_tables, 
+                               TRUE,  //first_alt
+                               disable_jbuf ? join->table_count :
+                                 first_loosescan_table + n_tables,
+                               record_count,
+                               read_time);
+    /*
+      We don't yet have any other strategies that could handle this
+      semi-join nest (the other options are Duplicate Elimination or
+      Materialization, which need at least the same set of tables in 
+      the join prefix to be considered) so unconditionally pick the 
+      LooseScan.
+    */
+    *strategy= SJ_OPT_LOOSE_SCAN;
+    *handled_fanout= first->table->emb_sj_nest->sj_inner_tables;
+    return TRUE;
+  }
+  return FALSE;
+}
 
-    }
+void Firstmatch_picker::set_from_prev(struct st_position *prev)
+{
+  if (prev->firstmatch_picker.is_used)
+    invalidate_firstmatch_prefix();
+  else
+  {
+    first_firstmatch_table= prev->firstmatch_picker.first_firstmatch_table;
+    first_firstmatch_rtbl=  prev->firstmatch_picker.first_firstmatch_rtbl;
+    firstmatch_need_tables= prev->firstmatch_picker.firstmatch_need_tables;
   }
+  is_used= FALSE;
+}
 
-  /* 5. Duplicate Weedout strategy handler */
+bool Firstmatch_picker::check_qep(JOIN *join,
+                                  uint idx,
+                                  table_map remaining_tables, 
+                                  const JOIN_TAB *new_join_tab,
+                                  double *record_count,
+                                  double *read_time,
+                                  table_map *handled_fanout,
+                                  sj_strategy_enum *strategy,
+                                  POSITION *loose_scan_pos)
+{
+  if (new_join_tab->emb_sj_nest &&
+      optimizer_flag(join->thd, OPTIMIZER_SWITCH_FIRSTMATCH) &&
+      !join->outer_join)
   {
+    const table_map outer_corr_tables=
+      new_join_tab->emb_sj_nest->nested_join->sj_corr_tables |
+      new_join_tab->emb_sj_nest->nested_join->sj_depends_on;
+    const table_map sj_inner_tables=
+      new_join_tab->emb_sj_nest->sj_inner_tables & ~join->const_table_map;
+
     /* 
-       Duplicate weedout can be applied after all ON-correlated and 
-       correlated 
+      Enter condition:
+       1. The next join tab belongs to semi-join nest
+          (verified for the encompassing code block above).
+       2. We're not in a duplicate producer range yet
+       3. All outer tables that
+           - the subquery is correlated with, or
+           - referred to from the outer_expr 
+          are in the join prefix
+       4. All inner tables are still part of remaining_tables.
     */
-    TABLE_LIST *nest;
-    if ((nest= new_join_tab->emb_sj_nest))
-    {
-      if (!pos->dupsweedout_tables)
-        pos->first_dupsweedout_table= idx;
-
-      pos->dupsweedout_tables |= nest->sj_inner_tables |
-                                 nest->nested_join->sj_depends_on |
-                                 nest->nested_join->sj_corr_tables;
-    }
-    
-    if (pos->dupsweedout_tables)
+    if (!join->cur_sj_inner_tables &&              // (2)
+        !(remaining_tables & outer_corr_tables) && // (3)
+        (sj_inner_tables ==                        // (4)
+         ((remaining_tables | new_join_tab->table->map) & sj_inner_tables)))
     {
-      /* we're in the process of constructing a DuplicateWeedout range */
-      TABLE_LIST *emb= new_join_tab->table->pos_in_table_list->embedding;
-      /* and we've entered an inner side of an outer join*/
-      if (emb && emb->on_expr)
-        pos->dupsweedout_tables |= emb->nested_join->used_tables;
+      /* Start tracking potential FirstMatch range */
+      first_firstmatch_table= idx;
+      firstmatch_need_tables= sj_inner_tables;
+      first_firstmatch_rtbl= remaining_tables;
     }
 
-    if (pos->dupsweedout_tables && 
-        !(remaining_tables &
-          ~new_join_tab->table->map & pos->dupsweedout_tables))
+    if (in_firstmatch_prefix())
     {
-      /*
-        Ok, reached a state where we could put a dups weedout point.
-        Walk back and calculate
-          - the join cost (this is needed as the accumulated cost may assume 
-            some other duplicate elimination method)
-          - extra fanout that will be removed by duplicate elimination
-          - duplicate elimination cost
-        There are two cases:
-          1. We have other strategy/ies to remove all of the duplicates.
-          2. We don't.
-        
-        We need to calculate the cost in case #2 also because we need to make
-        choice between this join order and others.
-      */
-      uint first_tab= pos->first_dupsweedout_table;
-      double dups_cost;
-      double prefix_rec_count;
-      double sj_inner_fanout= 1.0;
-      double sj_outer_fanout= 1.0;
-      uint temptable_rec_size;
-      if (first_tab == join->const_tables)
+      if (outer_corr_tables & first_firstmatch_rtbl)
       {
-        prefix_rec_count= 1.0;
-        temptable_rec_size= 0;
-        dups_cost= 0.0;
+        /*
+          Trying to add an sj-inner table whose sj-nest has an outer correlated 
+          table that was not in the prefix. This means FirstMatch can't be used.
+        */
+        invalidate_firstmatch_prefix();
       }
       else
       {
-        dups_cost= join->positions[first_tab - 1].prefix_cost.total_cost();
-        prefix_rec_count= join->positions[first_tab - 1].prefix_record_count;
-        temptable_rec_size= 8; /* This is not true but we'll make it so */
+        /* Record that we need all of this semi-join's inner tables, too */
+        firstmatch_need_tables|= sj_inner_tables;
       }
-      
-      table_map dups_removed_fanout= 0;
-      for (uint j= pos->first_dupsweedout_table; j <= idx; j++)
+    
+      if (in_firstmatch_prefix() && 
+          !(firstmatch_need_tables & remaining_tables))
       {
-        POSITION *p= join->positions + j;
-        dups_cost += p->read_time;
-        if (p->table->emb_sj_nest)
+        /*
+          Got a complete FirstMatch range. Calculate correct costs and fanout
+        */
+
+        if (idx == first_firstmatch_table && 
+            optimizer_flag(join->thd, OPTIMIZER_SWITCH_SEMIJOIN_WITH_CACHE))
         {
-          sj_inner_fanout *= p->records_read;
-          dups_removed_fanout |= p->table->table->map;
+          /* 
+            An important special case: only one inner table, and @@optimizer_switch
+            allows join buffering.
+             - read_time is the same (i.e. FirstMatch doesn't add any cost
+             - remove fanout added by the last table
+          */
+          if (*record_count)
+            *record_count /= join->positions[idx].records_read;
         }
         else
         {
-          sj_outer_fanout *= p->records_read;
-          temptable_rec_size += p->table->table->file->ref_length;
+          optimize_wo_join_buffering(join, first_firstmatch_table, idx,
+                                     remaining_tables, FALSE, idx,
+                                     record_count, 
+                                     read_time);
         }
+        /*
+          We ought to save the alternate POSITIONs produced by
+          optimize_wo_join_buffering but the problem is that providing save
+          space uses too much space. Instead, we will re-calculate the
+          alternate POSITIONs after we've picked the best QEP.
+        */
+        *handled_fanout= firstmatch_need_tables;
+        /* *record_count and *read_time were set by the above call */
+        *strategy= SJ_OPT_FIRST_MATCH;
+        return TRUE;
       }
+    }
+  }
+  return FALSE;
+}
 
-      /*
-        Add the cost of temptable use. The table will have sj_outer_fanout
-        records, and we will make 
-        - sj_outer_fanout table writes
-        - sj_inner_fanout*sj_outer_fanout  lookups.
 
-      */
-      double one_lookup_cost= get_tmp_table_lookup_cost(join->thd,
-                                                        sj_outer_fanout,
-                                                        temptable_rec_size);
-      double one_write_cost= get_tmp_table_write_cost(join->thd,
-                                                      sj_outer_fanout,
-                                                      temptable_rec_size);
+void Duplicate_weedout_picker::set_from_prev(POSITION *prev)
+{
+  if (prev->dups_weedout_picker.is_used)
+    set_empty();
+  else
+  {
+    dupsweedout_tables=      prev->dups_weedout_picker.dupsweedout_tables;
+    first_dupsweedout_table= prev->dups_weedout_picker.first_dupsweedout_table;
+  }
+  is_used= FALSE;
+}
+
 
-      double write_cost= join->positions[first_tab].prefix_record_count* 
-                         sj_outer_fanout * one_write_cost;
-      double full_lookup_cost= join->positions[first_tab].prefix_record_count* 
-                               sj_outer_fanout* sj_inner_fanout * 
-                               one_lookup_cost;
-      dups_cost += write_cost + full_lookup_cost;
+bool Duplicate_weedout_picker::check_qep(JOIN *join,
+                                         uint idx,
+                                         table_map remaining_tables, 
+                                         const JOIN_TAB *new_join_tab,
+                                         double *record_count,
+                                         double *read_time,
+                                         table_map *handled_fanout,
+                                         sj_strategy_enum *strategy,
+                                         POSITION *loose_scan_pos
+                                         )
+{
+  TABLE_LIST *nest;
+  if ((nest= new_join_tab->emb_sj_nest))
+  {
+    if (!dupsweedout_tables)
+      first_dupsweedout_table= idx;
+
+    dupsweedout_tables |= nest->sj_inner_tables |
+                          nest->nested_join->sj_depends_on |
+                          nest->nested_join->sj_corr_tables;
+  }
+  
+  if (dupsweedout_tables)
+  {
+    /* we're in the process of constructing a DuplicateWeedout range */
+    TABLE_LIST *emb= new_join_tab->table->pos_in_table_list->embedding;
+    /* and we've entered an inner side of an outer join*/
+    if (emb && emb->on_expr)
+      dupsweedout_tables |= emb->nested_join->used_tables;
+  }
+  
+  /* If this is the last table that we need for DuplicateWeedout range */
+  if (dupsweedout_tables && !(remaining_tables & ~new_join_tab->table->map &
+                              dupsweedout_tables))
+  {
+    /*
+      Ok, reached a state where we could put a dups weedout point.
+      Walk back and calculate
+        - the join cost (this is needed as the accumulated cost may assume 
+          some other duplicate elimination method)
+        - extra fanout that will be removed by duplicate elimination
+        - duplicate elimination cost
+      There are two cases:
+        1. We have other strategy/ies to remove all of the duplicates.
+        2. We don't.
       
-      /*
-        Use the strategy if 
-         * it is cheaper then what we've had, or
-         * we haven't picked any other semi-join strategy yet
-        The second part is necessary because this strategy is the last one
-        to consider (it needs "the most" tables in the prefix) and we can't
-        leave duplicate-producing tables not handled by any strategy.
-      */
-      if (dups_cost < *current_read_time || join->cur_dups_producing_tables)
+      We need to calculate the cost in case #2 also because we need to make
+      choice between this join order and others.
+    */
+    uint first_tab= first_dupsweedout_table;
+    double dups_cost;
+    double prefix_rec_count;
+    double sj_inner_fanout= 1.0;
+    double sj_outer_fanout= 1.0;
+    uint temptable_rec_size;
+    if (first_tab == join->const_tables)
+    {
+      prefix_rec_count= 1.0;
+      temptable_rec_size= 0;
+      dups_cost= 0.0;
+    }
+    else
+    {
+      dups_cost= join->positions[first_tab - 1].prefix_cost.total_cost();
+      prefix_rec_count= join->positions[first_tab - 1].prefix_record_count;
+      temptable_rec_size= 8; /* This is not true but we'll make it so */
+    }
+    
+    table_map dups_removed_fanout= 0;
+    double current_fanout= prefix_rec_count;
+    for (uint j= first_dupsweedout_table; j <= idx; j++)
+    {
+      POSITION *p= join->positions + j;
+      current_fanout *= p->records_read;
+      dups_cost += p->read_time + current_fanout / TIME_FOR_COMPARE;
+      if (p->table->emb_sj_nest)
       {
-        pos->sj_strategy= SJ_OPT_DUPS_WEEDOUT;
-        *current_read_time= dups_cost;
-        *current_record_count= prefix_rec_count * sj_outer_fanout;
-        join->cur_dups_producing_tables &= ~dups_removed_fanout;
+        sj_inner_fanout *= p->records_read;
+        dups_removed_fanout |= p->table->table->map;
+      }
+      else
+      {
+        sj_outer_fanout *= p->records_read;
+        temptable_rec_size += p->table->table->file->ref_length;
       }
     }
+
+    /*
+      Add the cost of temptable use. The table will have sj_outer_fanout
+      records, and we will make 
+      - sj_outer_fanout table writes
+      - sj_inner_fanout*sj_outer_fanout  lookups.
+
+    */
+    double one_lookup_cost= get_tmp_table_lookup_cost(join->thd,
+                                                      sj_outer_fanout,
+                                                      temptable_rec_size);
+    double one_write_cost= get_tmp_table_write_cost(join->thd,
+                                                    sj_outer_fanout,
+                                                    temptable_rec_size);
+
+    double write_cost= join->positions[first_tab].prefix_record_count* 
+                       sj_outer_fanout * one_write_cost;
+    double full_lookup_cost= join->positions[first_tab].prefix_record_count* 
+                             sj_outer_fanout* sj_inner_fanout * 
+                             one_lookup_cost;
+    dups_cost += write_cost + full_lookup_cost;
+    
+    *read_time= dups_cost;
+    *record_count= prefix_rec_count * sj_outer_fanout;
+    *handled_fanout= dups_removed_fanout;
+    *strategy= SJ_OPT_DUPS_WEEDOUT;
+    return TRUE;
   }
+  return FALSE;
 }
 
 
@@ -2831,11 +2981,11 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
     }
     else if (pos->sj_strategy == SJ_OPT_MATERIALIZE_SCAN)
     {
-      POSITION *first_inner= join->best_positions + pos->sjm_scan_last_inner;
+      POSITION *first_inner= join->best_positions + pos->sjmat_picker.sjm_scan_last_inner;
       SJ_MATERIALIZATION_INFO *sjm= first_inner->table->emb_sj_nest->sj_mat_info;
       sjm->is_used= TRUE;
       sjm->is_sj_scan= TRUE;
-      first= pos->sjm_scan_last_inner - sjm->tables + 1;
+      first= pos->sjmat_picker.sjm_scan_last_inner - sjm->tables + 1;
       memcpy(join->best_positions + first, 
              sjm->positions, sizeof(POSITION) * sjm->tables);
       join->best_positions[first].sj_strategy= SJ_OPT_MATERIALIZE_SCAN;
@@ -2873,7 +3023,7 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
  
     if (pos->sj_strategy == SJ_OPT_FIRST_MATCH)
     {
-      first= pos->first_firstmatch_table;
+      first= pos->firstmatch_picker.first_firstmatch_table;
       join->best_positions[first].sj_strategy= SJ_OPT_FIRST_MATCH;
       join->best_positions[first].n_sj_tables= tablenr - first + 1;
       POSITION dummy; // For loose scan paths
@@ -2906,7 +3056,7 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
 
     if (pos->sj_strategy == SJ_OPT_LOOSE_SCAN) 
     {
-      first= pos->first_loosescan_table;
+      first= pos->loosescan_picker.first_loosescan_table;
       POSITION *first_pos= join->best_positions + first;
       POSITION loose_scan_pos; // For loose scan paths
       double record_count= (first== join->const_tables)? 1.0: 
@@ -2945,7 +3095,7 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
         Duplicate Weedout starting at pos->first_dupsweedout_table, ending at
         this table.
       */
-      first= pos->first_dupsweedout_table;
+      first= pos->dups_weedout_picker.first_dupsweedout_table;
       join->best_positions[first].sj_strategy= SJ_OPT_DUPS_WEEDOUT;
       join->best_positions[first].n_sj_tables= tablenr - first + 1;
     }
@@ -2961,7 +3111,6 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
     if (tablenr != first)
       pos->sj_strategy= SJ_OPT_NONE;
     remaining_tables |= s->table->map;
-    //s->sj_strategy= pos->sj_strategy;
     join->join_tab[first].sj_strategy= join->best_positions[first].sj_strategy;
     join->join_tab[first].n_sj_tables= join->best_positions[first].n_sj_tables;
   }
@@ -3028,6 +3177,7 @@ bool setup_sj_materialization_part1(JOIN_TAB *sjm_tab)
                                      HA_POS_ERROR /*rows_limit */, 
                                      (char*)"sj-materialize")))
     DBUG_RETURN(TRUE); /* purecov: inspected */
+  sjm->table->map=  emb_sj_nest->nested_join->used_tables;
   sjm->table->file->extra(HA_EXTRA_WRITE_CACHE);
   sjm->table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
 
@@ -3337,12 +3487,8 @@ static bool is_cond_sj_in_equality(Item *item)
 
   SYNOPSIS
 
-    create_duplicate_weedout_tmp_table()
+    create_sj_weedout_tmp_table()
       thd                    Thread handle
-      uniq_tuple_length_arg  Length of the table's column
-      sjtbl                  Update sjtbl->[start_]recinfo values which 
-                             will be needed if we'll need to convert the 
-                             created temptable from HEAP to MyISAM/Maria.
 
   DESCRIPTION
     Create a temporary table to weed out duplicate rowid combinations. The
@@ -3367,9 +3513,8 @@ static bool is_cond_sj_in_equality(Item *item)
     NULL on error
 */
 
-TABLE *create_duplicate_weedout_tmp_table(THD *thd, 
-                                          uint uniq_tuple_length_arg,
-                                          SJ_TMP_TABLE *sjtbl)
+bool
+SJ_TMP_TABLE::create_sj_weedout_tmp_table(THD *thd)
 {
   MEM_ROOT *mem_root_save, own_root;
   TABLE *table;
@@ -3382,15 +3527,17 @@ TABLE *create_duplicate_weedout_tmp_table(THD *thd,
   uchar *group_buff;
   uchar *bitmaps;
   uint *blob_field;
-  ENGINE_COLUMNDEF *recinfo, *start_recinfo;
   bool using_unique_constraint=FALSE;
   bool use_packed_rows= FALSE;
   Field *field, *key_field;
   uint null_pack_length, null_count;
   uchar *null_flags;
   uchar *pos;
-  DBUG_ENTER("create_duplicate_weedout_tmp_table");
-  DBUG_ASSERT(!sjtbl->is_degenerate);
+  DBUG_ENTER("create_sj_weedout_tmp_table");
+  DBUG_ASSERT(!is_degenerate);
+
+  tmp_table= NULL;
+  uint uniq_tuple_length_arg= rowid_len + null_bytes;
   /*
     STEP 1: Get temporary table name
   */
@@ -3432,7 +3579,7 @@ TABLE *create_duplicate_weedout_tmp_table(THD *thd,
   {
     if (temp_pool_slot != MY_BIT_NONE)
       bitmap_lock_clear_bit(&temp_pool, temp_pool_slot);
-    DBUG_RETURN(NULL);
+    DBUG_RETURN(TRUE);
   }
   strmov(tmpname,path);
   
@@ -3636,20 +3783,19 @@ TABLE *create_duplicate_weedout_tmp_table(THD *thd,
     if (create_internal_tmp_table(table, keyinfo, start_recinfo, &recinfo, 0, 0))
       goto err;
   }
-  sjtbl->start_recinfo= start_recinfo;
-  sjtbl->recinfo=       recinfo;
   if (open_tmp_table(table))
     goto err;
 
   thd->mem_root= mem_root_save;
-  DBUG_RETURN(table);
+  tmp_table= table;
+  DBUG_RETURN(FALSE);
 
 err:
   thd->mem_root= mem_root_save;
   free_tmp_table(thd,table);                    /* purecov: inspected */
   if (temp_pool_slot != MY_BIT_NONE)
     bitmap_lock_clear_bit(&temp_pool, temp_pool_slot);
-  DBUG_RETURN(NULL);				/* purecov: inspected */
+  DBUG_RETURN(TRUE);				/* purecov: inspected */
 }
 
 
@@ -3657,25 +3803,25 @@ err:
   SemiJoinDuplicateElimination: Reset the temporary table
 */
 
-int do_sj_reset(SJ_TMP_TABLE *sj_tbl)
+int SJ_TMP_TABLE::sj_weedout_delete_rows()
 {
-  DBUG_ENTER("do_sj_reset");
-  if (sj_tbl->tmp_table)
+  DBUG_ENTER("SJ_TMP_TABLE::sj_weedout_delete_rows");
+  if (tmp_table)
   {
-    int rc= sj_tbl->tmp_table->file->ha_delete_all_rows();
+    int rc= tmp_table->file->ha_delete_all_rows();
     DBUG_RETURN(rc);
   }
-  sj_tbl->have_degenerate_row= FALSE;
+  have_degenerate_row= FALSE;
   DBUG_RETURN(0);
 }
 
+
 /*
   SemiJoinDuplicateElimination: Weed out duplicate row combinations
 
   SYNPOSIS
-    do_sj_dups_weedout()
+    sj_weedout_check_row()
       thd    Thread handle
-      sjtbl  Duplicate weedout table
 
   DESCRIPTION
     Try storing current record combination of outer tables (i.e. their
@@ -3688,47 +3834,47 @@ int do_sj_reset(SJ_TMP_TABLE *sj_tbl)
     0   The row combination is not a duplicate (continue)
 */
 
-int do_sj_dups_weedout(THD *thd, SJ_TMP_TABLE *sjtbl) 
+int SJ_TMP_TABLE::sj_weedout_check_row(THD *thd)
 {
   int error;
-  SJ_TMP_TABLE::TAB *tab= sjtbl->tabs;
-  SJ_TMP_TABLE::TAB *tab_end= sjtbl->tabs_end;
+  SJ_TMP_TABLE::TAB *tab= tabs;
+  SJ_TMP_TABLE::TAB *tab_end= tabs_end;
   uchar *ptr;
   uchar *nulls_ptr;
 
-  DBUG_ENTER("do_sj_dups_weedout");
+  DBUG_ENTER("SJ_TMP_TABLE::sj_weedout_check_row");
 
-  if (sjtbl->is_degenerate)
+  if (is_degenerate)
   {
-    if (sjtbl->have_degenerate_row) 
+    if (have_degenerate_row) 
       DBUG_RETURN(1);
 
-    sjtbl->have_degenerate_row= TRUE;
+    have_degenerate_row= TRUE;
     DBUG_RETURN(0);
   }
 
-  ptr= sjtbl->tmp_table->record[0] + 1;
+  ptr= tmp_table->record[0] + 1;
 
   /* Put the the rowids tuple into table->record[0]: */
 
   // 1. Store the length 
-  if (((Field_varstring*)(sjtbl->tmp_table->field[0]))->length_bytes == 1)
+  if (((Field_varstring*)(tmp_table->field[0]))->length_bytes == 1)
   {
-    *ptr= (uchar)(sjtbl->rowid_len + sjtbl->null_bytes);
+    *ptr= (uchar)(rowid_len + null_bytes);
     ptr++;
   }
   else
   {
-    int2store(ptr, sjtbl->rowid_len + sjtbl->null_bytes);
+    int2store(ptr, rowid_len + null_bytes);
     ptr += 2;
   }
 
   nulls_ptr= ptr;
   // 2. Zero the null bytes 
-  if (sjtbl->null_bytes)
+  if (null_bytes)
   {
-    bzero(ptr, sjtbl->null_bytes);
-    ptr += sjtbl->null_bytes; 
+    bzero(ptr, null_bytes);
+    ptr += null_bytes; 
   }
 
   // 3. Put the rowids
@@ -3748,21 +3894,91 @@ int do_sj_dups_weedout(THD *thd, SJ_TMP_TABLE *sjtbl)
     }
   }
 
-  error= sjtbl->tmp_table->file->ha_write_tmp_row(sjtbl->tmp_table->record[0]);
+  error= tmp_table->file->ha_write_tmp_row(tmp_table->record[0]);
   if (error)
   {
     /* create_internal_tmp_table_from_heap will generate error if needed */
-    if (!sjtbl->tmp_table->file->is_fatal_error(error, HA_CHECK_DUP))
+    if (!tmp_table->file->is_fatal_error(error, HA_CHECK_DUP))
       DBUG_RETURN(1); /* Duplicate */
-    if (create_internal_tmp_table_from_heap(thd, sjtbl->tmp_table,
-                                            sjtbl->start_recinfo,
-                                            &sjtbl->recinfo, error, 1))
+    if (create_internal_tmp_table_from_heap(thd, tmp_table, start_recinfo,
+                                            &recinfo, error, 1))
       DBUG_RETURN(-1);
   }
   DBUG_RETURN(0);
 }
 
 
+int init_dups_weedout(JOIN *join, uint first_table, int first_fanout_table, uint n_tables)
+{
+  THD *thd= join->thd;
+  DBUG_ENTER("init_dups_weedout");
+  SJ_TMP_TABLE::TAB sjtabs[MAX_TABLES];
+  SJ_TMP_TABLE::TAB *last_tab= sjtabs;
+  uint jt_rowid_offset= 0; // # tuple bytes are already occupied (w/o NULL bytes)
+  uint jt_null_bits= 0;    // # null bits in tuple bytes
+  /*
+    Walk through the range and remember
+     - tables that need their rowids to be put into temptable
+     - the last outer table
+  */
+  for (JOIN_TAB *j=join->join_tab + first_table; 
+       j < join->join_tab + first_table + n_tables; j++)
+  {
+    if (sj_table_is_included(join, j))
+    {
+      last_tab->join_tab= j;
+      last_tab->rowid_offset= jt_rowid_offset;
+      jt_rowid_offset += j->table->file->ref_length;
+      if (j->table->maybe_null)
+      {
+        last_tab->null_byte= jt_null_bits / 8;
+        last_tab->null_bit= jt_null_bits++;
+      }
+      last_tab++;
+      j->table->prepare_for_position();
+      j->keep_current_rowid= TRUE;
+    }
+  }
+
+  SJ_TMP_TABLE *sjtbl;
+  if (jt_rowid_offset) /* Temptable has at least one rowid */
+  {
+    size_t tabs_size= (last_tab - sjtabs) * sizeof(SJ_TMP_TABLE::TAB);
+    if (!(sjtbl= (SJ_TMP_TABLE*)thd->alloc(sizeof(SJ_TMP_TABLE))) ||
+        !(sjtbl->tabs= (SJ_TMP_TABLE::TAB*) thd->alloc(tabs_size)))
+      DBUG_RETURN(TRUE); /* purecov: inspected */
+    memcpy(sjtbl->tabs, sjtabs, tabs_size);
+    sjtbl->is_degenerate= FALSE;
+    sjtbl->tabs_end= sjtbl->tabs + (last_tab - sjtabs);
+    sjtbl->rowid_len= jt_rowid_offset;
+    sjtbl->null_bits= jt_null_bits;
+    sjtbl->null_bytes= (jt_null_bits + 7)/8;
+    if (sjtbl->create_sj_weedout_tmp_table(thd))
+      DBUG_RETURN(TRUE);
+    join->sj_tmp_tables.push_back(sjtbl->tmp_table);
+  }
+  else
+  {
+    /* 
+      This is a special case where the entire subquery predicate does 
+      not depend on anything at all, ie this is 
+        WHERE const IN (uncorrelated select)
+    */
+    if (!(sjtbl= (SJ_TMP_TABLE*)thd->alloc(sizeof(SJ_TMP_TABLE))))
+      DBUG_RETURN(TRUE); /* purecov: inspected */
+    sjtbl->tmp_table= NULL;
+    sjtbl->is_degenerate= TRUE;
+    sjtbl->have_degenerate_row= FALSE;
+  }
+
+  sjtbl->next_flush_table= join->join_tab[first_table].flush_weedout_table;
+  join->join_tab[first_table].flush_weedout_table= sjtbl;
+  join->join_tab[first_fanout_table].first_weedout_table= sjtbl;
+  join->join_tab[first_table + n_tables - 1].check_weed_out_table= sjtbl;
+  DBUG_RETURN(0);
+}
+
+
 /*
   Setup the strategies to eliminate semi-join duplicates.
   
@@ -3863,9 +4079,9 @@ int setup_semijoin_dups_elimination(JOIN *join, ulonglong options,
                                     uint no_jbuf_after)
 {
   uint i;
-  THD *thd= join->thd;
   DBUG_ENTER("setup_semijoin_dups_elimination");
-  
+
+
   POSITION *pos= join->best_positions + join->const_tables;
   for (i= join->const_tables ; i < join->top_join_tab_count; )
   {
@@ -3888,8 +4104,8 @@ int setup_semijoin_dups_elimination(JOIN *join, ulonglong options,
 
         /* Calculate key length */
         keylen= 0;
-        keyno= pos->loosescan_key;
-        for (uint kp=0; kp < pos->loosescan_parts; kp++)
+        keyno= pos->loosescan_picker.loosescan_key;
+        for (uint kp=0; kp < pos->loosescan_picker.loosescan_parts; kp++)
           keylen += tab->table->key_info[keyno].key_part[kp].store_length;
 
         tab->loosescan_key_len= keylen;
@@ -3906,6 +4122,7 @@ int setup_semijoin_dups_elimination(JOIN *join, ulonglong options,
           forwards, but do not destroy other duplicate elimination methods.
         */
         uint first_table= i;
+
         uint join_cache_level= join->thd->variables.join_cache_level;
         for (uint j= i; j < i + pos->n_sj_tables; j++)
         {
@@ -3925,74 +4142,21 @@ int setup_semijoin_dups_elimination(JOIN *join, ulonglong options,
           {
             /* Looks like we'll be using join buffer */
             first_table= join->const_tables;
-            break;
-          }
-        }
-
-        SJ_TMP_TABLE::TAB sjtabs[MAX_TABLES];
-        SJ_TMP_TABLE::TAB *last_tab= sjtabs;
-        uint jt_rowid_offset= 0; // # tuple bytes are already occupied (w/o NULL bytes)
-        uint jt_null_bits= 0;    // # null bits in tuple bytes
-        /*
-          Walk through the range and remember
-           - tables that need their rowids to be put into temptable
-           - the last outer table
-        */
-        for (JOIN_TAB *j=join->join_tab + first_table; 
-             j < join->join_tab + i + pos->n_sj_tables; j++)
-        {
-          if (sj_table_is_included(join, j))
-          {
-            last_tab->join_tab= j;
-            last_tab->rowid_offset= jt_rowid_offset;
-            jt_rowid_offset += j->table->file->ref_length;
-            if (j->table->maybe_null)
-            {
-              last_tab->null_byte= jt_null_bits / 8;
-              last_tab->null_bit= jt_null_bits++;
+            /* 
+              Make sure that possible sorting of rows from the head table 
+              is not to be employed.
+            */
+            if (join->get_sort_by_join_tab())
+	    {
+              join->simple_order= 0;
+              join->simple_group= 0;
+              join->need_tmp= join->test_if_need_tmp_table();
             }
-            last_tab++;
-            j->table->prepare_for_position();
-            j->keep_current_rowid= TRUE;
+            break;
           }
         }
 
-        SJ_TMP_TABLE *sjtbl;
-        if (jt_rowid_offset) /* Temptable has at least one rowid */
-        {
-          size_t tabs_size= (last_tab - sjtabs) * sizeof(SJ_TMP_TABLE::TAB);
-          if (!(sjtbl= (SJ_TMP_TABLE*)thd->alloc(sizeof(SJ_TMP_TABLE))) ||
-              !(sjtbl->tabs= (SJ_TMP_TABLE::TAB*) thd->alloc(tabs_size)))
-            DBUG_RETURN(TRUE); /* purecov: inspected */
-          memcpy(sjtbl->tabs, sjtabs, tabs_size);
-          sjtbl->is_degenerate= FALSE;
-          sjtbl->tabs_end= sjtbl->tabs + (last_tab - sjtabs);
-          sjtbl->rowid_len= jt_rowid_offset;
-          sjtbl->null_bits= jt_null_bits;
-          sjtbl->null_bytes= (jt_null_bits + 7)/8;
-          sjtbl->tmp_table= 
-            create_duplicate_weedout_tmp_table(thd, 
-                                               sjtbl->rowid_len + 
-                                               sjtbl->null_bytes,
-                                               sjtbl);
-          join->sj_tmp_tables.push_back(sjtbl->tmp_table);
-        }
-        else
-        {
-          /* 
-            This is a special case where the entire subquery predicate does 
-            not depend on anything at all, ie this is 
-              WHERE const IN (uncorrelated select)
-          */
-          if (!(sjtbl= (SJ_TMP_TABLE*)thd->alloc(sizeof(SJ_TMP_TABLE))))
-            DBUG_RETURN(TRUE); /* purecov: inspected */
-          sjtbl->tmp_table= NULL;
-          sjtbl->is_degenerate= TRUE;
-          sjtbl->have_degenerate_row= FALSE;
-        }
-        join->join_tab[first_table].flush_weedout_table= sjtbl;
-        join->join_tab[i + pos->n_sj_tables - 1].check_weed_out_table= sjtbl;
-
+        init_dups_weedout(join, first_table, i, i + pos->n_sj_tables - first_table);
         i+= pos->n_sj_tables;
         pos+= pos->n_sj_tables;
         break;
@@ -4074,6 +4238,8 @@ int clear_sj_tmp_tables(JOIN *join)
   {
     if ((res= table->file->ha_delete_all_rows()))
       return res; /* purecov: inspected */
+   free_io_cache(table);
+   filesort_free_buffers(table,0);
   }
 
   SJ_MATERIALIZATION_INFO *sjm;
@@ -4460,6 +4626,234 @@ enum_nested_loop_state join_tab_execution_startup(JOIN_TAB *tab)
 }
 
 
+/*
+  Create a dummy temporary table, useful only for the sake of having a 
+  TABLE* object with map,tablenr and maybe_null properties.
+  
+  This is used by non-mergeable semi-join materilization code to handle
+  degenerate cases where materialized subquery produced "Impossible WHERE" 
+  and thus wasn't materialized.
+*/
+
+TABLE *create_dummy_tmp_table(THD *thd)
+{
+  DBUG_ENTER("create_dummy_tmp_table");
+  TABLE *table;
+  TMP_TABLE_PARAM sjm_table_param;
+  sjm_table_param.init();
+  sjm_table_param.field_count= 1;
+  List<Item> sjm_table_cols;
+  Item *column_item= new Item_int(1);
+  sjm_table_cols.push_back(column_item);
+  if (!(table= create_tmp_table(thd, &sjm_table_param, 
+                                sjm_table_cols, (ORDER*) 0, 
+                                TRUE /* distinct */, 
+                                1, /*save_sum_fields*/
+                                thd->variables.option_bits | TMP_TABLE_ALL_COLUMNS, 
+                                HA_POS_ERROR /*rows_limit */, 
+                                (char*)"dummy", TRUE /* Do not open */)))
+  {
+    DBUG_RETURN(NULL);
+  }
+  DBUG_RETURN(table);
+}
+
+
+/*
+  A class that is used to catch one single tuple that is sent to the join
+  output, and save it in Item_cache element(s).
+
+  It is very similar to select_singlerow_subselect but doesn't require a 
+  Item_singlerow_subselect item.
+*/
+
+class select_value_catcher :public select_subselect
+{
+public:
+  select_value_catcher(Item_subselect *item_arg)
+    :select_subselect(item_arg)
+  {}
+  int send_data(List<Item> &items);
+  int setup(List<Item> *items);
+  bool assigned;  /* TRUE <=> we've caught a value */
+  uint n_elements; /* How many elements we get */
+  Item_cache **row; /* Array of cache elements */
+};
+
+
+int select_value_catcher::setup(List<Item> *items)
+{
+  assigned= FALSE;
+  n_elements= items->elements;
+ 
+  if (!(row= (Item_cache**) sql_alloc(sizeof(Item_cache*)*n_elements)))
+    return TRUE;
+  
+  Item *sel_item;
+  List_iterator<Item> li(*items);
+  for (uint i= 0; (sel_item= li++); i++)
+  {
+    if (!(row[i]= Item_cache::get_cache(sel_item)))
+      return TRUE;
+    row[i]->setup(sel_item);
+  }
+  return FALSE;
+}
+
+
+int select_value_catcher::send_data(List<Item> &items)
+{
+  DBUG_ENTER("select_value_catcher::send_data");
+  DBUG_ASSERT(!assigned);
+  DBUG_ASSERT(items.elements == n_elements);
+
+  if (unit->offset_limit_cnt)
+  {				          // Using limit offset,count
+    unit->offset_limit_cnt--;
+    DBUG_RETURN(0);
+  }
+
+  Item *val_item;
+  List_iterator_fast<Item> li(items);
+  for (uint i= 0; (val_item= li++); i++)
+  {
+    row[i]->store(val_item);
+    row[i]->cache_value();
+  }
+  assigned= TRUE;
+  DBUG_RETURN(0);
+}
+
+
+/*
+  Setup JTBM join tabs for execution
+*/
+
+bool setup_jtbm_semi_joins(JOIN *join, List<TABLE_LIST> *join_list, 
+                           Item **join_where)
+{
+  TABLE_LIST *table;
+  NESTED_JOIN *nested_join;
+  List_iterator<TABLE_LIST> li(*join_list);
+  DBUG_ENTER("setup_jtbm_semi_joins");
+  
+  while ((table= li++))
+  {
+    Item_in_subselect *item;
+    
+    if ((item= table->jtbm_subselect))
+    {
+      Item_in_subselect *subq_pred= item;
+      double rows;
+      double read_time;
+
+      /*
+        Perform optimization of the subquery, so that we know estmated
+        - cost of materialization process 
+        - how many records will be in the materialized temp.table
+      */
+      if (subq_pred->optimize(&rows, &read_time))
+        DBUG_RETURN(TRUE);
+
+      subq_pred->jtbm_read_time= read_time;
+      subq_pred->jtbm_record_count=rows;
+      JOIN *subq_join= subq_pred->unit->first_select()->join;
+
+      if (!subq_join->tables_list || !subq_join->table_count)
+      {
+        /*
+          A special case; subquery's join is degenerate, and it either produces
+          0 or 1 record. Examples of both cases:
+
+            select * from ot where col in (select ... from it where 2>3) 
+            select * from ot where col in (select min(it.key) from it)
+          
+          in this case, the subquery predicate has not been setup for
+          materialization. In particular, there is no materialized temp.table.
+          We'll now need to
+          1. Check whether 1 or 0 records are produced, setup this as a
+             constant join tab.
+          2. Create a dummy temporary table, because all of the join
+             optimization code relies on TABLE object being present (here we
+             follow a bad tradition started by derived tables)
+        */
+        DBUG_ASSERT(subq_pred->engine->engine_type() == 
+                    subselect_engine::SINGLE_SELECT_ENGINE);
+        subselect_single_select_engine *engine=
+          (subselect_single_select_engine*)subq_pred->engine;
+        select_value_catcher *new_sink;
+        if (!(new_sink= new select_value_catcher(subq_pred)))
+          DBUG_RETURN(TRUE);
+        if (new_sink->setup(&engine->select_lex->join->fields_list) ||
+            engine->select_lex->join->change_result(new_sink) ||
+            engine->exec())
+        {
+          DBUG_RETURN(TRUE);
+        }
+        subq_pred->is_jtbm_const_tab= TRUE;
+
+        if (new_sink->assigned)
+        {
+          subq_pred->jtbm_const_row_found= TRUE;
+          /* 
+            Subselect produced one row, which is saved in new_sink->row. 
+            Inject "left_expr[i] == row[i] equalities into parent's WHERE.
+          */
+          Item *eq_cond;
+          for (uint i= 0; i < subq_pred->left_expr->cols(); i++)
+          {
+            eq_cond= new Item_func_eq(subq_pred->left_expr->element_index(i),
+                                      new_sink->row[i]);
+            if (!eq_cond || eq_cond->fix_fields(join->thd, &eq_cond))
+              DBUG_RETURN(1);
+
+            (*join_where)= and_items(*join_where, eq_cond);
+          }
+        }
+        else
+        {
+          /* Subselect produced no rows. Just set the flag, */
+          subq_pred->jtbm_const_row_found= FALSE;
+        }
+
+        /* Set up a dummy TABLE*, optimizer code needs JOIN_TABs to have TABLE */
+        TABLE *dummy_table;
+        if (!(dummy_table= create_dummy_tmp_table(join->thd)))
+          DBUG_RETURN(1);
+        table->table= dummy_table;
+        table->table->pos_in_table_list= table;
+        setup_table_map(table->table, table, table->jtbm_table_no);
+      }
+      else
+      {
+        DBUG_ASSERT(subq_pred->test_set_strategy(SUBS_MATERIALIZATION));
+        subq_pred->is_jtbm_const_tab= FALSE;
+        subselect_hash_sj_engine *hash_sj_engine=
+          ((subselect_hash_sj_engine*)item->engine);
+        
+        table->table= hash_sj_engine->tmp_table;
+        table->table->pos_in_table_list= table;
+
+        setup_table_map(table->table, table, table->jtbm_table_no);
+
+        Item *sj_conds= hash_sj_engine->semi_join_conds;
+
+        (*join_where)= and_items(*join_where, sj_conds);
+        if (!(*join_where)->fixed)
+          (*join_where)->fix_fields(join->thd, join_where);
+      }
+    }
+
+    if ((nested_join= table->nested_join))
+    {
+      if (setup_jtbm_semi_joins(join, &nested_join->join_list, join_where))
+        DBUG_RETURN(TRUE);
+    }
+  }
+  DBUG_RETURN(FALSE);
+}
+
+
 /**
   Choose an optimal strategy to execute an IN/ALL/ANY subquery predicate
   based on cost.
@@ -4502,6 +4896,13 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
   enum_reopt_result reopt_result= REOPT_NONE;
   Item_in_subselect *in_subs;
 
+  /*
+    IN/ALL/ANY optimizations are not applicable for so called fake select
+    (this select exists only to filter results of union if it is needed).
+  */
+  if (select_lex == select_lex->master_unit()->fake_select_lex)
+    return 0;
+
   if (is_in_subquery())
   {
     in_subs= (Item_in_subselect*) unit->item;
@@ -4761,8 +5162,16 @@ bool JOIN::choose_tableless_subquery_plan()
          NULL.
       */
     }
-
-    if (subs_predicate->is_in_predicate())
+    
+    /*
+      For IN subqueries, use IN->EXISTS transfomation, unless the subquery 
+      has been converted to a JTBM semi-join. In that case, just leave
+      everything as-is, setup_jtbm_semi_joins() has special handling for cases
+      like this.
+    */
+    if (subs_predicate->is_in_predicate() && 
+        !(subs_predicate->substype() == Item_subselect::IN_SUBS && 
+          ((Item_in_subselect*)subs_predicate)->is_jtbm_merged))
     {
       Item_in_subselect *in_subs;
       in_subs= (Item_in_subselect*) subs_predicate;
diff --git a/sql/opt_subselect.h b/sql/opt_subselect.h
index 08af798948d..c211fc67c84 100644
--- a/sql/opt_subselect.h
+++ b/sql/opt_subselect.h
@@ -10,6 +10,8 @@ int check_and_do_in_subquery_rewrites(JOIN *join);
 bool convert_join_subqueries_to_semijoins(JOIN *join);
 int pull_out_semijoin_tables(JOIN *join);
 bool optimize_semijoin_nests(JOIN *join, table_map all_table_map);
+bool setup_jtbm_semi_joins(JOIN *join, List<TABLE_LIST> *join_list,  
+                           Item **join_where);
 
 // used by Loose_scan_opt
 ulonglong get_bound_sj_equalities(TABLE_LIST *sj_nest, 
@@ -40,7 +42,6 @@ ulonglong get_bound_sj_equalities(TABLE_LIST *sj_nest,
 
 class Loose_scan_opt
 {
-public:
   /* All methods must check this before doing anything else */
   bool try_loosescan;
 
@@ -71,6 +72,7 @@ public:
 
   uint best_max_loose_keypart;
 
+public:
   Loose_scan_opt():
     try_loosescan(FALSE),
     bound_sj_equalities(0),
@@ -263,8 +265,8 @@ public:
     {
       pos->records_read=    best_loose_scan_records;
       pos->key=             best_loose_scan_start_key;
-      pos->loosescan_key=   best_loose_scan_key;
-      pos->loosescan_parts= best_max_loose_keypart + 1;
+      pos->loosescan_picker.loosescan_key=   best_loose_scan_key;
+      pos->loosescan_picker.loosescan_parts= best_max_loose_keypart + 1;
       pos->use_join_buffer= FALSE;
       pos->table=           tab;
       // todo need ref_depend_map ?
@@ -277,8 +279,7 @@ public:
 };
 
 
-void advance_sj_state(JOIN *join, const table_map remaining_tables, 
-                      const JOIN_TAB *new_join_tab, uint idx, 
+void advance_sj_state(JOIN *join, const table_map remaining_tables, uint idx, 
                       double *current_record_count, double *current_read_time,
                       POSITION *loose_scan_pos);
 void restore_prev_sj_state(const table_map remaining_tables, 
@@ -289,10 +290,6 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join);
 bool setup_sj_materialization_part1(JOIN_TAB *sjm_tab);
 bool setup_sj_materialization_part2(JOIN_TAB *sjm_tab);
 
-TABLE *create_duplicate_weedout_tmp_table(THD *thd, uint uniq_tuple_length_arg,
-                                          SJ_TMP_TABLE *sjtbl);
-int do_sj_reset(SJ_TMP_TABLE *sj_tbl);
-int do_sj_dups_weedout(THD *thd, SJ_TMP_TABLE *sjtbl);
 
 /*
   Temporary table used by semi-join DuplicateElimination strategy
@@ -359,8 +356,11 @@ public:
   ENGINE_COLUMNDEF *start_recinfo;
   ENGINE_COLUMNDEF *recinfo;
 
-  /* Pointer to next table (next->start_idx > this->end_idx) */
-  SJ_TMP_TABLE *next; 
+  SJ_TMP_TABLE *next_flush_table; 
+
+  int sj_weedout_delete_rows();
+  int sj_weedout_check_row(THD *thd);
+  bool create_sj_weedout_tmp_table(THD *thd);
 };
 
 int setup_semijoin_dups_elimination(JOIN *join, ulonglong options, 
diff --git a/sql/opt_sum.cc b/sql/opt_sum.cc
index 5962e7de706..cda35895a3d 100644
--- a/sql/opt_sum.cc
+++ b/sql/opt_sum.cc
@@ -1,5 +1,5 @@
-/* Copyright (c) 2000, 2011 Oracle and/or its affiliates.
-   Copyright (c) 2010, 2011 Monty Program Ab
+/* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -301,7 +301,8 @@ int opt_sum_query(THD *thd,
       is_exact_count= FALSE;
       count= 1;                                 // ensure count != 0
     }
-    else if (tl->is_materialized_derived())
+    else if (tl->is_materialized_derived() || 
+             tl->jtbm_subselect)
     {
       /*
         Can't remove a derived table as it's number of rows is just an
diff --git a/sql/partition_info.cc b/sql/partition_info.cc
index 7c8a520879f..028afd7899e 100644
--- a/sql/partition_info.cc
+++ b/sql/partition_info.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2006, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/password.c b/sql/password.c
index a7e599d4777..f4ff3156bec 100644
--- a/sql/password.c
+++ b/sql/password.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/protocol.cc b/sql/protocol.cc
index eb9c3e34dbf..63b945f7078 100644
--- a/sql/protocol.cc
+++ b/sql/protocol.cc
@@ -1,5 +1,5 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2011, Monty Program Ab
+/* Copyright (c) 2000, 2010, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/records.cc b/sql/records.cc
index 1e74e6d7f30..5050064926b 100644
--- a/sql/records.cc
+++ b/sql/records.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/repl_failsafe.cc b/sql/repl_failsafe.cc
index e355be185db..abc22a00695 100644
--- a/sql/repl_failsafe.cc
+++ b/sql/repl_failsafe.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2001, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/rpl_injector.cc b/sql/rpl_injector.cc
index dfa5ef95e67..3aa3a6a212c 100644
--- a/sql/rpl_injector.cc
+++ b/sql/rpl_injector.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2006, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h
index 54623577e0b..5875290fa02 100644
--- a/sql/rpl_mi.h
+++ b/sql/rpl_mi.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2006, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc
index c893dc52587..d51e5a5ba4c 100644
--- a/sql/rpl_record.cc
+++ b/sql/rpl_record.cc
@@ -1,4 +1,6 @@
-/* Copyright (c) 2007, 2010, Oracle and/or its affiliates.
+/*
+   Copyright (c) 2007, 2010, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/rpl_record.h b/sql/rpl_record.h
index c857c07c33b..efe2a1da72c 100644
--- a/sql/rpl_record.h
+++ b/sql/rpl_record.h
@@ -1,4 +1,6 @@
-/* Copyright (c) 2007, 2010, Oracle and/or its affiliates.
+/*
+   Copyright (c) 2007, 2010, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc
index 2e5202d7cb6..299c032d02a 100644
--- a/sql/rpl_rli.cc
+++ b/sql/rpl_rli.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2006, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h
index 8ff70dff825..6048fe07ecc 100644
--- a/sql/rpl_rli.h
+++ b/sql/rpl_rli.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2005, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/rpl_utility.cc b/sql/rpl_utility.cc
index 4ab4542dd13..71fa5c8909c 100644
--- a/sql/rpl_utility.cc
+++ b/sql/rpl_utility.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2006, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/rpl_utility.h b/sql/rpl_utility.h
index eebef266754..1f5577d8b8b 100644
--- a/sql/rpl_utility.h
+++ b/sql/rpl_utility.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2006, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/set_var.cc b/sql/set_var.cc
index 895facee6e9..8e2f90b9bfd 100644
--- a/sql/set_var.cc
+++ b/sql/set_var.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2002, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2011, Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt
index 3d5d2eb9af5..0153c3f90b0 100644
--- a/sql/share/errmsg-utf8.txt
+++ b/sql/share/errmsg-utf8.txt
@@ -6558,3 +6558,4 @@ ER_CONNECTION_KILLED 70100
         eng "Connection was killed"
 ER_INTERNAL_ERROR
         eng "Internal error: '%-.192s'"
+
diff --git a/sql/slave.cc b/sql/slave.cc
index 91ddf7bf171..ff371b270b7 100644
--- a/sql/slave.cc
+++ b/sql/slave.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2009-2011, Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -637,7 +637,7 @@ terminate_slave_thread(THD *thd,
 
   while (*slave_running)                        // Should always be true
   {
-    int error;
+    int error __attribute__((unused));
     DBUG_PRINT("loop", ("killing slave thread"));
 
     mysql_mutex_lock(&thd->LOCK_thd_data);
diff --git a/sql/slave.h b/sql/slave.h
index 7bee83af744..e519a9fc3fa 100644
--- a/sql/slave.h
+++ b/sql/slave.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sp.cc b/sql/sp.cc
index 31cbe13a96e..29195234a5a 100644
--- a/sql/sp.cc
+++ b/sql/sp.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2002, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sp_head.cc b/sql/sp_head.cc
index e82f1b92312..10304247611 100644
--- a/sql/sp_head.cc
+++ b/sql/sp_head.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2002, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1027,12 +1027,23 @@ subst_spvars(THD *thd, sp_instr *instr, LEX_STRING *query_str)
   /*
     Allocate additional space at the end of the new query string for the
     query_cache_send_result_to_client function.
+
+    The query buffer layout is:
+       buffer :==
+            <statement>   The input statement(s)
+            '\0'          Terminating null char
+            <length>      Length of following current database name 2
+            <db_name>     Name of current database
+            <flags>       Flags struct
   */
-  buf_len= qbuf.length() + thd->db_length + 1 + QUERY_CACHE_FLAGS_SIZE + 1;
+  buf_len= (qbuf.length() + 1 + QUERY_CACHE_DB_LENGTH_SIZE + thd->db_length +
+            QUERY_CACHE_FLAGS_SIZE + 1);
   if ((pbuf= (char *) alloc_root(thd->mem_root, buf_len)))
   {
+    char *ptr= pbuf + qbuf.length();
     memcpy(pbuf, qbuf.ptr(), qbuf.length());
-    pbuf[qbuf.length()]= 0;
+    *ptr= 0;
+    int2store(ptr+1, thd->db_length);
   }
   else
     DBUG_RETURN(TRUE);
diff --git a/sql/sp_head.h b/sql/sp_head.h
index 96e119f23bc..c79b5dfbd0b 100644
--- a/sql/sp_head.h
+++ b/sql/sp_head.h
@@ -1,5 +1,6 @@
 /* -*- C++ -*- */
-/* Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2002, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/spatial.h b/sql/spatial.h
index bea9ade9598..0d0560656f0 100644
--- a/sql/spatial.h
+++ b/sql/spatial.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2002, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -100,6 +101,13 @@ struct MBR
     if (mbr->ymax > ymax)
       ymax= mbr->ymax;
   }
+  void buffer(double d)
+  {
+    xmin-= d;
+    ymin-= d;
+    xmax+= d;
+    ymax+= d;
+  }
 
   int equals(const MBR *mbr)
   {
diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc
index e4aa41b30ab..92747066121 100644
--- a/sql/sql_acl.cc
+++ b/sql/sql_acl.cc
@@ -8387,14 +8387,22 @@ static ulong parse_client_handshake_packet(MPVIO_EXT *mpvio,
   if (pkt_len < MIN_HANDSHAKE_SIZE)
     return packet_error;
 
+  /*
+    Protocol buffer is guaranteed to always end with \0. (see my_net_read())
+    As the code below depends on this, lets check that.
+  */
+  DBUG_ASSERT(net->read_pos[pkt_len] == 0);
+
   if (mpvio->connect_errors)
     reset_host_errors(thd->main_security_ctx.ip);
 
   ulong client_capabilities= uint2korr(net->read_pos);
   if (client_capabilities & CLIENT_PROTOCOL_41)
   {
-    client_capabilities|= ((ulonglong) uint2korr(net->read_pos + 2)) << 16;
-    thd->max_client_packet_length= uint4korr(net->read_pos + 4);
+    if (pkt_len < 32)
+      return packet_error;
+    client_capabilities|= ((ulong) uint2korr(net->read_pos+2)) << 16;
+    thd->max_client_packet_length= uint4korr(net->read_pos+4);
     DBUG_PRINT("info", ("client_character_set: %d", (uint) net->read_pos[8]));
     if (thd_init_client_charset(thd, (uint) net->read_pos[8]))
       return packet_error;
@@ -8403,8 +8411,10 @@ static ulong parse_client_handshake_packet(MPVIO_EXT *mpvio,
   }
   else
   {
-    thd->max_client_packet_length= uint3korr(net->read_pos + 2);
-    end= (char*) net->read_pos + 5;
+    if (pkt_len < 5)
+      return packet_error;
+    thd->max_client_packet_length= uint3korr(net->read_pos+2);
+    end= (char*) net->read_pos+5;
   }
 
   /* Disable those bits which are not supported by the client. */
diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc
index 8a2f475b6f2..f01e279333d 100644
--- a/sql/sql_admin.cc
+++ b/sql/sql_admin.cc
@@ -334,6 +334,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
 
     DBUG_PRINT("admin", ("table: '%s'.'%s'", table->db, table->table_name));
     strxmov(table_name, db, ".", table->table_name, NullS);
+    thd->open_options|= extra_open_options;
     table->lock_type= lock_type;
     /*
       To make code safe for re-execution we need to reset type of MDL
@@ -369,8 +370,6 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
           lex->sql_command == SQLCOM_ANALYZE ||
           lex->sql_command == SQLCOM_OPTIMIZE)
 	thd->prepare_derived_at_open= TRUE;
-
-      thd->open_options|= extra_open_options;
       if (!thd->locked_tables_mode && repair_table_use_frm)
       {
         /*
@@ -403,11 +402,11 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
 
         open_error= open_and_lock_tables(thd, table, TRUE, 0);
       }
-      thd->open_options&= ~extra_open_options;
       thd->prepare_derived_at_open= FALSE;
 
       table->next_global= save_next_global;
       table->next_local= save_next_local;
+      thd->open_options&= ~extra_open_options;
 
       /*
         If open_and_lock_tables() failed, close_thread_tables() will close
@@ -620,9 +619,17 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
       if (check_old_types == HA_ADMIN_NEEDS_ALTER ||
           check_for_upgrade == HA_ADMIN_NEEDS_ALTER)
       {
+        /* We use extra_open_options to be able to open crashed tables */
+        thd->open_options|= extra_open_options;
         result_code= admin_recreate_table(thd, table);
+        thd->open_options&= ~extra_open_options;
         goto send_result;
       }
+      if (check_old_types || check_for_upgrade)
+      {
+        /* If repair is not implemented for the engine, run ALTER TABLE */
+        need_repair_or_alter= 1;
+      }
     }
 
     DBUG_PRINT("admin", ("calling operator_func '%s'", operator_name));
diff --git a/sql/sql_analyse.cc b/sql/sql_analyse.cc
index d99337bd210..31e13882515 100644
--- a/sql/sql_analyse.cc
+++ b/sql/sql_analyse.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index e7abef13c75..2dc1893068f 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -918,6 +918,7 @@ void intern_close_table(TABLE *table)
   delete table->triggers;
   if (table->file)                              // Not true if placeholder
     (void) closefrm(table, 1);			// close file
+  table->alias.free();
   DBUG_VOID_RETURN;
 }
 
@@ -8160,7 +8161,7 @@ bool setup_tables(THD *thd, Name_resolution_context *context,
       Item *item= table_list->jtbm_subselect->optimizer;
       if (table_list->jtbm_subselect->optimizer->fix_fields(thd, &item))
       {
-        my_error(ER_TOO_MANY_TABLES,MYF(0),MAX_TABLES); /* psergey-todo: WHY ER_TOO_MANY_TABLES ???*/
+        my_error(ER_TOO_MANY_TABLES,MYF(0), static_cast<int>(MAX_TABLES)); /* psergey-todo: WHY ER_TOO_MANY_TABLES ???*/
         DBUG_RETURN(1);
       }
       DBUG_ASSERT(item == table_list->jtbm_subselect->optimizer);
@@ -8704,9 +8705,11 @@ fill_record(THD * thd, List<Item> &fields, List<Item> &values,
   Item *value, *fld;
   Item_field *field;
   TABLE *table= 0, *vcol_table= 0;
-  bool abort_on_warning_saved= thd->abort_on_warning;
+  bool save_abort_on_warning= thd->abort_on_warning;
+  bool save_no_errors= thd->no_errors;
   DBUG_ENTER("fill_record");
 
+  thd->no_errors= ignore_errors;
   /*
     Reset the table->auto_increment_field_not_null as it is valid for
     only one row.
@@ -8769,10 +8772,12 @@ fill_record(THD * thd, List<Item> &fields, List<Item> &values,
         goto err;
     }
   }
-  thd->abort_on_warning= abort_on_warning_saved;
+  thd->abort_on_warning= save_abort_on_warning;
+  thd->no_errors=        save_no_errors;
   DBUG_RETURN(thd->is_error());
 err:
-  thd->abort_on_warning= abort_on_warning_saved;
+  thd->abort_on_warning= save_abort_on_warning;
+  thd->no_errors=        save_no_errors;
   if (table)
     table->auto_increment_field_not_null= FALSE;
   DBUG_RETURN(TRUE);
diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc
index 5ddff949ce3..664590c34ac 100644
--- a/sql/sql_binlog.cc
+++ b/sql/sql_binlog.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2005, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_cache.cc b/sql/sql_cache.cc
index 6bc1d5204b6..7212f6996eb 100644
--- a/sql/sql_cache.cc
+++ b/sql/sql_cache.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -467,6 +468,7 @@ static void make_base_query(String *new_query,
   DBUG_ASSERT(query[query_length] == 0);
   DBUG_ASSERT(!is_white_space(query[0]));
 
+  new_query->length(0);           // Don't copy anything from old buffer
   if (new_query->realloc(query_length + additional_length))
   {
     /*
@@ -545,8 +547,11 @@ insert_space:
   }
   if (buffer == last_space)
     buffer--;                                   // Remove the last space
-  *buffer= 0;
+  *buffer= 0;                                   // End zero after query
   new_query->length((size_t) (buffer - new_query->ptr()));
+
+  /* Copy db_length */
+  memcpy(buffer+1, query_end+1, QUERY_CACHE_DB_LENGTH_SIZE);
 }
 
 
@@ -1482,7 +1487,8 @@ def_week_frmt: %lu, in_trans: %d, autocommit: %d",
     /* Key is query + database + flag */
     if (thd->db_length)
     {
-      memcpy((char*) (query + query_length + 1), thd->db, thd->db_length);
+      memcpy((char*) (query + query_length + 1 + QUERY_CACHE_DB_LENGTH_SIZE),
+             thd->db, thd->db_length);
       DBUG_PRINT("qcache", ("database: %s  length: %u",
 			    thd->db, (unsigned) thd->db_length)); 
     }
@@ -1490,8 +1496,8 @@ def_week_frmt: %lu, in_trans: %d, autocommit: %d",
     {
       DBUG_PRINT("qcache", ("No active database"));
     }
-    tot_length= query_length + thd->db_length + 1 +
-      QUERY_CACHE_FLAGS_SIZE;
+    tot_length= (query_length + thd->db_length + 1 + 
+                 QUERY_CACHE_DB_LENGTH_SIZE + QUERY_CACHE_FLAGS_SIZE);
     /*
       We should only copy structure (don't use it location directly)
       because of alignment issue
@@ -1654,7 +1660,7 @@ Query_cache::send_result_to_client(THD *thd, char *org_sql, uint query_length)
   Query_cache_block_table *block_table, *block_table_end;
   ulong tot_length;
   Query_cache_query_flags flags;
-  const char *sql, *sql_end;
+  const char *sql, *sql_end, *found_brace= 0;
   DBUG_ENTER("Query_cache::send_result_to_client");
 
   /*
@@ -1728,9 +1734,16 @@ Query_cache::send_result_to_client(THD *thd, char *org_sql, uint query_length)
       case '\n':
       case '\t':
       case ' ':
-      case '(':    // To handle (select a from t1) union (select a from t1);
         sql++;
         continue;
+      case '(':    // To handle (select a from t1) union (select a from t1);
+        if (!found_brace)
+        {
+          found_brace= sql;
+          sql++;
+          continue;
+        }
+        /* fall trough */
       default:
         break;
       }
@@ -1755,7 +1768,28 @@ Query_cache::send_result_to_client(THD *thd, char *org_sql, uint query_length)
     DBUG_PRINT("qcache", ("The statement has a SQL_NO_CACHE directive"));
     goto err;
   }
+  {
+    /*
+      We have allocated buffer space (in alloc_query) to hold the
+      SQL statement(s) + the current database name + a flags struct.
+      If the database name has changed during execution, which might
+      happen if there are multiple statements, we need to make
+      sure the new current database has a name with the same length
+      as the previous one.
+    */
+    size_t db_len= uint2korr(sql_end+1);
+    if (thd->db_length != db_len)
+    {
+      /*
+        We should probably reallocate the buffer in this case,
+        but for now we just leave it uncached
+      */
 
+      DBUG_PRINT("qcache", 
+                 ("Current database has changed since start of query"));
+      goto err;
+    }
+  }
   /*
     Try to obtain an exclusive lock on the query cache. If the cache is
     disabled or if a full cache flush is in progress, the attempt to
@@ -1772,8 +1806,11 @@ Query_cache::send_result_to_client(THD *thd, char *org_sql, uint query_length)
   Query_cache_block *query_block;
   if (thd->variables.query_cache_strip_comments)
   {
+    if (found_brace)
+      sql= found_brace;
     make_base_query(&thd->base_query, sql, (size_t) (sql_end - sql),
-                    thd->db_length + 1 + QUERY_CACHE_FLAGS_SIZE);
+                    thd->db_length + 1 + QUERY_CACHE_DB_LENGTH_SIZE +
+                    QUERY_CACHE_FLAGS_SIZE);
     sql=          thd->base_query.ptr();
     query_length= thd->base_query.length();
   }
@@ -1783,12 +1820,15 @@ Query_cache::send_result_to_client(THD *thd, char *org_sql, uint query_length)
     thd->base_query.set(sql, query_length, system_charset_info);
   }
 
-  tot_length= query_length + thd->db_length + 1 + QUERY_CACHE_FLAGS_SIZE;
+  tot_length= (query_length + 1 + QUERY_CACHE_DB_LENGTH_SIZE +
+               thd->db_length + QUERY_CACHE_FLAGS_SIZE);
+
   if (thd->db_length)
   {
-    memcpy((char*) (sql+query_length+1), thd->db, thd->db_length);
+    memcpy((uchar*) sql + query_length + 1 + QUERY_CACHE_DB_LENGTH_SIZE,
+           thd->db, thd->db_length);
     DBUG_PRINT("qcache", ("database: '%s'  length: %u",
-			  thd->db, (unsigned)thd->db_length));
+			  thd->db, (uint) thd->db_length));
   }
   else
   {
@@ -1910,7 +1950,7 @@ def_week_frmt: %lu, in_trans: %d, autocommit: %d",
       {
         DBUG_PRINT("qcache",
                    ("Temporary table detected: '%s.%s'",
-                    table_list.db, table_list.alias));
+                    tmptable->s->db.str, tmptable->alias.c_ptr()));
         unlock();
         /*
           We should not store result of this query because it contain
diff --git a/sql/sql_cache.h b/sql/sql_cache.h
index 87b861b34f8..9d713d012f5 100644
--- a/sql/sql_cache.h
+++ b/sql/sql_cache.h
@@ -545,6 +545,7 @@ struct Query_cache_query_flags
   MY_LOCALE *lc_time_names;
 };
 #define QUERY_CACHE_FLAGS_SIZE sizeof(Query_cache_query_flags)
+#define QUERY_CACHE_DB_LENGTH_SIZE 2
 #include "sql_cache.h"
 #define query_cache_abort(A) query_cache.abort(A)
 #define query_cache_end_of_result(A) query_cache.end_of_result(A)
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index cc58a131f00..5230663809d 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -1,5 +1,6 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1080,27 +1081,10 @@ MYSQL_ERROR* THD::raise_condition(uint sql_errno,
   {
     is_slave_error=  1; // needed to catch query errors during replication
 
-    /*
-      thd->lex->current_select == 0 if lex structure is not inited
-      (not query command (COM_QUERY))
-    */
-    if (lex->current_select &&
-        lex->current_select->no_error && !is_fatal_error)
+    if (! stmt_da->is_error())
     {
-      DBUG_PRINT("error",
-                 ("Error converted to warning: current_select: no_error %d  "
-                  "fatal_error: %d",
-                  (lex->current_select ?
-                   lex->current_select->no_error : 0),
-                  (int) is_fatal_error));
-    }
-    else
-    {
-      if (! stmt_da->is_error())
-      {
-        set_row_count_func(-1);
-        stmt_da->set_error_status(this, sql_errno, msg, sqlstate);
-      }
+      set_row_count_func(-1);
+      stmt_da->set_error_status(this, sql_errno, msg, sqlstate);
     }
   }
 
@@ -2870,7 +2854,8 @@ int select_singlerow_subselect::send_data(List<Item> &items)
   Item_singlerow_subselect *it= (Item_singlerow_subselect *)item;
   if (it->assigned())
   {
-    my_message(ER_SUBQUERY_NO_1_ROW, ER(ER_SUBQUERY_NO_1_ROW), MYF(0));
+    my_message(ER_SUBQUERY_NO_1_ROW, ER(ER_SUBQUERY_NO_1_ROW),
+               MYF(current_thd->lex->ignore ? ME_JUST_WARNING : 0));
     DBUG_RETURN(1);
   }
   if (unit->offset_limit_cnt)
@@ -4250,16 +4235,6 @@ void mark_transaction_to_rollback(THD *thd, bool all)
   {
     thd->is_fatal_sub_stmt_error= TRUE;
     thd->transaction_rollback_request= all;
-    /*
-      Aborted transactions can not be IGNOREd.
-      Switch off the IGNORE flag for the current
-      SELECT_LEX. This should allow my_error()
-      to report the error and abort the execution
-      flow, even in presence
-      of IGNORE clause.
-    */
-    if (thd->lex->current_select)
-      thd->lex->current_select->no_error= FALSE;
   }
 }
 /***************************************************************************
diff --git a/sql/sql_class.h b/sql/sql_class.h
index dd1617d7546..b39f0506ee1 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates.
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
    2009-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
diff --git a/sql/sql_connect.cc b/sql/sql_connect.cc
index 3c88b7a054d..265ef1e6e9f 100644
--- a/sql/sql_connect.cc
+++ b/sql/sql_connect.cc
@@ -1,5 +1,6 @@
 /*
-   Copyright (c) 2007, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2007, 2011, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -870,7 +871,6 @@ bool init_new_connection_handler_thread()
   return 0;
 }
 
-#ifndef EMBEDDED_LIBRARY
 /*
   Perform handshake, authorize client and update thd ACL variables.
 
@@ -883,6 +883,7 @@ bool init_new_connection_handler_thread()
      1  error
 */
 
+#ifndef EMBEDDED_LIBRARY
 static int check_connection(THD *thd)
 {
   uint connect_errors= 0;
diff --git a/sql/sql_cursor.cc b/sql/sql_cursor.cc
index 7350618adc0..c50cded7470 100644
--- a/sql/sql_cursor.cc
+++ b/sql/sql_cursor.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2005, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_db.cc b/sql/sql_db.cc
index 2301dae5fdf..052616f6965 100644
--- a/sql/sql_db.cc
+++ b/sql/sql_db.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc
index efce55cc18a..f48724236d4 100644
--- a/sql/sql_delete.cc
+++ b/sql/sql_delete.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -131,8 +132,6 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds,
     DBUG_RETURN(TRUE);
   }
 
-  select_lex->no_error= thd->lex->ignore;
-
   const_cond_result= const_cond && (!conds || conds->val_int());
   if (thd->is_error())
   {
@@ -359,16 +358,9 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds,
       }
       else
       {
-	table->file->print_error(error,MYF(0));
-	/*
-	  In < 4.0.14 we set the error number to 0 here, but that
-	  was not sensible, because then MySQL would not roll back the
-	  failed DELETE, and also wrote it to the binlog. For MyISAM
-	  tables a DELETE probably never should fail (?), but for
-	  InnoDB it can fail in a FOREIGN KEY error or an
-	  out-of-tablespace error.
-	*/
-        if (!select_lex->no_error)
+	table->file->print_error(error,
+                                 MYF(thd->lex->ignore ? ME_JUST_WARNING : 0));
+        if (thd->is_error())
         {
           error= 1;
           break;
@@ -673,7 +665,7 @@ multi_delete::initialize_tables(JOIN *join)
 
   for (JOIN_TAB *tab= first_linear_tab(join, WITH_CONST_TABLES); 
        tab; 
-       tab= next_linear_tab(join, tab, WITH_BUSH_ROOTS))
+       tab= next_linear_tab(join, tab, WITHOUT_BUSH_ROOTS))
   {
     if (tab->table->map & tables_to_delete_from)
     {
@@ -758,7 +750,7 @@ int multi_delete::send_data(List<Item> &values)
   TABLE_LIST *del_table;
   DBUG_ENTER("multi_delete::send_data");
 
-  bool ignore= thd->lex->current_select->no_error;
+  bool ignore= thd->lex->ignore;
 
   for (del_table= delete_tables;
        del_table;
@@ -909,11 +901,11 @@ int multi_delete::do_deletes()
        table_being_deleted= table_being_deleted->next_local, counter++)
   { 
     TABLE *table = table_being_deleted->table;
+    int local_error; 
     if (tempfiles[counter]->get(table))
       DBUG_RETURN(1);
 
-    int local_error= 
-      do_table_deletes(table, thd->lex->current_select->no_error);
+    local_error= do_table_deletes(table, thd->lex->ignore);
 
     if (thd->killed && !local_error)
       DBUG_RETURN(1);
diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc
index dee2f01fb15..01125bdb97b 100644
--- a/sql/sql_derived.cc
+++ b/sql/sql_derived.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2002, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_help.cc b/sql/sql_help.cc
index ca0e695b3fe..c352272e95c 100644
--- a/sql/sql_help.cc
+++ b/sql/sql_help.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2002, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc
index 70c66d6cf29..7d33f3e4c07 100644
--- a/sql/sql_insert.cc
+++ b/sql/sql_insert.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -78,6 +78,8 @@
 #include "sql_audit.h"
 #include "sql_derived.h"                        // mysql_handle_derived
 
+#include "debug_sync.h"
+
 #ifndef EMBEDDED_LIBRARY
 static bool delayed_get_table(THD *thd, MDL_request *grl_protection_request,
                               TABLE_LIST *table_list);
@@ -1595,6 +1597,8 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info)
 	error= HA_ERR_FOUND_DUPP_KEY;         /* Database can't find key */
 	goto err;
       }
+      DEBUG_SYNC(thd, "write_row_replace");
+
       /* Read all columns for the row we are going to replace */
       table->use_all_columns();
       /*
@@ -1685,11 +1689,12 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info)
           else
             error= 0;
           /*
-            If ON DUP KEY UPDATE updates a row instead of inserting one, it's
-            like a regular UPDATE statement: it should not affect the value of a
-            next SELECT LAST_INSERT_ID() or mysql_insert_id().
-            Except if LAST_INSERT_ID(#) was in the INSERT query, which is
-            handled separately by THD::arg_of_last_insert_id_function.
+            If ON DUP KEY UPDATE updates a row instead of inserting
+            one, it's like a regular UPDATE statement: it should not
+            affect the value of a next SELECT LAST_INSERT_ID() or
+            mysql_insert_id().  Except if LAST_INSERT_ID(#) was in the
+            INSERT query, which is handled separately by
+            THD::arg_of_last_insert_id_function.
           */
           insert_id_for_cur_row= table->file->insert_id_for_cur_row= 0;
           trg_error= (table->triggers &&
@@ -1764,11 +1769,12 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info)
     }
     
     /*
-        If more than one iteration of the above while loop is done, from the second 
-        one the row being inserted will have an explicit value in the autoinc field, 
-        which was set at the first call of handler::update_auto_increment(). This 
-        value is saved to avoid thd->insert_id_for_cur_row becoming 0. Use this saved
-        autoinc value.
+      If more than one iteration of the above while loop is done, from
+      the second one the row being inserted will have an explicit
+      value in the autoinc field, which was set at the first call of
+      handler::update_auto_increment(). This value is saved to avoid
+      thd->insert_id_for_cur_row becoming 0. Use this saved autoinc
+      value.
      */
     if (table->file->insert_id_for_cur_row == 0)
       table->file->insert_id_for_cur_row= insert_id_for_cur_row;
@@ -1784,6 +1790,7 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info)
   }
   else if ((error=table->file->ha_write_row(table->record[0])))
   {
+    DEBUG_SYNC(thd, "write_row_noreplace");
     if (!info->ignore ||
         table->file->is_fatal_error(error, HA_CHECK_DUP))
       goto err;
@@ -1807,9 +1814,6 @@ ok_or_after_trg_err:
 
 err:
   info->last_errno= error;
-  /* current_select is NULL if this is a delayed insert */
-  if (thd->lex->current_select)
-    thd->lex->current_select->no_error= 0;        // Give error
   table->file->print_error(error,MYF(0));
   
 before_trg_err:
@@ -3268,8 +3272,6 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u)
   */
   lex->current_select= &lex->select_lex;
 
-  /* Errors during check_insert_fields() should not be ignored. */
-  lex->current_select->no_error= FALSE;
   res= (setup_fields(thd, 0, values, MARK_COLUMNS_READ, 0, 0) ||
         check_insert_fields(thd, table_list, *fields, values,
                             !insert_into_view, 1, &map));
diff --git a/sql/sql_join_cache.cc b/sql/sql_join_cache.cc
index 76d01b076f0..3600bbbb65f 100644
--- a/sql/sql_join_cache.cc
+++ b/sql/sql_join_cache.cc
@@ -2368,7 +2368,7 @@ enum_nested_loop_state JOIN_CACHE::generate_full_extensions(uchar *rec_ptr)
     int res= 0;
 
     if (!join_tab->check_weed_out_table || 
-        !(res= do_sj_dups_weedout(join->thd, join_tab->check_weed_out_table)))
+        !(res= join_tab->check_weed_out_table->sj_weedout_check_row(join->thd)))
     {
       set_curr_rec_link(rec_ptr);
       rc= (join_tab->next_select)(join, join_tab+1, 0);
diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc
index a9d542756fb..723329379d0 100644
--- a/sql/sql_lex.cc
+++ b/sql/sql_lex.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1796,7 +1797,7 @@ void st_select_lex_node::init_query()
   options= 0;
   sql_cache= SQL_CACHE_UNSPECIFIED;
   linkage= UNSPECIFIED_TYPE;
-  no_error= no_table_names_allowed= 0;
+  no_table_names_allowed= 0;
   uncacheable= 0;
 }
 
@@ -1865,10 +1866,10 @@ void st_select_lex::init_query()
   exclude_from_table_unique_test= no_wrap_view_item= FALSE;
   nest_level= 0;
   link_next= 0;
-  m_non_agg_field_used= false;
-  m_agg_func_used= false;
   is_prep_leaf_list_saved= FALSE;
   bzero((char*) expr_cache_may_be_used, sizeof(expr_cache_may_be_used));
+  m_non_agg_field_used= false;
+  m_agg_func_used= false;
 }
 
 void st_select_lex::init_select()
@@ -1901,10 +1902,10 @@ void st_select_lex::init_select()
   non_agg_fields.empty();
   cond_value= having_value= Item::COND_UNDEF;
   inner_refs_list.empty();
-  m_non_agg_field_used= false;
-  m_agg_func_used= false;
   insert_tables= 0;
   merged_into= 0;
+  m_non_agg_field_used= false;
+  m_agg_func_used= false;
 }
 
 /*
@@ -3762,20 +3763,58 @@ void st_select_lex::set_explain_type()
   SELECT_LEX *first= master_unit()->first_select();
   /* drop UNCACHEABLE_EXPLAIN, because it is for internal usage only */
   uint8 is_uncacheable= (uncacheable & ~UNCACHEABLE_EXPLAIN);
+  
+  bool using_materialization= FALSE;
+  Item_subselect *parent_item;
+  if ((parent_item= master_unit()->item) &&
+      parent_item->substype() == Item_subselect::IN_SUBS)
+  {
+    Item_in_subselect *in_subs= (Item_in_subselect*)parent_item;
+    /*
+      Surprisingly, in_subs->is_set_strategy() can return FALSE here,
+      even for the last invocation of this function for the select.
+    */
+    if (in_subs->test_strategy(SUBS_MATERIALIZATION))
+      using_materialization= TRUE;
+  }
 
-  type= ((&master_unit()->thd->lex->select_lex == this) ?
-         (is_primary ? "PRIMARY" : "SIMPLE"):    
-         ((this == first) ?
-          ((linkage == DERIVED_TABLE_TYPE) ?
-           "DERIVED" :
-           ((is_uncacheable & UNCACHEABLE_DEPENDENT) ?
-            "DEPENDENT SUBQUERY" :
-            (is_uncacheable ? "UNCACHEABLE SUBQUERY" :
-             "SUBQUERY"))) :
-          ((is_uncacheable & UNCACHEABLE_DEPENDENT) ?
-           "DEPENDENT UNION":
-           is_uncacheable ? "UNCACHEABLE UNION":
-           "UNION")));
+  if (&master_unit()->thd->lex->select_lex == this)
+  {
+     type= is_primary ? "PRIMARY" : "SIMPLE";
+  }
+  else
+  {
+    if (this == first)
+    {
+      /* If we're a direct child of a UNION, we're the first sibling there */
+      if (linkage == DERIVED_TABLE_TYPE)
+        type= "DERIVED";
+      else if (using_materialization)
+        type= "MATERIALIZED";
+      else
+      {
+         if (is_uncacheable & UNCACHEABLE_DEPENDENT)
+           type= "DEPENDENT SUBQUERY";
+         else
+         {
+           type= is_uncacheable? "UNCACHEABLE SUBQUERY" :
+                                 "SUBQUERY";
+         }
+      }
+    }
+    else
+    {
+      /* This a non-first sibling in UNION */
+      if (is_uncacheable & UNCACHEABLE_DEPENDENT)
+        type= "DEPENDENT UNION";
+      else if (using_materialization)
+        type= "MATERIALIZED UNION";
+      else
+      {
+        type= is_uncacheable ? "UNCACHEABLE UNION": "UNION";
+      }
+    }
+  }
   options|= SELECT_DESCRIBE;
 }
 
@@ -3841,12 +3880,12 @@ bool st_select_lex::save_leaf_tables(THD *thd)
   {
     if (leaf_tables_exec.push_back(table))
       return 1;
-    table->tablenr_exec= table->table->tablenr;
-    table->map_exec= table->table->map;
+    table->tablenr_exec= table->get_tablenr();
+    table->map_exec= table->get_map();
     if (join && (join->select_options & SELECT_DESCRIBE))
       table->maybe_null_exec= 0;
     else
-      table->maybe_null_exec= table->table->maybe_null;
+      table->maybe_null_exec= table->table?  table->table->maybe_null: 0;
   }
   if (arena)
     thd->restore_active_arena(arena, &backup);
diff --git a/sql/sql_lex.h b/sql/sql_lex.h
index c776dfc5a8c..d4e94c8d2d8 100644
--- a/sql/sql_lex.h
+++ b/sql/sql_lex.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -529,7 +530,6 @@ public:
   uint8 uncacheable;
   enum sub_select_type linkage;
   bool no_table_names_allowed; /* used for global order by */
-  bool no_error; /* suppress error message (convert it to warnings) */
 
   static void *operator new(size_t size) throw ()
   {
diff --git a/sql/sql_list.h b/sql/sql_list.h
index 894edc4516d..7bd72cba359 100644
--- a/sql/sql_list.h
+++ b/sql/sql_list.h
@@ -1,6 +1,7 @@
 #ifndef INCLUDES_MYSQL_SQL_LIST_H
 #define INCLUDES_MYSQL_SQL_LIST_H
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_load.cc b/sql/sql_load.cc
index bc926118723..aab9ea76499 100644
--- a/sql/sql_load.cc
+++ b/sql/sql_load.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -190,7 +190,7 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list,
   */
   char *tdb= thd->db ? thd->db : db;		// Result is never null
   ulong skip_lines= ex->skip_lines;
-  bool transactional_table;
+  bool transactional_table __attribute__((unused));
   DBUG_ENTER("mysql_load");
 
   /*
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index ee2998e6e5e..4d98313c5ae 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -1,5 +1,6 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2011 Monty Program Ab
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -548,8 +549,10 @@ static void handle_bootstrap_impl(THD *thd)
 
     query= (char *) thd->memdup_w_gap(buff, length + 1,
                                       thd->db_length + 1 +
+                                      QUERY_CACHE_DB_LENGTH_SIZE +
                                       QUERY_CACHE_FLAGS_SIZE);
     thd->set_query_and_id(query, length, thd->charset(), next_query_id());
+    int2store(query + length + 1, 0);           // No db in bootstrap
     DBUG_PRINT("query",("%-.4096s",thd->query()));
 #if defined(ENABLED_PROFILING)
     thd->profiling.start_new_query();
@@ -1240,6 +1243,14 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
   case COM_REFRESH:
   {
     int not_used;
+
+    /*
+      Initialize thd->lex since it's used in many base functions, such as
+      open_tables(). Otherwise, it remains unitialized and may cause crash
+      during execution of COM_REFRESH.
+    */
+    lex_start(thd);
+    
     status_var_increment(thd->status_var.com_stat[SQLCOM_FLUSH]);
     ulong options= (ulong) (uchar) packet[0];
     if (trans_commit_implicit(thd))
@@ -1674,13 +1685,30 @@ bool alloc_query(THD *thd, const char *packet, uint packet_length)
     pos--;
     packet_length--;
   }
-  /* We must allocate some extra memory for query cache */
+  /* We must allocate some extra memory for query cache 
+
+    The query buffer layout is:
+       buffer :==
+            <statement>   The input statement(s)
+            '\0'          Terminating null char  (1 byte)
+            <length>      Length of following current database name (size_t)
+            <db_name>     Name of current database
+            <flags>       Flags struct
+  */
   if (! (query= (char*) thd->memdup_w_gap(packet,
                                           packet_length,
                                           1 + thd->db_length +
+                                          QUERY_CACHE_DB_LENGTH_SIZE +
                                           QUERY_CACHE_FLAGS_SIZE)))
       return TRUE;
   query[packet_length]= '\0';
+  /*
+    Space to hold the name of the current database is allocated.  We
+    also store this length, in case current database is changed during
+    execution.  We might need to reallocate the 'query' buffer
+  */
+  int2store(query + packet_length + 1, thd->db_length);
+    
   thd->set_query(query, packet_length);
 
   /* Reclaim some memory */
@@ -7562,8 +7590,8 @@ bool parse_sql(THD *thd,
   */
 
   DBUG_ASSERT(!mysql_parse_status ||
-              (mysql_parse_status && thd->is_error()) ||
-              (mysql_parse_status && thd->get_internal_handler()));
+              thd->is_error() ||
+              thd->get_internal_handler());
 
   /* Reset parser state. */
 
diff --git a/sql/sql_partition.h b/sql/sql_partition.h
index d2c15ae46d8..cc11d859903 100644
--- a/sql/sql_partition.h
+++ b/sql/sql_partition.h
@@ -1,7 +1,8 @@
 #ifndef SQL_PARTITION_INCLUDED
 #define SQL_PARTITION_INCLUDED
 
-/* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2006, 2010, Oracle and/or its affiliates.
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_plugin.h b/sql/sql_plugin.h
index f0b706bc322..aee04de7be4 100644
--- a/sql/sql_plugin.h
+++ b/sql/sql_plugin.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2005, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc
index f35d336fd65..f5a2f409dfc 100644
--- a/sql/sql_prepare.cc
+++ b/sql/sql_prepare.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2002, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2009-2011, Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -2515,9 +2515,6 @@ void reinit_stmt_before_use(THD *thd, LEX *lex)
       for (order= sl->order_list.first; order; order= order->next)
         order->item= &order->item_ptr;
       sl->handle_derived(lex, DT_REINIT);
-
-      /* clear the no_error flag for INSERT/UPDATE IGNORE */
-      sl->no_error= FALSE;
     }
     {
       SELECT_LEX_UNIT *unit= sl->master_unit();
diff --git a/sql/sql_priv.h b/sql/sql_priv.h
index 412686a49c9..b9017f1e5ab 100644
--- a/sql/sql_priv.h
+++ b/sql/sql_priv.h
@@ -197,15 +197,19 @@
                                     OPTIMIZER_SWITCH_INDEX_MERGE_SORT_UNION | \
                                     OPTIMIZER_SWITCH_INDEX_MERGE_INTERSECT | \
                                     OPTIMIZER_SWITCH_INDEX_COND_PUSHDOWN | \
+                                    OPTIMIZER_SWITCH_DERIVED_MERGE | \
+                                    OPTIMIZER_SWITCH_DERIVED_WITH_KEYS | \
                                     OPTIMIZER_SWITCH_TABLE_ELIMINATION | \
                                     OPTIMIZER_SWITCH_IN_TO_EXISTS | \
                                     OPTIMIZER_SWITCH_MATERIALIZATION | \
                                     OPTIMIZER_SWITCH_PARTIAL_MATCH_ROWID_MERGE|\
                                     OPTIMIZER_SWITCH_PARTIAL_MATCH_TABLE_SCAN|\
+                                    OPTIMIZER_SWITCH_OUTER_JOIN_WITH_CACHE | \
+                                    OPTIMIZER_SWITCH_SEMIJOIN_WITH_CACHE | \
                                     OPTIMIZER_SWITCH_JOIN_CACHE_INCREMENTAL | \
                                     OPTIMIZER_SWITCH_JOIN_CACHE_HASHED | \
                                     OPTIMIZER_SWITCH_JOIN_CACHE_BKA | \
-                                    OPTIMIZER_SWITCH_SUBQUERY_CACHE |\
+                                    OPTIMIZER_SWITCH_SUBQUERY_CACHE | \
                                     OPTIMIZER_SWITCH_SEMIJOIN | \
                                     OPTIMIZER_SWITCH_FIRSTMATCH | \
                                     OPTIMIZER_SWITCH_LOOSE_SCAN )
diff --git a/sql/sql_profile.cc b/sql/sql_profile.cc
index fe593dbae88..e743e474747 100644
--- a/sql/sql_profile.cc
+++ b/sql/sql_profile.cc
@@ -1,4 +1,6 @@
-/* Copyright (c) 2007, 2010, Oracle and/or its affiliates.
+/*
+   Copyright (c) 2007, 2010, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_rename.cc b/sql/sql_rename.cc
index 6cc08e64d86..6b0d1e980f9 100644
--- a/sql/sql_rename.cc
+++ b/sql/sql_rename.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc
index 4152c07da37..5bfb19f6828 100644
--- a/sql/sql_repl.cc
+++ b/sql/sql_repl.cc
@@ -1,5 +1,6 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2009-2011, Monty Program Ab
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 07daaa2bd5c..f22510bb2b7 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -80,7 +80,7 @@ static bool sort_and_filter_keyuse(THD *thd, DYNAMIC_ARRAY *keyuse,
                                    bool skip_unprefixed_keyparts);
 static int sort_keyuse(KEYUSE *a,KEYUSE *b);
 static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, KEYUSE *org_keyuse,
-			       table_map used_tables);
+			       bool allow_full_scan, table_map used_tables);
 void best_access_path(JOIN *join, JOIN_TAB *s, 
                              table_map remaining_tables, uint idx, 
                              bool disable_jbuf, double record_count,
@@ -106,7 +106,7 @@ C_MODE_END
 static bool find_best(JOIN *join,table_map rest_tables,uint index,
 		      double record_count,double read_time);
 static uint cache_record_length(JOIN *join,uint index);
-static bool get_best_combination(JOIN *join);
+bool get_best_combination(JOIN *join);
 static store_key *get_store_key(THD *thd,
 				KEYUSE *keyuse, table_map used_tables,
 				KEY_PART_INFO *key_part, uchar *key_buff,
@@ -451,6 +451,73 @@ fix_inner_refs(THD *thd, List<Item> &all_fields, SELECT_LEX *select,
 }
 
 /**
+   The following clauses are redundant for subqueries:
+
+   DISTINCT
+   GROUP BY   if there are no aggregate functions and no HAVING
+              clause
+
+   Because redundant clauses are removed both from JOIN and
+   select_lex, the removal is permanent. Thus, it only makes sense to
+   call this function for normal queries and on first execution of
+   SP/PS
+
+   @param subq_select_lex   select_lex that is part of a subquery 
+                            predicate. This object and the associated 
+                            join is modified.
+*/
+
+static
+void remove_redundant_subquery_clauses(st_select_lex *subq_select_lex)
+{
+  Item_subselect *subq_predicate= subq_select_lex->master_unit()->item;
+  /*
+    The removal should happen for IN, ALL, ANY and EXISTS subqueries,
+    which means all but single row subqueries. Example single row
+    subqueries: 
+       a) SELECT * FROM t1 WHERE t1.a = (<single row subquery>) 
+       b) SELECT a, (<single row subquery) FROM t1
+   */
+  if (subq_predicate->substype() == Item_subselect::SINGLEROW_SUBS)
+    return;
+
+  /* A subquery that is not single row should be one of IN/ALL/ANY/EXISTS. */
+  DBUG_ASSERT (subq_predicate->substype() == Item_subselect::EXISTS_SUBS ||
+               subq_predicate->is_in_predicate());
+
+  if (subq_select_lex->options & SELECT_DISTINCT)
+  {
+    subq_select_lex->join->select_distinct= false;
+    subq_select_lex->options&= ~SELECT_DISTINCT;
+  }
+
+  /*
+    Remove GROUP BY if there are no aggregate functions and no HAVING
+    clause
+  */
+  if (subq_select_lex->group_list.elements &&
+      !subq_select_lex->with_sum_func && !subq_select_lex->join->having)
+  {
+    subq_select_lex->join->group_list= NULL;
+    subq_select_lex->group_list.empty();
+  }
+
+  /*
+    TODO: This would prevent processing quries with ORDER BY ... LIMIT
+    therefore we disable this optimization for now.
+    Remove GROUP BY if there are no aggregate functions and no HAVING
+    clause
+  if (subq_select_lex->group_list.elements &&
+      !subq_select_lex->with_sum_func && !subq_select_lex->join->having)
+  {
+    subq_select_lex->join->group_list= NULL;
+    subq_select_lex->group_list.empty();
+  }
+  */
+}
+
+
+/**
   Function to setup clauses without sum functions.
 */
 inline int setup_without_group(THD *thd, Item **ref_pointer_array,
@@ -547,6 +614,22 @@ JOIN::prepare(Item ***rref_pointer_array,
                                     tables_list, select_lex->leaf_tables,
                                     FALSE, SELECT_ACL, SELECT_ACL, FALSE))
       DBUG_RETURN(-1);
+
+  /*
+    Permanently remove redundant parts from the query if
+      1) This is a subquery
+      2) This is the first time this query is optimized (since the
+         transformation is permanent
+      3) Not normalizing a view. Removal should take place when a
+         query involving a view is optimized, not when the view
+         is created
+  */
+  if (select_lex->master_unit()->item &&                               // 1)
+      select_lex->first_cond_optimization &&                           // 2)
+      !(thd->lex->context_analysis_only & CONTEXT_ANALYSIS_ONLY_VIEW)) // 3)
+  {
+    remove_redundant_subquery_clauses(select_lex);
+  }
   
   /*
     TRUE if the SELECT list mixes elements with and without grouping,
@@ -589,6 +672,9 @@ JOIN::prepare(Item ***rref_pointer_array,
       aggregate functions and non-aggregate fields, any non-aggregated field
       may produce a NULL value. Set all fields of each table as nullable before
       semantic analysis to take into account this change of nullability.
+
+      Note: this loop doesn't touch tables inside merged semi-joins, because
+      subquery-to-semijoin conversion has not been done yet. This is intended.
     */
     if (mixed_implicit_grouping)
       tbl->table->maybe_null= 1;
@@ -824,58 +910,6 @@ err:
 }
 
 
-void
-inject_jtbm_conds(JOIN *join, List<TABLE_LIST> *join_list, Item **join_where)
-{
-  TABLE_LIST *table;
-  NESTED_JOIN *nested_join;
-  List_iterator<TABLE_LIST> li(*join_list);
-  DBUG_ENTER("inject_jtbm_conds");
-
-  
-  while ((table= li++))
-  {
-    Item_in_subselect *item;
-    
-    if ((item= table->jtbm_subselect))
-    {
-      Item_in_subselect *subq_pred= item;
-      double rows;
-      double read_time;
-
-      //DBUG_ASSERT(subq_pred->test_set_strategy(SUBS_MATERIALIZATION));
-      subq_pred->optimize(&rows, &read_time);
-
-      subq_pred->jtbm_read_time= read_time;
-      subq_pred->jtbm_record_count=rows;
-      subq_pred->is_jtbm_merged= TRUE;
-
-      subselect_hash_sj_engine *hash_sj_engine=
-        ((subselect_hash_sj_engine*)item->engine);
-      
-      
-      //repeat of convert_subq_to_jtbm:
-      table->table= hash_sj_engine->tmp_table;
-      table->table->pos_in_table_list= table;
-
-      setup_table_map(table->table, table, table->jtbm_table_no);
-
-      Item *sj_conds= hash_sj_engine->semi_join_conds;
-
-      (*join_where)= and_items(*join_where, sj_conds);
-      if (!(*join_where)->fixed)
-        (*join_where)->fix_fields(join->thd, join_where);
-      //parent_join->select_lex->where= parent_join->conds;
-    }
-
-    if ((nested_join= table->nested_join))
-    {
-      inject_jtbm_conds(join, &nested_join->join_list, join_where);
-    }
-  }
-  DBUG_VOID_RETURN;
-}
-
 /**
   global select optimisation.
 
@@ -947,9 +981,6 @@ JOIN::optimize()
   select_limit= unit->select_limit_cnt;
   if (having || (select_options & OPTION_FOUND_ROWS))
     select_limit= HA_POS_ERROR;
-  // Ignore errors of execution if option IGNORE present
-  if (thd->lex->ignore)
-    thd->lex->current_select->no_error= 1;
 #ifdef HAVE_REF_TO_FIELDS			// Not done yet
   /* Add HAVING to WHERE if possible */
   if (having && !group_list && !sum_func_count)
@@ -1003,7 +1034,8 @@ JOIN::optimize()
       thd->restore_active_arena(arena, &backup);
   }
   
-  inject_jtbm_conds(this, join_list, &conds);
+  if (setup_jtbm_semi_joins(this, join_list, &conds))
+    DBUG_RETURN(1);
 
   conds= optimize_cond(this, conds, join_list, &cond_value, &cond_equal);
      
@@ -1450,7 +1482,10 @@ JOIN::optimize()
       DBUG_RETURN(1);
     }
     if (old_group_list && !group_list)
+    {
+      DBUG_ASSERT(group);
       select_distinct= 0;
+    }
   }
   if (!group_list && group)
   {
@@ -1458,6 +1493,7 @@ JOIN::optimize()
     simple_order=1;
     select_distinct= 0;                       // No need in distinct for 1 row
     group_optimized_away= 1;
+    implicit_grouping= TRUE;
   }
 
   calc_group_buffer(this, group_list);
@@ -1484,7 +1520,7 @@ JOIN::optimize()
   }
 
   // Can't use sort on head table if using join buffering
-  if (full_join)
+  if (full_join || hash_join)
   {
     TABLE *stable= (sort_by_table == (TABLE *) 1 ? 
       join_tab[const_tables].table : sort_by_table);
@@ -1501,22 +1537,7 @@ JOIN::optimize()
     }
   }
 
-  /*
-    Check if we need to create a temporary table.
-    This has to be done if all tables are not already read (const tables)
-    and one of the following conditions holds:
-    - We are using DISTINCT (simple distinct's are already optimized away)
-    - We are using an ORDER BY or GROUP BY on fields not in the first table
-    - We are using different ORDER BY and GROUP BY orders
-    - The user wants us to buffer the result.
-    When the WITH ROLLUP modifier is present, we cannot skip temporary table
-    creation for the DISTINCT clause just because there are only const tables.
-  */
-  need_tmp= ((const_tables != table_count &&
-	     ((select_distinct || !simple_order || !simple_group) ||
-	      (group_list && order) ||
-	      test(select_options & OPTION_BUFFER_RESULT))) ||
-             (rollup.state != ROLLUP::STATE_NONE && select_distinct));
+  need_tmp= test_if_need_tmp_table();
 
   /*
     If the hint FORCE INDEX FOR ORDER BY/GROUP BY is used for the table
@@ -1655,6 +1676,7 @@ JOIN::optimize()
   }
 
   error= 0;
+
   DBUG_RETURN(0);
 
 setup_subq_exit:
@@ -2903,7 +2925,7 @@ mysql_select(THD *thd, Item ***rref_pointer_array,
     if (!(join= new JOIN(thd, fields, select_options, result)))
 	DBUG_RETURN(TRUE);
     thd_proc_info(thd, "init");
-    thd->lex->used_tables=0;                         // Updated by setup_fields
+    thd->lex->used_tables=0;
     if ((err= join->prepare(rref_pointer_array, tables, wild_num,
                             conds, og_num, order, group, having, proc_param,
                             select_lex, unit)))
@@ -3048,6 +3070,14 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
     table->pos_in_table_list= tables;
     error= tables->fetch_number_of_rows();
 
+    DBUG_EXECUTE_IF("bug11747970_raise_error",
+                    {
+                      if (!error)
+                      {
+                        my_error(ER_UNKNOWN_ERROR, MYF(0));
+                        goto error;
+                      }
+                    });
     if (error)
     {
       table->file->print_error(error, MYF(0));
@@ -3132,6 +3162,14 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
       set_position(join,const_count++,s,(KEYUSE*) 0);
       no_rows_const_tables |= table->map;
     }
+    
+    /* SJ-Materialization handling: */
+    if (table->pos_in_table_list->jtbm_subselect &&
+        table->pos_in_table_list->jtbm_subselect->is_jtbm_const_tab)
+    {
+      set_position(join,const_count++,s,(KEYUSE*) 0);
+      no_rows_const_tables |= table->map;
+    }
   }
 
   stat_vector[i]=0;
@@ -3376,7 +3414,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
 	        s->type= JT_CONST;
 	        join->const_table_map|=table->map;
 	        set_position(join,const_count++,s,start_keyuse);
-	        if (create_ref_for_key(join, s, start_keyuse,
+	        if (create_ref_for_key(join, s, start_keyuse, FALSE,
 				       found_const_table_map))
                   goto error;
 	        if ((tmp=join_read_const_table(s,
@@ -4982,7 +5020,7 @@ void set_position(JOIN *join,uint idx,JOIN_TAB *table,KEYUSE *key)
   join->positions[idx].records_read=1.0;	/* This is a const table */
   join->positions[idx].ref_depend_map= 0;
 
-  join->positions[idx].loosescan_key= MAX_KEY; /* Not a LooseScan */
+//  join->positions[idx].loosescan_key= MAX_KEY; /* Not a LooseScan */
   join->positions[idx].sj_strategy= SJ_OPT_NONE;
   join->positions[idx].use_join_buffer= FALSE;
 
@@ -5080,6 +5118,8 @@ best_access_path(JOIN      *join,
   MY_BITMAP *eq_join_set= &s->table->eq_join_set;
   KEYUSE *hj_start_key= 0;
 
+  disable_jbuf= disable_jbuf || idx == join->const_tables;  
+
   Loose_scan_opt loose_scan_opt;
   DBUG_ENTER("best_access_path");
   
@@ -5476,7 +5516,9 @@ best_access_path(JOIN      *join,
     (1) s is inner table of semi-join -> join cache is allowed for semijoins
     (2) s is inner table of outer join -> join cache is allowed for outer joins
   */  
-  if (idx > join->const_tables && best_key == 0 && 
+  if (idx > join->const_tables && best_key == 0 &&
+      (join->allowed_join_cache_types & JOIN_CACHE_HASHED_BIT) &&
+      join->max_allowed_join_cache_level > 2 &&
      !bitmap_is_clear_all(eq_join_set) &&  !disable_jbuf &&
       (!s->emb_sj_nest ||                     
        join->allowed_semijoin_with_cache) &&    // (1)
@@ -5632,7 +5674,7 @@ best_access_path(JOIN      *join,
   pos->key=          best_key;
   pos->table=        s;
   pos->ref_depend_map= best_ref_depends_map;
-  pos->loosescan_key= MAX_KEY;
+  pos->loosescan_picker.loosescan_key= MAX_KEY;
   pos->use_join_buffer= best_uses_jbuf;
    
   loose_scan_opt.save_to_position(s, loose_scan_pos);
@@ -5938,22 +5980,22 @@ optimize_straight_join(JOIN *join, table_map join_tables)
 
     /* compute the cost of the new plan extended with 's' */
     record_count*= join->positions[idx].records_read;
-    read_time+=    join->positions[idx].read_time;
-    advance_sj_state(join, join_tables, s, idx, &record_count, &read_time,
+    read_time+= join->positions[idx].read_time +
+                record_count / (double) TIME_FOR_COMPARE;
+    advance_sj_state(join, join_tables, idx, &record_count, &read_time,
                      &loose_scan_pos);
 
     join_tables&= ~(s->table->map);
     ++idx;
   }
 
-  read_time+= record_count / (double) TIME_FOR_COMPARE;
   if (join->sort_by_table &&
       join->sort_by_table != join->positions[join->const_tables].table->table)
     read_time+= record_count;  // We have to make a temp table
   memcpy((uchar*) join->best_positions, (uchar*) join->positions,
          sizeof(POSITION)*idx);
   join->record_count= record_count;
-  join->best_read= read_time;
+  join->best_read= read_time - 0.001;
 }
 
 
@@ -6121,7 +6163,8 @@ greedy_search(JOIN      *join,
 
     /* compute the cost of the new plan extended with 'best_table' */
     record_count*= join->positions[idx].records_read;
-    read_time+=    join->positions[idx].read_time;
+    read_time+= join->positions[idx].read_time + 
+                record_count / (double) TIME_FOR_COMPARE;
 
     remaining_tables&= ~(best_table->table->map);
     --size_remain;
@@ -6229,7 +6272,7 @@ void JOIN::get_partial_cost_and_fanout(int end_tab_idx,
     if (tab->records_read && (cur_table_map & filter_map))
     {
       record_count *= tab->records_read;
-      read_time += tab->read_time;
+      read_time += tab->read_time + record_count / (double) TIME_FOR_COMPARE;
       if (tab->emb_sj_nest)
         sj_inner_fanout *= tab->records_read;
     }
@@ -6453,21 +6496,19 @@ best_extension_by_limited_search(JOIN      *join,
       /* Compute the cost of extending the plan with 's' */
 
       current_record_count= record_count * position->records_read;
-      current_read_time=    read_time + position->read_time;
+      current_read_time=read_time + position->read_time +
+                        current_record_count / (double) TIME_FOR_COMPARE;
 
-      advance_sj_state(join, remaining_tables, s, idx, &current_record_count,
+      advance_sj_state(join, remaining_tables, idx, &current_record_count,
                        &current_read_time, &loose_scan_pos);
 
       /* Expand only partial plans with lower cost than the best QEP so far */
-      if ((current_read_time +
-           current_record_count / (double) TIME_FOR_COMPARE) >= join->best_read)
+      if (current_read_time >= join->best_read)
       {
         DBUG_EXECUTE("opt", print_plan(join, idx+1,
                                        current_record_count,
                                        read_time,
-                                       (current_read_time +
-                                        current_record_count / 
-                                        (double) TIME_FOR_COMPARE),
+                                       current_read_time,
                                        "prune_by_cost"););
         restore_prev_nj_state(s);
         restore_prev_sj_state(remaining_tables, s, idx);
@@ -6526,13 +6567,12 @@ best_extension_by_limited_search(JOIN      *join,
           'join' is either the best partial QEP with 'search_depth' relations,
           or the best complete QEP so far, whichever is smaller.
         */
-        current_read_time+= current_record_count / (double) TIME_FOR_COMPARE;
         if (join->sort_by_table &&
             join->sort_by_table !=
             join->positions[join->const_tables].table->table)
           /* We have to make a temp table */
           current_read_time+= current_record_count;
-        if ((search_depth == 1) || (current_read_time < join->best_read))
+        if (current_read_time < join->best_read)
         {
           memcpy((uchar*) join->best_positions, (uchar*) join->positions,
                  sizeof(POSITION) * (idx + 1));
@@ -6612,7 +6652,7 @@ find_best(JOIN *join,table_map rest_tables,uint idx,double record_count,
       */
       double current_record_count=record_count*records;
       double current_read_time=read_time+best;
-      advance_sj_state(join, rest_tables, s, idx, &current_record_count, 
+      advance_sj_state(join, rest_tables, idx, &current_record_count, 
                        &current_read_time, &loose_scan_pos);
 
       if (best_record_count > current_record_count ||
@@ -7112,7 +7152,7 @@ static Item * const null_ptr= NULL;
     TRUE   Out of memory
 */
 
-static bool
+bool
 get_best_combination(JOIN *join)
 {
   uint tablenr;
@@ -7129,6 +7169,7 @@ get_best_combination(JOIN *join)
     DBUG_RETURN(TRUE);
 
   join->full_join=0;
+  join->hash_join= FALSE;
 
   used_tables= OUTER_REF_TABLE_BIT;		// Outer row is already read
 
@@ -7190,13 +7231,6 @@ get_best_combination(JOIN *join)
     
     *j= *join->best_positions[tablenr].table;
 
-#if 0
-/* SJ-Materialization is represented with join tab ranges */
-    if (j->sj_strategy == SJ_OPT_MATERIALIZE || 
-        j->sj_strategy == SJ_OPT_MATERIALIZE)
-      j->sj_strategy= SJ_OPT_NONE;  
-#endif
-
     j->bush_root_tab= sjm_nest_root;
 
     form=join->table[tablenr]=j->table;
@@ -7215,16 +7249,27 @@ get_best_combination(JOIN *join)
 
     if (j->type == JT_SYSTEM)
       goto loop_end;
-    if ( !(keyuse= join->best_positions[tablenr].key) || 
-        (join->best_positions[tablenr].sj_strategy == SJ_OPT_LOOSE_SCAN))
+    if ( !(keyuse= join->best_positions[tablenr].key))
     {
       j->type=JT_ALL;
-      j->index= join->best_positions[tablenr].loosescan_key;
       if (tablenr != join->const_tables)
 	join->full_join=1;
     }
-    else if (create_ref_for_key(join, j, keyuse, used_tables))
+
+    /*if (join->best_positions[tablenr].sj_strategy == SJ_OPT_LOOSE_SCAN)
+    {
+      DBUG_ASSERT(!keyuse || keyuse->key ==
+                             join->best_positions[tablenr].loosescan_picker.loosescan_key);
+      j->index= join->best_positions[tablenr].loosescan_picker.loosescan_key;
+    }*/
+    
+    if (keyuse && create_ref_for_key(join, j, keyuse, TRUE, used_tables))
       DBUG_RETURN(TRUE);                        // Something went wrong
+
+    if ((j->type == JT_REF || j->type == JT_EQ_REF) &&
+        is_hash_join_key_no(j->ref.key))
+      join->hash_join= TRUE; 
+
   loop_end:
     /* 
       Save records_read in JOIN_TAB so that select_describe()/etc don't have
@@ -7285,10 +7330,26 @@ static bool create_hj_key_for_table(JOIN *join, JOIN_TAB *join_tab,
 
   do
   {
-    if (!(~used_tables & keyuse->used_tables) &&
-	(first_keyuse || keyuse->keypart != (keyuse-1)->keypart))
-      key_parts++;
-    first_keyuse= FALSE;
+    if (!(~used_tables & keyuse->used_tables))
+    {
+      if (first_keyuse)
+      {
+        key_parts++;
+        first_keyuse= FALSE;
+      }
+      else
+      {
+        KEYUSE *curr= org_keyuse;
+        for( ; curr < keyuse; curr++)
+        {
+          if (curr->keypart == keyuse->keypart &&
+              !(~used_tables & curr->used_tables))
+            break;
+        }
+        if (curr == keyuse)
+           key_parts++;
+      }
+    }
     keyuse++;
   } while (keyuse->table == table && keyuse->is_for_hash_join());
   if (!key_parts)
@@ -7313,15 +7374,31 @@ static bool create_hj_key_for_table(JOIN *join, JOIN_TAB *join_tab,
   keyuse= org_keyuse;
   do
   {
-    if (!(~used_tables & keyuse->used_tables) &&
-        (first_keyuse || keyuse->keypart != (keyuse-1)->keypart))
-    {
-      Field *field= table->field[keyuse->keypart];
-      uint fieldnr= keyuse->keypart+1;
-      table->create_key_part_by_field(keyinfo, key_part_info, field, fieldnr);
-      first_keyuse= FALSE;
-      key_part_info++;
+    if (!(~used_tables & keyuse->used_tables))
+    { 
+      bool add_key_part= TRUE;
+      if (!first_keyuse)
+      {
+        for(KEYUSE *curr= org_keyuse; curr < keyuse; curr++)
+        {
+          if (curr->keypart == keyuse->keypart &&
+              !(~used_tables & curr->used_tables))
+	  {
+            keyuse->keypart= NO_KEYPART;
+            add_key_part= FALSE;
+            break;
+          }
+        }
+      }
+      if (add_key_part)
+      {
+        Field *field= table->field[keyuse->keypart];
+        uint fieldnr= keyuse->keypart+1;
+        table->create_key_part_by_field(keyinfo, key_part_info, field, fieldnr);
+        key_part_info++;
+      }
     }
+    first_keyuse= FALSE;
     keyuse++;
   } while (keyuse->table == table && keyuse->is_for_hash_join());
 
@@ -7359,7 +7436,8 @@ static bool are_tables_local(JOIN_TAB *jtab, table_map used_tables)
 }
 
 static bool create_ref_for_key(JOIN *join, JOIN_TAB *j,
-                               KEYUSE *org_keyuse, table_map used_tables)
+                               KEYUSE *org_keyuse, bool allow_full_scan, 
+                               table_map used_tables)
 {
   uint keyparts, length, key;
   TABLE *table;
@@ -7404,8 +7482,7 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j,
       {
         if  (are_tables_local(j, keyuse->val->used_tables()))
         {
-          if ((is_hash_join_key_no(key) && 
-              (keyparts == 0 || keyuse->keypart != (keyuse-1)->keypart)) ||
+          if ((is_hash_join_key_no(key) && keyuse->keypart != NO_KEYPART) ||
               (!is_hash_join_key_no(key) && keyparts == keyuse->keypart &&
                !(found_part_ref_or_null & keyuse->optimize)))
           {
@@ -7418,6 +7495,14 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j,
       keyuse++;
     } while (keyuse->table == table && keyuse->key == key);
   } /* not ftkey */
+  
+  if (!keyparts && allow_full_scan)
+  {
+    /* It's a LooseIndexScan strategy scanning whole index */
+    j->type= JT_ALL;
+    j->index= key;
+    DBUG_RETURN(FALSE);
+  }
 
   /* set up fieldref */
   j->ref.key_parts= keyparts;
@@ -7459,6 +7544,7 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j,
     for (i=0 ; i < keyparts ; keyuse++,i++)
     {
       while (((~used_tables) & keyuse->used_tables) || 
+             keyuse->keypart == NO_KEYPART ||
 	     (keyuse->keypart != 
               (is_hash_join_key_no(key) ?
                  keyinfo->key_part[i].field->field_index : i)) || 
@@ -8042,14 +8128,33 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
           DBUG_RETURN(1);	 // Impossible const condition
         }
 
-        COND *outer_ref_cond= make_cond_for_table(thd, cond, 
-                                                  OUTER_REF_TABLE_BIT,
-                                                  OUTER_REF_TABLE_BIT,
-                                                  -1, FALSE, FALSE);
-        if (outer_ref_cond)
-	{
-          add_cond_and_fix(thd, &outer_ref_cond, join->outer_ref_cond);
-          join->outer_ref_cond= outer_ref_cond;
+        if (join->table_count != join->const_tables)
+        {
+          COND *outer_ref_cond= make_cond_for_table(thd, cond,
+                                                    join->const_table_map |
+                                                    OUTER_REF_TABLE_BIT,
+                                                    OUTER_REF_TABLE_BIT,
+                                                    -1, FALSE, FALSE);
+          if (outer_ref_cond)
+          {
+            add_cond_and_fix(thd, &outer_ref_cond, join->outer_ref_cond);
+            join->outer_ref_cond= outer_ref_cond;
+          }
+        }
+        else
+        {
+          COND *pseudo_bits_cond=
+            make_cond_for_table(thd, cond,
+                                join->const_table_map |
+                                PSEUDO_TABLE_BITS,
+                                PSEUDO_TABLE_BITS,
+                                -1, FALSE, FALSE);
+          if (pseudo_bits_cond)
+          {
+            add_cond_and_fix(thd, &pseudo_bits_cond,
+                             join->pseudo_bits_cond);
+            join->pseudo_bits_cond= pseudo_bits_cond;
+          }
         }
       }
     }
@@ -8441,9 +8546,39 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
           COND *tmp_cond= make_cond_for_table(thd, on_expr, used_tables2,
                                               current_map, /*(tab - first_tab)*/ -1,
 					      FALSE, FALSE);
-          if (tab == first_inner_tab && tab->on_precond)
+          bool is_sjm_lookup_tab= FALSE;
+          if (tab->bush_children)
+          {
+            /*
+              'tab' is an SJ-Materialization tab, i.e. we have a join order 
+              like this:
+
+                ot1 sjm_tab LEFT JOIN ot2 ot3
+                         ^          ^
+                   'tab'-+          +--- left join we're adding triggers for
+
+              LEFT JOIN's ON expression may not have references to subquery
+              columns.  The subquery was in the WHERE clause, so IN-equality 
+              is in the WHERE clause, also.
+              However, equality propagation code may have propagated the
+              IN-equality into ON expression, and we may get things like
+
+                subquery_inner_table=const
+
+              in the ON expression. We must not check such conditions during
+              SJM-lookup, because 1) subquery_inner_table has no valid current
+              row (materialization temp.table has it instead), and 2) they
+              would be true anyway.
+            */
+            SJ_MATERIALIZATION_INFO *sjm=
+              tab->bush_children->start->emb_sj_nest->sj_mat_info;
+            if (sjm->is_used && !sjm->is_sj_scan)
+              is_sjm_lookup_tab= TRUE;
+          }
+
+          if (tab == first_inner_tab && tab->on_precond && !is_sjm_lookup_tab)
             add_cond_and_fix(thd, &tmp_cond, tab->on_precond);
-          if (tmp_cond)
+          if (tmp_cond && !is_sjm_lookup_tab)
           {
             JOIN_TAB *cond_tab= tab < first_inner_tab ? first_inner_tab : tab;
             Item **sel_cond_ref= tab < first_inner_tab ?
@@ -8872,8 +9007,7 @@ void revise_cache_usage(JOIN_TAB *join_tab)
     first_inner= join_tab->first_sj_inner_tab;
     for (tab= join_tab-1; tab >= first_inner; tab--)
     {
-      if (tab->first_sj_inner_tab == first_inner)
-        set_join_cache_denial(tab);
+      set_join_cache_denial(tab);
     }
   }
   else set_join_cache_denial(join_tab);
@@ -9157,11 +9291,12 @@ uint check_join_cache_usage(JOIN_TAB *tab,
   for (JOIN_TAB *first_inner= tab->first_inner; first_inner;
        first_inner= first_inner->first_upper)
   {
-    if (first_inner != tab && !first_inner->use_join_cache)
+    if (first_inner != tab && 
+        (!first_inner->use_join_cache || !(tab-1)->use_join_cache))
       goto no_join_cache;
   }
   if (tab->first_sj_inner_tab && tab->first_sj_inner_tab != tab &&
-      !tab->first_sj_inner_tab->use_join_cache)
+      (!tab->first_sj_inner_tab->use_join_cache || !(tab-1)->use_join_cache))
     goto no_join_cache;
   if (!prev_tab->use_join_cache)
   {
@@ -9692,8 +9827,6 @@ bool error_if_full_join(JOIN *join)
   {
     if (tab->type == JT_ALL && (!tab->select || !tab->select->quick))
     {
-      /* This error should not be ignored. */
-      join->select_lex->no_error= FALSE;
       my_message(ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE,
                  ER(ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE), MYF(0));
       return(1);
@@ -9734,9 +9867,22 @@ void JOIN_TAB::cleanup()
     if (table->pos_in_table_list && 
         table->pos_in_table_list->jtbm_subselect)
     {
-      end_read_record(&read_record);
-      table->pos_in_table_list->jtbm_subselect->cleanup();
-      table= NULL;
+      if (table->pos_in_table_list->jtbm_subselect->is_jtbm_const_tab)
+      {
+        free_tmp_table(join->thd, table);
+        table= NULL;
+      }
+      else
+      {
+        end_read_record(&read_record);
+        table->pos_in_table_list->jtbm_subselect->cleanup();
+        /* 
+          The above call freed the materializedd temptable. Set it to NULL so
+          that we don't attempt to touch it if JOIN_TAB::cleanup() is invoked
+          multiple times (it may be)
+        */
+        table=NULL;
+      }
       DBUG_VOID_RETURN;
     }
     /*
@@ -12047,7 +12193,7 @@ simplify_joins(JOIN *join, List<TABLE_LIST> *join_list, COND *conds, bool top,
     {
       if (!table->prep_on_expr)
         table->prep_on_expr= table->on_expr;
-      used_tables= table->table->map;
+      used_tables= table->get_map();
       if (conds)
         not_null_tables= conds->not_null_tables();
     }
@@ -12104,7 +12250,7 @@ simplify_joins(JOIN *join, List<TABLE_LIST> *join_list, COND *conds, bool top,
         table->embedding->on_expr_dep_tables|= table->on_expr->used_tables();
       }
       else
-        table->dep_tables&= ~table->table->map;
+        table->dep_tables&= ~table->get_map();
     }
 
     if (prev_table)
@@ -12117,7 +12263,7 @@ simplify_joins(JOIN *join, List<TABLE_LIST> *join_list, COND *conds, bool top,
         prev_table->dep_tables|= table->on_expr_dep_tables;
         table_map prev_used_tables= prev_table->nested_join ?
 	                            prev_table->nested_join->used_tables :
-	                            prev_table->table->map;
+	                            prev_table->get_map();
         /* 
           If on expression contains only references to inner tables
           we still make the inner tables dependent on the outer tables.
@@ -14189,10 +14335,22 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields,
       key_part_info->store_length= key_part_info->length;
 
       if ((*reg_field)->real_maybe_null())
+      {
         key_part_info->store_length+= HA_KEY_NULL_LENGTH;
-      if ((*reg_field)->type() == MYSQL_TYPE_BLOB || 
-          (*reg_field)->real_type() == MYSQL_TYPE_VARCHAR)
-        key_part_info->store_length+= HA_KEY_BLOB_LENGTH;
+        key_part_info->key_part_flag |= HA_NULL_PART;
+      }
+      if ((*reg_field)->type() == MYSQL_TYPE_BLOB ||
+          (*reg_field)->real_type() == MYSQL_TYPE_VARCHAR ||
+          (*reg_field)->type() == MYSQL_TYPE_GEOMETRY)
+      {
+        if ((*reg_field)->type() == MYSQL_TYPE_BLOB ||
+            (*reg_field)->type() == MYSQL_TYPE_GEOMETRY)
+          key_part_info->key_part_flag|= HA_BLOB_PART;
+        else
+          key_part_info->key_part_flag|= HA_VAR_LENGTH_PART;
+
+        key_part_info->store_length+=HA_KEY_BLOB_LENGTH;
+      }
 
       keyinfo->key_length+= key_part_info->store_length;
 
@@ -14224,6 +14382,9 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields,
       goto err;
   }
 
+  // Make empty record so random data is not written to disk
+  empty_record(table);
+
   thd->mem_root= mem_root_save;
 
   DBUG_RETURN(table);
@@ -14718,6 +14879,7 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo,
     goto err;
   }
   status_var_increment(table->in_use->status_var.created_tmp_disk_tables);
+  table->in_use->query_plan_flags|= QPLAN_TMP_DISK;
   share->db_record_offset= 1;
   table->created= TRUE;
   DBUG_RETURN(0);
@@ -15041,15 +15203,14 @@ do_select(JOIN *join,List<Item> *fields,TABLE *table,Procedure *procedure)
   {
     /*
       HAVING will be checked after processing aggregate functions,
-      But WHERE should checkd here (we alredy have read tables).
-      If there is join->exec_const_cond, and all tables are constant, then it
-      is equivalent to join->conds. exec_const_cond is already checked in the
-      beginning of JOIN::exec. If it is false, JOIN::exec returns zero
-      result already there, therefore execution reaches this point only if
-      exec_const_cond is TRUE. Since it is equvalent to join->conds, then
-      join->conds is also TRUE.
+      But WHERE should checked here (we alredy have read tables).
+      Notice that make_join_select() splits all conditions in this case
+      into two groups exec_const_cond and outer_ref_cond.
+      If join->table_count == join->const_tables then it is
+      sufficient to check only the condition pseudo_bits_cond.
     */
-    if (!join->conds || join->exec_const_cond || join->conds->val_int())
+    DBUG_ASSERT(join->outer_ref_cond == NULL);
+    if (!join->pseudo_bits_cond || join->pseudo_bits_cond->val_int())
     {
       error= (*end_select)(join, 0, 0);
       if (error == NESTED_LOOP_OK || error == NESTED_LOOP_QUERY_LIMIT)
@@ -15375,10 +15536,12 @@ sub_select(JOIN *join,JOIN_TAB *join_tab,bool end_of_records)
   int error;
   enum_nested_loop_state rc= NESTED_LOOP_OK;
   READ_RECORD *info= &join_tab->read_record;
-
-  if (join_tab->flush_weedout_table)
+   
+  for (SJ_TMP_TABLE *flush_dups_table= join_tab->flush_weedout_table;
+       flush_dups_table;
+       flush_dups_table= flush_dups_table->next_flush_table)
   {
-    do_sj_reset(join_tab->flush_weedout_table);
+    flush_dups_table->sj_weedout_delete_rows();
   }
 
   if (!join_tab->preread_init_done && join_tab->preread_init())
@@ -15427,7 +15590,7 @@ sub_select(JOIN *join,JOIN_TAB *join_tab,bool end_of_records)
         join_tab->loosescan_match_tab->found_match)
     {
       KEY *key= join_tab->table->key_info + join_tab->index;
-      key_copy(join_tab->loosescan_buf, info->record, key, 
+      key_copy(join_tab->loosescan_buf, join_tab->table->record[0], key, 
                join_tab->loosescan_key_len);
       skip_over= TRUE;
     }
@@ -15603,7 +15766,7 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab,
 
     if (join_tab->check_weed_out_table && found)
     {
-      int res= do_sj_dups_weedout(join->thd, join_tab->check_weed_out_table);
+      int res= join_tab->check_weed_out_table->sj_weedout_check_row(join->thd);
       if (res == -1)
         DBUG_RETURN(NESTED_LOOP_ERROR);
       else if (res == 1)
@@ -15730,7 +15893,7 @@ evaluate_null_complemented_join_record(JOIN *join, JOIN_TAB *join_tab)
   */
   if (join_tab->check_weed_out_table)
   {
-    int res= do_sj_dups_weedout(join->thd, join_tab->check_weed_out_table);
+    int res= join_tab->check_weed_out_table->sj_weedout_check_row(join->thd);
     if (res == -1)
       return NESTED_LOOP_ERROR;
     else if (res == 1)
@@ -15810,6 +15973,17 @@ join_read_const_table(JOIN_TAB *tab, POSITION *pos)
     /* Skip materialized derived tables/views. */
     DBUG_RETURN(0);
   }
+  else if (tab->table->pos_in_table_list->jtbm_subselect && 
+          tab->table->pos_in_table_list->jtbm_subselect->is_jtbm_const_tab)
+  {
+    /* Row will not be found */
+    int res;
+    if (tab->table->pos_in_table_list->jtbm_subselect->jtbm_const_row_found)
+      res= 0;
+    else
+      res= -1;
+    DBUG_RETURN(res);
+  }
   else if (tab->type == JT_SYSTEM)
   {
     if ((error=join_read_system(tab)))
@@ -17675,8 +17849,8 @@ find_field_in_item_list (Field *field, void *data)
 
   while ((item= li++))
   {
-    if (item->type() == Item::FIELD_ITEM &&
-        ((Item_field*) item)->field->eq(field))
+    if (item->real_item()->type() == Item::FIELD_ITEM &&
+	((Item_field*) (item->real_item()))->field->eq(field))
     {
       part_found= 1;
       break;
@@ -17810,7 +17984,7 @@ test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order,ha_rows select_limit,
           while (keyuse->key != new_ref_key && keyuse->table == tab->table)
             keyuse++;
 
-          if (create_ref_for_key(tab->join, tab, keyuse, 
+          if (create_ref_for_key(tab->join, tab, keyuse, FALSE,
                                  tab->join->const_table_map))
             goto use_filesort;
 
@@ -20665,7 +20839,7 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order,
       /* id */
       item_list.push_back(new Item_uint((uint32)select_id));
       /* select_type */
-      const char* stype= printing_materialize_nest? "SUBQUERY" : 
+      const char* stype= printing_materialize_nest? "MATERIALIZED" : 
                                                     join->select_lex->type;
       item_list.push_back(new Item_string(stype, strlen(stype), cs));
       
@@ -21062,7 +21236,7 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order,
           extra.append(STRING_WITH_LEN("; LooseScan"));
         }
 
-        if (tab->flush_weedout_table)
+        if (tab->first_weedout_table)
           extra.append(STRING_WITH_LEN("; Start temporary"));
         if (tab->check_weed_out_table)
           extra.append(STRING_WITH_LEN("; End temporary"));
@@ -21370,11 +21544,29 @@ void TABLE_LIST::print(THD *thd, table_map eliminated_tables, String *str,
   }
   else if (jtbm_subselect)
   {
-    str->append(STRING_WITH_LEN(" <materialize> ("));
-    subselect_hash_sj_engine *hash_engine;
-    hash_engine= (subselect_hash_sj_engine*)jtbm_subselect->engine;
-    hash_engine->materialize_engine->print(str, query_type);
-    str->append(')');
+    if (jtbm_subselect->engine->engine_type() ==
+          subselect_engine::SINGLE_SELECT_ENGINE)
+    {
+      /* 
+        We get here when conversion into materialization didn't finish (this
+        happens when
+        - The subquery is a degenerate case which produces 0 or 1 record
+        - subquery's optimization didn't finish because of @@max_join_size
+          limits
+        - ... maybe some other cases like this 
+      */
+      str->append(STRING_WITH_LEN(" <materialize> ("));
+      jtbm_subselect->engine->print(str, query_type);
+      str->append(')');
+    }
+    else
+    {
+      str->append(STRING_WITH_LEN(" <materialize> ("));
+      subselect_hash_sj_engine *hash_engine;
+      hash_engine= (subselect_hash_sj_engine*)jtbm_subselect->engine;
+      hash_engine->materialize_engine->print(str, query_type);
+      str->append(')');
+    }
   }
   else
   {
@@ -22015,7 +22207,8 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table,
           uint used_pk_parts= 0;
           if (used_key_parts > used_index_parts)
             used_pk_parts= used_key_parts-used_index_parts;
-          rec_per_key= keyinfo->rec_per_key[used_key_parts-1];
+          rec_per_key= used_key_parts ?
+                       keyinfo->rec_per_key[used_key_parts-1] : 1;
           /* Take into account the selectivity of the used pk prefix */
           if (used_pk_parts)
 	  {
diff --git a/sql/sql_select.h b/sql/sql_select.h
index 4b5e2903c1d..644828fa08c 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -2,7 +2,7 @@
 #define SQL_SELECT_INCLUDED
 
 /* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2011, Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -84,6 +84,8 @@ typedef struct keyuse_t {
   bool is_for_hash_join() { return is_hash_join_key_no(key); }
 } KEYUSE;
 
+#define NO_KEYPART ((uint)(-1))
+
 class store_key;
 
 const int NO_REF_PART= uint(-1);
@@ -165,6 +167,17 @@ enum enum_nested_loop_state
 };
 
 
+/* Possible sj_strategy values */
+enum sj_strategy_enum
+{
+  SJ_OPT_NONE=0,
+  SJ_OPT_DUPS_WEEDOUT=1,
+  SJ_OPT_LOOSE_SCAN  =2,
+  SJ_OPT_FIRST_MATCH =3,
+  SJ_OPT_MATERIALIZE =4,
+  SJ_OPT_MATERIALIZE_SCAN=5
+};
+
 /* Values for JOIN_TAB::packed_info */
 #define TAB_INFO_HAVE_VALUE 1
 #define TAB_INFO_USING_INDEX 2
@@ -278,7 +291,16 @@ typedef struct st_join_table {
   double        partial_join_cardinality;
 
   table_map	dependent,key_dependent;
-  uint		use_quick,index;
+  /*
+     1 - use quick select
+     2 - use "Range checked for each record"
+  */
+  uint		use_quick;
+  /*
+    Index to use. Note: this is valid only for 'index' access, but not range or
+    ref access.
+  */
+  uint          index;
   uint		status;				///< Save status for cache
   uint		used_fields;
   ulong         used_fieldlength;
@@ -326,6 +348,8 @@ typedef struct st_join_table {
   /* Variables for semi-join duplicate elimination */
   SJ_TMP_TABLE  *flush_weedout_table;
   SJ_TMP_TABLE  *check_weed_out_table;
+  /* for EXPLAIN only: */
+  SJ_TMP_TABLE  *first_weedout_table;
   
   /*
     If set, means we should stop join enumeration after we've got the first
@@ -370,7 +394,7 @@ typedef struct st_join_table {
     POSITION::sj_strategy field. This field is set up by the
     fix_semijoin_strategies_for_picked_join_order.
   */
-  uint sj_strategy;
+  enum sj_strategy_enum sj_strategy;
 
   uint n_sj_tables;
 
@@ -502,12 +526,221 @@ end_write_group(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
 		bool end_of_records);
 
 
+struct st_position;
+
+class Semi_join_strategy_picker
+{
+public:
+  /* Called when starting to build a new join prefix */
+  virtual void set_empty() = 0;
+
+  /* 
+    Update internal state after another table has been added to the join
+    prefix
+  */
+  virtual void set_from_prev(struct st_position *prev) = 0;
+  
+  virtual bool check_qep(JOIN *join,
+                         uint idx,
+                         table_map remaining_tables, 
+                         const JOIN_TAB *new_join_tab,
+                         double *record_count,
+                         double *read_time,
+                         table_map *handled_fanout,
+                         sj_strategy_enum *strategy,
+                         struct st_position *loose_scan_pos) = 0;
+
+  virtual void mark_used() = 0;
+
+  virtual ~Semi_join_strategy_picker() {} 
+};
+
+
+/*
+  Duplicate Weedout strategy optimization state
+*/
+
+class Duplicate_weedout_picker : public Semi_join_strategy_picker
+{
+  /* The first table that the strategy will need to handle */
+  uint  first_dupsweedout_table;
+
+  /*
+    Tables that we will need to have in the prefix to do the weedout step
+    (all inner and all outer that the involved semi-joins are correlated with)
+  */
+  table_map dupsweedout_tables;
+  
+  bool is_used;
+public:
+  void set_empty()
+  {
+    dupsweedout_tables= 0;
+    first_dupsweedout_table= MAX_TABLES;
+    is_used= FALSE;
+  }
+  void set_from_prev(struct st_position *prev);
+  
+  bool check_qep(JOIN *join,
+                 uint idx,
+                 table_map remaining_tables, 
+                 const JOIN_TAB *new_join_tab,
+                 double *record_count,
+                 double *read_time,
+                 table_map *handled_fanout,
+                 sj_strategy_enum *stratey,
+                 struct st_position *loose_scan_pos);
+
+  void mark_used() { is_used= TRUE; }
+  friend void fix_semijoin_strategies_for_picked_join_order(JOIN *join);
+};
+
+
+class Firstmatch_picker : public Semi_join_strategy_picker
+{
+  /*
+    Index of the first inner table that we intend to handle with this
+    strategy
+  */
+  uint first_firstmatch_table;
+  /*
+    Tables that were not in the join prefix when we've started considering 
+    FirstMatch strategy.
+  */
+  table_map first_firstmatch_rtbl;
+  /* 
+    Tables that need to be in the prefix before we can calculate the cost
+    of using FirstMatch strategy.
+   */
+  table_map firstmatch_need_tables;
+
+  bool is_used;
+
+  bool in_firstmatch_prefix() { return (first_firstmatch_table != MAX_TABLES); }
+  void invalidate_firstmatch_prefix() { first_firstmatch_table= MAX_TABLES; }
+public:
+  void set_empty()
+  {
+    invalidate_firstmatch_prefix();
+    is_used= FALSE;
+  }
+
+  void set_from_prev(struct st_position *prev);
+  bool check_qep(JOIN *join,
+                 uint idx,
+                 table_map remaining_tables, 
+                 const JOIN_TAB *new_join_tab,
+                 double *record_count,
+                 double *read_time,
+                 table_map *handled_fanout,
+                 sj_strategy_enum *strategy,
+                 struct st_position *loose_scan_pos);
+
+  void mark_used() { is_used= TRUE; }
+  friend void fix_semijoin_strategies_for_picked_join_order(JOIN *join);
+};
+
+
+class LooseScan_picker : public Semi_join_strategy_picker
+{
+  /* The first (i.e. driving) table we're doing loose scan for */
+  uint        first_loosescan_table;
+  /* 
+     Tables that need to be in the prefix before we can calculate the cost
+     of using LooseScan strategy.
+  */
+  table_map   loosescan_need_tables;
+
+  /*
+    keyno  -  Planning to do LooseScan on this key. If keyuse is NULL then 
+              this is a full index scan, otherwise this is a ref+loosescan
+              scan (and keyno matches the KEUSE's)
+    MAX_KEY - Not doing a LooseScan
+  */
+  uint loosescan_key;  // final (one for strategy instance )
+  uint loosescan_parts; /* Number of keyparts to be kept distinct */
+  
+  bool is_used;
+public:
+  void set_empty()
+  {
+    first_loosescan_table= MAX_TABLES; 
+    is_used= FALSE;
+  }
+
+  void set_from_prev(struct st_position *prev);
+  bool check_qep(JOIN *join,
+                 uint idx,
+                 table_map remaining_tables, 
+                 const JOIN_TAB *new_join_tab,
+                 double *record_count,
+                 double *read_time,
+                 table_map *handled_fanout,
+                 sj_strategy_enum *strategy,
+                 struct st_position *loose_scan_pos);
+  void mark_used() { is_used= TRUE; }
+
+  friend class Loose_scan_opt;
+  friend void best_access_path(JOIN      *join,
+                               JOIN_TAB  *s,
+                               table_map remaining_tables,
+                               uint      idx,
+                               bool      disable_jbuf,
+                               double    record_count,
+                               struct st_position *pos,
+                               struct st_position *loose_scan_pos);
+  friend bool get_best_combination(JOIN *join);
+  friend int setup_semijoin_dups_elimination(JOIN *join, ulonglong options,
+                                             uint no_jbuf_after);
+  friend void fix_semijoin_strategies_for_picked_join_order(JOIN *join);
+};
+
+
+class Sj_materialization_picker : public Semi_join_strategy_picker
+{
+  bool is_used;
+
+  /* The last inner table (valid once we're after it) */
+  uint      sjm_scan_last_inner;
+  /*
+    Tables that we need to have in the prefix to calculate the correct cost.
+    Basically, we need all inner tables and outer tables mentioned in the
+    semi-join's ON expression so we can correctly account for fanout.
+  */
+  table_map sjm_scan_need_tables;
+
+public:
+  void set_empty()
+  {
+    sjm_scan_need_tables= 0;
+    LINT_INIT(sjm_scan_last_inner);
+    is_used= FALSE;
+  }
+  void set_from_prev(struct st_position *prev);
+  bool check_qep(JOIN *join,
+                 uint idx,
+                 table_map remaining_tables, 
+                 const JOIN_TAB *new_join_tab,
+                 double *record_count,
+                 double *read_time,
+                 table_map *handled_fanout,
+                 sj_strategy_enum *strategy,
+                 struct st_position *loose_scan_pos);
+  void mark_used() { is_used= TRUE; }
+
+  friend void fix_semijoin_strategies_for_picked_join_order(JOIN *join);
+};
+
+
 /**
   Information about a position of table within a join order. Used in join
   optimization.
 */
 typedef struct st_position
 {
+  /* The table that's put into join order */
+  JOIN_TAB *table;
+
   /*
     The "fanout": number of output rows that will be produced (after
     pushed down selection condition is applied) per each row combination of
@@ -521,7 +754,10 @@ typedef struct st_position
     number the access method will be invoked.
   */
   double read_time;
-  JOIN_TAB *table;
+
+  /* Cumulative cost and record count for the join prefix */
+  COST_VECT prefix_cost;
+  double    prefix_record_count;
 
   /*
     NULL  -  'index' or 'range' or 'index_merge' or 'ALL' access is used.
@@ -531,14 +767,13 @@ typedef struct st_position
 
   /* If ref-based access is used: bitmap of tables this table depends on  */
   table_map ref_depend_map;
-
-  bool use_join_buffer; 
-  
-  
-  /* These form a stack of partial join order costs and output sizes */
-  COST_VECT prefix_cost;
-  double    prefix_record_count;
-
+ 
+  /*
+    TRUE <=> join buffering will be used. At the moment this is based on 
+    *very* imprecise guesses made in best_access_path(). 
+  */
+  bool use_join_buffer;
+ 
   /*
     Current optimization state: Semi-join strategy to be used for this
     and preceding join tables.
@@ -551,7 +786,8 @@ typedef struct st_position
     this applies to. The values of covered_preceding_positions->sj_strategy
     must be ignored.
   */
-  uint sj_strategy;
+  enum sj_strategy_enum sj_strategy;
+  
   /*
     Valid only after fix_semijoin_strategies_for_picked_join_order() call:
     if sj_strategy!=SJ_OPT_NONE, this is the number of subsequent tables that
@@ -559,67 +795,15 @@ typedef struct st_position
   */
   uint n_sj_tables;
 
-/* LooseScan strategy members */
-
-  /* The first (i.e. driving) table we're doing loose scan for */
-  uint        first_loosescan_table;
-  /* 
-     Tables that need to be in the prefix before we can calculate the cost
-     of using LooseScan strategy.
-  */
-  table_map   loosescan_need_tables;
-
-  /*
-    keyno  -  Planning to do LooseScan on this key. If keyuse is NULL then 
-              this is a full index scan, otherwise this is a ref+loosescan
-              scan (and keyno matches the KEUSE's)
-    MAX_KEY - Not doing a LooseScan
-  */
-  uint loosescan_key;  // final (one for strategy instance )
-  uint loosescan_parts; /* Number of keyparts to be kept distinct */
-  
-/* FirstMatch strategy */
-  /*
-    Index of the first inner table that we intend to handle with this
-    strategy
-  */
-  uint first_firstmatch_table;
-  /*
-    Tables that were not in the join prefix when we've started considering 
-    FirstMatch strategy.
-  */
-  table_map first_firstmatch_rtbl;
-  /* 
-    Tables that need to be in the prefix before we can calculate the cost
-    of using FirstMatch strategy.
-   */
-  table_map firstmatch_need_tables;
-
-  bool in_firstmatch_prefix() { return (first_firstmatch_table != MAX_TABLES); }
-  void invalidate_firstmatch_prefix() { first_firstmatch_table= MAX_TABLES; }
-
-/* Duplicate Weedout strategy */
-  /* The first table that the strategy will need to handle */
-  uint  first_dupsweedout_table;
-  /*
-    Tables that we will need to have in the prefix to do the weedout step
-    (all inner and all outer that the involved semi-joins are correlated with)
-  */
-  table_map dupsweedout_tables;
-
-/* SJ-Materialization-Scan strategy */
-  /* The last inner table (valid once we're after it) */
-  uint      sjm_scan_last_inner;
-  /*
-    Tables that we need to have in the prefix to calculate the correct cost.
-    Basically, we need all inner tables and outer tables mentioned in the
-    semi-join's ON expression so we can correctly account for fanout.
-  */
-  table_map sjm_scan_need_tables;
-
   table_map prefix_dups_producing_tables;
-} POSITION;
 
+  table_map inner_tables_handled_with_other_sjs;
+   
+  Duplicate_weedout_picker  dups_weedout_picker;
+  Firstmatch_picker         firstmatch_picker;
+  LooseScan_picker          loosescan_picker;
+  Sj_materialization_picker sjmat_picker;
+} POSITION;
 
 typedef struct st_rollup
 {
@@ -631,18 +815,6 @@ typedef struct st_rollup
 } ROLLUP;
 
 
-#define SJ_OPT_NONE 0
-#define SJ_OPT_DUPS_WEEDOUT 1
-#define SJ_OPT_LOOSE_SCAN   2
-#define SJ_OPT_FIRST_MATCH  3
-#define SJ_OPT_MATERIALIZE  4
-#define SJ_OPT_MATERIALIZE_SCAN  5
-
-inline bool sj_is_materialize_strategy(uint strategy)
-{
-  return strategy >= SJ_OPT_MATERIALIZE;
-}
-
 class JOIN_TAB_RANGE: public Sql_alloc
 {
 public:
@@ -750,6 +922,7 @@ public:
   */
   bool     sort_and_group; 
   bool     first_record,full_join, no_field_update;
+  bool     hash_join;
   bool	   do_send_rows;
   table_map const_table_map;
   /*
@@ -810,7 +983,7 @@ public:
     they produce.
   */
   table_map cur_dups_producing_tables;
-
+  
   /* We also maintain a stack of join optimization states in * join->positions[] */
 /******* Join optimization state members end *******/
   /*
@@ -935,6 +1108,7 @@ public:
   COND *conds;                            // ---"---
   Item *conds_history;                    // store WHERE for explain
   COND *outer_ref_cond;       ///<part of conds containing only outer references
+  COND *pseudo_bits_cond;     // part of conds containing special bita
   TABLE_LIST *tables_list;           ///<hold 'tables' parameter of mysql_select
   List<TABLE_LIST> *join_list;       ///< list of joined tables in reverse order
   COND_EQUAL *cond_equal;
@@ -1061,7 +1235,7 @@ public:
     rollup.state= ROLLUP::STATE_NONE;
 
     no_const_tables= FALSE;
-    outer_ref_cond= 0;
+    outer_ref_cond= pseudo_bits_cond= NULL;
     in_to_exists_where= NULL;
     in_to_exists_having= NULL;
   }
@@ -1151,6 +1325,25 @@ public:
     return test(allowed_join_cache_types & JOIN_CACHE_HASHED_BIT) &&
            max_allowed_join_cache_level > JOIN_CACHE_HASHED_BIT;
   }
+  /*
+    Check if we need to create a temporary table.
+    This has to be done if all tables are not already read (const tables)
+    and one of the following conditions holds:
+    - We are using DISTINCT (simple distinct's are already optimized away)
+    - We are using an ORDER BY or GROUP BY on fields not in the first table
+    - We are using different ORDER BY and GROUP BY orders
+    - The user wants us to buffer the result.
+    When the WITH ROLLUP modifier is present, we cannot skip temporary table
+    creation for the DISTINCT clause just because there are only const tables.
+  */
+  bool test_if_need_tmp_table()
+  {
+    return ((const_tables != table_count &&
+	    ((select_distinct || !simple_order || !simple_group) ||
+	     (group_list && order) ||
+	     test(select_options & OPTION_BUFFER_RESULT))) ||
+            (rollup.state != ROLLUP::STATE_NONE && select_distinct));
+  }
   bool choose_subquery_plan(table_map join_tables);
   void get_partial_cost_and_fanout(int end_tab_idx,
                                    table_map filter_map,
diff --git a/sql/sql_show.cc b/sql/sql_show.cc
index d098bca4776..d64c7a6df52 100644
--- a/sql/sql_show.cc
+++ b/sql/sql_show.cc
@@ -688,6 +688,21 @@ public:
 };
 
 
+/*
+  Return CREATE command for table or view
+
+  @param thd	     Thread handler
+  @param table_list  Table / view
+
+  @return
+  @retval 0      OK
+  @retval 1      Error
+
+  @notes
+  table_list->db and table_list->table_name are kept unchanged to
+  not cause problems with SP.
+*/
+
 bool
 mysqld_show_create(THD *thd, TABLE_LIST *table_list)
 {
@@ -3442,8 +3457,8 @@ fill_schema_table_by_open(THD *thd, bool is_show_fields_or_keys,
   LEX_STRING db_name, table_name;
   TABLE_LIST *table_list;
   bool result= true;
-
   DBUG_ENTER("fill_schema_table_by_open");
+
   /*
     When a view is opened its structures are allocated on a permanent
     statement arena and linked into the LEX tree for the current statement
@@ -4216,7 +4231,6 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond)
             }
 
             DEBUG_SYNC(thd, "before_open_in_get_all_tables");
-
             if (fill_schema_table_by_open(thd, FALSE,
                                           table, schema_table,
                                           db_name, table_name,
diff --git a/sql/sql_string.cc b/sql/sql_string.cc
index 683511c3da4..c4f5f315b08 100644
--- a/sql/sql_string.cc
+++ b/sql/sql_string.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_string.h b/sql/sql_string.h
index e0f9af9615b..86af507918c 100644
--- a/sql/sql_string.h
+++ b/sql/sql_string.h
@@ -1,7 +1,9 @@
 #ifndef SQL_STRING_INCLUDED
 #define SQL_STRING_INCLUDED
 
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index abd29ea6a02..349cc7f3045 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -3431,7 +3431,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info,
           }
           if (!f_is_geom(sql_field->pack_flag))
           {
-            my_error(ER_SPATIAL_MUST_HAVE_GEOM_COL, MYF(0));
+            my_error(ER_WRONG_ARGUMENTS, MYF(0), "SPATIAL INDEX");
             DBUG_RETURN(TRUE);
           }
         }
@@ -5873,7 +5873,7 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name,
   TABLE *table_for_fast_alter_partition= NULL;
   bool partition_changed= FALSE;
 #endif
-  bool need_lock_for_indexes= TRUE;
+  bool need_lock_for_indexes __attribute__((unused)) = TRUE;
   KEY  *key_info_buffer;
   uint index_drop_count= 0;
   uint *index_drop_buffer= NULL;
@@ -6287,7 +6287,7 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name,
     need_copy_table= ALTER_TABLE_DATA_CHANGED;
   else
   {
-    enum_alter_table_change_level need_copy_table_res;
+    enum_alter_table_change_level need_copy_table_res=ALTER_TABLE_METADATA_ONLY;
     /* Check how much the tables differ. */
     if (mysql_compare_tables(table, alter_info,
                              create_info, order_num,
diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc
index 0ecc43b512f..dacdaca3fa2 100644
--- a/sql/sql_trigger.cc
+++ b/sql/sql_trigger.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2004, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -2258,7 +2258,8 @@ void Table_triggers_list::mark_fields_used(trg_event_type event)
 void Table_triggers_list::set_parse_error_message(char *error_message)
 {
   m_has_unparseable_trigger= true;
-  strcpy(m_parse_error_message, error_message);
+  strnmov(m_parse_error_message, error_message,
+          sizeof(m_parse_error_message)-1);
 }
 
 
diff --git a/sql/sql_trigger.h b/sql/sql_trigger.h
index 84637b79818..47b1d19ae54 100644
--- a/sql/sql_trigger.h
+++ b/sql/sql_trigger.h
@@ -2,7 +2,7 @@
 #define SQL_TRIGGER_INCLUDED
 
 /*
-   Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2004, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_udf.cc b/sql/sql_udf.cc
index 7eb0cd7ebf7..e41598f943c 100644
--- a/sql/sql_udf.cc
+++ b/sql/sql_udf.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_union.cc b/sql/sql_union.cc
index 9f24ed45842..584ebd904a8 100644
--- a/sql/sql_union.cc
+++ b/sql/sql_union.cc
@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/sql_view.cc b/sql/sql_view.cc
index 94dffad822e..d39ec82aad1 100644
--- a/sql/sql_view.cc
+++ b/sql/sql_view.cc
@@ -847,7 +847,7 @@ static int mysql_register_view(THD *thd, TABLE_LIST *view,
 
     thd->variables.sql_mode|= sql_mode;
   }
-  DBUG_PRINT("info", ("View: %s", view_query.c_ptr_safe()));
+  DBUG_PRINT("info", ("View: %.*s", view_query.length(), view_query.ptr()));
 
   /* fill structure */
   view->source= thd->lex->create_view_select;
@@ -1283,6 +1283,37 @@ bool mysql_make_view(THD *thd, File_parser *parser, TABLE_LIST *table,
     if (!table->prelocking_placeholder &&
         (old_lex->sql_command == SQLCOM_SELECT && old_lex->describe))
     {
+      /*
+        The user we run EXPLAIN as (either the connected user who issued
+        the EXPLAIN statement, or the definer of a SUID stored routine
+        which contains the EXPLAIN) should have both SHOW_VIEW_ACL and
+        SELECT_ACL on the view being opened as well as on all underlying
+        views since EXPLAIN will disclose their structure. This user also
+        should have SELECT_ACL on all underlying tables of the view since
+        this EXPLAIN will disclose information about the number of rows in it.
+
+        To perform this privilege check we create auxiliary TABLE_LIST object
+        for the view in order a) to avoid trashing "table->grant" member for
+        original table list element, which contents can be important at later
+        stage for column-level privilege checking b) get TABLE_LIST object
+        with "security_ctx" member set to 0, i.e. forcing check_table_access()
+        to use active user's security context.
+
+        There is no need for creating similar copies of TABLE_LIST elements
+        for underlying tables since they just have been constructed and thus
+        have TABLE_LIST::security_ctx == 0 and fresh TABLE_LIST::grant member.
+
+        Finally at this point making sure we have SHOW_VIEW_ACL on the views
+        will suffice as we implicitly require SELECT_ACL anyway.
+      */
+        
+      TABLE_LIST view_no_suid;
+      bzero(static_cast<void *>(&view_no_suid), sizeof(TABLE_LIST));
+      view_no_suid.db= table->db;
+      view_no_suid.table_name= table->table_name;
+
+      DBUG_ASSERT(view_tables == NULL || view_tables->security_ctx == NULL);
+
       if (check_table_access(thd, SELECT_ACL, view_tables, FALSE,
                              UINT_MAX, TRUE) &&
           check_table_access(thd, SHOW_VIEW_ACL, table, FALSE, UINT_MAX, TRUE))
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index c070f0ecf7c..eb53c8a00cb 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -3339,7 +3339,7 @@ static Sys_var_ulong Sys_join_cache_level(
        "numbers are used for plain join buffers while even numbers are used "
        "for linked buffers",
        SESSION_VAR(join_cache_level), CMD_LINE(REQUIRED_ARG),
-       VALID_RANGE(0, 8), DEFAULT(1), BLOCK_SIZE(1));
+       VALID_RANGE(0, 8), DEFAULT(2), BLOCK_SIZE(1));
 
 static Sys_var_ulong Sys_mrr_buffer_size(
        "mrr_buffer_size",
diff --git a/sql/table.cc b/sql/table.cc
index fc0ee45add9..53e5872bfe4 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2011, Oracle and/or its affiliates.
-   Copyright (c) 2009-2011, Monty Program Ab
+   Copyright (c) 2008-2011 Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -6327,6 +6327,8 @@ bool TABLE_LIST::init_derived(THD *thd, bool init_view)
 int TABLE_LIST::fetch_number_of_rows()
 {
   int error= 0;
+  if (jtbm_subselect)
+    return 0;
   if (is_materialized_derived() && !fill_me)
 
   {
diff --git a/sql/table.h b/sql/table.h
index b22cdb6e76b..a327d625387 100644
--- a/sql/table.h
+++ b/sql/table.h
@@ -1598,6 +1598,26 @@ struct TABLE_LIST
   select_union  *derived_result;
   /* Stub used for materialized derived tables. */
   table_map	map;                    /* ID bit of table (1,2,4,8,16...) */
+  table_map get_map()
+  {
+    return jtbm_subselect? table_map(1) << jtbm_table_no : table->map;
+  }
+  uint get_tablenr()
+  {
+    return jtbm_subselect? jtbm_table_no : table->tablenr;
+  }
+  void set_tablenr(uint new_tablenr)
+  {
+    if (jtbm_subselect)
+    {
+      jtbm_table_no= new_tablenr;
+    }
+    if (table)
+    {
+      table->tablenr= new_tablenr;
+      table->map= table_map(1) << new_tablenr;
+    }
+  }
   /*
     Reference from aux_tables to local list entry of main select of
     multi-delete statement:
diff --git a/sql/thr_malloc.cc b/sql/thr_malloc.cc
index ba534e52b99..cedcbefc26f 100644
--- a/sql/thr_malloc.cc
+++ b/sql/thr_malloc.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/tztime.cc b/sql/tztime.cc
index aa945687675..9fae9f3fedd 100644
--- a/sql/tztime.cc
+++ b/sql/tztime.cc
@@ -1,4 +1,5 @@
-/* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2004, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/udf_example.c b/sql/udf_example.c
index d68f49e1729..36a5eafb704 100644
--- a/sql/udf_example.c
+++ b/sql/udf_example.c
@@ -1,4 +1,5 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/unireg.cc b/sql/unireg.cc
index b5faac3b797..c9b0f91d9f7 100644
--- a/sql/unireg.cc
+++ b/sql/unireg.cc
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2011, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
diff --git a/sql/unireg.h b/sql/unireg.h
index f1066dbccb8..50aaa103b34 100644
--- a/sql/unireg.h
+++ b/sql/unireg.h
@@ -1,7 +1,8 @@
 #ifndef UNIREG_INCLUDED
 #define UNIREG_INCLUDED
 
-/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+/*
+   Copyright (c) 2000, 2010, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by