merged 5.0-bugteam -> 5.1-bugtteam

3 files changed, 168 insertions, 9 deletions

diff --git a/mysql-test/r/greedy_optimizer.result b/mysql-test/r/greedy_optimizer.result
index 9e8c66722e3..c0012c297d1 100644
--- a/mysql-test/r/greedy_optimizer.result
+++ b/mysql-test/r/greedy_optimizer.result
@@ -655,3 +655,80 @@ show status like 'Last_query_cost';
 Variable_name	Value
 Last_query_cost	794.837037
 drop table t1,t2,t3,t4,t5,t6,t7;
+CREATE TABLE t1 (a int, b int, d int, i int);
+INSERT INTO t1 VALUES (1,1,1,1);
+CREATE TABLE t2 (b int, c int, j int);
+INSERT INTO t2 VALUES (1,1,1);
+CREATE TABLE t2_1 (j int);
+INSERT INTO t2_1 VALUES (1);
+CREATE TABLE t3 (c int, f int);
+INSERT INTO t3 VALUES (1,1);
+CREATE TABLE t3_1 (f int);
+INSERT INTO t3_1 VALUES (1);
+CREATE TABLE t4 (d int, e int, k int);
+INSERT INTO t4 VALUES (1,1,1);
+CREATE TABLE t4_1 (k int);
+INSERT INTO t4_1 VALUES (1);
+CREATE TABLE t5 (g int, d int, h int, l int);
+INSERT INTO t5 VALUES (1,1,1,1);
+CREATE TABLE t5_1 (l int);
+INSERT INTO t5_1 VALUES (1);
+SET optimizer_search_depth = 3;
+SELECT 1
+FROM t1
+LEFT JOIN (
+t2 JOIN t3 ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+t4 JOIN t5 ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+1
+1
+SELECT 1
+FROM t1
+LEFT JOIN (
+t2 LEFT JOIN (t3 JOIN t3_1 ON t3.f = t3_1.f) ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+t4 JOIN t5 ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+1
+1
+SELECT 1
+FROM t1
+LEFT JOIN (
+(t2 JOIN t2_1 ON t2.j = t2_1.j) JOIN t3 ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+t4 JOIN t5 ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+1
+1
+SELECT 1
+FROM t1
+LEFT JOIN (
+t2 JOIN t3 ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+(t4 JOIN t4_1 ON t4.k = t4_1.k) LEFT JOIN t5 ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+1
+1
+SELECT 1
+FROM t1
+LEFT JOIN (
+t2 JOIN t3 ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+t4 LEFT JOIN (t5 JOIN t5_1 ON t5.l = t5_1.l) ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+1
+1
+SET optimizer_search_depth = DEFAULT;
+DROP TABLE t1,t2,t2_1,t3,t3_1,t4,t4_1,t5,t5_1;
+End of 5.0 tests
diff --git a/mysql-test/t/greedy_optimizer.test b/mysql-test/t/greedy_optimizer.test
index b73f70c6a3e..5131c97f122 100644
--- a/mysql-test/t/greedy_optimizer.test
+++ b/mysql-test/t/greedy_optimizer.test
@@ -311,3 +311,76 @@ explain select t1.c11 from t7, t6, t5, t4, t3, t2, t1 where t1.c11 = t2.c21 and
 show status like 'Last_query_cost';
 
 drop table t1,t2,t3,t4,t5,t6,t7;
+
+
+#
+# Bug # 38795: Automatic search depth and nested join's results in server
+# crash
+#
+
+CREATE TABLE t1 (a int, b int, d int, i int); INSERT INTO t1 VALUES (1,1,1,1);
+CREATE TABLE t2 (b int, c int, j int); INSERT INTO t2 VALUES (1,1,1);
+CREATE TABLE t2_1 (j int); INSERT INTO t2_1 VALUES (1);
+CREATE TABLE t3 (c int, f int); INSERT INTO t3 VALUES (1,1);
+CREATE TABLE t3_1 (f int); INSERT INTO t3_1 VALUES (1);
+CREATE TABLE t4 (d int, e int, k int); INSERT INTO t4 VALUES (1,1,1);
+CREATE TABLE t4_1 (k int); INSERT INTO t4_1 VALUES (1);
+CREATE TABLE t5 (g int, d int, h int, l int); INSERT INTO t5 VALUES (1,1,1,1);
+CREATE TABLE t5_1 (l int); INSERT INTO t5_1 VALUES (1);
+
+SET optimizer_search_depth = 3;
+
+SELECT 1
+FROM t1
+LEFT JOIN (
+  t2 JOIN t3 ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+  t4 JOIN t5 ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+
+SELECT 1
+FROM t1
+LEFT JOIN (
+  t2 LEFT JOIN (t3 JOIN t3_1 ON t3.f = t3_1.f) ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+  t4 JOIN t5 ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+
+SELECT 1
+FROM t1
+LEFT JOIN (
+ (t2 JOIN t2_1 ON t2.j = t2_1.j) JOIN t3 ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+  t4 JOIN t5 ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+
+SELECT 1
+FROM t1
+LEFT JOIN (
+  t2 JOIN t3 ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+  (t4 JOIN t4_1 ON t4.k = t4_1.k) LEFT JOIN t5 ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+
+SELECT 1
+FROM t1
+LEFT JOIN (
+  t2 JOIN t3 ON t3.c = t2.c
+) ON t2.b = t1.b
+LEFT JOIN (
+  t4 LEFT JOIN (t5 JOIN t5_1 ON t5.l = t5_1.l) ON t5.d = t4.d
+) ON t4.d = t1.d
+;
+
+SET optimizer_search_depth = DEFAULT;
+DROP TABLE t1,t2,t2_1,t3,t3_1,t4,t4_1,t5,t5_1;
+
+--echo End of 5.0 tests
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 49c0958055e..49cacbe9272 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -109,7 +109,7 @@ static COND* substitute_for_best_equal_field(COND *cond,
                                              void *table_join_idx);
 static COND *simplify_joins(JOIN *join, List<TABLE_LIST> *join_list,
                             COND *conds, bool top);
-static bool check_interleaving_with_nj(JOIN_TAB *last, JOIN_TAB *next);
+static bool check_interleaving_with_nj(JOIN_TAB *next);
 static void restore_prev_nj_state(JOIN_TAB *last);
 static void reset_nj_counters(List<TABLE_LIST> *join_list);
 static uint build_bitmap_for_nested_joins(List<TABLE_LIST> *join_list,
@@ -4889,6 +4889,18 @@ greedy_search(JOIN      *join,
     */
     join->positions[idx]= best_pos;
 
+    /*
+      Update the interleaving state after extending the current partial plan
+      with a new table.
+      We are doing this here because best_extension_by_limited_search reverts
+      the interleaving state to the one of the non-extended partial plan 
+      on exit.
+    */
+    IF_DBUG(bool is_interleave_error= )
+    check_interleaving_with_nj (best_table);
+    /* This has been already checked by best_extension_by_limited_search */
+    DBUG_ASSERT(!is_interleave_error);
+
     /* find the position of 'best_table' in 'join->best_ref' */
     best_idx= idx;
     JOIN_TAB *pos= join->best_ref[best_idx];
@@ -4906,7 +4918,7 @@ greedy_search(JOIN      *join,
     --size_remain;
     ++idx;
 
-    DBUG_EXECUTE("opt", print_plan(join, join->tables,
+    DBUG_EXECUTE("opt", print_plan(join, idx,
                                    record_count, read_time, read_time,
                                    "extended"););
   } while (TRUE);
@@ -5064,7 +5076,7 @@ best_extension_by_limited_search(JOIN      *join,
     table_map real_table_bit= s->table->map;
     if ((remaining_tables & real_table_bit) && 
         !(remaining_tables & s->dependent) && 
-        (!idx || !check_interleaving_with_nj(join->positions[idx-1].table, s)))
+        (!idx || !check_interleaving_with_nj(s)))
     {
       double current_record_count, current_read_time;
 
@@ -5210,7 +5222,7 @@ find_best(JOIN *join,table_map rest_tables,uint idx,double record_count,
   {
     table_map real_table_bit=s->table->map;
     if ((rest_tables & real_table_bit) && !(rest_tables & s->dependent) &&
-        (!idx|| !check_interleaving_with_nj(join->positions[idx-1].table, s)))
+        (!idx|| !check_interleaving_with_nj(s)))
     {
       double records, best;
       best_access_path(join, s, thd, rest_tables, idx, record_count, 
@@ -8790,9 +8802,6 @@ static void reset_nj_counters(List<TABLE_LIST> *join_list)
              the partial join order.
   @endverbatim
 
-  @param join       Join being processed
-  @param last_tab   Last table in current partial join order (this function is
-                    not called for empty partial join orders)
   @param next_tab   Table we're going to extend the current partial join with
 
   @retval
@@ -8802,10 +8811,10 @@ static void reset_nj_counters(List<TABLE_LIST> *join_list)
     TRUE   Requested join order extension not allowed.
 */
 
-static bool check_interleaving_with_nj(JOIN_TAB *last_tab, JOIN_TAB *next_tab)
+static bool check_interleaving_with_nj(JOIN_TAB *next_tab)
 {
   TABLE_LIST *next_emb= next_tab->table->pos_in_table_list->embedding;
-  JOIN *join= last_tab->join;
+  JOIN *join= next_tab->join;
 
   if (join->cur_embedding_map & ~next_tab->embedding_map)
   {