Merge in LPBUG#602574

6 files changed, 134 insertions, 22 deletions

diff --git a/mysql-test/r/subselect_sj.result b/mysql-test/r/subselect_sj.result
index f24b294fa90..02ca5a17830 100644
--- a/mysql-test/r/subselect_sj.result
+++ b/mysql-test/r/subselect_sj.result
@@ -1075,3 +1075,43 @@ DROP TABLE t1;
 DROP TABLE t2;
 DROP TABLE t3;
 # End of Bug#48623
+# 
+# LPBUG#602574: RQG: sql_select.cc:5385: bool greedy_search(JOIN*, table_map, uint, 
+#               uint): Assertion `join->best_read <
+#
+set @save_optimizer_switch=@@optimizer_switch;
+set optimizer_switch='materialization=off';
+CREATE TABLE t1 (
+varchar_key varchar(1) DEFAULT NULL,
+KEY varchar_key (varchar_key)
+);
+CREATE TABLE t2 (
+varchar_key varchar(1) DEFAULT NULL,
+KEY varchar_key (varchar_key)
+);
+INSERT INTO t2 VALUES
+(NULL),(NULL),(NULL),(NULL),('a'),('a'),('a'),('b'),('b'),('b'),('b'),('c'),
+('c'),('c'),('c'),('c'),('c'),('c'),('d'),('d'),('d'),('d'),('d'),('d'),('e'),
+('e'),('e'),('e'),('e'),('e'),('f'),('f'),('f'),('g'),('g'),('h'),('h'),('h'),
+('h'),('i'),('j'),('j'),('j'),('k'),('k'),('l'),('l'),('m'),('m'),('m'),('m'),
+('n'),('n'),('n'),('o'),('o'),('o'),('p'),('p'),('p'),('q'),('q'),('q'),('r'),
+('r'),('r'),('r'),('s'),('s'),('s'),('s'),('t'),('t'),('t'),('t'),('u'),('u'),
+('u'),('u'),('v'),('v'),('v'),('v'),('w'),('w'),('w'),('w'),('w'),('w'),('x'),
+('x'),('x'),('y'),('y'),('y'),('y'),('z'),('z'),('z'),('z');
+CREATE TABLE t3 (
+varchar_key varchar(1) DEFAULT NULL,
+KEY varchar_key (varchar_key)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t3 VALUES
+(NULL),('c'),('d'),('e'),('f'),('h'),('j'),('k'),('k'),('m'),('m'),('m'),
+('n'),('o'),('r'),('t'),('t'),('u'),('w'),('y');
+SELECT varchar_key FROM t3 
+WHERE (SELECT varchar_key FROM t3 
+WHERE (varchar_key,varchar_key) 
+IN (SELECT t1.varchar_key, t2 .varchar_key 
+FROM t1 RIGHT JOIN t2 ON t1.varchar_key  
+)  
+);
+varchar_key
+set optimizer_switch=@save_optimizer_switch;
+DROP TABLE t1, t2, t3;
diff --git a/mysql-test/r/subselect_sj_jcl6.result b/mysql-test/r/subselect_sj_jcl6.result
index 40687d301d3..9c60424740a 100644
--- a/mysql-test/r/subselect_sj_jcl6.result
+++ b/mysql-test/r/subselect_sj_jcl6.result
@@ -1079,6 +1079,46 @@ DROP TABLE t1;
 DROP TABLE t2;
 DROP TABLE t3;
 # End of Bug#48623
+# 
+# LPBUG#602574: RQG: sql_select.cc:5385: bool greedy_search(JOIN*, table_map, uint, 
+#               uint): Assertion `join->best_read <
+#
+set @save_optimizer_switch=@@optimizer_switch;
+set optimizer_switch='materialization=off';
+CREATE TABLE t1 (
+varchar_key varchar(1) DEFAULT NULL,
+KEY varchar_key (varchar_key)
+);
+CREATE TABLE t2 (
+varchar_key varchar(1) DEFAULT NULL,
+KEY varchar_key (varchar_key)
+);
+INSERT INTO t2 VALUES
+(NULL),(NULL),(NULL),(NULL),('a'),('a'),('a'),('b'),('b'),('b'),('b'),('c'),
+('c'),('c'),('c'),('c'),('c'),('c'),('d'),('d'),('d'),('d'),('d'),('d'),('e'),
+('e'),('e'),('e'),('e'),('e'),('f'),('f'),('f'),('g'),('g'),('h'),('h'),('h'),
+('h'),('i'),('j'),('j'),('j'),('k'),('k'),('l'),('l'),('m'),('m'),('m'),('m'),
+('n'),('n'),('n'),('o'),('o'),('o'),('p'),('p'),('p'),('q'),('q'),('q'),('r'),
+('r'),('r'),('r'),('s'),('s'),('s'),('s'),('t'),('t'),('t'),('t'),('u'),('u'),
+('u'),('u'),('v'),('v'),('v'),('v'),('w'),('w'),('w'),('w'),('w'),('w'),('x'),
+('x'),('x'),('y'),('y'),('y'),('y'),('z'),('z'),('z'),('z');
+CREATE TABLE t3 (
+varchar_key varchar(1) DEFAULT NULL,
+KEY varchar_key (varchar_key)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t3 VALUES
+(NULL),('c'),('d'),('e'),('f'),('h'),('j'),('k'),('k'),('m'),('m'),('m'),
+('n'),('o'),('r'),('t'),('t'),('u'),('w'),('y');
+SELECT varchar_key FROM t3 
+WHERE (SELECT varchar_key FROM t3 
+WHERE (varchar_key,varchar_key) 
+IN (SELECT t1.varchar_key, t2 .varchar_key 
+FROM t1 RIGHT JOIN t2 ON t1.varchar_key  
+)  
+);
+varchar_key
+set optimizer_switch=@save_optimizer_switch;
+DROP TABLE t1, t2, t3;
 #
 # BUG#49129: Wrong result with IN-subquery with join_cache_level=6 and firstmatch=off
 #
diff --git a/mysql-test/t/subselect_sj.test b/mysql-test/t/subselect_sj.test
index 33f3e936482..a7a6476b969 100644
--- a/mysql-test/t/subselect_sj.test
+++ b/mysql-test/t/subselect_sj.test
@@ -935,3 +935,47 @@ DROP TABLE t2;
 DROP TABLE t3;
 
 --echo # End of Bug#48623
+
+--echo # 
+--echo # LPBUG#602574: RQG: sql_select.cc:5385: bool greedy_search(JOIN*, table_map, uint, 
+--echo #               uint): Assertion `join->best_read <
+--echo #
+set @save_optimizer_switch=@@optimizer_switch;
+set optimizer_switch='materialization=off';
+CREATE TABLE t1 (
+  varchar_key varchar(1) DEFAULT NULL,
+  KEY varchar_key (varchar_key)
+);
+
+CREATE TABLE t2 (
+  varchar_key varchar(1) DEFAULT NULL,
+  KEY varchar_key (varchar_key)
+);
+INSERT INTO t2 VALUES
+  (NULL),(NULL),(NULL),(NULL),('a'),('a'),('a'),('b'),('b'),('b'),('b'),('c'),
+  ('c'),('c'),('c'),('c'),('c'),('c'),('d'),('d'),('d'),('d'),('d'),('d'),('e'),
+  ('e'),('e'),('e'),('e'),('e'),('f'),('f'),('f'),('g'),('g'),('h'),('h'),('h'),
+  ('h'),('i'),('j'),('j'),('j'),('k'),('k'),('l'),('l'),('m'),('m'),('m'),('m'),
+  ('n'),('n'),('n'),('o'),('o'),('o'),('p'),('p'),('p'),('q'),('q'),('q'),('r'),
+  ('r'),('r'),('r'),('s'),('s'),('s'),('s'),('t'),('t'),('t'),('t'),('u'),('u'),
+  ('u'),('u'),('v'),('v'),('v'),('v'),('w'),('w'),('w'),('w'),('w'),('w'),('x'),
+  ('x'),('x'),('y'),('y'),('y'),('y'),('z'),('z'),('z'),('z');
+
+CREATE TABLE t3 (
+  varchar_key varchar(1) DEFAULT NULL,
+  KEY varchar_key (varchar_key)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t3 VALUES
+  (NULL),('c'),('d'),('e'),('f'),('h'),('j'),('k'),('k'),('m'),('m'),('m'),
+  ('n'),('o'),('r'),('t'),('t'),('u'),('w'),('y');
+ 
+SELECT varchar_key FROM t3 
+WHERE (SELECT varchar_key FROM t3 
+       WHERE (varchar_key,varchar_key) 
+         IN (SELECT t1.varchar_key, t2 .varchar_key 
+             FROM t1 RIGHT JOIN t2 ON t1.varchar_key  
+            )  
+      );
+set optimizer_switch=@save_optimizer_switch;
+DROP TABLE t1, t2, t3;
+
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index a58313ec9fd..5973c95d101 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -1519,11 +1519,10 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
           Got a complete FirstMatch range.
             Calculate correct costs and fanout
         */
-        double reopt_cost, reopt_rec_count, sj_inner_fanout;
         optimize_wo_join_buffering(join, pos->first_firstmatch_table, idx,
                                    remaining_tables, FALSE, idx,
-                                   &reopt_rec_count, &reopt_cost, 
-                                   &sj_inner_fanout);
+                                   current_record_count, 
+                                   current_read_time);
         /*
           We don't yet know what are the other strategies, so pick the
           FirstMatch.
@@ -1534,8 +1533,6 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
           alternate POSITIONs after we've picked the best QEP.
         */
         pos->sj_strategy= SJ_OPT_FIRST_MATCH;
-        *current_read_time=    reopt_cost;
-        *current_record_count= reopt_rec_count / sj_inner_fanout;
         handled_by_fm_or_ls=  pos->firstmatch_need_tables;
       }
     }
@@ -1584,7 +1581,6 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
       first=join->positions + pos->first_loosescan_table; 
       uint n_tables= my_count_bits(first->table->emb_sj_nest->sj_inner_tables);
       /* Got a complete LooseScan range. Calculate its cost */
-      double reopt_cost, reopt_rec_count, sj_inner_fanout;
       /*
         The same problem as with FirstMatch - we need to save POSITIONs
         somewhere but reserving space for all cases would require too
@@ -1594,8 +1590,8 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
                                  remaining_tables, 
                                  TRUE,  //first_alt
                                  pos->first_loosescan_table + n_tables,
-                                 &reopt_rec_count, 
-                                 &reopt_cost, &sj_inner_fanout);
+                                 current_record_count,
+                                 current_read_time);
       /*
         We don't yet have any other strategies that could handle this
         semi-join nest (the other options are Duplicate Elimination or
@@ -1604,8 +1600,6 @@ void advance_sj_state(JOIN *join, table_map remaining_tables,
         LooseScan.
       */
       pos->sj_strategy= SJ_OPT_LOOSE_SCAN;
-      *current_read_time=    reopt_cost;
-      *current_record_count= reopt_rec_count / sj_inner_fanout;
       handled_by_fm_or_ls= first->table->emb_sj_nest->sj_inner_tables;
     }
   }
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 7c51d0e25bd..b4cf6a959ec 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -10348,8 +10348,6 @@ static void restore_prev_nj_state(JOIN_TAB *last)
                               table
       reopt_rec_count     OUT New output record count
       reopt_cost          OUT New join prefix cost
-      sj_inner_fanout     OUT Fanout in the [first_tab; last_tab] range that
-                              is produced by semi-join-inner tables.
 
   DESCRIPTION
     Given a join prefix [0; ... first_tab], change the access to the tables
@@ -10366,10 +10364,9 @@ static void restore_prev_nj_state(JOIN_TAB *last)
 void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab, 
                                 table_map last_remaining_tables, 
                                 bool first_alt, uint no_jbuf_before,
-                                double *reopt_rec_count, double *reopt_cost,
-                                double *sj_inner_fanout)
+                                double *outer_rec_count, double *reopt_cost)
 {
-  double cost, rec_count, inner_fanout= 1.0;
+  double cost, rec_count;
   table_map reopt_remaining_tables= last_remaining_tables;
   uint i;
 
@@ -10384,6 +10381,7 @@ void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab,
     rec_count= 1;
   }
 
+  *outer_rec_count= rec_count;
   for (i= first_tab; i <= last_tab; i++)
     reopt_remaining_tables |= join->positions[i].table->table->map;
 
@@ -10409,13 +10407,10 @@ void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab,
     rec_count *= pos.records_read;
     cost += pos.read_time;
 
-    if (rs->emb_sj_nest)
-      inner_fanout *= pos.records_read;
+    if (!rs->emb_sj_nest)
+      *outer_rec_count *= pos.records_read;
   }
-
-  *reopt_rec_count= rec_count;
   *reopt_cost= cost;
-  *sj_inner_fanout= inner_fanout;
 }
 
 
diff --git a/sql/sql_select.h b/sql/sql_select.h
index 885cff0163c..1ec79796ebf 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -1982,8 +1982,7 @@ void get_partial_join_cost(JOIN *join, uint n_tables, double *read_time_arg,
 void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab, 
                                 table_map last_remaining_tables, 
                                 bool first_alt, uint no_jbuf_before,
-                                double *reopt_rec_count, double *reopt_cost,
-                                double *sj_inner_fanout);
+                                double *outer_rec_count, double *reopt_cost);
 Item_equal *find_item_equal(COND_EQUAL *cond_equal, Field *field,
                             bool *inherited_fl);
 bool test_if_ref(COND *root_cond,