MDEV-5898 FOUND_ROWS() return incorrect value when using DISTINCT

revert the fix for MDEV-5549, use a different approach.

5 files changed, 79 insertions, 23 deletions

diff --git a/mysql-test/r/select_found.result b/mysql-test/r/select_found.result
index ee6d309f005..07c7b7939af 100644
--- a/mysql-test/r/select_found.result
+++ b/mysql-test/r/select_found.result
@@ -291,3 +291,36 @@ select found_rows();
 found_rows()
 1
 drop table t1, t2;
+create table t1 (
+a1 int auto_increment primary key,
+b1  datetime,
+c1 int
+);
+insert t1 (a1) values (null);
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+update t1 set c1=a1 % 2;
+create table t2 (
+a2 int,
+b2 int,
+c2  char(16) default '',
+primary key (a2, b2)
+);
+insert t2 select a1, 1, 'ok' from t1;
+insert t2 select a1, 2, 'ko' from t1;
+insert t2 select a1, 3, 'ko' from t1;
+insert t2 select a1, 4, 'ok' from t1;
+insert t2 select a1, 5, 'ok' from t1;
+select sql_calc_found_rows distinct a1,b1,c2 from t1 join t2 on a2=a1 
+where a1 <= 256 and c1=0 and c2='ok' order by b1 desc, a1 desc limit 46;
+select found_rows();
+found_rows()
+128
+drop table t1, t2;
diff --git a/mysql-test/t/select_found.test b/mysql-test/t/select_found.test
index d6bca6b19b1..684ae54b517 100644
--- a/mysql-test/t/select_found.test
+++ b/mysql-test/t/select_found.test
@@ -208,3 +208,43 @@ select f1 from t1,t2 where f1=f3 and f2=3 order by f1;
 select found_rows();
 drop table t1, t2;
 
+create table t1 (
+  a1 int auto_increment primary key,
+  b1  datetime,
+  c1 int
+);
+
+insert t1 (a1) values (null);
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+insert t1 (a1) select null from t1;
+update t1 set c1=a1 % 2;
+
+create table t2 (
+  a2 int,
+  b2 int,
+  c2  char(16) default '',
+  primary key (a2, b2)
+);
+
+insert t2 select a1, 1, 'ok' from t1;
+insert t2 select a1, 2, 'ko' from t1;
+insert t2 select a1, 3, 'ko' from t1;
+insert t2 select a1, 4, 'ok' from t1;
+insert t2 select a1, 5, 'ok' from t1;
+
+--disable_result_log
+select sql_calc_found_rows distinct a1,b1,c2 from t1 join t2 on a2=a1 
+  where a1 <= 256 and c1=0 and c2='ok' order by b1 desc, a1 desc limit 46;
+--enable_result_log
+
+select found_rows();
+
+drop table t1, t2;
+
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 12b9bb5aadc..776ec064365 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -188,7 +188,6 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
   my_b_clear(&buffpek_pointers);
   buffpek=0;
   error= 1;
-  *found_rows= HA_POS_ERROR;
 
   param.init_for_filesort(sortlength(thd, sortorder, s_length,
                                      &multi_byte_charset),
@@ -687,7 +686,7 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
   ref_pos= ref_buff;
   quick_select=select && select->quick;
   record=0;
-  *found_rows= pq ? 0 : HA_POS_ERROR; // don't count unless pq is used
+  *found_rows= 0;
   flag= ((file->ha_table_flags() & HA_REC_NOT_IN_SEQ) || quick_select);
   if (flag)
     ref_pos= &file->ref[0];
@@ -807,14 +806,9 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
 
     if (write_record)
     {
+      (*found_rows)++;
       if (pq)
       {
-        /*
-          only count rows when pq is used - otherwise there might be
-          other filters *after* the filesort, we don't know the final row
-          count here
-        */
-        (*found_rows)++;
         pq->push(ref_pos);
         idx= pq->num_elements();
       }
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index fee8b96ad0e..1ea738d976c 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -3064,7 +3064,8 @@ void JOIN::exec_inner()
                                    Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF);
   error= do_select(curr_join, curr_fields_list, NULL, procedure);
   thd->limit_found_rows= curr_join->send_records;
-  if (curr_join->order && curr_join->filesort_found_rows)
+  if (curr_join->order && curr_join->sortorder &&
+      curr_join->select_options & OPTION_FOUND_ROWS)
   {
     /* Use info provided by filesort. */
     DBUG_ASSERT(curr_join->table_count > curr_join->const_tables);
@@ -18562,7 +18563,7 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
         records are read. Because of optimization in some cases it can
         provide only select_limit_cnt+1 records.
       */
-      if (join->order && join->filesort_found_rows &&
+      if (join->order && join->sortorder &&
           join->select_options & OPTION_FOUND_ROWS)
       {
         DBUG_PRINT("info", ("filesort NESTED_LOOP_QUERY_LIMIT"));
@@ -18584,7 +18585,6 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
 	  /* Join over all rows in table;  Return number of found rows */
 	  TABLE *table=jt->table;
 
-	  join->select_options ^= OPTION_FOUND_ROWS;
 	  if (table->sort.record_pointers ||
 	      (table->sort.io_cache && my_b_inited(table->sort.io_cache)))
 	  {
@@ -20417,11 +20417,7 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order,
                             select, filesort_limit, 0,
                             &examined_rows, &found_rows);
   table->sort.found_records= filesort_retval;
-  if (found_rows != HA_POS_ERROR)
-  {
-    tab->records= found_rows;                     // For SQL_CALC_ROWS
-    join->filesort_found_rows= true;
-  }
+  tab->records= found_rows;                     // For SQL_CALC_ROWS
 
   if (quick_created)
   {
diff --git a/sql/sql_select.h b/sql/sql_select.h
index bc6aa1eda1e..271199e3d51 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -1119,12 +1119,6 @@ public:
     restore_no_rows_in_result() in ::reinit()
   */
   bool no_rows_in_result_called;
-
-  /**
-    This is set if SQL_CALC_ROWS was calculated by filesort()
-    and should be taken from the appropriate JOIN_TAB
-  */
-  bool filesort_found_rows;
   
   /**
     Copy of this JOIN to be used with temporary tables.
@@ -1341,7 +1335,6 @@ public:
     emb_sjm_nest= NULL;
     sjm_lookup_tables= 0;
 
-    filesort_found_rows= false;
     exec_saved_explain= false;
     /* 
       The following is needed because JOIN::cleanup(true) may be called for