revert the fix for MDEV-5898, restore the fix for MDEV-5549.

simplify test case for MDEV-5898

3 files changed, 22 insertions, 6 deletions

diff --git a/sql/filesort.cc b/sql/filesort.cc
index 0d554df6e18..b1eb973086b 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -686,7 +686,7 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
   ref_pos= ref_buff;
   quick_select=select && select->quick;
   record=0;
-  *found_rows= 0;
+  *found_rows= pq ? 0 : HA_POS_ERROR; // don't count unless pq is used
   flag= ((file->ha_table_flags() & HA_REC_NOT_IN_SEQ) || quick_select);
   if (flag)
     ref_pos= &file->ref[0];
@@ -806,9 +806,14 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
 
     if (write_record)
     {
-      (*found_rows)++;
       if (pq)
       {
+        /*
+          only count rows when pq is used - otherwise there might be
+          other filters *after* the filesort, we don't know the final row
+          count here
+        */
+        (*found_rows)++;
         pq->push(ref_pos);
         idx= pq->num_elements();
       }
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 6cf8cfc32b5..727560dd173 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -3074,8 +3074,7 @@ void JOIN::exec_inner()
                                    Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF);
   error= do_select(curr_join, curr_fields_list, NULL, procedure);
   thd->limit_found_rows= curr_join->send_records;
-  if (curr_join->order && curr_join->sortorder &&
-      curr_join->select_options & OPTION_FOUND_ROWS)
+  if (curr_join->order && curr_join->filesort_found_rows)
   {
     /* Use info provided by filesort. */
     DBUG_ASSERT(curr_join->table_count > curr_join->const_tables);
@@ -18702,7 +18701,7 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
         records are read. Because of optimization in some cases it can
         provide only select_limit_cnt+1 records.
       */
-      if (join->order && join->sortorder &&
+      if (join->order && join->filesort_found_rows &&
           join->select_options & OPTION_FOUND_ROWS)
       {
         DBUG_PRINT("info", ("filesort NESTED_LOOP_QUERY_LIMIT"));
@@ -18724,6 +18723,7 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
 	  /* Join over all rows in table;  Return number of found rows */
 	  TABLE *table=jt->table;
 
+          join->select_options ^= OPTION_FOUND_ROWS;
 	  if (table->sort.record_pointers ||
 	      (table->sort.io_cache && my_b_inited(table->sort.io_cache)))
 	  {
@@ -20556,7 +20556,11 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order,
                             select, filesort_limit, 0,
                             &examined_rows, &found_rows);
   table->sort.found_records= filesort_retval;
-  tab->records= found_rows;                     // For SQL_CALC_ROWS
+  if (found_rows != HA_POS_ERROR)
+  {
+    tab->records= found_rows;                     // For SQL_CALC_ROWS
+    join->filesort_found_rows= true;
+  }
 
   if (quick_created)
   {
diff --git a/sql/sql_select.h b/sql/sql_select.h
index 271199e3d51..bc6aa1eda1e 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -1119,6 +1119,12 @@ public:
     restore_no_rows_in_result() in ::reinit()
   */
   bool no_rows_in_result_called;
+
+  /**
+    This is set if SQL_CALC_ROWS was calculated by filesort()
+    and should be taken from the appropriate JOIN_TAB
+  */
+  bool filesort_found_rows;
   
   /**
     Copy of this JOIN to be used with temporary tables.
@@ -1335,6 +1341,7 @@ public:
     emb_sjm_nest= NULL;
     sjm_lookup_tables= 0;
 
+    filesort_found_rows= false;
     exec_saved_explain= false;
     /* 
       The following is needed because JOIN::cleanup(true) may be called for