MDEV-5549 Wrong row counter in found_rows() result

only let filesort() count rows for SQL_CALC_ROWS if it's using priority queue

3 files changed, 21 insertions, 7 deletions

diff --git a/sql/filesort.cc b/sql/filesort.cc
index 2b0a91bf5d0..183edf67466 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -595,7 +595,7 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
   ref_pos= ref_buff;
   quick_select=select && select->quick;
   record=0;
-  *found_rows= 0;
+  *found_rows= pq ? 0 : HA_POS_ERROR; // don't count unless pq is used
   flag= ((file->ha_table_flags() & HA_REC_NOT_IN_SEQ) || quick_select);
   if (flag)
     ref_pos= &file->ref[0];
@@ -714,9 +714,14 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
 
     if (write_record)
     {
-       ++(*found_rows);
       if (pq)
       {
+        /*
+          only count rows when pq is used - otherwise there might be
+          other filters *after* the filesort, we don't know the final row
+          count here
+        */
+        (*found_rows)++;
         pq->push(ref_pos);
         idx= pq->num_elements();
       }
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index c109ee10877..1a1a39ad166 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -3065,8 +3065,7 @@ void JOIN::exec_inner()
                                    Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF);
   error= do_select(curr_join, curr_fields_list, NULL, procedure);
   thd->limit_found_rows= curr_join->send_records;
-  if (curr_join->order &&
-      curr_join->sortorder)
+  if (curr_join->order && curr_join->filesort_found_rows)
   {
     /* Use info provided by filesort. */
     DBUG_ASSERT(curr_join->table_count > curr_join->const_tables);
@@ -18535,8 +18534,7 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
         records are read. Because of optimization in some cases it can
         provide only select_limit_cnt+1 records.
       */
-      if (join->order &&
-          join->sortorder &&
+      if (join->order && join->filesort_found_rows &&
           join->select_options & OPTION_FOUND_ROWS)
       {
         DBUG_PRINT("info", ("filesort NESTED_LOOP_QUERY_LIMIT"));
@@ -20394,7 +20392,11 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order,
                             select, filesort_limit, 0,
                             &examined_rows, &found_rows);
   table->sort.found_records= filesort_retval;
-  tab->records= found_rows;                     // For SQL_CALC_ROWS
+  if (found_rows != HA_POS_ERROR)
+  {
+    tab->records= found_rows;                     // For SQL_CALC_ROWS
+    join->filesort_found_rows= true;
+  }
 
   if (quick_created)
   {
diff --git a/sql/sql_select.h b/sql/sql_select.h
index c413d0ca023..bd0433eb5ee 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -1119,6 +1119,12 @@ public:
     restore_no_rows_in_result() in ::reinit()
   */
   bool no_rows_in_result_called;
+
+  /**
+    This is set if SQL_CALC_ROWS was calculated by filesort()
+    and should be taken from the appropriate JOIN_TAB
+  */
+  bool filesort_found_rows;
   
   /**
     Copy of this JOIN to be used with temporary tables.
@@ -1334,6 +1340,7 @@ public:
     emb_sjm_nest= NULL;
     sjm_lookup_tables= 0;
 
+    filesort_found_rows= false;
     exec_saved_explain= false;
     /* 
       The following is needed because JOIN::cleanup(true) may be called for