MDEV-6995: EXPLAIN JSON and ORDER BY, GROUP BY, etc

- Make ANALYZE correctly remember and report filesort() calls
- Temp.table use is collected but only basic info is reported.

6 files changed, 339 insertions, 46 deletions

diff --git a/mysql-test/r/analyze_stmt_orderby.result b/mysql-test/r/analyze_stmt_orderby.result
index 42a38037a0e..91f033ac3b9 100644
--- a/mysql-test/r/analyze_stmt_orderby.result
+++ b/mysql-test/r/analyze_stmt_orderby.result
@@ -203,6 +203,10 @@ ANALYZE
     "r_loops": 1,
     "r_total_time_ms": "REPLACED",
     "filesort": {
+      "r_loops": 1,
+      "r_limit": 4,
+      "r_used_priority_queue": true,
+      "r_output_rows": 4,
       "temporary_table": {
         "table": {
           "table_name": "t0",
@@ -318,4 +322,45 @@ ANALYZE
   }
 }
 drop table t2;
+create table t2 (
+a int,
+b int,
+c int
+);
+insert into t2 
+select 
+a.a+10*b.a+100*c.a, 
+b.a+10*c.a, 
+c.a
+from t0 a, t0 b, t0 c;
+analyze format=json 
+select MAX(b) from t2 where mod(a,2)=0 group by c;
+ANALYZE
+{
+  "query_block": {
+    "select_id": 1,
+    "r_loops": 1,
+    "r_total_time_ms": "REPLACED",
+    "filesort": {
+      "r_loops": 1,
+      "r_used_priority_queue": false,
+      "r_output_rows": 10,
+      "r_buffer_size": "REPLACED",
+      "temporary_table": {
+        "table": {
+          "table_name": "t2",
+          "access_type": "ALL",
+          "r_loops": 1,
+          "rows": 1000,
+          "r_rows": 1000,
+          "r_total_time_ms": "REPLACED",
+          "filtered": 100,
+          "r_filtered": 50,
+          "attached_condition": "((t2.a % 2) = 0)"
+        }
+      }
+    }
+  }
+}
+drop table t2;
 drop table t0, t1;
diff --git a/mysql-test/t/analyze_stmt_orderby.test b/mysql-test/t/analyze_stmt_orderby.test
index baddef3566c..965271ceded 100644
--- a/mysql-test/t/analyze_stmt_orderby.test
+++ b/mysql-test/t/analyze_stmt_orderby.test
@@ -70,4 +70,24 @@ analyze format=json
 select * from t0,t2 where t2.a=t0.a order by t0.a limit 4;
 
 drop table t2;
+
+
+create table t2 (
+  a int,
+  b int,
+  c int
+);
+insert into t2 
+select 
+  a.a+10*b.a+100*c.a, 
+  b.a+10*c.a, 
+  c.a
+from t0 a, t0 b, t0 c;
+
+--replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ /"r_buffer_size": "[^"]+"/"r_buffer_size": "REPLACED"/
+analyze format=json 
+select MAX(b) from t2 where mod(a,2)=0 group by c;
+
+drop table t2;
 drop table t0, t1;
+
diff --git a/sql/sql_analyze_stmt.cc b/sql/sql_analyze_stmt.cc
index 4874c33a544..d2eb46298f1 100644
--- a/sql/sql_analyze_stmt.cc
+++ b/sql/sql_analyze_stmt.cc
@@ -23,7 +23,7 @@
 #include "sql_select.h"
 #include "my_json_writer.h"
 
-void Filesort_tracker::print_json(Json_writer *writer)
+void Filesort_tracker::print_json_members(Json_writer *writer)
 {
   const char *varied_str= "(varied across executions)";
   writer->add_member("r_loops").add_ll(r_loops);
@@ -60,3 +60,56 @@ void Filesort_tracker::print_json(Json_writer *writer)
   }
 }
 
+
+/* 
+  Report that we are doing a filesort. 
+    @return 
+      Tracker object to be used with filesort
+*/
+
+Filesort_tracker *Sort_and_group_tracker::report_sorting()
+{
+  DBUG_ASSERT(cur_action < MAX_QEP_ACTIONS);
+
+  if (total_actions)
+  {
+    /* This is not the first execution. Check */
+    if (qep_actions[cur_action] != EXPL_ACTION_FILESORT)
+    {
+      varied_executions= true;
+      cur_action++;
+      if (!dummy_fsort_tracker)
+        dummy_fsort_tracker= new (current_thd->mem_root) Filesort_tracker();
+      return dummy_fsort_tracker;
+    }
+    return qep_actions_data[cur_action++].filesort_tracker;
+  }
+
+  Filesort_tracker *fs_tracker= new(current_thd->mem_root)Filesort_tracker();
+  qep_actions_data[cur_action].filesort_tracker= fs_tracker;
+  qep_actions[cur_action++]= EXPL_ACTION_FILESORT;
+
+  return fs_tracker;
+}
+
+
+void Sort_and_group_tracker::report_tmp_table(TABLE *tbl)
+{
+  DBUG_ASSERT(cur_action < MAX_QEP_ACTIONS);
+  if (total_actions)
+  {
+    /* This is not the first execution. Check if the steps match.  */
+    // todo: should also check that tmp.table kinds are the same.
+    if (qep_actions[cur_action] != EXPL_ACTION_TEMPTABLE)
+      varied_executions= true;
+  }
+
+  if (!varied_executions)
+  {
+    qep_actions[cur_action]= EXPL_ACTION_TEMPTABLE;
+    // qep_actions_data[cur_action]= ....
+  }
+  
+  cur_action++;
+}
+
diff --git a/sql/sql_analyze_stmt.h b/sql/sql_analyze_stmt.h
index abd4fc1ea4b..56d1a8668e3 100644
--- a/sql/sql_analyze_stmt.h
+++ b/sql/sql_analyze_stmt.h
@@ -214,7 +214,7 @@ public:
   }
   
   /* Functions to get the statistics */
-  void print_json(Json_writer *writer);
+  void print_json_members(Json_writer *writer);
   
   ulonglong get_r_loops() { return r_loops; }
   double get_avg_examined_rows() 
@@ -283,6 +283,7 @@ typedef enum
 
 typedef enum 
 {
+  EXPL_ACTION_EOF, /* not-an-action */
   EXPL_ACTION_FILESORT,
   EXPL_ACTION_TEMPTABLE,
   EXPL_ACTION_REMOVE_DUPS,
@@ -333,8 +334,117 @@ class Sort_and_group_tracker : public Sql_alloc
 {
   enum { MAX_QEP_ACTIONS = 5 };
 
+  /* Query actions in the order they were made. */
+  enum_qep_action qep_actions[MAX_QEP_ACTIONS];
+  
+  /* Number for the next action */
+  int cur_action;
+
+  /*
+    Non-zero means there was already an execution which had
+    #total_actions actions
+  */
+  int total_actions;
+
+  int get_n_actions()
+  {
+    return total_actions? total_actions: cur_action;
+  }
+
+  /*
+    TRUE<=>there were executions which took different sort/buffer/de-duplicate
+    routes. The counter values are not meaningful.
+  */
+  bool varied_executions;
+
+  /* Details about query actions */
+  union 
+  {
+    Filesort_tracker *filesort_tracker;
+    enum_tmp_table_use tmp_table;
+  } 
+  qep_actions_data[MAX_QEP_ACTIONS];
+  
+  Filesort_tracker *dummy_fsort_tracker;
+
+public:
+  Sort_and_group_tracker() : 
+    cur_action(0), total_actions(0), varied_executions(false),
+    dummy_fsort_tracker(NULL)
+  {}
+
+  /*************** Reporting interface ***************/
+  /* Report that join execution is started */
+  void report_join_start()
+  {
+    if (!total_actions && cur_action != 0)
+    {
+      /* This is a second execution */
+      total_actions= cur_action;
+    }
+    cur_action= 0;
+  }
+
+  /* 
+    Report that a temporary table is created. The next step is to write to the
+    this tmp. table
+  */
+  void report_tmp_table(TABLE *tbl);
+
+  /* 
+    Report that we are doing a filesort. 
+      @return 
+        Tracker object to be used with filesort
+  */
+  Filesort_tracker *report_sorting();
+  
+  friend class Iterator;
+  /*************** Statistics retrieval interface ***************/
+  bool had_varied_executions() { return varied_executions; }
+
+  class Iterator 
+  {
+    Sort_and_group_tracker *owner;
+    int idx;
+  public:
+    Iterator(Sort_and_group_tracker *owner_arg) : 
+      owner(owner_arg), idx(owner_arg->get_n_actions() - 1)
+    {}
+
+    enum_qep_action get_next(Filesort_tracker **tracker/*,
+                             enum_tmp_table_use *tmp_table_use*/)
+    {
+      /* Walk back through the array... */
+      if (idx < 0)
+        return EXPL_ACTION_EOF;
+      switch (owner->qep_actions[idx])
+      {
+        case EXPL_ACTION_FILESORT:
+          *tracker= owner->qep_actions_data[idx].filesort_tracker;
+          break;
+        case EXPL_ACTION_TEMPTABLE:
+          //*tmp_table_use= tmp_table_kind[tmp_table_idx++];
+          break;
+        default:
+          break;
+      }
+      return owner->qep_actions[idx--];
+    }
+
+    bool is_last_element() { return idx == -1; }
+  };
+};
+
+#if 0
+class Sort_and_group_tracker : public Sql_alloc
+{
+  enum { MAX_QEP_ACTIONS = 5 };
+
   /* Query actions in the order they were made */
   enum_qep_action qep_actions[MAX_QEP_ACTIONS];
+
+  /* Index in filesort_tracker or tmp_table_kind arrays */
+  int qep_action_idx[MAX_QEP_ACTIONS];
   uint n_actions;
   
   /* 
@@ -348,7 +458,7 @@ class Sort_and_group_tracker : public Sql_alloc
   enum_tmp_table_use tmp_table_kind[2];
   int cur_tmp_table;
 
-  friend class Explain_select;
+  //friend class Explain_select;
 
 public:
   Sort_and_group_tracker() : 
@@ -366,7 +476,10 @@ public:
     cur_tmp_table= 0;
   }
 
-  /* Report that a temporary table is created. */
+  /* 
+    Report that a temporary table is created. The next step is to write to the
+    this tmp. table
+  */
   void report_tmp_table(TABLE *tbl)
   {
     DBUG_ASSERT(n_actions < MAX_QEP_ACTIONS);
@@ -385,8 +498,44 @@ public:
     DBUG_ASSERT(cur_tracker < 2);
     return &filesort_tracker[cur_tracker++];
   }
-
+  
+  friend class Iterator;
   /*************** Statistics retrieval interface ***************/
+  // need to iterate over steps
+#if 0
+  class Iterator 
+  {
+    Sort_and_group_tracker *owner;
+    uint idx;
+    int fs_tracker_idx;
+    //int tmp_table_idx;
+  public:
+    Iterator(Sort_and_group_tracker *owner_arg) : 
+      owner(owner_arg), idx(0), fs_tracker_idx(0)//, tmp_table_idx(0)
+    {}
+
+    enum_qep_action get_next(Filesort_tracker **tracker/*,
+                             enum_tmp_table_use *tmp_table_use*/)
+    {
+      /* Walk back through the array... */
+      if (idx >= owner->n_actions)
+        return EXPL_ACTION_EOF;
+      switch (owner->qep_actions[idx]) 
+      {
+        case EXPL_ACTION_FILESORT:
+          *tracker= &owner->filesort_tracker[fs_tracker_idx++];
+          break;
+        case EXPL_ACTION_TEMPTABLE:
+          //*tmp_table_use= tmp_table_kind[tmp_table_idx++];
+          break;
+        default:
+          break;
+      }
+      return owner->qep_actions[idx++];
+    }
+  };
+#endif
   //enum_tmp_table_use get_tmp_table_type() { return join_result_tmp_table; }
 };
+#endif 
 
diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc
index 92f221513f2..21959423084 100644
--- a/sql/sql_explain.cc
+++ b/sql/sql_explain.cc
@@ -770,47 +770,62 @@ void Explain_select::print_explain_json(Explain_query *query,
     }
      
     Filesort_tracker *first_table_sort= NULL;
+    bool first_table_sort_used= false;
     int started_objects= 0;
 
     if (is_analyze)
     {
       /* ANALYZE has collected this part of query plan independently */
-      for (int i= ops_tracker.n_actions-1; i >= 0; i--)
+      if (ops_tracker.had_varied_executions())
       {
-        if (ops_tracker.qep_actions[i] == EXPL_ACTION_FILESORT)
+        writer->add_member("varied-sort-and-tmp").start_object();
+        started_objects++;
+      }
+      else
+      {
+        Sort_and_group_tracker::Iterator iter(&ops_tracker);
+        enum_qep_action action;
+        Filesort_tracker *fs_tracker;
+
+        while ((action= iter.get_next(&fs_tracker)) != EXPL_ACTION_EOF)
         {
-          if (i == 0)
+          if (action == EXPL_ACTION_FILESORT)
           {
-            /* filesort operation was the first in the pipeline */
-            first_table_sort= &ops_tracker.filesort_tracker[0];
-            break;
+            if (iter.is_last_element())
+            {
+              first_table_sort= fs_tracker;
+              break;
+            }
+            writer->add_member("filesort").start_object();
+            started_objects++;
+            fs_tracker->print_json_members(writer);
+          }
+          else if (action == EXPL_ACTION_TEMPTABLE)
+          {
+            writer->add_member("temporary_table").start_object();
+            started_objects++;
+            /*
+            if (tmp == EXPL_TMP_TABLE_BUFFER)
+              func= "buffer";
+            else if (tmp == EXPL_TMP_TABLE_GROUP)
+              func= "group-by";
+            else
+              func= "distinct";
+            writer->add_member("function").add_str(func);
+           */
+          }
+          else if (action == EXPL_ACTION_REMOVE_DUPS)
+          {
+            writer->add_member("duplicate_removal").start_object();
+            started_objects++;
           }
-          writer->add_member("filesort").start_object();
-          started_objects++;
-        }
-        else if (ops_tracker.qep_actions[i] == EXPL_ACTION_TEMPTABLE)
-        {
-          writer->add_member("temporary_table").start_object();
-          started_objects++;
-          /*
-          if (tmp == EXPL_TMP_TABLE_BUFFER)
-            func= "buffer";
-          else if (tmp == EXPL_TMP_TABLE_GROUP)
-            func= "group-by";
           else
-            func= "distinct";
-          writer->add_member("function").add_str(func);
-         */
+            DBUG_ASSERT(0);
         }
-        else if (ops_tracker.qep_actions[i] == EXPL_ACTION_REMOVE_DUPS)
-        {
-          writer->add_member("duplicate_removal").start_object();
-          started_objects++;
-        }
-        else
-          DBUG_ASSERT(0);
       }
 
+      if (first_table_sort)
+        first_table_sort_used= true;
     }
     else
     {
@@ -828,13 +843,15 @@ void Explain_select::print_explain_json(Explain_query *query,
       }
       else
       {
+        fprintf(stderr, "Weird!\n");
         if (using_filesort)
-          first_table_sort= &ops_tracker.filesort_tracker[0];
+          first_table_sort_used= true;
       }
     }
-      
+    
     Explain_basic_join::print_explain_json_interns(query, writer, is_analyze,
-                                                   first_table_sort);
+                                                   first_table_sort,
+                                                   first_table_sort_used);
 
     for (;started_objects; started_objects--)
       writer->end_object();
@@ -852,7 +869,7 @@ void Explain_basic_join::print_explain_json(Explain_query *query,
   writer->add_member("query_block").start_object();
   writer->add_member("select_id").add_ll(select_id);
   
-  print_explain_json_interns(query, writer, is_analyze, NULL);
+  print_explain_json_interns(query, writer, is_analyze, NULL, false);
 
   writer->end_object();
 }
@@ -862,7 +879,8 @@ void Explain_basic_join::
 print_explain_json_interns(Explain_query *query, 
                            Json_writer *writer, 
                            bool is_analyze,
-                           Filesort_tracker *first_table_sort)
+                           Filesort_tracker *first_table_sort,
+                           bool first_table_sort_used)
 {
   Json_writer_nesting_guard guard(writer);
   for (uint i=0; i< n_join_tabs; i++)
@@ -871,7 +889,11 @@ print_explain_json_interns(Explain_query *query,
       writer->add_member("duplicates_removal").start_object();
 
     join_tabs[i]->print_explain_json(query, writer, is_analyze,
-                                     (i==0)? first_table_sort : NULL);
+                                     first_table_sort,
+                                     first_table_sort_used);
+
+    first_table_sort= NULL;
+    first_table_sort_used= false;
 
     if (join_tabs[i]->end_dups_weedout)
       writer->end_object();
@@ -1314,11 +1336,12 @@ void add_json_keyset(Json_writer *writer, const char *elem_name,
 void Explain_table_access::print_explain_json(Explain_query *query,
                                               Json_writer *writer,
                                               bool is_analyze,
-                                              Filesort_tracker *fs_tracker)
+                                              Filesort_tracker *fs_tracker,
+                                              bool first_table_sort_used)
 {
   Json_writer_nesting_guard guard(writer);
   
-  if (fs_tracker)
+  if (first_table_sort_used)
   {
     /* filesort was invoked on this join tab before doing the join with the rest */
     writer->add_member("read_sorted_file").start_object();
@@ -1346,7 +1369,7 @@ void Explain_table_access::print_explain_json(Explain_query *query,
     }
     writer->add_member("filesort").start_object();
     if (is_analyze)
-      fs_tracker->print_json(writer);
+      fs_tracker->print_json_members(writer);
   }
 
   if (bka_type.is_using_jbuf())
@@ -1537,7 +1560,7 @@ void Explain_table_access::print_explain_json(Explain_query *query,
     writer->end_object();
   }
 
-  if (fs_tracker)
+  if (first_table_sort_used)
   {
     writer->end_object(); // filesort
     writer->end_object(); // read_sorted_file
@@ -1976,7 +1999,7 @@ void Explain_update::print_explain_json(Explain_query *query,
   {
     writer->add_member("filesort").start_object();
     if (is_analyze)
-      filesort_tracker->print_json(writer);
+      filesort_tracker->print_json_members(writer);
     doing_buffering= true;
   }
 
diff --git a/sql/sql_explain.h b/sql/sql_explain.h
index 61720ca8d76..dd2b5783b41 100644
--- a/sql/sql_explain.h
+++ b/sql/sql_explain.h
@@ -171,7 +171,8 @@ public:
 
   void print_explain_json_interns(Explain_query *query, Json_writer *writer,
                                   bool is_analyze, 
-                                  Filesort_tracker *first_table_sort);
+                                  Filesort_tracker *first_table_sort,
+                                  bool first_table_sort_used);
 
   /* A flat array of Explain structs for tables. */
   Explain_table_access** join_tabs;
@@ -704,7 +705,9 @@ public:
                     uint select_id, const char *select_type,
                     bool using_temporary, bool using_filesort);
   void print_explain_json(Explain_query *query, Json_writer *writer,
-                          bool is_analyze, Filesort_tracker *fs_tracker);
+                          bool is_analyze, 
+                          Filesort_tracker *fs_tracker,
+                          bool first_table_sort_used);
 
 private:
   void append_tag_name(String *str, enum explain_extra_tag tag);