Merge 10.4 into 10.5

6 files changed, 147 insertions, 66 deletions

diff --git a/sql/handler.h b/sql/handler.h
index 47bab7f75e2..673283b9bd5 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -2579,6 +2579,9 @@ public:
   /** true when InnoDB should abort the alter when table is not empty */
   bool error_if_not_empty;
 
+  /** True when DDL should avoid downgrading the MDL */
+  bool mdl_exclusive_after_prepare= false;
+
   Alter_inplace_info(HA_CREATE_INFO *create_info_arg,
                      Alter_info *alter_info_arg,
                      KEY *key_info_arg, uint key_count_arg,
diff --git a/sql/my_json_writer.cc b/sql/my_json_writer.cc
index 8e6f0942857..0397f87dd77 100644
--- a/sql/my_json_writer.cc
+++ b/sql/my_json_writer.cc
@@ -18,6 +18,14 @@
 #include "sql_string.h"
 #include "my_json_writer.h"
 
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+bool Json_writer::named_item_expected() const
+{
+  return named_items_expectation.size()
+      && named_items_expectation.back();
+}
+#endif
+
 void Json_writer::append_indent()
 {
   if (!document_start)
@@ -26,9 +34,21 @@ void Json_writer::append_indent()
     output.append(' ');
 }
 
-void Json_writer::start_object()
+inline void Json_writer::on_start_object()
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  if(!fmt_helper.is_making_writer_calls())
+  {
+    VALIDITY_ASSERT(got_name == named_item_expected());
+    named_items_expectation.push_back(true);
+  }
+#endif
   fmt_helper.on_start_object();
+}
+
+void Json_writer::start_object()
+{
+  on_start_object();
 
   if (!element_started)
     start_element();
@@ -38,10 +58,22 @@ void Json_writer::start_object()
   first_child=true;
   element_started= false;
   document_start= false;
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  got_name= false;
+#endif
 }
 
 void Json_writer::start_array()
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  if(!fmt_helper.is_making_writer_calls())
+  {
+    VALIDITY_ASSERT(got_name == named_item_expected());
+    named_items_expectation.push_back(false);
+    got_name= false;
+  }
+#endif
+
   if (fmt_helper.on_start_array())
     return;
 
@@ -58,6 +90,12 @@ void Json_writer::start_array()
 
 void Json_writer::end_object()
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  VALIDITY_ASSERT(named_item_expected());
+  named_items_expectation.pop_back();
+  VALIDITY_ASSERT(!got_name);
+  got_name= false;
+#endif
   indent_level-=INDENT_SIZE;
   if (!first_child)
     append_indent();
@@ -68,6 +106,11 @@ void Json_writer::end_object()
 
 void Json_writer::end_array()
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  VALIDITY_ASSERT(!named_item_expected());
+  named_items_expectation.pop_back();
+  got_name= false;
+#endif
   if (fmt_helper.on_end_array())
     return;
   indent_level-=INDENT_SIZE;
@@ -80,31 +123,25 @@ void Json_writer::end_array()
 Json_writer& Json_writer::add_member(const char *name)
 {
   size_t len= strlen(name);
-  if (fmt_helper.on_add_member(name, len))
-    return *this; // handled
-
-  // assert that we are in an object
-  DBUG_ASSERT(!element_started);
-  start_element();
-
-  output.append('"');
-  output.append(name, len);
-  output.append("\": ", 3);
-  return *this;
+  return add_member(name, len);
 }
 
 Json_writer& Json_writer::add_member(const char *name, size_t len)
 {
-  if (fmt_helper.on_add_member(name, len))
-    return *this; // handled
-
-  // assert that we are in an object
-  DBUG_ASSERT(!element_started);
-  start_element();
+  if (!fmt_helper.on_add_member(name, len))
+  {
+    // assert that we are in an object
+    DBUG_ASSERT(!element_started);
+    start_element();
 
-  output.append('"');
-  output.append(name, len);
-  output.append("\": ");
+    output.append('"');
+    output.append(name, len);
+    output.append("\": ", 3);
+  }
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  if (!fmt_helper.is_making_writer_calls())
+    got_name= true;
+#endif
   return *this;
 }
 
@@ -200,19 +237,14 @@ void Json_writer::add_null()
 void Json_writer::add_unquoted_str(const char* str)
 {
   size_t len= strlen(str);
-  if (fmt_helper.on_add_str(str, len))
-    return;
-
-  if (!element_started)
-    start_element();
-
-  output.append(str, len);
-  element_started= false;
+  add_unquoted_str(str, len);
 }
 
 void Json_writer::add_unquoted_str(const char* str, size_t len)
 {
-  if (fmt_helper.on_add_str(str, len))
+  VALIDITY_ASSERT(fmt_helper.is_making_writer_calls() ||
+                  got_name == named_item_expected());
+  if (on_add_str(str, len))
     return;
 
   if (!element_started)
@@ -222,19 +254,19 @@ void Json_writer::add_unquoted_str(const char* str, size_t len)
   element_started= false;
 }
 
+inline bool Json_writer::on_add_str(const char *str, size_t num_bytes)
+{
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  got_name= false;
+#endif
+  bool helped= fmt_helper.on_add_str(str, num_bytes);
+  return helped;
+}
+
 void Json_writer::add_str(const char *str)
 {
   size_t len= strlen(str);
-  if (fmt_helper.on_add_str(str, len))
-    return;
-
-  if (!element_started)
-    start_element();
-
-  output.append('"');
-  output.append(str, len);
-  output.append('"');
-  element_started= false;
+  add_str(str, len);
 }
 
 /*
@@ -243,7 +275,9 @@ void Json_writer::add_str(const char *str)
 
 void Json_writer::add_str(const char* str, size_t num_bytes)
 {
-  if (fmt_helper.on_add_str(str, num_bytes))
+  VALIDITY_ASSERT(fmt_helper.is_making_writer_calls() ||
+                  got_name == named_item_expected());
+  if (on_add_str(str, num_bytes))
     return;
 
   if (!element_started)
diff --git a/sql/my_json_writer.h b/sql/my_json_writer.h
index 9686984ba9b..07ab433abda 100644
--- a/sql/my_json_writer.h
+++ b/sql/my_json_writer.h
@@ -17,9 +17,19 @@
 #define JSON_WRITER_INCLUDED
 
 #include "my_base.h"
-#include "sql_select.h"
 
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST) || defined ENABLED_JSON_WRITER_CONSISTENCY_CHECKS
 #include <vector>
+#endif
+
+#ifdef JSON_WRITER_UNIT_TEST
+#include "sql_string.h"
+// Also, mock objects are defined in my_json_writer-t.cc
+#define VALIDITY_ASSERT(x) if (!(x)) this->invalid_json= true;
+#else
+#include "sql_select.h"
+#define VALIDITY_ASSERT(x) DBUG_ASSERT(x)
+#endif
 
 class Opt_trace_stmt;
 class Opt_trace_context;
@@ -96,9 +106,18 @@ public:
   bool on_end_array();
   void on_start_object();
   // on_end_object() is not needed.
-   
+
   bool on_add_str(const char *str, size_t num_bytes);
 
+  /*
+    Returns true if the helper is flushing its buffer and is probably
+    making calls back to its Json_writer. (The Json_writer uses this
+    function to avoid re-doing the processing that it has already done
+    before making a call to fmt_helper)
+  */
+  bool is_making_writer_calls() const { return state == DISABLED; }
+
+private:
   void flush_on_one_line();
   void disable_and_flush();
 };
@@ -185,6 +204,25 @@ private:
 
 class Json_writer
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  /*
+    In debug mode, Json_writer will fail and assertion if one attempts to
+    produce an invalid JSON document (e.g. JSON array having named elements).
+  */
+  std::vector<bool> named_items_expectation;
+
+  bool named_item_expected() const;
+
+  bool got_name;
+
+#ifdef JSON_WRITER_UNIT_TEST
+public:
+  // When compiled for unit test, creating invalid JSON will set this to true
+  // instead of an assertion.
+  bool invalid_json= false;
+#endif
+#endif
+
 public:
   /* Add a member. We must be in an object. */
   Json_writer& add_member(const char *name);
@@ -208,6 +246,10 @@ public:
 private:
   void add_unquoted_str(const char* val);
   void add_unquoted_str(const char* val, size_t len);
+
+  bool on_add_str(const char *str, size_t num_bytes);
+  void on_start_object();
+
 public:
   /* Start a child object */
   void start_object();
@@ -225,6 +267,9 @@ public:
   size_t get_truncated_bytes() { return output.get_truncated_bytes(); }
 
   Json_writer() : 
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+    got_name(false),
+#endif
     indent_level(0), document_start(true), element_started(false), 
     first_child(true)
   {
@@ -312,6 +357,9 @@ public:
 /* A common base for Json_writer_object and Json_writer_array */
 class Json_writer_struct
 {
+  Json_writer_struct(const Json_writer_struct&)= delete;
+  Json_writer_struct& operator=(const Json_writer_struct&)= delete;
+
 #ifdef ENABLED_JSON_WRITER_CONSISTENCY_CHECKS
   static thread_local std::vector<bool> named_items_expectation;
 #endif
@@ -372,24 +420,18 @@ private:
     my_writer->add_member(name);
   }
 public:
-  explicit Json_writer_object(THD *thd)
-  : Json_writer_struct(thd, true)
-  {
-#ifdef ENABLED_JSON_WRITER_CONSISTENCY_CHECKS
-    DBUG_ASSERT(!named_item_expected());
-#endif
-    if (unlikely(my_writer))
-      my_writer->start_object();
-  }
-
-  explicit Json_writer_object(THD* thd, const char *str)
+  explicit Json_writer_object(THD* thd, const char *str= nullptr)
   : Json_writer_struct(thd, true)
   {
 #ifdef ENABLED_JSON_WRITER_CONSISTENCY_CHECKS
     DBUG_ASSERT(named_item_expected());
 #endif
     if (unlikely(my_writer))
-      my_writer->add_member(str).start_object();
+    {
+      if (str)
+        my_writer->add_member(str);
+      my_writer->start_object();
+    }
   }
 
   ~Json_writer_object()
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 9ec8781bc30..b573dc25ef0 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -370,7 +370,7 @@ TRP_ROR_INTERSECT *get_best_covering_ror_intersect(PARAM *param,
                                                    double read_time);
 static
 TABLE_READ_PLAN *get_best_disjunct_quick(PARAM *param, SEL_IMERGE *imerge,
-                                         double read_time);
+                                         double read_time, bool named_trace= false);
 static
 TABLE_READ_PLAN *merge_same_index_scans(PARAM *param, SEL_IMERGE *imerge,
                                         TRP_INDEX_MERGE *imerge_trp,
@@ -5085,7 +5085,7 @@ double get_sweep_read_cost(const PARAM *param, ha_rows records)
 
 static
 TABLE_READ_PLAN *get_best_disjunct_quick(PARAM *param, SEL_IMERGE *imerge,
-                                         double read_time)
+                                         double read_time, bool named_trace)
 {
   SEL_TREE **ptree;
   TRP_INDEX_MERGE *imerge_trp= NULL;
@@ -5133,7 +5133,8 @@ TABLE_READ_PLAN *get_best_disjunct_quick(PARAM *param, SEL_IMERGE *imerge,
                                              n_child_scans)))
     DBUG_RETURN(NULL);
 
-  Json_writer_object trace_best_disjunct(thd);
+  const char* trace_best_disjunct_obj_name= named_trace ? "best_disjunct_quick" : nullptr;
+  Json_writer_object trace_best_disjunct(thd, trace_best_disjunct_obj_name);
   Json_writer_array to_merge(thd, "indexes_to_merge");
   /*
     Collect best 'range' scan for each of disjuncts, and, while doing so,
@@ -5489,7 +5490,7 @@ TABLE_READ_PLAN *merge_same_index_scans(PARAM *param, SEL_IMERGE *imerge,
   DBUG_ASSERT(imerge->trees_next>imerge->trees);
 
   if (imerge->trees_next-imerge->trees > 1)
-    trp= get_best_disjunct_quick(param, imerge, read_time);
+    trp= get_best_disjunct_quick(param, imerge, read_time, true);
   else
   {
     /*
@@ -5677,7 +5678,7 @@ void print_keyparts(THD *thd, KEY *key, uint key_parts)
   DBUG_ASSERT(thd->trace_started());
 
   KEY_PART_INFO *part= key->key_part;
-  Json_writer_array keyparts= Json_writer_array(thd, "keyparts");
+  Json_writer_array keyparts(thd, "keyparts");
   for(uint i= 0; i < key_parts; i++, part++)
     keyparts.add(part->field->field_name);
 }
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index dd981a8fdcb..99b6af1ef3c 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -3827,9 +3827,9 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
       Json_writer_array semijoin_plan(thd, "join_order");
       for (i= first + sjm->tables; i <= tablenr; i++)
       {
+        Json_writer_object trace_one_table(thd);
         if (unlikely(thd->trace_started()))
         {
-          Json_writer_object trace_one_table(thd);
           trace_one_table.add_table_name(join->best_positions[i].table);
         }
         best_access_path(join, join->best_positions[i].table, rem_tables,
@@ -3866,9 +3866,9 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
       Json_writer_array semijoin_plan(thd, "join_order");
       for (idx= first; idx <= tablenr; idx++)
       {
+        Json_writer_object trace_one_table(thd);
         if (unlikely(thd->trace_started()))
         {
-          Json_writer_object trace_one_table(thd);
           trace_one_table.add_table_name(join->best_positions[idx].table);
         }
         if (join->best_positions[idx].use_join_buffer)
@@ -3905,9 +3905,9 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
       Json_writer_array semijoin_plan(thd, "join_order");
       for (idx= first; idx <= tablenr; idx++)
       {
+        Json_writer_object trace_one_table(thd);
         if (unlikely(thd->trace_started()))
         {
-          Json_writer_object trace_one_table(thd);
           trace_one_table.add_table_name(join->best_positions[idx].table);
         }
         if (join->best_positions[idx].use_join_buffer || (idx == first))
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index b433d6b7af6..42effe31465 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -8001,15 +8001,16 @@ static bool mysql_inplace_alter_table(THD *thd,
     lock for prepare phase under LOCK TABLES in the same way as when
     exclusive lock is required for duration of the whole statement.
   */
-  if (inplace_supported == HA_ALTER_INPLACE_EXCLUSIVE_LOCK ||
-      ((inplace_supported == HA_ALTER_INPLACE_COPY_NO_LOCK ||
+  if (!ha_alter_info->mdl_exclusive_after_prepare &&
+      (inplace_supported == HA_ALTER_INPLACE_EXCLUSIVE_LOCK ||
+       ((inplace_supported == HA_ALTER_INPLACE_COPY_NO_LOCK ||
         inplace_supported == HA_ALTER_INPLACE_COPY_LOCK ||
         inplace_supported == HA_ALTER_INPLACE_NOCOPY_NO_LOCK ||
         inplace_supported == HA_ALTER_INPLACE_NOCOPY_LOCK ||
         inplace_supported == HA_ALTER_INPLACE_INSTANT) &&
        (thd->locked_tables_mode == LTM_LOCK_TABLES ||
         thd->locked_tables_mode == LTM_PRELOCKED_UNDER_LOCK_TABLES)) ||
-      alter_info->requested_lock == Alter_info::ALTER_TABLE_LOCK_EXCLUSIVE)
+      alter_info->requested_lock == Alter_info::ALTER_TABLE_LOCK_EXCLUSIVE))
   {
     if (wait_while_table_is_used(thd, table, HA_EXTRA_FORCE_REOPEN))
       goto cleanup;