Merge 10.6 into 10.7

9 files changed, 189 insertions, 94 deletions

diff --git a/sql/handler.h b/sql/handler.h
index 3303b8fe9e0..fe61666bf20 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -2625,6 +2625,9 @@ public:
   /** true when InnoDB should abort the alter when table is not empty */
   const bool error_if_not_empty;
 
+  /** True when DDL should avoid downgrading the MDL */
+  bool mdl_exclusive_after_prepare= false;
+
   Alter_inplace_info(HA_CREATE_INFO *create_info_arg,
                      Alter_info *alter_info_arg,
                      KEY *key_info_arg, uint key_count_arg,
diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc
index 5cf48f10ad8..2e4992a021e 100644
--- a/sql/log_event_server.cc
+++ b/sql/log_event_server.cc
@@ -5621,14 +5621,16 @@ int Rows_log_event::do_apply_event(rpl_group_info *rgi)
       }
     }
 
-#if defined(WITH_WSREP) && defined(HAVE_QUERY_CACHE)
+#ifdef HAVE_QUERY_CACHE
     /*
       Moved invalidation right before the call to rows_event_stmt_cleanup(),
       to avoid query cache being polluted with stale entries,
     */
-    if (WSREP(thd) && wsrep_thd_is_applying(thd))
+# ifdef WITH_WSREP
+    if (!WSREP(thd) && !wsrep_thd_is_applying(thd))
+# endif /* WITH_WSREP */
       query_cache.invalidate_locked_for_write(thd, rgi->tables_to_lock);
-#endif /* WITH_WSREP && HAVE_QUERY_CACHE */
+#endif /* HAVE_QUERY_CACHE */
   }
 
   table= m_table= rgi->m_table_map.get_table(m_table_id);
diff --git a/sql/my_json_writer.cc b/sql/my_json_writer.cc
index 55e71bc1b52..9470ba57855 100644
--- a/sql/my_json_writer.cc
+++ b/sql/my_json_writer.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2014, 2020, MariaDB Corporation.
+/* Copyright (C) 2014, 2021, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -18,6 +18,14 @@
 #include "sql_string.h"
 #include "my_json_writer.h"
 
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+bool Json_writer::named_item_expected() const
+{
+  return named_items_expectation.size()
+      && named_items_expectation.back();
+}
+#endif
+
 void Json_writer::append_indent()
 {
   if (!document_start)
@@ -26,9 +34,21 @@ void Json_writer::append_indent()
     output.append(' ');
 }
 
-void Json_writer::start_object()
+inline void Json_writer::on_start_object()
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  if(!fmt_helper.is_making_writer_calls())
+  {
+    VALIDITY_ASSERT(got_name == named_item_expected());
+    named_items_expectation.push_back(true);
+  }
+#endif
   fmt_helper.on_start_object();
+}
+
+void Json_writer::start_object()
+{
+  on_start_object();
 
   if (!element_started)
     start_element();
@@ -38,10 +58,22 @@ void Json_writer::start_object()
   first_child=true;
   element_started= false;
   document_start= false;
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  got_name= false;
+#endif
 }
 
 void Json_writer::start_array()
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  if(!fmt_helper.is_making_writer_calls())
+  {
+    VALIDITY_ASSERT(got_name == named_item_expected());
+    named_items_expectation.push_back(false);
+    got_name= false;
+  }
+#endif
+
   if (fmt_helper.on_start_array())
     return;
 
@@ -58,6 +90,12 @@ void Json_writer::start_array()
 
 void Json_writer::end_object()
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  VALIDITY_ASSERT(named_item_expected());
+  named_items_expectation.pop_back();
+  VALIDITY_ASSERT(!got_name);
+  got_name= false;
+#endif
   indent_level-=INDENT_SIZE;
   if (!first_child)
     append_indent();
@@ -68,6 +106,11 @@ void Json_writer::end_object()
 
 void Json_writer::end_array()
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  VALIDITY_ASSERT(!named_item_expected());
+  named_items_expectation.pop_back();
+  got_name= false;
+#endif
   if (fmt_helper.on_end_array())
     return;
   indent_level-=INDENT_SIZE;
@@ -80,31 +123,25 @@ void Json_writer::end_array()
 Json_writer& Json_writer::add_member(const char *name)
 {
   size_t len= strlen(name);
-  if (fmt_helper.on_add_member(name, len))
-    return *this; // handled
-
-  // assert that we are in an object
-  DBUG_ASSERT(!element_started);
-  start_element();
-
-  output.append('"');
-  output.append(name, len);
-  output.append("\": ", 3);
-  return *this;
+  return add_member(name, len);
 }
 
 Json_writer& Json_writer::add_member(const char *name, size_t len)
 {
-  if (fmt_helper.on_add_member(name, len))
-    return *this; // handled
-
-  // assert that we are in an object
-  DBUG_ASSERT(!element_started);
-  start_element();
+  if (!fmt_helper.on_add_member(name, len))
+  {
+    // assert that we are in an object
+    DBUG_ASSERT(!element_started);
+    start_element();
 
-  output.append('"');
-  output.append(name, len);
-  output.append(STRING_WITH_LEN("\": "));
+    output.append('"');
+    output.append(name, len);
+    output.append(STRING_WITH_LEN("\": "));
+  }
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  if (!fmt_helper.is_making_writer_calls())
+    got_name= true;
+#endif
   return *this;
 }
 
@@ -200,19 +237,14 @@ void Json_writer::add_null()
 void Json_writer::add_unquoted_str(const char* str)
 {
   size_t len= strlen(str);
-  if (fmt_helper.on_add_str(str, len))
-    return;
-
-  if (!element_started)
-    start_element();
-
-  output.append(str, len);
-  element_started= false;
+  add_unquoted_str(str, len);
 }
 
 void Json_writer::add_unquoted_str(const char* str, size_t len)
 {
-  if (fmt_helper.on_add_str(str, len))
+  VALIDITY_ASSERT(fmt_helper.is_making_writer_calls() ||
+                  got_name == named_item_expected());
+  if (on_add_str(str, len))
     return;
 
   if (!element_started)
@@ -222,19 +254,19 @@ void Json_writer::add_unquoted_str(const char* str, size_t len)
   element_started= false;
 }
 
+inline bool Json_writer::on_add_str(const char *str, size_t num_bytes)
+{
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  got_name= false;
+#endif
+  bool helped= fmt_helper.on_add_str(str, num_bytes);
+  return helped;
+}
+
 void Json_writer::add_str(const char *str)
 {
   size_t len= strlen(str);
-  if (fmt_helper.on_add_str(str, len))
-    return;
-
-  if (!element_started)
-    start_element();
-
-  output.append('"');
-  output.append(str, len);
-  output.append('"');
-  element_started= false;
+  add_str(str, len);
 }
 
 /*
@@ -243,7 +275,9 @@ void Json_writer::add_str(const char *str)
 
 void Json_writer::add_str(const char* str, size_t num_bytes)
 {
-  if (fmt_helper.on_add_str(str, num_bytes))
+  VALIDITY_ASSERT(fmt_helper.is_making_writer_calls() ||
+                  got_name == named_item_expected());
+  if (on_add_str(str, num_bytes))
     return;
 
   if (!element_started)
diff --git a/sql/my_json_writer.h b/sql/my_json_writer.h
index d82313f996f..e2d60df4bf5 100644
--- a/sql/my_json_writer.h
+++ b/sql/my_json_writer.h
@@ -17,9 +17,20 @@
 #define JSON_WRITER_INCLUDED
 
 #include "my_base.h"
-#include "sql_select.h"
 
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST) || defined ENABLED_JSON_WRITER_CONSISTENCY_CHECKS
 #include <vector>
+#endif
+
+#ifdef JSON_WRITER_UNIT_TEST
+#include "sql_string.h"
+constexpr uint FAKE_SELECT_LEX_ID= UINT_MAX;
+// Also, mock objects are defined in my_json_writer-t.cc
+#define VALIDITY_ASSERT(x) if (!(x)) this->invalid_json= true;
+#else
+#include "sql_select.h"
+#define VALIDITY_ASSERT(x) DBUG_ASSERT(x)
+#endif
 
 class Opt_trace_stmt;
 class Opt_trace_context;
@@ -96,9 +107,18 @@ public:
   bool on_end_array();
   void on_start_object();
   // on_end_object() is not needed.
-   
+
   bool on_add_str(const char *str, size_t num_bytes);
 
+  /*
+    Returns true if the helper is flushing its buffer and is probably
+    making calls back to its Json_writer. (The Json_writer uses this
+    function to avoid re-doing the processing that it has already done
+    before making a call to fmt_helper)
+  */
+  bool is_making_writer_calls() const { return state == DISABLED; }
+
+private:
   void flush_on_one_line();
   void disable_and_flush();
 };
@@ -185,6 +205,25 @@ private:
 
 class Json_writer
 {
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+  /*
+    In debug mode, Json_writer will fail and assertion if one attempts to
+    produce an invalid JSON document (e.g. JSON array having named elements).
+  */
+  std::vector<bool> named_items_expectation;
+
+  bool named_item_expected() const;
+
+  bool got_name;
+
+#ifdef JSON_WRITER_UNIT_TEST
+public:
+  // When compiled for unit test, creating invalid JSON will set this to true
+  // instead of an assertion.
+  bool invalid_json= false;
+#endif
+#endif
+
 public:
   /* Add a member. We must be in an object. */
   Json_writer& add_member(const char *name);
@@ -208,6 +247,10 @@ public:
 private:
   void add_unquoted_str(const char* val);
   void add_unquoted_str(const char* val, size_t len);
+
+  bool on_add_str(const char *str, size_t num_bytes);
+  void on_start_object();
+
 public:
   /* Start a child object */
   void start_object();
@@ -225,6 +268,9 @@ public:
   size_t get_truncated_bytes() { return output.get_truncated_bytes(); }
 
   Json_writer() : 
+#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
+    got_name(false),
+#endif
     indent_level(0), document_start(true), element_started(false), 
     first_child(true)
   {
@@ -312,6 +358,9 @@ public:
 /* A common base for Json_writer_object and Json_writer_array */
 class Json_writer_struct
 {
+  Json_writer_struct(const Json_writer_struct&)= delete;
+  Json_writer_struct& operator=(const Json_writer_struct&)= delete;
+
 #ifdef ENABLED_JSON_WRITER_CONSISTENCY_CHECKS
   static thread_local std::vector<bool> named_items_expectation;
 #endif
@@ -372,24 +421,18 @@ private:
     my_writer->add_member(name);
   }
 public:
-  explicit Json_writer_object(THD *thd)
-  : Json_writer_struct(thd, true)
-  {
-#ifdef ENABLED_JSON_WRITER_CONSISTENCY_CHECKS
-    DBUG_ASSERT(!named_item_expected());
-#endif
-    if (unlikely(my_writer))
-      my_writer->start_object();
-  }
-
-  explicit Json_writer_object(THD* thd, const char *str)
+  explicit Json_writer_object(THD* thd, const char *str= nullptr)
   : Json_writer_struct(thd, true)
   {
 #ifdef ENABLED_JSON_WRITER_CONSISTENCY_CHECKS
     DBUG_ASSERT(named_item_expected());
 #endif
     if (unlikely(my_writer))
-      my_writer->add_member(str).start_object();
+    {
+      if (str)
+        my_writer->add_member(str);
+      my_writer->start_object();
+    }
   }
 
   ~Json_writer_object()
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 93a548b476e..808e590435a 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -7655,7 +7655,7 @@ static int mysql_init_variables(void)
   disable_log_notes= 0;
   mqh_used= 0;
   cleanup_done= 0;
-  test_flags= select_errors= dropping_tables= ha_open_options=0;
+  select_errors= dropping_tables= ha_open_options=0;
   THD_count::count= CONNECT::count= 0;
   slave_open_temp_tables= 0;
   opt_endinfo= using_udf_functions= 0;
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index afe82c71289..9dc1bcf733f 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -370,7 +370,7 @@ TRP_ROR_INTERSECT *get_best_covering_ror_intersect(PARAM *param,
                                                    double read_time);
 static
 TABLE_READ_PLAN *get_best_disjunct_quick(PARAM *param, SEL_IMERGE *imerge,
-                                         double read_time);
+                                         double read_time, bool named_trace= false);
 static
 TABLE_READ_PLAN *merge_same_index_scans(PARAM *param, SEL_IMERGE *imerge,
                                         TRP_INDEX_MERGE *imerge_trp,
@@ -5084,7 +5084,7 @@ double get_sweep_read_cost(const PARAM *param, ha_rows records)
 
 static
 TABLE_READ_PLAN *get_best_disjunct_quick(PARAM *param, SEL_IMERGE *imerge,
-                                         double read_time)
+                                         double read_time, bool named_trace)
 {
   SEL_TREE **ptree;
   TRP_INDEX_MERGE *imerge_trp= NULL;
@@ -5132,7 +5132,8 @@ TABLE_READ_PLAN *get_best_disjunct_quick(PARAM *param, SEL_IMERGE *imerge,
                                              n_child_scans)))
     DBUG_RETURN(NULL);
 
-  Json_writer_object trace_best_disjunct(thd);
+  const char* trace_best_disjunct_obj_name= named_trace ? "best_disjunct_quick" : nullptr;
+  Json_writer_object trace_best_disjunct(thd, trace_best_disjunct_obj_name);
   Json_writer_array to_merge(thd, "indexes_to_merge");
   /*
     Collect best 'range' scan for each of disjuncts, and, while doing so,
@@ -5488,7 +5489,7 @@ TABLE_READ_PLAN *merge_same_index_scans(PARAM *param, SEL_IMERGE *imerge,
   DBUG_ASSERT(imerge->trees_next>imerge->trees);
 
   if (imerge->trees_next-imerge->trees > 1)
-    trp= get_best_disjunct_quick(param, imerge, read_time);
+    trp= get_best_disjunct_quick(param, imerge, read_time, true);
   else
   {
     /*
@@ -5676,7 +5677,7 @@ void print_keyparts(THD *thd, KEY *key, uint key_parts)
   DBUG_ASSERT(thd->trace_started());
 
   KEY_PART_INFO *part= key->key_part;
-  Json_writer_array keyparts= Json_writer_array(thd, "keyparts");
+  Json_writer_array keyparts(thd, "keyparts");
   for(uint i= 0; i < key_parts; i++, part++)
     keyparts.add(part->field->field_name);
 }
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index d91557c5be2..725b8b054ba 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -3839,9 +3839,9 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
       Json_writer_array semijoin_plan(thd, "join_order");
       for (i= first + sjm->tables; i <= tablenr; i++)
       {
+        Json_writer_object trace_one_table(thd);
         if (unlikely(thd->trace_started()))
         {
-          Json_writer_object trace_one_table(thd);
           trace_one_table.add_table_name(join->best_positions[i].table);
         }
         best_access_path(join, join->best_positions[i].table, rem_tables,
@@ -3878,9 +3878,9 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
       Json_writer_array semijoin_plan(thd, "join_order");
       for (idx= first; idx <= tablenr; idx++)
       {
+        Json_writer_object trace_one_table(thd);
         if (unlikely(thd->trace_started()))
         {
-          Json_writer_object trace_one_table(thd);
           trace_one_table.add_table_name(join->best_positions[idx].table);
         }
         if (join->best_positions[idx].use_join_buffer)
@@ -3917,9 +3917,9 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
       Json_writer_array semijoin_plan(thd, "join_order");
       for (idx= first; idx <= tablenr; idx++)
       {
+        Json_writer_object trace_one_table(thd);
         if (unlikely(thd->trace_started()))
         {
-          Json_writer_object trace_one_table(thd);
           trace_one_table.add_table_name(join->best_positions[idx].table);
         }
         if (join->best_positions[idx].use_join_buffer || (idx == first))
diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc
index d299769c666..445bc97a3b5 100644
--- a/sql/sql_cte.cc
+++ b/sql/sql_cte.cc
@@ -1048,6 +1048,15 @@ st_select_lex_unit *With_element::clone_parsed_spec(LEX *old_lex,
   lex_start(thd);
   lex->clone_spec_offset= unparsed_spec_offset;
   lex->with_cte_resolution= true;
+  /*
+    There's no need to add SPs/SFs referenced in the clone to the global
+    list of the SPs/SFs used in the query as they were added when the first
+    reference to the cloned CTE was parsed. Yet the recursive call of the
+    parser must to know that they were already included into the list.
+  */
+  lex->sroutines= old_lex->sroutines;
+  lex->sroutines_list_own_last= old_lex->sroutines_list_own_last;
+  lex->sroutines_list_own_elements= old_lex->sroutines_list_own_elements;
 
   /*
     The specification of a CTE is to be parsed as a regular query.
@@ -1093,6 +1102,29 @@ st_select_lex_unit *With_element::clone_parsed_spec(LEX *old_lex,
     goto err;
 
   /*
+    The unit of the specification that just has been parsed is included
+    as a slave of the select that contained in its from list the table
+    reference for which the unit has been created.
+  */
+  lex->unit.include_down(with_table->select_lex);
+  lex->unit.set_slave(with_select);
+  lex->unit.cloned_from= spec;
+
+  /*
+    Now all references to the CTE defined outside of the cloned specification
+    has to be resolved. Additionally if old_lex->only_cte_resolution == false
+    for the table references that has not been resolved requests for mdl_locks
+    has to be set.
+  */
+  lex->only_cte_resolution= old_lex->only_cte_resolution;
+  if (lex->resolve_references_to_cte(lex->query_tables,
+                                     lex->query_tables_last))
+  {
+    res= NULL;
+    goto err;
+  }
+
+  /*
     The global chain of TABLE_LIST objects created for the specification that
     just has been parsed is added to such chain that contains the reference
     to the CTE whose specification is parsed right after the TABLE_LIST object
@@ -1116,32 +1148,11 @@ st_select_lex_unit *With_element::clone_parsed_spec(LEX *old_lex,
       old_lex->query_tables_last= lex->query_tables_last;
     }
   }
+  old_lex->sroutines_list_own_last= lex->sroutines_list_own_last;
+  old_lex->sroutines_list_own_elements= lex->sroutines_list_own_elements;
   res= &lex->unit;
   res->with_element= this;
   
-  /*
-    The unit of the specification that just has been parsed is included
-    as a slave of the select that contained in its from list the table
-    reference for which the unit has been created.
-  */
-  lex->unit.include_down(with_table->select_lex);
-  lex->unit.set_slave(with_select);
-  lex->unit.cloned_from= spec;
-
-  /*
-    Now all references to the CTE defined outside of the cloned specification
-    has to be resolved. Additionally if old_lex->only_cte_resolution == false
-    for the table references that has not been resolved requests for mdl_locks
-    has to be set.
-  */
-  lex->only_cte_resolution= old_lex->only_cte_resolution;
-  if (lex->resolve_references_to_cte(lex->query_tables,
-                                     lex->query_tables_last))
-  {
-    res= NULL;
-    goto err;
-  }
-
   last_clone_select= lex->all_selects_list;
   while (last_clone_select->next_select_in_list())
     last_clone_select= last_clone_select->next_select_in_list();
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 7a7a56c1336..b07efb29bba 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -7346,15 +7346,16 @@ static bool mysql_inplace_alter_table(THD *thd,
     lock for prepare phase under LOCK TABLES in the same way as when
     exclusive lock is required for duration of the whole statement.
   */
-  if (inplace_supported == HA_ALTER_INPLACE_EXCLUSIVE_LOCK ||
-      ((inplace_supported == HA_ALTER_INPLACE_COPY_NO_LOCK ||
+  if (!ha_alter_info->mdl_exclusive_after_prepare &&
+      (inplace_supported == HA_ALTER_INPLACE_EXCLUSIVE_LOCK ||
+       ((inplace_supported == HA_ALTER_INPLACE_COPY_NO_LOCK ||
         inplace_supported == HA_ALTER_INPLACE_COPY_LOCK ||
         inplace_supported == HA_ALTER_INPLACE_NOCOPY_NO_LOCK ||
         inplace_supported == HA_ALTER_INPLACE_NOCOPY_LOCK ||
         inplace_supported == HA_ALTER_INPLACE_INSTANT) &&
        (thd->locked_tables_mode == LTM_LOCK_TABLES ||
         thd->locked_tables_mode == LTM_PRELOCKED_UNDER_LOCK_TABLES)) ||
-      alter_info->requested_lock == Alter_info::ALTER_TABLE_LOCK_EXCLUSIVE)
+      alter_info->requested_lock == Alter_info::ALTER_TABLE_LOCK_EXCLUSIVE))
   {
     if (wait_while_table_is_used(thd, table, HA_EXTRA_FORCE_REOPEN))
       goto cleanup;