Added new MDL_BACKUP locks for all backup stages

Part of MDEV-5336 Implement LOCK FOR BACKUP

- Added new locks to MDL_BACKUP for all stages of backup locks and
  a new MDL lock needed for backup stages.
- Renamed MDL_BACKUP_STMT to MDL_BACKUP_DDL
- flush_tables() takes a new parameter that decides what should be flushed.
- InnoDB, Aria (transactional tables with checksums), Blackhole, Federated
  and Federatedx tables are marked to be safe for online backup. We are
  using MDL_BACKUP_TRANS_DML instead of MDL_BACKUP_DML locks for these
  which allows any DML's to proceed for these tables during the whole
  backup process until BACKUP STAGE COMMIT which will block the final
  commit.

1 files changed, 145 insertions, 66 deletions

diff --git a/sql/mdl.cc b/sql/mdl.cc
index b096f944fb5..40ef4c04676 100644
--- a/sql/mdl.cc
+++ b/sql/mdl.cc
@@ -109,12 +109,23 @@ static const LEX_STRING lock_types[]=
 
 static const LEX_STRING backup_lock_types[]=
 {
+  { C_STRING_WITH_LEN("MDL_BACKUP_START") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_FLUSH") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_WAIT_FLUSH") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_WAIT_DDL") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_WAIT_COMMIT") },
   { C_STRING_WITH_LEN("MDL_BACKUP_FTWRL1") },
   { C_STRING_WITH_LEN("MDL_BACKUP_FTWRL2") },
-  { C_STRING_WITH_LEN("MDL_BACKUP_STMT") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_DML") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_TRANS_DML") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_SYS_DML") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_DDL") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_BLOCK_DDL") },
+  { C_STRING_WITH_LEN("MDL_BACKUP_ALTER_COPY") },
   { C_STRING_WITH_LEN("MDL_BACKUP_COMMIT") }
 };
 
+
 #ifdef HAVE_PSI_INTERFACE
 void MDL_key::init_psi_keys()
 {
@@ -300,8 +311,6 @@ Deadlock_detection_visitor::opt_change_victim_to(MDL_context *new_victim)
   and compatibility matrices.
 */
 
-#define MDL_BIT(A) static_cast<MDL_lock::bitmap_t>(1U << A)
-
 /**
   The lock context. Created internally for an acquired lock.
   For a given name, there exists only one MDL_lock instance,
@@ -316,7 +325,7 @@ Deadlock_detection_visitor::opt_change_victim_to(MDL_context *new_victim)
 class MDL_lock
 {
 public:
-  typedef unsigned short bitmap_t;
+  typedef mdl_bitmap_t bitmap_t;
 
   class Ticket_list
   {
@@ -416,9 +425,10 @@ public:
     { return m_waiting_incompatible; }
     virtual bool needs_notification(const MDL_ticket *ticket) const
     {
-      return ticket->get_type() == MDL_SHARED_NO_WRITE ||
-             ticket->get_type() == MDL_SHARED_NO_READ_WRITE ||
-             ticket->get_type() == MDL_EXCLUSIVE;
+      return (MDL_BIT(ticket->get_type()) &
+              (MDL_BIT(MDL_SHARED_NO_WRITE) |
+               MDL_BIT(MDL_SHARED_NO_READ_WRITE) |
+               MDL_BIT(MDL_EXCLUSIVE)));
     }
 
     /**
@@ -459,17 +469,19 @@ public:
     { return m_waiting_incompatible; }
     virtual bool needs_notification(const MDL_ticket *ticket) const
     {
-      return ticket->get_type() == MDL_BACKUP_FTWRL1;
+      return (MDL_BIT(ticket->get_type()) & MDL_BIT(MDL_BACKUP_FTWRL1));
     }
 
     /**
-      Threads having aninsert delayed thread may hold STMT lock. We
-      need to kill such threads in order to get backup lock for FTWRL
-      or BACKUP statements. We do this my calling code outside of MDL.
+       Insert delayed threads may hold DML or TRANS_DML lock.
+       We need to kill such threads in order to get lock for FTWRL statements.
+       We do this by calling code outside of MDL.
     */
     virtual bool conflicting_locks(const MDL_ticket *ticket) const
     {
-      return ticket->get_type() == MDL_BACKUP_STMT;
+      return (MDL_BIT(ticket->get_type()) &
+              (MDL_BIT(MDL_BACKUP_DML) |
+               MDL_BIT(MDL_BACKUP_TRANS_DML)));
     }
 
     /*
@@ -1488,41 +1500,41 @@ MDL_lock::MDL_scoped_lock::m_waiting_incompatible[MDL_TYPE_END]=
   The first array specifies if particular type of request can be satisfied
   if there is granted lock of certain type.
 
-     Request  |  Granted requests for lock            |
-      type    | S  SH  SR  SW  SU  SRO  SNW  SNRW  X  |
-    ----------+---------------------------------------+
-    S         | +   +   +   +   +   +    +    +    -  |
-    SH        | +   +   +   +   +   +    +    +    -  |
-    SR        | +   +   +   +   +   +    +    -    -  |
-    SW        | +   +   +   +   +   -    -    -    -  |
-    SU        | +   +   +   +   -   +    -    -    -  |
-    SRO       | +   +   +   -   +   +    +    -    -  |
-    SNW       | +   +   +   -   -   +    -    -    -  |
-    SNRW      | +   +   -   -   -   -    -    -    -  |
-    X         | -   -   -   -   -   -    -    -    -  |
-    SU -> X   | -   -   -   -   0   -    0    0    0  |
-    SNW -> X  | -   -   -   0   0   -    0    0    0  |
-    SNRW -> X | -   -   0   0   0   0    0    0    0  |
+     Request  |  Granted requests for lock         |
+      type    | S  SH  SR  SW  SU  SRO SNW SNRW X  |
+    ----------+------------------------------------+
+    S         | +   +   +   +   +   +   +   +   -  |
+    SH        | +   +   +   +   +   +   +   +   -  |
+    SR        | +   +   +   +   +   +   +   -   -  |
+    SW        | +   +   +   +   +   -   -   -   -  |
+    SU        | +   +   +   +   -   +   -   -   -  |
+    SRO       | +   +   +   -   +   +   +   -   -  |
+    SNW       | +   +   +   -   -   +   -   -   -  |
+    SNRW      | +   +   -   -   -   -   -   -   -  |
+    X         | -   -   -   -   -   -   -   -   -  |
+    SU -> X   | -   -   -   -   0   -   0   0   0  |
+    SNW -> X  | -   -   -   0   0   -   0   0   0  |
+    SNRW -> X | -   -   0   0   0   0   0   0   0  |
 
   The second array specifies if particular type of request can be satisfied
   if there is waiting request for the same lock of certain type. In other
   words it specifies what is the priority of different lock types.
 
-     Request  |  Pending requests for lock           |
-      type    | S  SH  SR  SW  SU  SRO  SNW  SNRW  X |
-    ----------+--------------------------------------+
-    S         | +   +   +   +   +   +    +     +   - |
-    SH        | +   +   +   +   +   +    +     +   + |
-    SR        | +   +   +   +   +   +    +     -   - |
-    SW        | +   +   +   +   +   +    -     -   - |
-    SU        | +   +   +   +   +   +    +     +   - |
-    SRO       | +   +   +   -   +   +    +     -   - |
-    SNW       | +   +   +   +   +   +    +     +   - |
-    SNRW      | +   +   +   +   +   +    +     +   - |
-    X         | +   +   +   +   +   +    +     +   + |
-    SU -> X   | +   +   +   +   +   +    +     +   + |
-    SNW -> X  | +   +   +   +   +   +    +     +   + |
-    SNRW -> X | +   +   +   +   +   +    +     +   + |
+     Request  |  Pending requests for lock        |
+      type    | S  SH  SR  SW  SU  SRO SNW SNRW X |
+    ----------+-----------------------------------+
+    S         | +   +   +   +   +   +   +   +   - |
+    SH        | +   +   +   +   +   +   +   +   + |
+    SR        | +   +   +   +   +   +   +   -   - |
+    SW        | +   +   +   +   +   +   -   -   - |
+    SU        | +   +   +   +   +   +   +   +   - |
+    SRO       | +   +   +   -   +   +   +   -   - |
+    SNW       | +   +   +   +   +   +   +   +   - |
+    SNRW      | +   +   +   +   +   +   +   +   - |
+    X         | +   +   +   +   +   +   +   +   + |
+    SU -> X   | +   +   +   +   +   +   +   +   + |
+    SNW -> X  | +   +   +   +   +   +   +   +   + |
+    SNRW -> X | +   +   +   +   +   +   +   +   + |
 
   Here: "+" -- means that request can be satisfied
         "-" -- means that request can't be satisfied and should wait
@@ -1590,49 +1602,106 @@ MDL_lock::MDL_object_lock::m_waiting_incompatible[MDL_TYPE_END]=
   The first array specifies if particular type of request can be satisfied
   if there is granted backup lock of certain type.
 
-             | Type of active  |
-     Request |   backup lock   |
-      type   | F1  F2   S   C  |
-    ---------+-----------------+
-    FTWRL1   |  +   +   -   +  |
-    FTWRL2   |  +   +   -   -  |
-    STMT     |  -   -   +   +  |
-    COMMIT   |  +   -   +   +  |
+     Request  |           Type of active backup lock                    |
+      type    | S0  S1  S2  S3  S4  F1  F2   D  TD  SD   DD  BL  AC  C  |
+    ----------+---------------------------------------------------------+
+    S0        |  -   -   -   -   -   +   +   +   +   +   +   +   +   +  |
+    S1        |  -   +   +   +   +   +   +   +   +   +   +   +   +   +  |
+    S2        |  -   +   +   +   +   +   +   -   +   +   +   +   +   +  |
+    S3        |  -   +   +   +   +   +   +   -   +   +   -   +   +   +  |
+    S4        |  -   +   +   +   +   +   +   -   +   -   -   +   +   -  |
+    FTWRL1    |  +   +   +   +   +   +   +   -   -   -   -   +   -   +  |
+    FTWRL2    |  +   +   +   +   +   +   +   -   -   -   -   +   -   -  |
+    D         |  +   -   -   -   -   -   -   +   +   +   +   +   +   +  |
+    TD        |  +   +   +   +   +   -   -   +   +   +   +   +   +   +  |
+    SD        |  +   +   +   +   -   -   -   +   +   +   +   +   +   +  |
+    DDL       |  +   +   +   -   -   -   -   +   +   +   +   -   +   +  |
+    BLOCK_DDL |  +   +   +   +   +   +   +   +   +   +   -   +   +   +  |
+    ALTER_COP |  +   +   +   +   +   -   -   +   +   +   +   +   +   +  |
+    COMMIT    |  +   +   +   +   -   +   -   +   +   +   +   +   +   +  |
 
   The second array specifies if particular type of request can be satisfied
   if there is already waiting request for the backup lock of certain type.
   I.e. it specifies what is the priority of different lock types.
 
-             |     Pending     |
-     Request |   backup lock   |
-      type   | F1  F2   S   C  |
-    ---------+-----------------+
-    FTWRL1   |  +   +   +   +  |
-    FTWRL2   |  +   +   +   +  |
-    STMT     |  -   -   +   +  |
-    COMMIT   |  +   -   +   +  |
+     Request  |               Pending backup lock                       |
+      type    | S0  S1  S2  S3  S4  F1  F2   D  TD  SD   DD  BL  AC  C  |
+    ----------+---------------------------------------------------------+
+    S0        |  -   -   -   -   -   +   +   +   +   +   +   +   +   +  |
+    S1        |  +   +   +   +   +   +   +   +   +   +   +   +   +   +  |
+    S2        |  +   +   +   +   +   +   +   +   +   +   +   +   +   +  |
+    S3        |  +   +   +   +   +   +   +   +   +   +   +   +   +   +  |
+    S4        |  +   +   +   +   +   +   +   +   +   +   +   +   +   +  |
+    FTWRL1    |  +   +   +   +   +   +   +   +   +   +   +   +   +   +  |
+    FTWRL2    |  +   +   +   +   +   +   +   +   +   +   +   +   +   +  |
+    D         |  +   -   -   -   -   -   -   +   +   +   +   +   +   +  |
+    TD        |  +   +   +   +   +   -   -   +   +   +   +   +   +   +  |
+    SD        |  +   +   +   +   -   -   -   +   +   +   +   +   +   +  |
+    DDL       |  +   +   +   -   -   -   -   +   +   +   +   -   +   +  |
+    BLOCK_DDL |  +   +   +   +   +   +   +   +   +   +   +   +   +   +  |
+    ALTER_COP |  +   +   +   +   +   -   -   +   +   +   +   +   +   +  |
+    COMMIT    |  +   +   +   +   -   +   -   +   +   +   +   +   +   +  |
 
   Here: "+" -- means that request can be satisfied
         "-" -- means that request can't be satisfied and should wait
 */
 
+/*
+  NOTE: If you add a new MDL_BACKUP_XXX level lock, you have to also add it
+  to MDL_BACKUP_START in the two arrays below!
+*/
+
 const MDL_lock::bitmap_t
 MDL_lock::MDL_backup_lock::m_granted_incompatible[MDL_BACKUP_END]=
 {
-  MDL_BIT(MDL_BACKUP_STMT),
-  MDL_BIT(MDL_BACKUP_STMT) | MDL_BIT(MDL_BACKUP_COMMIT),
+  /* MDL_BACKUP_START */
+  MDL_BIT(MDL_BACKUP_START) | MDL_BIT(MDL_BACKUP_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_DDL) | MDL_BIT(MDL_BACKUP_WAIT_COMMIT),
+  MDL_BIT(MDL_BACKUP_START),
+  MDL_BIT(MDL_BACKUP_START) | MDL_BIT(MDL_BACKUP_DML),
+  MDL_BIT(MDL_BACKUP_START) | MDL_BIT(MDL_BACKUP_DML) | MDL_BIT(MDL_BACKUP_DDL),
+  MDL_BIT(MDL_BACKUP_START) | MDL_BIT(MDL_BACKUP_DML) | MDL_BIT(MDL_BACKUP_SYS_DML) | MDL_BIT(MDL_BACKUP_DDL) | MDL_BIT(MDL_BACKUP_COMMIT),
+
+  /* MDL_BACKUP_FTWRL1 */
+  MDL_BIT(MDL_BACKUP_DML) | MDL_BIT(MDL_BACKUP_TRANS_DML) | MDL_BIT(MDL_BACKUP_SYS_DML) | MDL_BIT(MDL_BACKUP_DDL) | MDL_BIT(MDL_BACKUP_ALTER_COPY),
+  MDL_BIT(MDL_BACKUP_DML) | MDL_BIT(MDL_BACKUP_TRANS_DML) | MDL_BIT(MDL_BACKUP_SYS_DML) | MDL_BIT(MDL_BACKUP_DDL) | MDL_BIT(MDL_BACKUP_ALTER_COPY) | MDL_BIT(MDL_BACKUP_COMMIT),
+  /* MDL_BACKUP_DML */
+  MDL_BIT(MDL_BACKUP_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_DDL) | MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2),
+  MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2),
+  MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2),
+  /* MDL_BACKUP_DDL */
+  MDL_BIT(MDL_BACKUP_WAIT_DDL) | MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2) | MDL_BIT(MDL_BACKUP_BLOCK_DDL),
+  /* MDL_BACKUP_BLOCK_DDL */
+  MDL_BIT(MDL_BACKUP_DDL),
   MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2),
-  MDL_BIT(MDL_BACKUP_FTWRL2)
+  /* MDL_BACKUP_COMMIT */
+  MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL2)
 };
 
 
 const MDL_lock::bitmap_t
 MDL_lock::MDL_backup_lock::m_waiting_incompatible[MDL_BACKUP_END]=
 {
+  /* MDL_BACKUP_START */
+  MDL_BIT(MDL_BACKUP_START) | MDL_BIT(MDL_BACKUP_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_DDL) | MDL_BIT(MDL_BACKUP_WAIT_COMMIT),
   0,
   0,
+  0,
+  0,
+  /* MDL_BACKUP_FTWRL1 */
+  0,
+  0,
+
+  /* MDL_BACKUP_DML */
+  MDL_BIT(MDL_BACKUP_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_FLUSH) | MDL_BIT(MDL_BACKUP_WAIT_DDL) | MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2),
+  MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2),
+  MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2),
+  /* MDL_BACKUP_DDL */
+  MDL_BIT(MDL_BACKUP_WAIT_DDL) | MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2) | MDL_BIT(MDL_BACKUP_BLOCK_DDL),
+  /* MDL_BACKUP_BLOCK_DDL */
+  0,
   MDL_BIT(MDL_BACKUP_FTWRL1) | MDL_BIT(MDL_BACKUP_FTWRL2),
-  MDL_BIT(MDL_BACKUP_FTWRL2)
+  /* MDL_BACKUP_COMMIT */
+  MDL_BIT(MDL_BACKUP_WAIT_COMMIT) | MDL_BIT(MDL_BACKUP_FTWRL2)
 };
 
 
@@ -2454,8 +2523,14 @@ MDL_context::upgrade_shared_lock(MDL_ticket *mdl_ticket,
   /*
     Do nothing if already upgraded. Used when we FLUSH TABLE under
     LOCK TABLES and a table is listed twice in LOCK TABLES list.
+
+    In BACKUP namespace upgrade must always happen. Even though
+    MDL_BACKUP_START is not stronger than MDL_BACKUP_FLUSH from
+    has_stronger_or_equal_type(), the latter effectively blocks
+    new MDL_BACKUP_DML while the former doesn't.
   */
-  if (mdl_ticket->has_stronger_or_equal_type(new_type))
+  if (mdl_ticket->has_stronger_or_equal_type(new_type) &&
+      mdl_ticket->get_key()->mdl_namespace() != MDL_key::BACKUP)
     DBUG_RETURN(FALSE);
 
   mdl_xlock_request.init(&mdl_ticket->m_lock->key, new_type,
@@ -2856,9 +2931,13 @@ void MDL_ticket::downgrade_lock(enum_mdl_type type)
   if (m_type == type || !has_stronger_or_equal_type(type))
     return;
 
-  /* Only allow downgrade from EXCLUSIVE and SHARED_NO_WRITE. */
-  DBUG_ASSERT(m_type == MDL_EXCLUSIVE ||
-              m_type == MDL_SHARED_NO_WRITE);
+  /* Only allow downgrade in some specific known cases */
+  DBUG_ASSERT((get_key()->mdl_namespace() != MDL_key::BACKUP &&
+               (m_type == MDL_EXCLUSIVE ||
+                m_type == MDL_SHARED_NO_WRITE)) ||
+              (get_key()->mdl_namespace() == MDL_key::BACKUP &&
+               (m_type == MDL_BACKUP_DDL ||
+                m_type == MDL_BACKUP_WAIT_FLUSH)));
 
   mysql_prlock_wrlock(&m_lock->m_rwlock);
   /*