MDEV-29831 Galera crashes when running CoR for a locked table afterbb-10.11-midenok-MDEV-25292

	   setting the minimum memory for a user session.

Failure happens when finalize_atomic_replace() was already finished
and we removed the table from locked tables list.
finalize_locked_tables() doesn't know about that, it doesn't add back
last deleted lock because operation_failed == true.
reopen_tables() doesn't reopen table and as a result we have NULL in
pos_in_locked_tables->table.

The fix adds the knowledge that the locked_tables_count changed since
the start of the command. And if that happened we
add_back_last_deleted_lock(). That makes MDEV-29544 fix with
locked_tables_decremented deprecated.

Alternative fix would add atomic_replace_finished to Atomic_info and
updated it on successful finalize_atomic_replace(). Then the condition
would look like this:

  if (atomic_replace_finished || !operation_failed)
  {
    /*
      Add back the deleted table and re-created table as a locked table
      This should always work as we have a meta lock on the table.
    */
    thd->locked_tables_list.add_back_last_deleted_lock(pos_in_locked_tables);
  }

6 files changed, 63 insertions, 12 deletions

diff --git a/mysql-test/suite/galera/r/create.result b/mysql-test/suite/galera/r/create.result
index b434052727f..a219d60cc17 100644
--- a/mysql-test/suite/galera/r/create.result
+++ b/mysql-test/suite/galera/r/create.result
@@ -100,6 +100,26 @@ CREATE TABLE t0 (x INT,y INT);
 LOCK TABLES t0 WRITE;
 CREATE OR REPLACE TABLE t0 SELECT 1;
 DROP TABLE t0;
+#
+# MDEV-29831 Galera crashes when running CoR for a locked table after
+#            setting the minimum memory for a user session
+#
+connection node_1;
+SET @max_mem_save= @@session.max_session_mem_used;
+SET SESSION max_session_mem_used= 8192;
+CREATE TABLE t1 (a INT, KEY(a)) ENGINE=InnoDB;
+SET autocommit=OFF;
+CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, DATA INT) ENGINE=MEMORY;
+INSERT INTO t2(id) VALUES ('11');
+ERROR HY000: The MariaDB server is running with the --max-session-mem-used=8192 option so it cannot execute this statement
+ALTER TABLE t1 ADD COLUMN f3 INT NOT NULL DEFAULT 10;
+ERROR HY000: The MariaDB server is running with the --max-session-mem-used=8192 option so it cannot execute this statement
+LOCK TABLE t1 WRITE, t2 READ;
+CREATE OR REPLACE TABLE t1 SELECT 1;
+ERROR 70100: Query execution was interrupted
+UNLOCK TABLES;
+DROP TABLES t2, t1;
+SET SESSION max_session_mem_used= @max_mem_save;
 disconnect node_2;
 disconnect node_1;
 # End of tests
diff --git a/mysql-test/suite/galera/t/create.test b/mysql-test/suite/galera/t/create.test
index 7ebee7e2e47..9024dd46253 100644
--- a/mysql-test/suite/galera/t/create.test
+++ b/mysql-test/suite/galera/t/create.test
@@ -94,6 +94,26 @@ LOCK TABLES t0 WRITE;
 CREATE OR REPLACE TABLE t0 SELECT 1;
 DROP TABLE t0;
 
+--echo #
+--echo # MDEV-29831 Galera crashes when running CoR for a locked table after
+--echo #            setting the minimum memory for a user session
+--echo #
+--connection node_1
+SET @max_mem_save= @@session.max_session_mem_used;
+SET SESSION max_session_mem_used= 8192;
+CREATE TABLE t1 (a INT, KEY(a)) ENGINE=InnoDB;
+SET autocommit=OFF;
+CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, DATA INT) ENGINE=MEMORY;
+--error ER_OPTION_PREVENTS_STATEMENT
+INSERT INTO t2(id) VALUES ('11');
+--error ER_OPTION_PREVENTS_STATEMENT
+ALTER TABLE t1 ADD COLUMN f3 INT NOT NULL DEFAULT 10;
+LOCK TABLE t1 WRITE, t2 READ;
+--error ER_QUERY_INTERRUPTED
+CREATE OR REPLACE TABLE t1 SELECT 1;
+UNLOCK TABLES;
+DROP TABLES t2, t1;
+SET SESSION max_session_mem_used= @max_mem_save;
+
 --source include/galera_end.inc
 --echo # End of tests
-
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index ac1f921f62a..4b9602c2538 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -2627,6 +2627,7 @@ void Locked_tables_list::reset()
   m_locked_tables_last= &m_locked_tables;
   m_reopen_array= NULL;
   m_locked_tables_count= 0;
+  m_locked_tables_original= 0;
   some_table_marked_for_reopen= 0;
 }
 
diff --git a/sql/sql_class.h b/sql/sql_class.h
index a0d097c527f..35717b51a63 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -2100,6 +2100,7 @@ private:
     an exact number of TABLE objects.
   */
   uint m_locked_tables_count;
+  uint m_locked_tables_original;
 public:
   bool some_table_marked_for_reopen;
 
@@ -2108,6 +2109,7 @@ public:
     m_locked_tables_last(&m_locked_tables),
     m_reopen_array(NULL),
     m_locked_tables_count(0),
+    m_locked_tables_original(0),
     some_table_marked_for_reopen(0)
   {
     init_sql_alloc(key_memory_locked_table_list, &m_locked_tables_root,
@@ -2132,6 +2134,18 @@ public:
                     MYSQL_LOCK *lock);
   void add_back_last_deleted_lock(TABLE_LIST *dst_table_list);
   void mark_table_for_reopen(TABLE *table);
+  void start_command()
+  {
+    m_locked_tables_original= m_locked_tables_count;
+  }
+  uint locked_count() const
+  {
+    return m_locked_tables_count;
+  }
+  uint original_count() const
+  {
+    return m_locked_tables_original;
+  }
 };
 
 
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index e217e56e1cd..52ef3e624ad 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -3853,6 +3853,9 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt)
   }
 #endif /* WITH_WSREP */
 
+  if (thd->locked_tables_mode)
+    thd->locked_tables_list.start_command();
+
   switch (lex->sql_command) {
 
   case SQLCOM_SHOW_EVENTS:
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index cbe0707ec83..77c4f7a9b48 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -4428,7 +4428,6 @@ bool HA_CREATE_INFO::finalize_atomic_replace(THD *thd, TABLE_LIST *orig_table)
   LEX_CSTRING cpath;
   char path[FN_REFLEN + 1];
   cpath.str= path;
-  bool locked_tables_decremented= false;
 
   DBUG_ASSERT(is_atomic_replace());
 
@@ -4479,7 +4478,6 @@ bool HA_CREATE_INFO::finalize_atomic_replace(THD *thd, TABLE_LIST *orig_table)
                                 HA_EXTRA_PREPARE_FOR_DROP, NULL);
       table= NULL;
       orig_table->table= NULL;
-      locked_tables_decremented= true;
     }
 
     param.rename_flags= FN_TO_IS_TMP;
@@ -4517,8 +4515,6 @@ bool HA_CREATE_INFO::finalize_atomic_replace(THD *thd, TABLE_LIST *orig_table)
                                DDL_RENAME_PHASE_TRIGGER,
                                DDL_LOG_FLAG_FROM_IS_TMP))
       {
-        if (locked_tables_decremented)
-          thd->locked_tables_list.add_back_last_deleted_lock(pos_in_locked_tables);
         return true;
       }
 
@@ -4530,16 +4526,12 @@ bool HA_CREATE_INFO::finalize_atomic_replace(THD *thd, TABLE_LIST *orig_table)
           ddl_log_drop_table(ddl_log_state_rm, old_hton, &cpath,
                             &db, &table_name, 0))
       {
-        if (locked_tables_decremented)
-          thd->locked_tables_list.add_back_last_deleted_lock(pos_in_locked_tables);
         return true;
       }
 
       debug_crash_here("ddl_log_replace_broken_4");
       if (ddl_log_commit_atomic_block(ddl_log_state_rm))
       {
-        if (locked_tables_decremented)
-          thd->locked_tables_list.add_back_last_deleted_lock(pos_in_locked_tables);
         return true;
       }
 
@@ -4567,8 +4559,6 @@ bool HA_CREATE_INFO::finalize_atomic_replace(THD *thd, TABLE_LIST *orig_table)
                                 &dummy))
   {
     thd->variables.option_bits= option_bits_save;
-    if (locked_tables_decremented)
-      thd->locked_tables_list.add_back_last_deleted_lock(pos_in_locked_tables);
     return true;
   }
   thd->variables.option_bits= option_bits_save;
@@ -4630,6 +4620,8 @@ bool HA_CREATE_INFO::finalize_locked_tables(THD *thd, bool operation_failed)
   DBUG_ASSERT(pos_in_locked_tables);
   DBUG_ASSERT(thd->locked_tables_mode);
   DBUG_ASSERT(thd->variables.option_bits & OPTION_TABLE_LOCK);
+  const uint locked_count= thd->locked_tables_list.locked_count();
+  const uint orig_count= thd->locked_tables_list.original_count();
 
 #ifdef WITH_WSREP
   /*
@@ -4644,8 +4636,9 @@ bool HA_CREATE_INFO::finalize_locked_tables(THD *thd, bool operation_failed)
   }
 #endif
 
-  if (!operation_failed)
+  if (locked_count != orig_count)
   {
+    DBUG_ASSERT(locked_count < orig_count);
     /*
       Add back the deleted table and re-created table as a locked table
       This should always work as we have a meta lock on the table.