MDEV-14290 Binlog rotate crashes when two commit_checkpoint_notify capable engines.

The crash (sometimes assert) in MYSQL_BIN_LOG::mark_xid_done was caused by a
fact that log.cc:binlog_background_thread_queue could become a cyclic list.
This possibility becomes real with two checkpoint capable engines that
may execute TC_LOG_BINLOG::commit_checkpoint_notify() in succession before
binlog_background thread gets control and eventually finds a freed memory
while otherwise endlessly looping in while(queue).

It is fixed with counting the notificaion kind instead of en-listing the same notificaion kind in commit_checkpoint_notify as formerly. The while(queue) of binlog background thread is refined to pay attention to the new counter. In effectno more access to free memory is possible.

5 files changed, 246 insertions, 3 deletions

diff --git a/sql/log.cc b/sql/log.cc
index 93d7a00c840..3aff3b2c40d 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -3583,6 +3583,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name,
         new_xid_list_entry->binlog_name= name_mem;
         new_xid_list_entry->binlog_name_len= len;
         new_xid_list_entry->xid_count= 0;
+        new_xid_list_entry->notify_count= 0;
 
         /*
           Find the name for the Initial binlog checkpoint.
@@ -9678,9 +9679,20 @@ void
 TC_LOG_BINLOG::commit_checkpoint_notify(void *cookie)
 {
   xid_count_per_binlog *entry= static_cast<xid_count_per_binlog *>(cookie);
+  bool found_entry= false;
   mysql_mutex_lock(&LOCK_binlog_background_thread);
-  entry->next_in_queue= binlog_background_thread_queue;
-  binlog_background_thread_queue= entry;
+  /* count the same notification kind from different engines */
+  for (xid_count_per_binlog *link= binlog_background_thread_queue;
+       link && !found_entry; link= link->next_in_queue)
+  {
+    if ((found_entry= (entry == link)))
+      entry->notify_count++;
+  }
+  if (!found_entry)
+  {
+    entry->next_in_queue= binlog_background_thread_queue;
+    binlog_background_thread_queue= entry;
+  }
   mysql_cond_signal(&COND_binlog_background_thread);
   mysql_mutex_unlock(&LOCK_binlog_background_thread);
 }
@@ -9775,13 +9787,16 @@ binlog_background_thread(void *arg __attribute__((unused)))
       );
     while (queue)
     {
+      long count= queue->notify_count;
       THD_STAGE_INFO(thd, stage_binlog_processing_checkpoint_notify);
       DEBUG_SYNC(thd, "binlog_background_thread_before_mark_xid_done");
       /* Set the thread start time */
       thd->set_time();
       /* Grab next pointer first, as mark_xid_done() may free the element. */
       next= queue->next_in_queue;
-      mysql_bin_log.mark_xid_done(queue->binlog_id, true);
+      queue->notify_count= 0;
+      for (long i= 0; i <= count; i++)
+        mysql_bin_log.mark_xid_done(queue->binlog_id, true);
       queue= next;
 
       DBUG_EXECUTE_IF("binlog_background_checkpoint_processed",
diff --git a/sql/log.h b/sql/log.h
index 8484d54c5c2..97e257829b7 100644
--- a/sql/log.h
+++ b/sql/log.h
@@ -580,6 +580,7 @@ public:
     ulong binlog_id;
     /* Total prepared XIDs and pending checkpoint requests in this binlog. */
     long xid_count;
+    long notify_count;
     /* For linking in requests to the binlog background thread. */
     xid_count_per_binlog *next_in_queue;
     xid_count_per_binlog();   /* Give link error if constructor used. */
diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_binlog_xid_count.result b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_binlog_xid_count.result
new file mode 100644
index 00000000000..9b46a5b5227
--- /dev/null
+++ b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_binlog_xid_count.result
@@ -0,0 +1,204 @@
+CREATE TABLE `t` (
+`a` text DEFAULT NULL
+) ENGINE=ROCKSDB;
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+INSERT INTO t SET a=repeat('a', 4096);
+INSERT INTO t SET a=repeat('a', 4096/2);
+DROP TABLE t;
diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_binlog_xid_count-master.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_binlog_xid_count-master.opt
new file mode 100644
index 00000000000..ed50a8a3deb
--- /dev/null
+++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_binlog_xid_count-master.opt
@@ -0,0 +1,3 @@
+--innodb --max-binlog-size=4096
+
+
diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_binlog_xid_count.test b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_binlog_xid_count.test
new file mode 100644
index 00000000000..7667f153cde
--- /dev/null
+++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_binlog_xid_count.test
@@ -0,0 +1,20 @@
+--source include/have_rocksdb.inc
+--source include/have_binlog_format_row.inc
+
+CREATE TABLE `t` (
+  `a` text DEFAULT NULL
+) ENGINE=ROCKSDB;
+
+
+--let $size=`SELECT @@GLOBAL.max_binlog_size`
+--let $loop_cnt= 100
+while ($loop_cnt)
+{
+  --eval INSERT INTO t SET a=repeat('a', $size)
+  --eval INSERT INTO t SET a=repeat('a', $size/2)
+
+  --dec $loop_cnt
+}
+
+# Cleanup
+DROP TABLE t;