MDEV-26864 Race condition between transaction commit and undo log truncation

trx_commit_in_memory(): Do not release the rseg reference before
trx_undo_commit_cleanup() has been invoked and the current transaction
is truly done with the rollback segment. The purpose of the reference
count is to prevent data races with trx_purge_truncate_history().

This is based on
mysql/mysql-server@ac79aa1522f33e6eb912133a81fa2614db764c9c.

1 files changed, 10 insertions, 6 deletions

diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index 3e558a7181d..6669d15a31b 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -1775,12 +1775,7 @@ trx_commit_in_memory(
 
 	ut_ad(!trx->rsegs.m_redo.update_undo);
 
-	if (trx_rseg_t*	rseg = trx->rsegs.m_redo.rseg) {
-		mutex_enter(&rseg->mutex);
-		ut_ad(rseg->trx_ref_count > 0);
-		--rseg->trx_ref_count;
-		mutex_exit(&rseg->mutex);
-
+	if (ut_d(trx_rseg_t* rseg =) trx->rsegs.m_redo.rseg) {
 		if (trx_undo_t*& insert = trx->rsegs.m_redo.insert_undo) {
 			ut_ad(insert->rseg == rseg);
 			trx_undo_commit_cleanup(insert, false);
@@ -1849,6 +1844,15 @@ trx_commit_in_memory(
 
 	ut_ad(!trx->rsegs.m_noredo.undo);
 
+	/* Only after trx_undo_commit_cleanup() it is safe to release
+	our rseg reference. */
+	if (trx_rseg_t* rseg = trx->rsegs.m_redo.rseg) {
+		mutex_enter(&rseg->mutex);
+		ut_ad(rseg->trx_ref_count > 0);
+		--rseg->trx_ref_count;
+		mutex_exit(&rseg->mutex);
+	}
+
 	/* Free all savepoints, starting from the first. */
 	trx_named_savept_t*	savep = UT_LIST_GET_FIRST(trx->trx_savepoints);