BUG#25032066 PREPARED TRANSACTION SHOULD NOT BE ROLLED BACK BY HIGH PRIORITY TRANSACTION

Problem :
---------
1. delete_all_rows() and rnd_init() are not returning error
after async rollback in 5.7. This results in assert in
innodb in next call.

2. High priority transaction is rolling back prepared transaction.
This is because TRX_FORCE_ROLLBACK_DISABLE is getting set only for
first entry [TrxInInnoDB].

Solution :
----------
1. return DB_FORCED_ABORT error after rollback.
2. check and disable rollback in TrxInInnodb::enter always.

Reviewed-by: Sunny Bains <sunny.bains@oracle.com>

RB: 13777

2 files changed, 24 insertions, 28 deletions

diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index c1c7330943e..e039722deba 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -10471,13 +10471,7 @@ ha_innobase::rnd_init(
 	bool	scan)	/*!< in: true if table/index scan FALSE otherwise */
 {
 	TrxInInnoDB	trx_in_innodb(m_prebuilt->trx);
-
-	if (trx_in_innodb.is_aborted()) {
-
-		return(innobase_rollback(ht, m_user_thd, false));
-	}
-
-	int	err;
+	int		err;
 
 	/* Store the active index value so that we can restore the original
 	value after a scan */
diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h
index b08148578dc..9e5248e7b86 100644
--- a/storage/innobase/include/trx0trx.h
+++ b/storage/innobase/include/trx0trx.h
@@ -1468,10 +1468,30 @@ private:
 			return;
 		}
 
-		/* Avoid excessive mutex acquire/release */
-
 		ut_ad(!is_async_rollback(trx));
 
+		/* If it hasn't already been marked for async rollback.
+		and it will be committed/rolled back. */
+		if (disable) {
+
+			trx_mutex_enter(trx);
+			if (!is_forced_rollback(trx)
+			    && is_started(trx)
+			    && !trx_is_autocommit_non_locking(trx)) {
+
+				ut_ad(trx->killed_by == 0);
+
+				/* This transaction has crossed the point of
+				no return and cannot be rolled back
+				asynchronously now. It must commit or rollback
+				synhronously. */
+
+				trx->in_innodb |= TRX_FORCE_ROLLBACK_DISABLE;
+			}
+			trx_mutex_exit(trx);
+		}
+
+		/* Avoid excessive mutex acquire/release */
 		++trx->in_depth;
 
 		/* If trx->in_depth is greater than 1 then
@@ -1489,25 +1509,7 @@ private:
 
 		wait(trx);
 
-		ut_ad((trx->in_innodb & TRX_FORCE_ROLLBACK_MASK)
-		      < (TRX_FORCE_ROLLBACK_MASK - 1));
-
-		/* If it hasn't already been marked for async rollback.
-		and it will be committed/rolled back. */
-
-		if (!is_forced_rollback(trx)
-		    && disable
-		    && is_started(trx)
-		    && !trx_is_autocommit_non_locking(trx)) {
-
-			ut_ad(trx->killed_by == 0);
-
-			/* This transaction has crossed the point of no
-			return and cannot be rolled back asynchronously
-			now. It must commit or rollback synhronously. */
-
-			trx->in_innodb |= TRX_FORCE_ROLLBACK_DISABLE;
-		}
+		ut_ad((trx->in_innodb & TRX_FORCE_ROLLBACK_MASK) == 0);
 
 		++trx->in_innodb;