Bug#12547647 UPDATE LOGGING COULD EXCEED LOG PAGE SIZE - take 2

The original fix was accidentally pushed to mysql-5.1 after the 5.1.59
clone-off in bzr revision id
marko.makela@oracle.com-20110829081642-z0w992a0mrc62s6w with thne fix
of Bug#12704861 Corruption after a crash during BLOB update.
It was pushed separately to mysql-5.5 in bzr revision id
marko.makela@oracle.com-20110901184804-2901f6qmuro3jas8.

trx_undo_report_row_operation(): If the page for which the undo log
was too big was empty, commit and start the mini-transaction before
acquiring the rollback segment mutex and freeing the undo page. This
is necessary, because the mini-transaction may be holding lower-order
latches in the levels SYNC_FSP and SYNC_FSP_PAGE.

trx_undo_erase_page_end(): Erase also empty pages, because
trx_undo_report_row_operation() needs to commit the mini-transaction
before freeing the empty page.

rb:756 approved by Sunny Bains

2 files changed, 20 insertions, 17 deletions

diff --git a/storage/innodb_plugin/ChangeLog b/storage/innodb_plugin/ChangeLog
index 96b6a47085a..328cafff58f 100644
--- a/storage/innodb_plugin/ChangeLog
+++ b/storage/innodb_plugin/ChangeLog
@@ -1,3 +1,8 @@
+2011-09-06	The InnoDB Team
+
+	* include/trx0undo.h, trx/trx0rec.c, trx/trx0undo.c:
+	Fix Bug#12547647 UPDATE LOGGING COULD EXCEED LOG PAGE SIZE
+
 2011-08-29	The InnoDB Team
 
 	* btr/btr0btr.c, btr/btr0cur.c, fsp/fsp0fsp.c,
@@ -7,11 +12,6 @@
 	Fix Bug#12704861 Corruption after a crash during BLOB update
 	and other regressions from the fix of Bug#12612184
 
-2011-08-23	The InnoDB Team
-
-	* include/trx0undo.h, trx/trx0rec.c, trx/trx0undo.c:
-	Fix Bug#12547647 UPDATE LOGGING COULD EXCEED LOG PAGE SIZE
-
 2011-08-15	The InnoDB Team
 
 	* btr/btr0btr.c, btr/btr0cur.c, btr/btr0pcur.c, btr/btr0sea.c,
diff --git a/storage/innodb_plugin/trx/trx0rec.c b/storage/innodb_plugin/trx/trx0rec.c
index a729a39d0cc..2db98e029df 100644
--- a/storage/innodb_plugin/trx/trx0rec.c
+++ b/storage/innodb_plugin/trx/trx0rec.c
@@ -1099,7 +1099,7 @@ trx_undo_rec_get_partial_row(
 /***********************************************************************//**
 Erases the unused undo log page end.
 @return TRUE if the page contained something, FALSE if it was empty */
-static __attribute__((nonnull, warn_unused_result))
+static __attribute__((nonnull))
 ibool
 trx_undo_erase_page_end(
 /*====================*/
@@ -1110,16 +1110,11 @@ trx_undo_erase_page_end(
 
 	first_free = mach_read_from_2(undo_page + TRX_UNDO_PAGE_HDR
 				      + TRX_UNDO_PAGE_FREE);
-	if (first_free == TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE) {
-		/* This was an empty page to begin with.
-		Do nothing here; the caller should free the page. */
-		return(FALSE);
-	}
 	memset(undo_page + first_free, 0xff,
 	       (UNIV_PAGE_SIZE - FIL_PAGE_DATA_END) - first_free);
 
 	mlog_write_initial_log_record(undo_page, MLOG_UNDO_ERASE_END, mtr);
-	return(TRUE);
+	return(first_free != TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE);
 }
 
 /***********************************************************//**
@@ -1141,11 +1136,7 @@ trx_undo_parse_erase_page_end(
 		return(ptr);
 	}
 
-	if (!trx_undo_erase_page_end(page, mtr)) {
-		/* The function trx_undo_erase_page_end() should not
-		have done anything to an empty page. */
-		ut_ad(0);
-	}
+	trx_undo_erase_page_end(page, mtr);
 
 	return(ptr);
 }
@@ -1290,6 +1281,18 @@ trx_undo_report_row_operation(
 				undo page. Discard the freshly allocated
 				page and return an error. */
 
+				/* When we remove a page from an undo
+				log, this is analogous to a
+				pessimistic insert in a B-tree, and we
+				must reserve the counterpart of the
+				tree latch, which is the rseg
+				mutex. We must commit the mini-transaction
+				first, because it may be holding lower-level
+				latches, such as SYNC_FSP and SYNC_FSP_PAGE. */
+
+				mtr_commit(&mtr);
+				mtr_start(&mtr);
+
 				mutex_enter(&rseg->mutex);
 				trx_undo_free_last_page(trx, undo, &mtr);
 				mutex_exit(&rseg->mutex);