MDEV-15030 Add ASAN instrumentation to trx_t Pool

Pool::mem_free(): Poison the freed memory. Assert that it was
fully initialized, because the reuse of trx_t objects will
assume that the objects were previously initialized.

Pool::~Pool(), Pool::get(): Unpoison the allocated memory,
and mark it initialized.

trx_free(): After invoking Pool::mem_free(), unpoison
trx_t::mutex and trx_t::undo_mutex, because MutexMonitor
will access these even for freed trx_t objects.

2 files changed, 27 insertions, 1 deletions

diff --git a/storage/innobase/include/ut0pool.h b/storage/innobase/include/ut0pool.h
index f60608bf6c6..5b5afe7cd80 100644
--- a/storage/innobase/include/ut0pool.h
+++ b/storage/innobase/include/ut0pool.h
@@ -86,6 +86,11 @@ struct Pool {
 		for (Element* elem = m_start; elem != m_last; ++elem) {
 
 			ut_ad(elem->m_pool == this);
+			/* Unpoison the memory for AddressSanitizer */
+			MEM_UNDEFINED(&elem->m_type, sizeof elem->m_type);
+			/* Declare the contents as initialized for Valgrind;
+			we checked this in mem_free(). */
+			UNIV_MEM_VALID(&elem->m_type, sizeof elem->m_type);
 			Factory::destroy(&elem->m_type);
 		}
 
@@ -122,7 +127,18 @@ struct Pool {
 
 		m_lock_strategy.exit();
 
-		return(elem != NULL ? &elem->m_type : 0);
+		if (elem) {
+			/* Unpoison the memory for AddressSanitizer */
+			MEM_UNDEFINED(&elem->m_type, sizeof elem->m_type);
+			/* Declare the memory initialized for Valgrind.
+			The trx_t that are released to the pool are
+			actually initialized; we checked that by
+			UNIV_MEM_ASSERT_RW() in mem_free() below. */
+			UNIV_MEM_VALID(&elem->m_type, sizeof elem->m_type);
+			return &elem->m_type;
+		}
+
+		return NULL;
 	}
 
 	/** Add the object to the pool.
@@ -133,8 +149,10 @@ struct Pool {
 		byte*		p = reinterpret_cast<byte*>(ptr + 1);
 
 		elem = reinterpret_cast<Element*>(p - sizeof(*elem));
+		UNIV_MEM_ASSERT_RW(&elem->m_type, sizeof elem->m_type);
 
 		elem->m_pool->put(elem);
+		MEM_NOACCESS(&elem->m_type, sizeof elem->m_type);
 	}
 
 protected:
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index 1c058a3ede5..deacc244154 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -445,6 +445,14 @@ trx_free(trx_t*& trx)
 	ut_ad(trx->will_lock == 0);
 
 	trx_pools->mem_free(trx);
+	/* Unpoison the memory for innodb_monitor_set_option;
+	it is operating also on the freed transaction objects. */
+	MEM_UNDEFINED(&trx->mutex, sizeof trx->mutex);
+	MEM_UNDEFINED(&trx->undo_mutex, sizeof trx->undo_mutex);
+	/* Declare the contents as initialized for Valgrind;
+	we checked that it was initialized in trx_pools->mem_free(trx). */
+	UNIV_MEM_VALID(&trx->mutex, sizeof trx->mutex);
+	UNIV_MEM_VALID(&trx->undo_mutex, sizeof trx->undo_mutex);
 
 	trx = NULL;
 }