MDEV-22206 Assertion "heap_no == ULINT_UNDEFINED" in trx0i_s.cc

commit d09aec7a15ab4be539d2b110742af544fa6b139f (MDEV-19940)
caused a regression. We made wait_lock_get_heap_no() return
uint16_t instead of ulint, and we mostly replaced the previous
magic value ULINT_UNDEFINED with 0. But, we failed to adjust
some assertions. Furthermore, 0 is a valid although rare value
for record locks. (Record locks can be temporarily stored on
page infimum in some operations that involve multiple leaf pages.)

Let us use 0xFFFF as the magic value. Valid heap numbers
are limited to less than 9362 = innodb_page_size/(5+1+1)
when using a minimal 1-byte PRIMARY KEY and a
secondary index on a NULL or '' column.

1 files changed, 13 insertions, 13 deletions

diff --git a/storage/innobase/trx/trx0i_s.cc b/storage/innobase/trx/trx0i_s.cc
index 27329448091..fb797f05f79 100644
--- a/storage/innobase/trx/trx0i_s.cc
+++ b/storage/innobase/trx/trx0i_s.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2007, 2015, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2019, MariaDB Corporation.
+Copyright (c) 2017, 2020, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -180,12 +180,12 @@ code in handler/i_s.cc. */
 trx_i_s_cache_t*	trx_i_s_cache = &trx_i_s_cache_static;
 
 /** @return the heap number of a record lock
-@retval 0 for table locks */
-static uint16_t wait_lock_get_heap_no(const lock_t* lock)
+@retval 0xFFFF for table locks */
+static uint16_t wait_lock_get_heap_no(const lock_t *lock)
 {
-	return lock_get_type(lock) == LOCK_REC
-		? static_cast<uint16_t>(lock_rec_find_set_bit(lock))
-		: uint16_t{0};
+  return lock_get_type(lock) == LOCK_REC
+    ? static_cast<uint16_t>(lock_rec_find_set_bit(lock))
+    : uint16_t{0xFFFF};
 }
 
 /*******************************************************************//**
@@ -820,7 +820,7 @@ fold_lock(
 /*======*/
 	const lock_t*	lock,	/*!< in: lock object to fold */
 	ulint		heap_no)/*!< in: lock's record number
-				or ULINT_UNDEFINED if the lock
+				or 0xFFFF if the lock
 				is a table lock */
 {
 #ifdef TEST_LOCK_FOLD_ALWAYS_DIFFERENT
@@ -832,7 +832,7 @@ fold_lock(
 
 	switch (lock_get_type(lock)) {
 	case LOCK_REC:
-		ut_a(heap_no != ULINT_UNDEFINED);
+		ut_a(heap_no != 0xFFFF);
 
 		ret = ut_fold_ulint_pair((ulint) lock->trx->id,
 					 lock->un_member.rec_lock.space);
@@ -847,7 +847,7 @@ fold_lock(
 		/* this check is actually not necessary for continuing
 		correct operation, but something must have gone wrong if
 		it fails. */
-		ut_a(heap_no == ULINT_UNDEFINED);
+		ut_a(heap_no == 0xFFFF);
 
 		ret = (ulint) lock_get_table_id(lock);
 
@@ -870,7 +870,7 @@ locks_row_eq_lock(
 	const i_s_locks_row_t*	row,	/*!< in: innodb_locks row */
 	const lock_t*		lock,	/*!< in: lock object */
 	ulint			heap_no)/*!< in: lock's record number
-					or ULINT_UNDEFINED if the lock
+					or 0xFFFF if the lock
 					is a table lock */
 {
 	ut_ad(i_s_locks_row_validate(row));
@@ -879,7 +879,7 @@ locks_row_eq_lock(
 #else
 	switch (lock_get_type(lock)) {
 	case LOCK_REC:
-		ut_a(heap_no != ULINT_UNDEFINED);
+		ut_a(heap_no != 0xFFFF);
 
 		return(row->lock_trx_id == lock->trx->id
 		       && row->lock_space == lock->un_member.rec_lock.space
@@ -890,7 +890,7 @@ locks_row_eq_lock(
 		/* this check is actually not necessary for continuing
 		correct operation, but something must have gone wrong if
 		it fails. */
-		ut_a(heap_no == ULINT_UNDEFINED);
+		ut_a(heap_no == 0xFFFF);
 
 		return(row->lock_trx_id == lock->trx->id
 		       && row->lock_table_id == lock_get_table_id(lock));
@@ -914,7 +914,7 @@ search_innodb_locks(
 	trx_i_s_cache_t*	cache,	/*!< in: cache */
 	const lock_t*		lock,	/*!< in: lock to search for */
 	uint16_t		heap_no)/*!< in: lock's record number
-					or ULINT_UNDEFINED if the lock
+					or 0xFFFF if the lock
 					is a table lock */
 {
 	i_s_hash_chain_t*	hash_chain;