A fix and a test case for

Bug#41756 "Strange error messages about locks from InnoDB".

In JT_EQ_REF (join_read_key()) access method,
don't try to unlock rows in the handler, unless certain that
a) they were locked
b) they are not used.

Unlocking of rows is done by the logic of the nested join loop,
and is unaware of the possible caching that the access method may
have. This could lead to double unlocking, when a row
was unlocked first after reading into the cache, and then
when taken from cache, as well as to unlocking of rows which
were actually used (but taken from cache).

Delegate part of the unlocking logic to the access method,
and in JT_EQ_REF count how many times a record was actually
used in the join. Unlock it only if it's usage count is 0.

Implemented review comments.


mysql-test/r/bug41756.result:
  Add result file (Bug#41756)
mysql-test/t/bug41756-master.opt:
  Use --innodb-locks-unsafe-for-binlog, as in 5.0 just
  using read_committed isolation is not sufficient to 
  reproduce the bug.
mysql-test/t/bug41756.test:
  Add a test file (Bug#41756)
sql/item_subselect.cc:
  Complete struct READ_RECORD initialization with a new
  member to unlock records.
sql/records.cc:
  Extend READ_RECORD API with a method to unlock read records.
sql/sql_select.cc:
  In JT_EQ_REF (join_read_key()) access method,
  don't try to unlock rows in the handler, unless certain that
  a) they were locked
  b) they are not used.
sql/sql_select.h:
  Add members to TABLE_REF to count TABLE_REF buffer usage count.
sql/structs.h:
  Update declarations.

3 files changed, 432 insertions, 0 deletions

diff --git a/mysql-test/r/bug41756.result b/mysql-test/r/bug41756.result
new file mode 100644
index 00000000000..7c97b2ac690
--- /dev/null
+++ b/mysql-test/r/bug41756.result
@@ -0,0 +1,277 @@
+#
+# Bug#41756 Strange error messages about locks from InnoDB
+#
+drop table if exists t1;
+# In the default transaction isolation mode, and/or with
+# innodb_locks_unsafe_for_binlog=OFF, handler::unlock_row()
+# in InnoDB does nothing.
+# Thus in order to reproduce the condition that led to the
+# warning, one needs to relax isolation by either
+# setting a weaker tx_isolation value, or by turning on
+# the unsafe replication switch.
+# For testing purposes, choose to tweak the isolation level,
+# since it's settable at runtime, unlike
+# innodb_locks_unsafe_for_binlog, which is
+# only a command-line switch.
+#
+set @@session.tx_isolation="read-committed";
+# Prepare data. We need a table with a unique index,
+# for join_read_key to be used. The other column
+# allows to control what passes WHERE clause filter.
+create table t1 (a int primary key, b int) engine=innodb;
+# Let's make sure t1 has sufficient amount of rows
+# to exclude JT_ALL access method when reading it,
+# i.e. make sure that JT_EQ_REF(a) is always preferred.
+insert into t1 values (1,1), (2,null), (3,1), (4,1),
+(5,1), (6,1), (7,1), (8,1), (9,1), (10,1),
+(11,1), (12,1);
+#
+# Demonstrate that for the SELECT statement
+# used later in the test JT_EQ_REF access method is used.
+#
+explain
+select 1 from t1 natural join (select 2 as a, 1 as b union all
+select 2 as a, 2 as b) as t2 for update;
+id	1
+select_type	PRIMARY
+table	<derived2>
+type	ALL
+possible_keys	NULL
+key	NULL
+key_len	NULL
+ref	NULL
+rows	2
+Extra	
+id	1
+select_type	PRIMARY
+table	t1
+type	eq_ref
+possible_keys	PRIMARY
+key	PRIMARY
+key_len	4
+ref	t2.a
+rows	1
+Extra	Using where
+id	2
+select_type	DERIVED
+table	NULL
+type	NULL
+possible_keys	NULL
+key	NULL
+key_len	NULL
+ref	NULL
+rows	NULL
+Extra	No tables used
+id	3
+select_type	UNION
+table	NULL
+type	NULL
+possible_keys	NULL
+key	NULL
+key_len	NULL
+ref	NULL
+rows	NULL
+Extra	No tables used
+id	NULL
+select_type	UNION RESULT
+table	<union2,3>
+type	ALL
+possible_keys	NULL
+key	NULL
+key_len	NULL
+ref	NULL
+rows	NULL
+Extra	
+#
+# Demonstrate that the reported SELECT statement
+# no longer produces warnings.
+#
+select 1 from t1 natural join (select 2 as a, 1 as b union all
+select 2 as a, 2 as b) as t2 for update;
+1
+commit;
+# 
+# Demonstrate that due to lack of inter-sweep "reset" function,
+# we keep some non-matching records locked, even though we know
+# we could unlock them.
+# To do that, show that if there is only one distinct value
+# for a in t2 (a=2), we will keep record (2,null) in t1 locked.
+# But if we add another value for "a" to t2, say 6,
+# join_read_key cache will be pruned at least once, 
+# and thus record (2, null) in t1 will get unlocked.
+#
+begin;
+select 1 from t1 natural join (select 2 as a, 1 as b union all
+select 2 as a, 2 as b) as t2 for update;
+1
+#
+# Switching to connection con1
+# We should be able to delete all records from t1 except (2, null),
+# since they were not locked.
+begin;
+delete from t1 where a in (1,3,4);
+delete from t1 where a in (5,6,7);
+delete from t1 where a in (8,9,10);
+delete from t1 where a in (11,12);
+# 
+# Record (2, null) is locked. This is actually unnecessary, 
+# because the previous select returned no rows. 
+# Just demonstrate the effect.
+#
+delete from t1;
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+rollback;
+#
+# Switching to connection default
+#
+# Show that the original contents of t1 is intact:
+select * from t1;
+a	b
+1	1
+2	NULL
+3	1
+4	1
+5	1
+6	1
+7	1
+8	1
+9	1
+10	1
+11	1
+12	1
+commit;
+#
+# Have a one more record in t2 to show that 
+# if join_read_key cache is purned, the current
+# row under the cursor is unlocked (provided, this row didn't 
+# match the partial WHERE clause, of course).
+# Sic: the result of this test dependent on the order of retrieval
+# of records --echo # from the derived table, if !
+# We use DELETE to disable the JOIN CACHE. This DELETE modifies no
+# records. It also should leave no InnoDB row locks.
+#
+begin;
+delete t1.* from t1 natural join (select 2 as a, 2 as b union all
+select 0 as a, 0 as b) as t2;
+# Demonstrate that nothing was deleted form t1
+select * from t1;
+a	b
+1	1
+2	NULL
+3	1
+4	1
+5	1
+6	1
+7	1
+8	1
+9	1
+10	1
+11	1
+12	1
+#
+# Switching to connection con1
+begin;
+# Since there is another distinct record in the derived table
+# the previous matching record in t1 -- (2,null) -- was unlocked.
+delete from t1;
+# We will need the contents of the table again.
+rollback;
+select * from t1;
+a	b
+1	1
+2	NULL
+3	1
+4	1
+5	1
+6	1
+7	1
+8	1
+9	1
+10	1
+11	1
+12	1
+commit;
+#
+# Switching to connection default
+commit;
+begin;
+#
+# Before this patch, we could wrongly unlock a record
+# that was cached and later used in a join. Demonstrate that
+# this is no longer the case.
+# Sic: this test is also order-dependent (i.e. the
+# the bug would show up only if the first record in the union
+# is retreived and processed first.
+#
+# Verify that JT_EQ_REF is used.
+explain
+select 1 from t1 natural join (select 3 as a, 3 as b union all
+select 3 as a, 1 as b) as t2 for update;
+id	1
+select_type	PRIMARY
+table	t1
+type	ALL
+possible_keys	PRIMARY
+key	NULL
+key_len	NULL
+ref	NULL
+rows	1
+Extra	
+id	1
+select_type	PRIMARY
+table	<derived2>
+type	ALL
+possible_keys	NULL
+key	NULL
+key_len	NULL
+ref	NULL
+rows	2
+Extra	Using where
+id	2
+select_type	DERIVED
+table	NULL
+type	NULL
+possible_keys	NULL
+key	NULL
+key_len	NULL
+ref	NULL
+rows	NULL
+Extra	No tables used
+id	3
+select_type	UNION
+table	NULL
+type	NULL
+possible_keys	NULL
+key	NULL
+key_len	NULL
+ref	NULL
+rows	NULL
+Extra	No tables used
+id	NULL
+select_type	UNION RESULT
+table	<union2,3>
+type	ALL
+possible_keys	NULL
+key	NULL
+key_len	NULL
+ref	NULL
+rows	NULL
+Extra	
+# Lock the record.
+select 1 from t1 natural join (select 3 as a, 3 as b union all
+select 3 as a, 2 as b) as t2 for update;
+1
+# Switching to connection con1
+#
+# We should not be able to delete record (3,1) from t1,
+# (previously it was possible).
+#
+delete from t1 where a=3;
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+# Switching to connection default
+commit;
+set @@session.tx_isolation=default;
+drop table t1;
+#
+# End of 5.0 tests
+#
diff --git a/mysql-test/t/bug41756-master.opt b/mysql-test/t/bug41756-master.opt
new file mode 100644
index 00000000000..50c7c0e0ffa
--- /dev/null
+++ b/mysql-test/t/bug41756-master.opt
@@ -0,0 +1,2 @@
+--innodb_lock_wait_timeout=1
+--innodb-locks-unsafe-for-binlog
diff --git a/mysql-test/t/bug41756.test b/mysql-test/t/bug41756.test
new file mode 100644
index 00000000000..0d635a6a9f6
--- /dev/null
+++ b/mysql-test/t/bug41756.test
@@ -0,0 +1,153 @@
+--source include/have_innodb.inc
+--echo #
+--echo # Bug#41756 Strange error messages about locks from InnoDB
+--echo #
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+--echo # In the default transaction isolation mode, and/or with
+--echo # innodb_locks_unsafe_for_binlog=OFF, handler::unlock_row()
+--echo # in InnoDB does nothing.
+--echo # Thus in order to reproduce the condition that led to the
+--echo # warning, one needs to relax isolation by either
+--echo # setting a weaker tx_isolation value, or by turning on
+--echo # the unsafe replication switch.
+--echo # For testing purposes, choose to tweak the isolation level,
+--echo # since it's settable at runtime, unlike
+--echo # innodb_locks_unsafe_for_binlog, which is
+--echo # only a command-line switch.
+--echo #
+set @@session.tx_isolation="read-committed";
+
+--echo # Prepare data. We need a table with a unique index,
+--echo # for join_read_key to be used. The other column
+--echo # allows to control what passes WHERE clause filter.
+create table t1 (a int primary key, b int) engine=innodb;
+--echo # Let's make sure t1 has sufficient amount of rows
+--echo # to exclude JT_ALL access method when reading it,
+--echo # i.e. make sure that JT_EQ_REF(a) is always preferred.
+insert into t1 values (1,1), (2,null), (3,1), (4,1),
+                      (5,1), (6,1), (7,1), (8,1), (9,1), (10,1),
+                      (11,1), (12,1);
+--echo #
+--echo # Demonstrate that for the SELECT statement
+--echo # used later in the test JT_EQ_REF access method is used.
+--echo #
+--vertical_results
+explain
+select 1 from t1 natural join (select 2 as a, 1 as b union all
+                               select 2 as a, 2 as b) as t2 for update;
+--horizontal_results
+--echo #
+--echo # Demonstrate that the reported SELECT statement
+--echo # no longer produces warnings.
+--echo #
+select 1 from t1 natural join (select 2 as a, 1 as b union all
+                               select 2 as a, 2 as b) as t2 for update;
+commit;
+--echo # 
+--echo # Demonstrate that due to lack of inter-sweep "reset" function,
+--echo # we keep some non-matching records locked, even though we know
+--echo # we could unlock them.
+--echo # To do that, show that if there is only one distinct value
+--echo # for a in t2 (a=2), we will keep record (2,null) in t1 locked.
+--echo # But if we add another value for "a" to t2, say 6,
+--echo # join_read_key cache will be pruned at least once, 
+--echo # and thus record (2, null) in t1 will get unlocked.
+--echo #
+begin;
+select 1 from t1 natural join (select 2 as a, 1 as b union all
+                               select 2 as a, 2 as b) as t2 for update;
+connect (con1,localhost,root,,);
+--echo #
+--echo # Switching to connection con1
+connection con1;
+--echo # We should be able to delete all records from t1 except (2, null),
+--echo # since they were not locked.
+begin;
+delete from t1 where a in (1,3,4);
+delete from t1 where a in (5,6,7);
+delete from t1 where a in (8,9,10);
+delete from t1 where a in (11,12);
+--echo # 
+--echo # Record (2, null) is locked. This is actually unnecessary, 
+--echo # because the previous select returned no rows. 
+--echo # Just demonstrate the effect.
+--echo #
+--error ER_LOCK_WAIT_TIMEOUT
+delete from t1;
+rollback;
+--echo #
+--echo # Switching to connection default
+connection default;
+--echo #
+--echo # Show that the original contents of t1 is intact:
+select * from t1;
+commit;
+--echo #
+--echo # Have a one more record in t2 to show that 
+--echo # if join_read_key cache is purned, the current
+--echo # row under the cursor is unlocked (provided, this row didn't 
+--echo # match the partial WHERE clause, of course).
+--echo # Sic: the result of this test dependent on the order of retrieval
+--echo # of records --echo # from the derived table, if !
+--echo # We use DELETE to disable the JOIN CACHE. This DELETE modifies no
+--echo # records. It also should leave no InnoDB row locks.
+--echo #
+begin;
+delete t1.* from t1 natural join (select 2 as a, 2 as b union all
+                                  select 0 as a, 0 as b) as t2;
+--echo # Demonstrate that nothing was deleted form t1
+select * from t1;
+--echo #
+--echo # Switching to connection con1
+connection con1;
+begin;
+--echo # Since there is another distinct record in the derived table
+--echo # the previous matching record in t1 -- (2,null) -- was unlocked.
+delete from t1;
+--echo # We will need the contents of the table again.
+rollback;
+select * from t1;
+commit;
+--echo #
+--echo # Switching to connection default
+connection default;
+commit;
+begin;
+--echo #
+--echo # Before this patch, we could wrongly unlock a record
+--echo # that was cached and later used in a join. Demonstrate that
+--echo # this is no longer the case.
+--echo # Sic: this test is also order-dependent (i.e. the
+--echo # the bug would show up only if the first record in the union
+--echo # is retreived and processed first.
+--echo #
+--echo # Verify that JT_EQ_REF is used.
+--vertical_results
+explain
+select 1 from t1 natural join (select 3 as a, 3 as b union all
+                               select 3 as a, 1 as b) as t2 for update;
+--horizontal_results
+--echo # Lock the record.
+select 1 from t1 natural join (select 3 as a, 3 as b union all
+                               select 3 as a, 2 as b) as t2 for update;
+--echo # Switching to connection con1
+connection con1;
+--echo #
+--echo # We should not be able to delete record (3,1) from t1,
+--echo # (previously it was possible).
+--echo #
+--error ER_LOCK_WAIT_TIMEOUT
+delete from t1 where a=3;
+--echo # Switching to connection default
+connection default;
+commit;
+
+disconnect con1;
+set @@session.tx_isolation=default;
+drop table t1;
+
+--echo #
+--echo # End of 5.0 tests
+--echo #