MDEV-17614 INSERT on dup key update is replication unsafe

Problem:-
When mysql executes INSERT ON DUPLICATE KEY INSERT, the storage engine checks
if the inserted row would generate a duplicate key error. If yes, it returns
the existing row to mysql, mysql updates it and sends it back to the storage
engine.When the table has more than one unique or primary key, this statement
is sensitive to the order in which the storage engines checks the keys.
Depending on this order, the storage engine may determine different rows
to mysql, and hence mysql can update different rows.The order that the
storage engine checks keys is not deterministic. For example, InnoDB checks
keys in an order that depends on the order in which indexes were added to
the table. The first added index is checked first. So if master and slave
have added indexes in different orders, then slave may go out of sync.

Solution:-
Make INSERT...ON DUPLICATE KEY UPDATE unsafe while using stmt or mixed format
When there is more then one unique key.
Although there is two exception.
  1. Auto Increment key is not counted because Innodb will get gap lock for
    failed Insert and concurrent insert will get a next increment value. But if
    user supplies auto inc value it can be unsafe.
  2. Count only unique keys for which insertion is performed.

So this patch also addresses the bug id #72921

6 files changed, 225 insertions, 60 deletions

diff --git a/mysql-test/suite/rpl/r/rpl_known_bugs_detection.result b/mysql-test/suite/rpl/r/rpl_known_bugs_detection.result
index ea738b710fd..adef091ea3e 100644
--- a/mysql-test/suite/rpl/r/rpl_known_bugs_detection.result
+++ b/mysql-test/suite/rpl/r/rpl_known_bugs_detection.result
@@ -1,26 +1,6 @@
 call mtr.add_suppression("Unsafe statement written to the binary log using statement format");
 include/master-slave.inc
 [connection master]
-call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT.");
-CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b INT,
-UNIQUE(b));
-INSERT INTO t1(b) VALUES(1),(1),(2) ON DUPLICATE KEY UPDATE t1.b=10;
-SELECT * FROM t1;
-a	b
-1	10
-2	2
-call mtr.add_suppression("Slave SQL.*suffer.*http:..bugs.mysql.com.bug.php.id=24432");
-include/wait_for_slave_sql_error.inc [errno=1105]
-Last_SQL_Error = 'Error 'master may suffer from http://bugs.mysql.com/bug.php?id=24432 so slave stops; check error log on slave for more info' on query. Default database: 'test'. Query: 'INSERT INTO t1(b) VALUES(1),(1),(2) ON DUPLICATE KEY UPDATE t1.b=10''
-SELECT * FROM t1;
-a	b
-stop slave;
-include/wait_for_slave_to_stop.inc
-reset slave;
-reset master;
-drop table t1;
-start slave;
-include/wait_for_slave_to_start.inc
 CREATE TABLE t1 (
 id bigint(20) unsigned NOT NULL auto_increment,
 field_1 int(10) unsigned NOT NULL,
diff --git a/mysql-test/suite/rpl/r/rpl_mdev_17614.result b/mysql-test/suite/rpl/r/rpl_mdev_17614.result
new file mode 100644
index 00000000000..28de23e28c9
--- /dev/null
+++ b/mysql-test/suite/rpl/r/rpl_mdev_17614.result
@@ -0,0 +1,98 @@
+include/master-slave.inc
+[connection master]
+call mtr.add_suppression("Unsafe statement written to the binary log using statement format");
+CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY , b INT,
+UNIQUE(b),  c int) engine=innodb;
+INSERT INTO t1 VALUES (1, 1, 1);
+BEGIN;
+INSERT INTO t1 VALUES (2, 1, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+Warnings:
+Note	1592	Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. INSERT... ON DUPLICATE KEY UPDATE  on a table with more than one UNIQUE KEY is unsafe
+INSERT INTO t1 VALUES(2, 2, 3) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+Warnings:
+Note	1592	Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. INSERT... ON DUPLICATE KEY UPDATE  on a table with more than one UNIQUE KEY is unsafe
+COMMIT;
+SELECT * FROM t1;
+a	b	c
+1	1	2
+2	2	3
+include/wait_for_slave_sql_error.inc [errno=1062]
+Last_SQL_Error = 'Error 'Duplicate entry '1' for key 'b'' on query. Default database: 'test'. Query: 'INSERT INTO t1 VALUES (2, 1, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c)''
+#Different value from server
+SELECT * FROM t1;
+a	b	c
+1	1	1
+2	2	3
+stop slave;
+include/wait_for_slave_to_stop.inc
+reset slave;
+reset master;
+drop table t1;
+start slave;
+include/wait_for_slave_to_start.inc
+CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY auto_increment, b INT,
+UNIQUE(b),  c int) engine=innodb;
+INSERT INTO t1 VALUES (default, 1, 1);
+BEGIN;
+INSERT INTO t1 VALUES (default, 1, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+INSERT INTO t1 VALUES(default, 2, 3) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+COMMIT;
+SELECT * FROM t1;
+a	b	c
+1	1	2
+3	2	3
+#same data as master
+SELECT * FROM t1;
+a	b	c
+1	1	2
+3	2	3
+drop table t1;
+CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY, b INT,
+UNIQUE(b),  c int, d int ) engine=innodb;
+INSERT INTO t1 VALUES (1, 1, 1, 1);
+BEGIN;
+INSERT INTO t1 VALUES (2, NULL, 2, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+Warnings:
+Note	1592	Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. INSERT... ON DUPLICATE KEY UPDATE  on a table with more than one UNIQUE KEY is unsafe
+INSERT INTO t1 VALUES(3, NULL, 2, 3) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+Warnings:
+Note	1592	Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. INSERT... ON DUPLICATE KEY UPDATE  on a table with more than one UNIQUE KEY is unsafe
+COMMIT;
+SELECT * FROM t1;
+a	b	c	d
+1	1	1	1
+2	NULL	2	2
+3	NULL	2	3
+#same data as master
+SELECT * FROM t1;
+a	b	c	d
+1	1	1	1
+2	NULL	2	2
+3	NULL	2	3
+drop table t1;
+CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY auto_increment, b INT,
+UNIQUE(b),  c int) engine=innodb;
+INSERT INTO t1 VALUES (1, 1, 1);
+BEGIN;
+INSERT INTO t1 VALUES (2, 1, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+INSERT INTO t1 VALUES(2, 2, 3) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+COMMIT;
+SELECT * FROM t1;
+a	b	c
+1	1	2
+2	2	3
+include/wait_for_slave_sql_error.inc [errno=1062]
+Last_SQL_Error = 'Error 'Duplicate entry '1' for key 'b'' on query. Default database: 'test'. Query: 'INSERT INTO t1 VALUES (2, 1, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c)''
+#Different value from server
+SELECT * FROM t1;
+a	b	c
+1	1	1
+2	2	3
+stop slave;
+include/wait_for_slave_to_stop.inc
+reset slave;
+reset master;
+drop table t1;
+start slave;
+include/wait_for_slave_to_start.inc
+include/rpl_end.inc
diff --git a/mysql-test/suite/rpl/r/rpl_unsafe_statements.result b/mysql-test/suite/rpl/r/rpl_unsafe_statements.result
index 2efb3eba2b1..55ff09f6fad 100644
--- a/mysql-test/suite/rpl/r/rpl_unsafe_statements.result
+++ b/mysql-test/suite/rpl/r/rpl_unsafe_statements.result
@@ -1,5 +1,6 @@
 include/master-slave.inc
 [connection master]
+call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT");
 CREATE TABLE t1(id INT AUTO_INCREMENT, i INT, PRIMARY KEY (id)) ENGINE=INNODB;
 CREATE TABLE t2(id INT AUTO_INCREMENT, i INT, PRIMARY KEY (id)) ENGINE=INNODB;
 CREATE TRIGGER trig1 AFTER INSERT ON t1
@@ -43,9 +44,13 @@ include/diff_tables.inc [master:t1, slave:t1]
 DROP TABLE t1;
 CREATE TABLE t1(i INT, j INT, UNIQUE KEY(i), UNIQUE KEY(j)) ENGINE=INNODB;
 INSERT INTO t1 (i,j) VALUES (1,2) ON DUPLICATE KEY UPDATE j=j+1;
+Warnings:
+Note	1592	Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. INSERT... ON DUPLICATE KEY UPDATE  on a table with more than one UNIQUE KEY is unsafe
 START TRANSACTION;
 LOCK TABLES t1 WRITE;
 INSERT INTO t1 (i,j) VALUES (1,2) ON DUPLICATE KEY UPDATE j=j+1;
+Warnings:
+Note	1592	Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. INSERT... ON DUPLICATE KEY UPDATE  on a table with more than one UNIQUE KEY is unsafe
 UNLOCK TABLES;
 COMMIT;
 include/diff_tables.inc [master:t1, slave:t1]
diff --git a/mysql-test/suite/rpl/t/rpl_known_bugs_detection.test b/mysql-test/suite/rpl/t/rpl_known_bugs_detection.test
index ab263ece407..5ea056d5f14 100644
--- a/mysql-test/suite/rpl/t/rpl_known_bugs_detection.test
+++ b/mysql-test/suite/rpl/t/rpl_known_bugs_detection.test
@@ -14,45 +14,6 @@ source include/have_binlog_checksum_off.inc;
 
 source include/master-slave.inc;
 
-call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT.");
-
-#
-# This is to test that slave properly detects if
-# master may suffer from:
-# BUG#24432 "INSERT... ON DUPLICATE KEY UPDATE skips auto_increment values"
-# (i.e. on master, INSERT ON DUPLICATE KEY UPDATE is used and manipulates
-# an auto_increment column, and is binlogged statement-based).
-#
-
-# testcase with INSERT VALUES
-CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b INT,
-UNIQUE(b));
-sync_slave_with_master;
-connection master;
-INSERT INTO t1(b) VALUES(1),(1),(2) ON DUPLICATE KEY UPDATE t1.b=10;
-SELECT * FROM t1;
-connection slave;
-
-# show the error message 
-#1105 = ER_UNKNOWN_ERROR
---let $slave_sql_errno= 1105
---let $show_slave_sql_error= 1
-call mtr.add_suppression("Slave SQL.*suffer.*http:..bugs.mysql.com.bug.php.id=24432");
---source include/wait_for_slave_sql_error.inc
-# show that it was not replicated
-SELECT * FROM t1;
-
-# restart replication for the next testcase
-stop slave;
---source include/wait_for_slave_to_stop.inc
-reset slave;
-connection master;
-reset master;
-drop table t1;
-connection slave;
-start slave;
---source include/wait_for_slave_to_start.inc
-
 # testcase with INSERT SELECT
 connection master;
 CREATE TABLE t1 (
diff --git a/mysql-test/suite/rpl/t/rpl_mdev_17614.test b/mysql-test/suite/rpl/t/rpl_mdev_17614.test
new file mode 100644
index 00000000000..9b86c8c15b5
--- /dev/null
+++ b/mysql-test/suite/rpl/t/rpl_mdev_17614.test
@@ -0,0 +1,121 @@
+source include/have_debug.inc;
+source include/have_innodb.inc;
+-- source include/have_binlog_format_statement.inc
+source include/master-slave.inc;
+# MDEV-17614
+# INSERT on dup key update is replication unsafe
+# There can be three case
+#  1. 2 unique key, Replication is unsafe.
+#  2. 2 unique key , with one auto increment key, Safe to replicate because Innodb will acquire gap lock
+#  3. n no of unique keys (n>1) but insert is only in 1 unique key
+#  4. 2 unique key , with one auto increment key(but user gives auto inc value), unsafe to replicate
+
+# Case 1
+call mtr.add_suppression("Unsafe statement written to the binary log using statement format");
+CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY , b INT,
+UNIQUE(b),  c int) engine=innodb;
+sync_slave_with_master;
+connection master;
+INSERT INTO t1 VALUES (1, 1, 1);
+BEGIN;
+INSERT INTO t1 VALUES (2, 1, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+  --connection master1
+  INSERT INTO t1 VALUES(2, 2, 3) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+--connection master
+COMMIT;
+SELECT * FROM t1;
+--connection slave
+# show the error message
+--let $slave_sql_errno= 1062
+--let $show_slave_sql_error= 1
+--source include/wait_for_slave_sql_error.inc
+--echo #Different value from server
+SELECT * FROM t1;
+
+# restart replication for the next testcase
+stop slave;
+--source include/wait_for_slave_to_stop.inc
+reset slave;
+connection master;
+reset master;
+drop table t1;
+connection slave;
+start slave;
+--source include/wait_for_slave_to_start.inc
+# Case 2
+--connection master
+CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY auto_increment, b INT,
+UNIQUE(b),  c int) engine=innodb;
+sync_slave_with_master;
+connection master;
+INSERT INTO t1 VALUES (default, 1, 1);
+BEGIN;
+INSERT INTO t1 VALUES (default, 1, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+  --connection master1
+  INSERT INTO t1 VALUES(default, 2, 3) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+--connection master
+COMMIT;
+SELECT * FROM t1;
+--sync_slave_with_master
+--echo #same data as master
+SELECT * FROM t1;
+
+connection master;
+drop table t1;
+--sync_slave_with_master
+
+# Case 3
+--connection master
+CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY, b INT,
+UNIQUE(b),  c int, d int ) engine=innodb;
+sync_slave_with_master;
+connection master;
+INSERT INTO t1 VALUES (1, 1, 1, 1);
+BEGIN;
+INSERT INTO t1 VALUES (2, NULL, 2, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+  --connection master1
+  INSERT INTO t1 VALUES(3, NULL, 2, 3) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+--connection master
+COMMIT;
+SELECT * FROM t1;
+--sync_slave_with_master
+--echo #same data as master
+SELECT * FROM t1;
+connection master;
+drop table t1;
+--sync_slave_with_master
+
+# Case 4
+--connection master
+CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY auto_increment, b INT,
+UNIQUE(b),  c int) engine=innodb;
+sync_slave_with_master;
+connection master;
+INSERT INTO t1 VALUES (1, 1, 1);
+BEGIN;
+INSERT INTO t1 VALUES (2, 1, 2) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+  --connection master1
+  INSERT INTO t1 VALUES(2, 2, 3) ON DUPLICATE KEY UPDATE b=VALUES(b), c=VALUES(c);
+--connection master
+COMMIT;
+SELECT * FROM t1;
+--connection slave
+# show the error message
+--let $slave_sql_errno= 1062
+--let $show_slave_sql_error= 1
+--source include/wait_for_slave_sql_error.inc
+--echo #Different value from server
+SELECT * FROM t1;
+
+# restart replication for the next testcase
+stop slave;
+--source include/wait_for_slave_to_stop.inc
+reset slave;
+connection master;
+reset master;
+drop table t1;
+connection slave;
+start slave;
+--source include/wait_for_slave_to_start.inc
+
+--source include/rpl_end.inc
diff --git a/mysql-test/suite/rpl/t/rpl_unsafe_statements.test b/mysql-test/suite/rpl/t/rpl_unsafe_statements.test
index cbb4b54a220..aa0bd076398 100644
--- a/mysql-test/suite/rpl/t/rpl_unsafe_statements.test
+++ b/mysql-test/suite/rpl/t/rpl_unsafe_statements.test
@@ -24,7 +24,7 @@
 --source include/have_innodb.inc
 --source include/have_binlog_format_mixed.inc
 --source include/master-slave.inc
-
+call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT");
 # Case-1: BINLOG_STMT_UNSAFE_AUTOINC_COLUMNS
 #  Statement is unsafe because it invokes a trigger or a
 #  stored function that inserts into an AUTO_INCREMENT column.