MDEV-17891 Assertion failure upon attempt to replace into a full table

Problem: Assertion `transactional_table || !changed ||
thd->transaction.stmt.modified_non_trans_table' failed due REPLACE into a
versioned table.

It is not specific to system versioning/pertitioning/heap, but this
combination makes it much easier to reproduce.

The thing is to make first ha_update_row call succeed to make
info->deleted != 0. And then make REPLACE fail by any reason.

In this scenario we overflow versioned partition, so next ha_update_row
succeeds, but corresponding ha_write_row fails to insert history record.

Fix: modified_non_trans_table is set in one missed place

5 files changed, 114 insertions, 0 deletions

diff --git a/mysql-test/suite/versioning/r/partition.result b/mysql-test/suite/versioning/r/partition.result
index 06aa0ded32d..b152cbd89e9 100644
--- a/mysql-test/suite/versioning/r/partition.result
+++ b/mysql-test/suite/versioning/r/partition.result
@@ -530,6 +530,7 @@ set timestamp=1523466002.799571;
 insert into t1 values (11),(12);
 set timestamp=1523466004.169435;
 delete from t1 where pk in (11, 12);
+set timestamp= default;
 #
 # MDEV-18136 Server crashes in Item_func_dyncol_create::prepare_arguments
 #
@@ -682,6 +683,41 @@ create table t1 (a int) with system versioning partition by system_time
 (partition p1 history, partition pn current);
 alter table t1 add partition (partition p2);
 ERROR HY000: Wrong partitioning type, expected type: `SYSTEM_TIME`
+# MDEV-17891 Assertion failures in select_insert::abort_result_set and
+#            mysql_load upon attempt to replace into a full table
+set @@max_heap_table_size= 1024*1024;
+create or replace table t1 (
+pk integer auto_increment,
+primary key (pk),
+f varchar(45000)
+) with system versioning engine=memory
+partition by system_time interval 1 year (partition p1 history,
+partition pn current);
+# fill the table until full
+insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
+insert into t1 (f) select f from t1;
+ERROR HY000: The table 't1' is full
+# leave space for exactly one record in current partition
+delete from t1 where pk = 1;
+# copy all data into history partition
+replace into t1 select * from t1;
+replace into t1 select * from t1;
+ERROR HY000: The table 't1' is full
+create or replace table t1 (
+pk integer auto_increment,
+primary key (pk),
+f varchar(45000)
+) with system versioning engine=memory
+partition by system_time interval 1 year (partition p1 history,
+partition pn current);
+insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
+select * from t1 into outfile 'load.data';
+load data infile 'load.data' replace into table t1;
+load data infile 'load.data' replace into table t1;
+ERROR HY000: The table 't1' is full
+load data infile 'load.data' replace into table t1;
+ERROR HY000: The table 't1' is full
+set @@max_heap_table_size= 1048576;
 drop table t1;
 #
 # MDEV-22178 Assertion `info->alias.str' failed in partition_info::check_partition_info instead of ER_VERS_WRONG_PARTS
diff --git a/mysql-test/suite/versioning/r/replace.result b/mysql-test/suite/versioning/r/replace.result
index bda61f118b0..57a992cce49 100644
--- a/mysql-test/suite/versioning/r/replace.result
+++ b/mysql-test/suite/versioning/r/replace.result
@@ -48,3 +48,16 @@ INSERT INTO t1 () VALUES (),(),(),(),(),();
 UPDATE IGNORE t1 SET f = 1;
 REPLACE t1 SELECT * FROM t1;
 DROP TABLE t1;
+# MDEV-22540 ER_DUP_ENTRY upon REPLACE or Assertion failed
+set timestamp=1589245268.41934;
+create table t1 (a int primary key) with system versioning;
+insert into t1 values (1),(2);
+connect  con1,localhost,root,,test;
+set timestamp=1589245268.52093;
+replace into t1 values (1),(2);
+connection default;
+replace into t1 values (1),(2);
+connection con1;
+replace into t1 values (1),(2);
+ERROR 23000: Duplicate entry '1' for key 'PRIMARY'
+drop table t1;
diff --git a/mysql-test/suite/versioning/t/partition.test b/mysql-test/suite/versioning/t/partition.test
index 454e4068e15..d131bcfcc21 100644
--- a/mysql-test/suite/versioning/t/partition.test
+++ b/mysql-test/suite/versioning/t/partition.test
@@ -482,6 +482,7 @@ set timestamp=1523466002.799571;
 insert into t1 values (11),(12);
 set timestamp=1523466004.169435;
 delete from t1 where pk in (11, 12);
+set timestamp= default;
 
 --echo #
 --echo # MDEV-18136 Server crashes in Item_func_dyncol_create::prepare_arguments
@@ -649,7 +650,51 @@ create table t1 (a int) with system versioning partition by system_time
 --error ER_PARTITION_WRONG_TYPE
 alter table t1 add partition (partition p2);
 
+--echo # MDEV-17891 Assertion failures in select_insert::abort_result_set and
+--echo #            mysql_load upon attempt to replace into a full table
+
+--let $max_heap_table_size_orig= `select @@max_heap_table_size;`
+set @@max_heap_table_size= 1024*1024;
+create or replace table t1 (
+  pk integer auto_increment,
+  primary key (pk),
+  f varchar(45000)
+) with system versioning engine=memory
+  partition by system_time interval 1 year (partition p1 history,
+                                            partition pn current);
+
+--echo # fill the table until full
+insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
+--error ER_RECORD_FILE_FULL
+insert into t1 (f) select f from t1;
+--echo # leave space for exactly one record in current partition
+delete from t1 where pk = 1;
+--echo # copy all data into history partition
+replace into t1 select * from t1;
+--error ER_RECORD_FILE_FULL
+replace into t1 select * from t1;
+
+create or replace table t1 (
+  pk integer auto_increment,
+  primary key (pk),
+  f varchar(45000)
+) with system versioning engine=memory
+  partition by system_time interval 1 year (partition p1 history,
+                                            partition pn current);
+
+insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
+
+select * from t1 into outfile 'load.data';
+load data infile 'load.data' replace into table t1;
+--error ER_RECORD_FILE_FULL
+load data infile 'load.data' replace into table t1;
+--error ER_RECORD_FILE_FULL
+load data infile 'load.data' replace into table t1;
+
 # Cleanup
+--let $datadir= `select @@datadir`
+--remove_file $datadir/test/load.data
+eval set @@max_heap_table_size= $max_heap_table_size_orig;
 drop table t1;
 
 --echo #
diff --git a/mysql-test/suite/versioning/t/replace.test b/mysql-test/suite/versioning/t/replace.test
index 392c0ffcf35..83489f4a4b9 100644
--- a/mysql-test/suite/versioning/t/replace.test
+++ b/mysql-test/suite/versioning/t/replace.test
@@ -59,4 +59,22 @@ UPDATE IGNORE t1 SET f = 1;
 REPLACE t1 SELECT * FROM t1;
 DROP TABLE t1;
 
+--echo # MDEV-22540 ER_DUP_ENTRY upon REPLACE or Assertion failed
+set timestamp=1589245268.41934;
+create table t1 (a int primary key) with system versioning;
+insert into t1 values (1),(2);
+
+--connect (con1,localhost,root,,test)
+set timestamp=1589245268.52093;
+replace into t1 values (1),(2);
+
+--connection default
+replace into t1 values (1),(2);
+
+--connection con1
+--error ER_DUP_ENTRY
+replace into t1 values (1),(2);
+
+drop table t1;
+
 --source suite/versioning/common_finish.inc
diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc
index 8115b3c6397..9db6acf73f8 100644
--- a/sql/sql_insert.cc
+++ b/sql/sql_insert.cc
@@ -1959,6 +1959,8 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info)
           if (likely(!error))
           {
             info->deleted++;
+            if (!table->file->has_transactions())
+              thd->transaction.stmt.modified_non_trans_table= TRUE;
             if (table->versioned(VERS_TIMESTAMP))
             {
               store_record(table, record[2]);