MDEV-10259 mysqld crash with certain statement length and...

order with Galera and encrypt-tmp-files=1

Problem:- If trans_cache (IO_CACHE) uses encrypted tmp file
then on next DML server will crash.

Case:-
 Lets take a case , we have a table t1 , We try to do 2 inserts in t1
  1. A really long insert so that trans_cache has to use temp_file
  2. Just a small insert

Analysis:- Actually server crashes from inside of galera
library.
/lib64/libc.so.6(abort+0x175)[0x7fb5ba779dc5]
/usr/lib64/galera/libgalera_smm.so(_ZN6galera3FSMINS_9TrxHandle5State...
mysys/stacktrace.c:247(my_print_stacktrace)[0x7fb5a714940e]
sql/signal_handler.cc:160(handle_fatal_signal)[0x7fb5a715c1bd]
sql/wsrep_hton.cc:257(wsrep_rollback)[0x7fb5bcce923a]
sql/wsrep_hton.cc:268(wsrep_rollback)[0x7fb5bcce9368]
sql/handler.cc:1658(ha_rollback_trans(THD*, bool))[0x7fb5bcd4f41a]
sql/handler.cc:1483(ha_commit_trans(THD*, bool))[0x7fb5bcd4f804]

but actual issue is not in galera but in mariadb, because for 2nd
insert we should never call rollback. We are calling rollback because
log_and_order fails it fails because write_cache fails , It fails
because after reinit_io_cache(trans_cache) , my_b_bytes_in_cache says 0
so we look into tmp_file for data , which is obviously wrong since temp
was used for previous insert and it no longer exist.
wsrep_write_cache_inc() reads the IO_CACHE in a loop, filling it with
my_b_fill() until it returns "0 bytes read". Later
MYSQL_BIN_LOG::write_cache() does the same.  wsrep_write_cache_inc()
assumes that reading a zero bytes past EOF leaves the old data in the
cache

Solution:- There is two issue in my_b_encr_read
1st we should never equal read_end to info->buffer. I mean this
does not make sense read_end should always point to end of buffer.
2nd For most of the case(apart from async IO_CACHE) info->pos_in_file
should be equal to info->buffer position wrt to temp file , since
in this case we are not changing info->buffer it should remain
unchanged.

1 files changed, 71 insertions, 1 deletions

diff --git a/unittest/sql/mf_iocache-t.cc b/unittest/sql/mf_iocache-t.cc
index 31f98562521..1b04f8eb0d3 100644
--- a/unittest/sql/mf_iocache-t.cc
+++ b/unittest/sql/mf_iocache-t.cc
@@ -187,10 +187,76 @@ void mdev9044()
   close_cached_file(&info);
 }
 
+/* 2 Reads (with my_b_fill) in cache makes second read to fail */
+void mdev10259()
+{
+  int res;
+  uchar buf[200];
+  memset(buf, FILL, sizeof(buf));
+
+  diag("MDEV-10259- mysqld crash with certain statement length and order with"
+         " Galera and encrypt-tmp-files=1");
+
+  init_io_cache_encryption();
+
+  res= open_cached_file(&info, 0, 0, CACHE_SIZE, 0);
+  ok(res == 0, "open_cached_file" INFO_TAIL);
+
+  res= my_b_write(&info, buf, sizeof(buf));
+  ok(res == 0 && info.pos_in_file == 0, "200 write" INFO_TAIL);
+
+  res= my_b_flush_io_cache(&info, 1);
+  ok(res == 0, "flush" INFO_TAIL);
+
+  ulong saved_pos=  my_b_tell(&info);
+  res= reinit_io_cache(&info, READ_CACHE, 0, 0, 0);
+  ok(res == 0, "reinit READ_CACHE" INFO_TAIL);
+
+  res= my_b_fill(&info);
+  ok(res == 200, "fill" INFO_TAIL);
+
+  res= my_b_fill(&info);
+  ok(res == 0, "fill" INFO_TAIL);
+
+  res= my_b_fill(&info);
+  ok(res == 0, "fill" INFO_TAIL);
+
+  res= reinit_io_cache(&info, WRITE_CACHE, saved_pos, 0, 0);
+  ok(res == 0, "reinit WRITE_CACHE" INFO_TAIL);
+
+  res= reinit_io_cache(&info, READ_CACHE, 0, 0, 0);
+  ok(res == 0, "reinit READ_CACHE" INFO_TAIL);
+
+  ok(200 == my_b_bytes_in_cache(&info),"my_b_bytes_in_cache == 200");
+
+  res= my_b_fill(&info);
+  ok(res == 0, "fill" INFO_TAIL);
+
+  res= my_b_fill(&info);
+  ok(res == 0, "fill" INFO_TAIL);
+
+  res= my_b_fill(&info);
+  ok(res == 0, "fill" INFO_TAIL);
+
+  res= reinit_io_cache(&info, WRITE_CACHE, saved_pos, 0, 0);
+  ok(res == 0, "reinit WRITE_CACHE" INFO_TAIL);
+
+  res= reinit_io_cache(&info, READ_CACHE, 0, 0, 0);
+  ok(res == 0, "reinit READ_CACHE" INFO_TAIL);
+
+  ok(200 == my_b_bytes_in_cache(&info),"my_b_bytes_in_cache == 200");
+
+  res= my_b_read(&info, buf, sizeof(buf)) || data_bad(buf, sizeof(buf));
+  ok(res == 0 && info.pos_in_file == 0, "large read" INFO_TAIL);
+
+  close_cached_file(&info);
+
+}
+
 int main(int argc __attribute__((unused)),char *argv[])
 {
   MY_INIT(argv[0]);
-  plan(29);
+  plan(46);
 
   /* temp files with and without encryption */
   encrypt_tmp_files= 1;
@@ -202,6 +268,10 @@ int main(int argc __attribute__((unused)),char *argv[])
   /* regression tests */
   mdev9044();
 
+  encrypt_tmp_files= 1;
+  mdev10259();
+  encrypt_tmp_files= 0;
+
   my_end(0);
   return exit_status();
 }