MDEV-22556: Incorrect result for window function when using encrypt-tmp-files=ON10.4-mdev22555

The issue here is that end_of_file for encrypted temporary IO_CACHE (used by filesort) is updated
using lseek.
Encryption adds storage overhead and hides it from the caller by recalculating offsets and lengths.
Two different IO_CACHE cannot possibly modify the same file
because the encryption key is randomly generated and stored in the IO_CACHE.
So when the tempfiles are encrypted DO NOT use lseek to change end_of_file.

Further observations about updating end_of_file using lseek
1) The end_of_file update is only used for binlog index files
2) The whole point is to update file length when the file was modified via a different file descriptor.
3) The temporary IO_CACHE files can never be modified via a different file descriptor.
4) For encrypted temporary IO_CACHE, end_of_file should not be updated with lseek

4 files changed, 51 insertions, 2 deletions

diff --git a/mysql-test/suite/encryption/r/tempfiles_encrypted.result b/mysql-test/suite/encryption/r/tempfiles_encrypted.result
new file mode 100644
index 00000000000..d08cb33e0fc
--- /dev/null
+++ b/mysql-test/suite/encryption/r/tempfiles_encrypted.result
@@ -0,0 +1,18 @@
+#
+# Tests when the temporary files are encrypted
+#
+select @@encrypt_tmp_files;
+@@encrypt_tmp_files
+1
+#
+# MDEV-22556: Incorrect result for window function when using encrypt-tmp-files=ON
+#
+set @save_sort_buffer_size=@@sort_buffer_size;
+set sort_buffer_size= 2000;
+create table t1( a DECIMAL(12,0) DEFAULT NULL, b VARCHAR(20) DEFAULT NULL, c DECIMAL(12,0) DEFAULT NULL)engine=INNODB;
+insert into t1 select seq, seq, seq from seq_1_to_5000;
+select count(*) from (select a, b, c, ROW_NUMBER() OVER (PARTITION BY a) FROM t1)q;
+count(*)
+5000
+set @@sort_buffer_size=@save_sort_buffer_size;
+drop table t1;
diff --git a/mysql-test/suite/encryption/t/tempfiles_encrypted.opt b/mysql-test/suite/encryption/t/tempfiles_encrypted.opt
new file mode 100644
index 00000000000..f515f7b8689
--- /dev/null
+++ b/mysql-test/suite/encryption/t/tempfiles_encrypted.opt
@@ -0,0 +1,5 @@
+--encrypt-tmp_files=ON
+--loose-file-key-management-encryption-algorithm=aes_cbc
+--file-key-management=FORCE
+--loose-file-key-management-filename=$MTR_SUITE_DIR/t/filekeys-data.enc
+--loose-file-key-management-filekey=FILE:$MTR_SUITE_DIR/t/filekeys-data.key
diff --git a/mysql-test/suite/encryption/t/tempfiles_encrypted.test b/mysql-test/suite/encryption/t/tempfiles_encrypted.test
new file mode 100644
index 00000000000..96b981c1c06
--- /dev/null
+++ b/mysql-test/suite/encryption/t/tempfiles_encrypted.test
@@ -0,0 +1,23 @@
+--echo #
+--echo # Tests when the temporary files are encrypted
+--echo #
+
+source include/have_file_key_management_plugin.inc;
+source include/have_sequence.inc;
+source include/have_innodb.inc;
+ 
+select @@encrypt_tmp_files;
+
+--echo #
+--echo # MDEV-22556: Incorrect result for window function when using encrypt-tmp-files=ON
+--echo #
+
+set @save_sort_buffer_size=@@sort_buffer_size;
+set sort_buffer_size= 2000;
+create table t1( a DECIMAL(12,0) DEFAULT NULL, b VARCHAR(20) DEFAULT NULL, c DECIMAL(12,0) DEFAULT NULL)engine=INNODB;
+insert into t1 select seq, seq, seq from seq_1_to_5000;
+select count(*) from (select a, b, c, ROW_NUMBER() OVER (PARTITION BY a) FROM t1)q;
+
+set @@sort_buffer_size=@save_sort_buffer_size;
+ 
+drop table t1;
diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c
index 54b89007b4c..51e8fe1a02f 100644
--- a/mysys/mf_iocache.c
+++ b/mysys/mf_iocache.c
@@ -504,8 +504,11 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type,
       if (info->type == WRITE_CACHE)
 	info->end_of_file= my_b_tell(info);
       else
-	info->end_of_file= mysql_file_seek(info->file, 0L, MY_SEEK_END,
-					   MYF(0));
+      {
+        if (!(info->myflags & MY_ENCRYPT))
+          info->end_of_file= mysql_file_seek(info->file, 0L,
+                                             MY_SEEK_END, MYF(0));
+      }
     }
     /* flush cache if we want to reuse it */
     if (!clear_cache && my_b_flush_io_cache(info,1))