MDEV-22390: Assertion `m_next_rec_ptr >= m_rawmem' failed in Filesort_buffer::spaceleft | SIGSEGV in __memmove_avx_unaligned_erms from my_b_write

Make sure that the sort_buffer that is allocated has atleast space for MERGEBUFF2 keys.
The issue here was that the record length is quite high and sort buffer size is very small,
due to which we end up with zero number of keys in the sort buffer. The Sort_param::max_keys_per_buffer
was zero in such a case, due to which we were flushing empty sort_buffer to the disk.

4 files changed, 78 insertions, 2 deletions

diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result
index d35012e0de5..ffb37c9309f 100644
--- a/mysql-test/r/order_by.result
+++ b/mysql-test/r/order_by.result
@@ -3302,3 +3302,57 @@ a	b	c
 SET @@sort_buffer_size= @save_sort_buffer_size;
 SET @@max_sort_length= @save_max_sort_length;
 DROP TABLE t1;
+#
+# MDEV-22390: Assertion `m_next_rec_ptr >= m_rawmem' failed in Filesort_buffer::spaceleft |
+# SIGSEGV in __memmove_avx_unaligned_erms from my_b_write (on optimized)
+#
+SET @save_max_sort_length= @@max_sort_length;
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET @save_max_length_for_sort_data= @@max_length_for_sort_data;
+SET max_sort_length=8;
+SET sort_buffer_size=1024;
+SET max_length_for_sort_data=7000;
+CREATE TABLE t1(a VARCHAR(64), b VARCHAR(2048))DEFAULT CHARSET=utf8;
+INSERT INTO t1 SELECT seq,seq from seq_1_to_100;
+ANALYZE FORMAT=JSON SELECT * FROM t1 ORDER BY a LIMIT 5;
+ANALYZE
+{
+  "query_block": {
+    "select_id": 1,
+    "r_loops": 1,
+    "r_total_time_ms": "REPLACED",
+    "read_sorted_file": {
+      "r_rows": 5,
+      "filesort": {
+        "r_loops": 1,
+        "r_total_time_ms": "REPLACED",
+        "r_limit": 5,
+        "r_used_priority_queue": false,
+        "r_output_rows": 35,
+        "r_sort_passes": 1,
+        "r_buffer_size": "REPLACED",
+        "table": {
+          "table_name": "t1",
+          "access_type": "ALL",
+          "r_loops": 1,
+          "rows": 100,
+          "r_rows": 100,
+          "r_total_time_ms": "REPLACED",
+          "filtered": 100,
+          "r_filtered": 100
+        }
+      }
+    }
+  }
+}
+SELECT * FROM t1 ORDER BY a LIMIT 5;
+a	b
+1	1
+10	10
+100	100
+11	11
+12	12
+SET max_sort_length= @save_max_sort_length;
+SET sort_buffer_size= @save_sort_buffer_size;
+SET max_length_for_sort_data= @save_max_length_for_sort_data;
+DROP TABLE t1;
diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test
index 2fbacb10b68..3a30e0b6c76 100644
--- a/mysql-test/t/order_by.test
+++ b/mysql-test/t/order_by.test
@@ -2174,3 +2174,25 @@ SELECT * FROM t1 ORDER BY a,b;
 SET @@sort_buffer_size= @save_sort_buffer_size;
 SET @@max_sort_length= @save_max_sort_length;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-22390: Assertion `m_next_rec_ptr >= m_rawmem' failed in Filesort_buffer::spaceleft |
+--echo # SIGSEGV in __memmove_avx_unaligned_erms from my_b_write (on optimized)
+--echo #
+
+SET @save_max_sort_length= @@max_sort_length;
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET @save_max_length_for_sort_data= @@max_length_for_sort_data;
+SET max_sort_length=8;
+SET sort_buffer_size=1024;
+# needed to make sure we use addon fields
+SET max_length_for_sort_data=7000;
+CREATE TABLE t1(a VARCHAR(64), b VARCHAR(2048))DEFAULT CHARSET=utf8;
+INSERT INTO t1 SELECT seq,seq from seq_1_to_100;
+--source include/analyze-format.inc
+ANALYZE FORMAT=JSON SELECT * FROM t1 ORDER BY a LIMIT 5;
+SELECT * FROM t1 ORDER BY a LIMIT 5;
+SET max_sort_length= @save_max_sort_length;
+SET sort_buffer_size= @save_sort_buffer_size;
+SET max_length_for_sort_data= @save_max_length_for_sort_data;
+DROP TABLE t1;
diff --git a/sql/filesort.cc b/sql/filesort.cc
index bb3e73343ad..90edd39b1da 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -256,7 +256,8 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
     while (memory_available >= min_sort_memory)
     {
       ulonglong keys= memory_available / (param.rec_length + sizeof(char*));
-      param.max_keys_per_buffer= (uint) MY_MIN(num_rows, keys);
+      param.max_keys_per_buffer= MY_MAX(MERGEBUFF2,
+                                        (uint) MY_MIN(num_rows, keys));
       if (table_sort.get_sort_keys())
       {
         // If we have already allocated a buffer, it better have same size!
diff --git a/sql/filesort_utils.cc b/sql/filesort_utils.cc
index f1a164dd9fe..287edddf619 100644
--- a/sql/filesort_utils.cc
+++ b/sql/filesort_utils.cc
@@ -97,7 +97,6 @@ uchar **Filesort_buffer::alloc_sort_buffer(uint num_records, uint record_length)
   if (m_idx_array.is_null())
   {
     sort_buff_sz= ((size_t)num_records) * (record_length + sizeof(uchar*));
-    set_if_bigger(sort_buff_sz, record_length * MERGEBUFF2); 
     uchar **sort_keys=
       (uchar**) my_malloc(sort_buff_sz, MYF(MY_THREAD_SPECIFIC));
     m_idx_array= Idx_array(sort_keys, num_records);