MDEV-22715: SIGSEGV in radixsort_for_str_ptr and in native_compare/my_qsort2 (optimized builds)

For DECIMAL[(M[,D])] datatype max_sort_length was not being honoured which was leading to buffer
overflow while making the sort key. The fix to this problem would be to create sort keys for decimals
with atmost max_sort_key bytes

Important:
The minimum value of max_sort_length has been raised to 8 (previously was 4),
so fixed size datatypes like DOUBLE and BIGINIT are not truncated for
lower values of max_sort_length.

23 files changed, 135 insertions, 47 deletions

diff --git a/mysql-test/include/ctype_utf8mb4.inc b/mysql-test/include/ctype_utf8mb4.inc
index 71993a946b3..aa06cd97e4c 100644
--- a/mysql-test/include/ctype_utf8mb4.inc
+++ b/mysql-test/include/ctype_utf8mb4.inc
@@ -1576,7 +1576,7 @@ drop table t1;
 --echo #
 --echo # Check strnxfrm() with odd length
 --echo #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 eval create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci) engine $engine;
 insert into t1 values ('a'),('b'),('c');
diff --git a/mysql-test/r/ctype_utf16.result b/mysql-test/r/ctype_utf16.result
index c826a93809d..7c3d18a6cfc 100644
--- a/mysql-test/r/ctype_utf16.result
+++ b/mysql-test/r/ctype_utf16.result
@@ -1489,7 +1489,7 @@ ab
 ab
 AE
 AE
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 s1
 ab
diff --git a/mysql-test/r/ctype_utf16le.result b/mysql-test/r/ctype_utf16le.result
index e1a71e02bb6..d129323bae0 100644
--- a/mysql-test/r/ctype_utf16le.result
+++ b/mysql-test/r/ctype_utf16le.result
@@ -1762,7 +1762,7 @@ ab
 ab
 AE
 AE
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 s1
 ab
diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result
index 4a44147e945..a4f7571d972 100644
--- a/mysql-test/r/ctype_utf32.result
+++ b/mysql-test/r/ctype_utf32.result
@@ -1503,7 +1503,7 @@ ab
 ab
 AE
 AE
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 s1
 ab
diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result
index 9ff12870d75..8b35305a79b 100644
--- a/mysql-test/r/ctype_utf8.result
+++ b/mysql-test/r/ctype_utf8.result
@@ -6807,10 +6807,10 @@ DFFFDFFF9CFF9DFF9EFF
 #
 # Checking strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8 collate utf8_general_ci);
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;
diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result
index fa1cf3b1d19..d7bc36119ec 100644
--- a/mysql-test/r/ctype_utf8mb4.result
+++ b/mysql-test/r/ctype_utf8mb4.result
@@ -2364,10 +2364,10 @@ drop table t1;
 #
 # Check strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci);
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;
diff --git a/mysql-test/r/ctype_utf8mb4_heap.result b/mysql-test/r/ctype_utf8mb4_heap.result
index ef134641ff8..9933c659ae3 100644
--- a/mysql-test/r/ctype_utf8mb4_heap.result
+++ b/mysql-test/r/ctype_utf8mb4_heap.result
@@ -2196,10 +2196,10 @@ drop table t1;
 #
 # Check strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci) engine heap;
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;
diff --git a/mysql-test/r/ctype_utf8mb4_innodb.result b/mysql-test/r/ctype_utf8mb4_innodb.result
index 9c58dc87126..ff8fe980247 100644
--- a/mysql-test/r/ctype_utf8mb4_innodb.result
+++ b/mysql-test/r/ctype_utf8mb4_innodb.result
@@ -2324,10 +2324,10 @@ drop table t1;
 #
 # Check strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci) engine InnoDB;
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;
diff --git a/mysql-test/r/ctype_utf8mb4_myisam.result b/mysql-test/r/ctype_utf8mb4_myisam.result
index 18bec51358d..b2e717a0737 100644
--- a/mysql-test/r/ctype_utf8mb4_myisam.result
+++ b/mysql-test/r/ctype_utf8mb4_myisam.result
@@ -2324,10 +2324,10 @@ drop table t1;
 #
 # Check strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci) engine MyISAM;
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;
diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result
index 380687554d7..d35012e0de5 100644
--- a/mysql-test/r/order_by.result
+++ b/mysql-test/r/order_by.result
@@ -3237,3 +3237,68 @@ o	15
 p	16
 set @@sort_buffer_size= @save_sort_buffer_size;
 drop table t1;
+#
+# MDEV-22715: SIGSEGV in radixsort_for_str_ptr and in native_compare/my_qsort2 (optimized builds)
+#
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET @save_max_sort_length= @@max_sort_length;
+SET max_sort_length=8;
+SET sort_buffer_size=1024;
+CREATE TABLE t1(a INT, b DECIMAL(65), c BLOB);
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_25;
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_25;
+SELECT * FROM t1 ORDER BY a,b;
+a	b	c
+1	1	1
+1	1	1
+2	2	2
+2	2	2
+3	3	3
+3	3	3
+4	4	4
+4	4	4
+5	5	5
+5	5	5
+6	6	6
+6	6	6
+7	7	7
+7	7	7
+8	8	8
+8	8	8
+9	9	9
+9	9	9
+10	10	10
+10	10	10
+11	11	11
+11	11	11
+12	12	12
+12	12	12
+13	13	13
+13	13	13
+14	14	14
+14	14	14
+15	15	15
+15	15	15
+16	16	16
+16	16	16
+17	17	17
+17	17	17
+18	18	18
+18	18	18
+19	19	19
+19	19	19
+20	20	20
+20	20	20
+21	21	21
+21	21	21
+22	22	22
+22	22	22
+23	23	23
+23	23	23
+24	24	24
+24	24	24
+25	25	25
+25	25	25
+SET @@sort_buffer_size= @save_sort_buffer_size;
+SET @@max_sort_length= @save_max_sort_length;
+DROP TABLE t1;
diff --git a/mysql-test/suite/sys_vars/r/max_sort_length_basic.result b/mysql-test/suite/sys_vars/r/max_sort_length_basic.result
index a8876b2c81e..b48b045897c 100644
--- a/mysql-test/suite/sys_vars/r/max_sort_length_basic.result
+++ b/mysql-test/suite/sys_vars/r/max_sort_length_basic.result
@@ -27,14 +27,14 @@ SELECT @@session.max_sort_length = 1024;
 @@session.max_sort_length = 1024
 1
 '#--------------------FN_DYNVARS_084_03-------------------------#'
-SET @@global.max_sort_length = 4;
+SET @@global.max_sort_length = 8;
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
-SET @@global.max_sort_length = 5;
+8
+SET @@global.max_sort_length = 9;
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-5
+9
 SET @@global.max_sort_length = 8388608;
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
@@ -48,14 +48,14 @@ SELECT @@global.max_sort_length;
 @@global.max_sort_length
 65536
 '#--------------------FN_DYNVARS_084_04-------------------------#'
-SET @@session.max_sort_length = 4;
+SET @@session.max_sort_length = 8;
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-4
-SET @@session.max_sort_length = 5;
+8
+SET @@session.max_sort_length = 9;
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-5
+9
 SET @@session.max_sort_length = 8388608;
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
@@ -74,13 +74,13 @@ Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '-1024'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = 3;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '3'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = 8388609;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '8388609'
@@ -92,17 +92,17 @@ Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '0'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = 65530.34;
 ERROR 42000: Incorrect argument type to variable 'max_sort_length'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = test;
 ERROR 42000: Incorrect argument type to variable 'max_sort_length'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@session.max_sort_length = 8388610;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '8388610'
@@ -114,19 +114,19 @@ Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '-1'
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-4
+8
 SET @@session.max_sort_length = 3;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '3'
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-4
+8
 SET @@session.max_sort_length = 0;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '0'
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-4
+8
 SET @@session.max_sort_length = 65530.34;
 ERROR 42000: Incorrect argument type to variable 'max_sort_length'
 SET @@session.max_sort_length = 10737418241;
@@ -158,13 +158,13 @@ Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '1'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = FALSE;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '0'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 '#---------------------FN_DYNVARS_084_09----------------------#'
 SET @@global.max_sort_length = 2048;
 SELECT @@max_sort_length = @@global.max_sort_length;
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
index 9ca6995d7ef..865b5a8ca2f 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
@@ -2033,7 +2033,7 @@ DEFAULT_VALUE	1024
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The number of bytes to use when sorting BLOB or TEXT values (only the first max_sort_length bytes of each value are used; the rest are ignored)
-NUMERIC_MIN_VALUE	4
+NUMERIC_MIN_VALUE	8
 NUMERIC_MAX_VALUE	8388608
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
index 9288912eb57..21f1f905de8 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
@@ -2229,7 +2229,7 @@ DEFAULT_VALUE	1024
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The number of bytes to use when sorting BLOB or TEXT values (only the first max_sort_length bytes of each value are used; the rest are ignored)
-NUMERIC_MIN_VALUE	4
+NUMERIC_MIN_VALUE	8
 NUMERIC_MAX_VALUE	8388608
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
diff --git a/mysql-test/suite/sys_vars/t/max_sort_length_basic.test b/mysql-test/suite/sys_vars/t/max_sort_length_basic.test
index 9c3b88d3c3c..fcd6db017f1 100644
--- a/mysql-test/suite/sys_vars/t/max_sort_length_basic.test
+++ b/mysql-test/suite/sys_vars/t/max_sort_length_basic.test
@@ -74,9 +74,9 @@ SELECT @@session.max_sort_length = 1024;
 # Change the value of max_sort_length to a valid value for GLOBAL Scope #
 #########################################################################
 
-SET @@global.max_sort_length = 4;
+SET @@global.max_sort_length = 8;
 SELECT @@global.max_sort_length;
-SET @@global.max_sort_length = 5;
+SET @@global.max_sort_length = 9;
 SELECT @@global.max_sort_length;
 SET @@global.max_sort_length = 8388608;
 SELECT @@global.max_sort_length;
@@ -90,10 +90,10 @@ SELECT @@global.max_sort_length;
 # Change the value of max_sort_length to a valid value for SESSION Scope #
 ##########################################################################
 
-SET @@session.max_sort_length = 4;
+SET @@session.max_sort_length = 8;
 SELECT @@session.max_sort_length;
 
-SET @@session.max_sort_length = 5;
+SET @@session.max_sort_length = 9;
 SELECT @@session.max_sort_length;
 
 SET @@session.max_sort_length = 8388608;
diff --git a/mysql-test/t/ctype_utf16.test b/mysql-test/t/ctype_utf16.test
index 1febe3d8aea..28c31d639aa 100644
--- a/mysql-test/t/ctype_utf16.test
+++ b/mysql-test/t/ctype_utf16.test
@@ -721,7 +721,7 @@ CREATE TABLE t1 AS SELECT repeat('a',2) as s1 LIMIT 0;
 SHOW CREATE TABLE t1;
 INSERT INTO t1 VALUES ('ab'),('AE'),('ab'),('AE');
 SELECT * FROM t1 ORDER BY s1;
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 DROP TABLE t1;
 SET max_sort_length=DEFAULT;
diff --git a/mysql-test/t/ctype_utf16le.test b/mysql-test/t/ctype_utf16le.test
index 5e29408cbd0..2a8aaeafe59 100644
--- a/mysql-test/t/ctype_utf16le.test
+++ b/mysql-test/t/ctype_utf16le.test
@@ -683,7 +683,7 @@ CREATE TABLE t1 AS SELECT REPEAT('a',2) as s1 LIMIT 0;
 SHOW CREATE TABLE t1;
 INSERT INTO t1 VALUES ('ab'),('AE'),('ab'),('AE');
 SELECT * FROM t1 ORDER BY s1;
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 DROP TABLE t1;
 SET max_sort_length=DEFAULT;
diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test
index bf822291a21..c7835e3c74f 100644
--- a/mysql-test/t/ctype_utf32.test
+++ b/mysql-test/t/ctype_utf32.test
@@ -778,7 +778,7 @@ CREATE TABLE t1 AS SELECT repeat('a',2) as s1 LIMIT 0;
 SHOW CREATE TABLE t1;
 INSERT INTO t1 VALUES ('ab'),('AE'),('ab'),('AE');
 SELECT * FROM t1 ORDER BY s1;
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 DROP TABLE t1;
 SET max_sort_length=DEFAULT;
diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test
index 51b6d63ee2f..7abcc693926 100644
--- a/mysql-test/t/ctype_utf8.test
+++ b/mysql-test/t/ctype_utf8.test
@@ -1756,7 +1756,7 @@ set @@collation_connection=utf8_bin;
 --echo #
 --echo # Checking strnxfrm() with odd length
 --echo #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 create table t1 (a varchar(128) character set utf8 collate utf8_general_ci);
 insert into t1 values ('a'),('b'),('c');
diff --git a/mysql-test/t/ctype_utf8mb4.test b/mysql-test/t/ctype_utf8mb4.test
index ffc098ff938..91256d6e096 100644
--- a/mysql-test/t/ctype_utf8mb4.test
+++ b/mysql-test/t/ctype_utf8mb4.test
@@ -1511,7 +1511,7 @@ drop table t1;
 --echo #
 --echo # Check strnxfrm() with odd length
 --echo #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci);
 insert into t1 values ('a'),('b'),('c');
diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test
index 999c7314139..2fbacb10b68 100644
--- a/mysql-test/t/order_by.test
+++ b/mysql-test/t/order_by.test
@@ -12,6 +12,8 @@ call mtr.add_suppression("Out of sort memory; increase server sort buffer size")
 # Test old ORDER BY bug
 #
 
+--source include/have_sequence.inc
+
 CREATE TABLE t1 (
   id int(6) DEFAULT '0' NOT NULL,
   idservice int(5),
@@ -2157,3 +2159,18 @@ select * from t1 order  by b;
 set @@sort_buffer_size= @save_sort_buffer_size;
 drop table t1;
 
+--echo #
+--echo # MDEV-22715: SIGSEGV in radixsort_for_str_ptr and in native_compare/my_qsort2 (optimized builds)
+--echo #
+
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET @save_max_sort_length= @@max_sort_length;
+SET max_sort_length=8;
+SET sort_buffer_size=1024;
+CREATE TABLE t1(a INT, b DECIMAL(65), c BLOB);
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_25;
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_25;
+SELECT * FROM t1 ORDER BY a,b;
+SET @@sort_buffer_size= @save_sort_buffer_size;
+SET @@max_sort_length= @save_max_sort_length;
+DROP TABLE t1;
diff --git a/sql/field.cc b/sql/field.cc
index 291e2134dcf..f65c6f88bb9 100644
--- a/sql/field.cc
+++ b/sql/field.cc
@@ -3295,10 +3295,9 @@ int Field_new_decimal::cmp(const uchar *a,const uchar*b)
 }
 
 
-void Field_new_decimal::sort_string(uchar *buff,
-                                    uint length __attribute__((unused)))
+void Field_new_decimal::sort_string(uchar *buff, uint length)
 {
-  memcpy(buff, ptr, bin_size);
+  memcpy(buff, ptr, length);
 }
 
 
diff --git a/sql/field.h b/sql/field.h
index 60849ea8099..5e8f39d903e 100644
--- a/sql/field.h
+++ b/sql/field.h
@@ -1099,6 +1099,13 @@ public:
 
   void make_sort_key(uchar *buff, uint length);
   virtual void make_field(Send_field *);
+
+  /*
+    Some implementations actually may write up to 8 bytes regardless of what
+    size was requested. This is due to the minimum value of the system variable
+    max_sort_length.
+  */
+
   virtual void sort_string(uchar *buff,uint length)=0;
   virtual bool optimize_range(uint idx, uint part);
   virtual void free() {}
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index 9f87c846fd0..7dc1d1ab6e8 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -2178,7 +2178,7 @@ static Sys_var_ulong Sys_max_sort_length(
        "the first max_sort_length bytes of each value are used; the rest "
        "are ignored)",
        SESSION_VAR(max_sort_length), CMD_LINE(REQUIRED_ARG),
-       VALID_RANGE(4, 8192*1024L), DEFAULT(1024), BLOCK_SIZE(1));
+       VALID_RANGE(8, 8192*1024L), DEFAULT(1024), BLOCK_SIZE(1));
 
 static Sys_var_ulong Sys_max_sp_recursion_depth(
        "max_sp_recursion_depth",