MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple

cmp_item_sort_string::store_value() did not cache the string returned
from item->val_str(), whose result can point to various private members
such as Item_char_typecast::tmp_value.

- cmp_item_sort_string::store_value() remembered the pointer returned
  from item->val_str() poiting to tmp_value into cmp_item_string::value_res.
- Later, cmp_item_real::store_value() was called, which called
  Item_str_func::val_real(), which called Item_char_typecast::val_str(&tmp)
  using a local stack variable "String tmp". Item_char_typecast::tmp_value
  was overwritten and become a link to "tmp":
  tmp_value.Ptr freed its own buffer and set to point to the buffer
  owned by "tmp".
- On return from Item_str_func::val_real(), "String tmp" was destructed,
  but "tmp_value" still pointed to the buffer owned by "tmp",
  So tmp_value.Ptr became invalid.
- Then cmp_item_sort_string() passed cmp_item_string::value_res to sortcmp().
  At this point, value_res still pointed to an invalid value of
  Item_char_typecast::tmp_value.

Fix:
changing cmp_item_sort_string::store_value() to force copying
to cmp_item_string::value if item->val_str(&value) returned
a different pointer (instead of &value).

4 files changed, 45 insertions, 0 deletions

diff --git a/mysql-test/r/ctype_latin1.result b/mysql-test/r/ctype_latin1.result
index 85035982cf9..66c5a37750d 100644
--- a/mysql-test/r/ctype_latin1.result
+++ b/mysql-test/r/ctype_latin1.result
@@ -8208,5 +8208,18 @@ Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t1`.`c` AS `c`,`test`.`t1`.`d` AS `d` from `test`.`t1` where ((coalesce(`test`.`t1`.`c`,0) = '3 ') and (coalesce(`test`.`t1`.`d`,0) = '3 '))
 DROP TABLE t1;
 #
+# MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple
+#
+SET NAMES latin1;
+SELECT CONVERT(1, CHAR) IN ('100', 10, '101');
+CONVERT(1, CHAR) IN ('100', 10, '101')
+0
+SELECT CONVERT(1, CHAR) IN ('100', 10, '1');
+CONVERT(1, CHAR) IN ('100', 10, '1')
+1
+SELECT CONVERT(1, CHAR) IN ('100', '10', '1');
+CONVERT(1, CHAR) IN ('100', '10', '1')
+1
+#
 # End of 10.1 tests
 #
diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result
index 5a77ea3aea5..3da74d12877 100644
--- a/mysql-test/r/ctype_utf8.result
+++ b/mysql-test/r/ctype_utf8.result
@@ -10535,5 +10535,18 @@ CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1)
 Warnings:
 Warning	1977	Cannot convert 'utf8' character 0xC499 to 'latin1'
 #
+# MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple
+#
+SET NAMES utf8;
+SELECT CONVERT(1, CHAR) IN ('100', 10, '101');
+CONVERT(1, CHAR) IN ('100', 10, '101')
+0
+SELECT CONVERT(1, CHAR) IN ('100', 10, '1');
+CONVERT(1, CHAR) IN ('100', 10, '1')
+1
+SELECT CONVERT(1, CHAR) IN ('100', '10', '1');
+CONVERT(1, CHAR) IN ('100', '10', '1')
+1
+#
 # End of 10.1 tests
 #
diff --git a/mysql-test/t/ctype_latin1.test b/mysql-test/t/ctype_latin1.test
index 1ee48eed18c..3455b0437c1 100644
--- a/mysql-test/t/ctype_latin1.test
+++ b/mysql-test/t/ctype_latin1.test
@@ -390,5 +390,14 @@ SELECT * FROM t1 WHERE COALESCE(c,0)='3 ' AND COALESCE(d,0)=COALESCE(c,0);
 DROP TABLE t1;
 
 --echo #
+--echo # MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple
+--echo #
+
+SET NAMES latin1;
+SELECT CONVERT(1, CHAR) IN ('100', 10, '101');
+SELECT CONVERT(1, CHAR) IN ('100', 10, '1');
+SELECT CONVERT(1, CHAR) IN ('100', '10', '1');
+
+--echo #
 --echo # End of 10.1 tests
 --echo #
diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test
index e013109d0a7..be17eb461be 100644
--- a/mysql-test/t/ctype_utf8.test
+++ b/mysql-test/t/ctype_utf8.test
@@ -2025,5 +2025,15 @@ SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1);
 
 
 --echo #
+--echo # MDEV-15005 ASAN: stack-buffer-overflow in my_strnncollsp_simple
+--echo #
+
+SET NAMES utf8;
+SELECT CONVERT(1, CHAR) IN ('100', 10, '101');
+SELECT CONVERT(1, CHAR) IN ('100', 10, '1');
+SELECT CONVERT(1, CHAR) IN ('100', '10', '1');
+
+
+--echo #
 --echo # End of 10.1 tests
 --echo #