From 1ade062c5d8263bee84376a543de883850d1489f Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <sergey@mariadb.com>
Date: Wed, 29 Mar 2023 19:42:21 +0300
Subject: MDEV-30964: MAX_SEL_ARG memory exhaustion is not visible in the
 optimizer trace

Add printing
---
 mysql-test/main/opt_trace.result | 41 ++++++++++++++++++++++++++++++++++++++++
 mysql-test/main/opt_trace.test   | 33 ++++++++++++++++++++++++++++++++
 sql/opt_range.cc                 |  7 +++++++
 3 files changed, 81 insertions(+)

diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result
index 2eef0da62bb..f3dad759d88 100644
--- a/mysql-test/main/opt_trace.result
+++ b/mysql-test/main/opt_trace.result
@@ -8522,5 +8522,46 @@ SELECT a FROM t1 WHERE (a,b) in (SELECT @c,@d);
 a
 DROP TABLE t1;
 #
+# MDEV-30964: MAX_SEL_ARG memory exhaustion is not visible in the optimizer trace
+#
+create table t1 (
+c1 int,
+c2 int,
+c3 int,
+c4 int,
+c5 int,
+c6 int,
+c7 int,
+c8 int,
+key(c1,c2,c3,c4,c5,c6,c7,c8)
+);
+insert into t1 () values (),(),();
+explain select *
+from t1
+where
+(c1 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16) and c2=1) and
+c3 in (1,2,3,4,5,6,7,8,9,10) and
+c4 in (1,2,3,4,5,6,7,8,9,10) and
+c5 in (1,2,3,4,5,6,7,8,9,10) and
+c6 in (1,2,3,4);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	c1	c1	40	NULL	3	Using where; Using index
+select
+json_detailed(json_extract(trace, '$**.setup_range_conditions'))
+from
+information_schema.optimizer_trace;
+json_detailed(json_extract(trace, '$**.setup_range_conditions'))
+[
+    [
+        {
+            "sel_arg_alloc_limit_hit": 
+            {
+                "alloced_sel_args": 16001
+            }
+        }
+    ]
+]
+drop table t1;
+#
 # End of 10.4 tests
 #
diff --git a/mysql-test/main/opt_trace.test b/mysql-test/main/opt_trace.test
index 0785d828a07..dab0dc6cdff 100644
--- a/mysql-test/main/opt_trace.test
+++ b/mysql-test/main/opt_trace.test
@@ -677,6 +677,39 @@ INSERT INTO t1 VALUES (0,0);
 SELECT a FROM t1 WHERE (a,b) in (SELECT @c,@d);
 DROP TABLE t1;
 
+
+--echo #
+--echo # MDEV-30964: MAX_SEL_ARG memory exhaustion is not visible in the optimizer trace
+--echo #
+create table t1 (
+  c1 int,
+  c2 int,
+  c3 int,
+  c4 int,
+  c5 int,
+  c6 int,
+  c7 int,
+  c8 int,
+  key(c1,c2,c3,c4,c5,c6,c7,c8)
+);
+insert into t1 () values (),(),();
+
+explain select *
+from t1
+where
+  (c1 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16) and c2=1) and
+  c3 in (1,2,3,4,5,6,7,8,9,10) and
+  c4 in (1,2,3,4,5,6,7,8,9,10) and
+  c5 in (1,2,3,4,5,6,7,8,9,10) and
+  c6 in (1,2,3,4);
+
+select
+  json_detailed(json_extract(trace, '$**.setup_range_conditions'))
+from
+  information_schema.optimizer_trace;
+
+drop table t1;
+
 --echo #
 --echo # End of 10.4 tests
 --echo #
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index a2bbe1447e9..a6ae0429a8b 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -2853,6 +2853,13 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
         Json_writer_array trace_range_summary(thd,
                                               "setup_range_conditions");
         tree= cond->get_mm_tree(&param, &cond);
+        if (thd->trace_started() && 
+            param.alloced_sel_args >= SEL_ARG::MAX_SEL_ARGS)
+        {
+          Json_writer_object wrapper(thd);
+          Json_writer_object obj(thd, "sel_arg_alloc_limit_hit");
+          obj.add("alloced_sel_args", param.alloced_sel_args);
+        }
       }
       if (tree)
       {
-- 
cgit v1.2.1