MDEV-21958: Query having many NOT-IN clauses running forever

Basic variant of the fix: do not consider conditions in form

  unique_key NOT IN (c1,c2...)

to be sargable. If there are only a few constants, the condition
is not selective. If there are a lot constants, the overhead of
processing such a huge range list is not worth it.

1 files changed, 24 insertions, 0 deletions

diff --git a/mysql-test/main/range.test b/mysql-test/main/range.test
index 65f580698c5..fab6c8530c2 100644
--- a/mysql-test/main/range.test
+++ b/mysql-test/main/range.test
@@ -2207,6 +2207,30 @@ let $a= `select group_concat(a) from t2`;
 eval analyze SELECT * FROM t1 where a in ($a);
 drop table t1,ten,t2;
 
+--echo #
+--echo # MDEV-21958: Query having many NOT-IN clauses running forever
+--echo #
+create table t2 (
+  pk int primary key,
+  key1 int,
+  col1 int,
+  key (key1, pk)
+);
+
+insert into t2 (pk, key1) values (1,1),(2,2),(3,3),(4,4),(5,5);
+
+set @tmp_21958=@@optimizer_trace;
+set optimizer_trace=1;
+explain select * from t2 where key1 in (1,2,3) and pk not in (1,2,3);
+
+--echo # This should show only ranges in form "(1) <= (key1) <= (1)"
+--echo #  ranges over "pk" should not be constructed.
+select json_detailed(JSON_EXTRACT(trace, '$**.ranges'))
+from information_schema.optimizer_trace;
+set optimizer_trace=@tmp_21958;
+
+drop table t2;
+
 --echo # End of 10.4 tests
 
 set global innodb_stats_persistent= @innodb_stats_persistent_save;