MDEV-30596: Assertion 'pushed_rowid_filter != __null ...' failedbb-11.0-mdev30596

ha_partition doesn't forward Rowid Filter API calls to the storage
engines handling partitions.
An attempt to use rowid filtering with caused either
- Rowid Filter being shown in EXPLAIN but not actually used
- Assertion failure when subquery code tried to disable/enable rowid
  filter, which was not present.

Fixed by returning correct flags from ha_partition::index_flags()

3 files changed, 88 insertions, 2 deletions

diff --git a/mysql-test/main/partition.result b/mysql-test/main/partition.result
index 0d37d4cb168..57aaa227030 100644
--- a/mysql-test/main/partition.result
+++ b/mysql-test/main/partition.result
@@ -2820,3 +2820,41 @@ DROP TABLE t1,t2;
 #
 # End of 10.1 tests
 #
+#
+# MDEV-30596: Assertion 'pushed_rowid_filter != __null ...' failed
+#
+create table t1 (a int);
+insert into t1 values (NULL),(1),(2);
+create table t2 (a int);
+insert into t2 select seq from seq_1_to_1000;
+create table t3 (
+a1 int,
+a2 int,
+b int,
+c int,
+filler1 char(200),
+filler2 char(200),
+key(a1,a2),
+key(b)
+) partition by hash(a1) partitions 2;
+insert into t3 select seq/100, seq/100, seq, seq, seq, seq from seq_1_to_10000;
+analyze table t3 persistent for all;
+Table	Op	Msg_type	Msg_text
+test.t3	analyze	status	Engine-independent statistics collected
+test.t3	analyze	status	OK
+set @tmp_os= @@optimizer_switch;
+set optimizer_switch='materialization=off';
+# Must not show "Using rowid filter":
+explain
+select *
+from t1
+where
+t1.a not in (select straight_join t3.a1
+from t2, t3
+where t3.b < 3000 and t3.a2=t2.a);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	3	Using where
+2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	1000	Using where
+2	DEPENDENT SUBQUERY	t3	ref_or_null	a1,b	a1	10	func,test.t2.a	198	Using where; Full scan on NULL key
+set optimizer_switch=@tmp_os;
+drop table t1,t2,t3;
diff --git a/mysql-test/main/partition.test b/mysql-test/main/partition.test
index 5f4c5f659ca..d96031a6b2b 100644
--- a/mysql-test/main/partition.test
+++ b/mysql-test/main/partition.test
@@ -3021,7 +3021,44 @@ INSERT INTO t2 VALUES (1),(2);
 UPDATE t1 SET a = 7 WHERE a = ( SELECT b FROM t2 ) ORDER BY a LIMIT 6;
 
 DROP TABLE t1,t2;
-
 --echo #
 --echo # End of 10.1 tests
 --echo #
+
+--echo #
+--echo # MDEV-30596: Assertion 'pushed_rowid_filter != __null ...' failed
+--echo #
+--source include/have_sequence.inc
+create table t1 (a int);
+insert into t1 values (NULL),(1),(2);
+create table t2 (a int);
+insert into t2 select seq from seq_1_to_1000;
+
+create table t3 (
+  a1 int,
+  a2 int,
+  b int,
+  c int,
+  filler1 char(200),
+  filler2 char(200),
+  key(a1,a2),
+  key(b)
+) partition by hash(a1) partitions 2;
+insert into t3 select seq/100, seq/100, seq, seq, seq, seq from seq_1_to_10000;
+analyze table t3 persistent for all;
+
+set @tmp_os= @@optimizer_switch;
+set optimizer_switch='materialization=off';
+
+--echo # Must not show "Using rowid filter":
+explain
+select *
+from t1
+where
+  t1.a not in (select straight_join t3.a1
+               from t2, t3
+               where t3.b < 3000 and t3.a2=t2.a);
+set optimizer_switch=@tmp_os;
+
+drop table t1,t2,t3;
+
diff --git a/sql/ha_partition.h b/sql/ha_partition.h
index 37529073391..cfa281da4ef 100644
--- a/sql/ha_partition.h
+++ b/sql/ha_partition.h
@@ -1304,7 +1304,18 @@ public:
       The following code is not safe if you are using different
       storage engines or different index types per partition.
     */
-    return m_file[0]->index_flags(inx, part, all_parts);
+    ulong part_flags= m_file[0]->index_flags(inx, part, all_parts);
+
+    /*
+      The underlying storage engine might support Rowid Filtering. But
+      ha_partition does not forward the needed SE API calls, so the feature
+      will not be used.
+
+      Note: It's the same with IndexConditionPushdown, except for its variant
+      of IndexConditionPushdown+BatchedKeyAccess (that one works). Because of
+      that, we do not clear HA_DO_INDEX_COND_PUSHDOWN here.
+    */
+    return part_flags & ~HA_DO_RANGE_FILTER_PUSHDOWN;
   }
 
   /**