MDEV-30098 Server crashes in ha_myisam::index_read_map with index_merge_sort_intersection=on

Fixes also
MDEV-30104 Server crashes in handler_rowid_filter_check upon ANALYZE TABLE

cancel_pushed_rowid_filter() didn't inform the handler that rowid_filter
was canceled.

3 files changed, 81 insertions, 1 deletions

diff --git a/mysql-test/main/myisam.result b/mysql-test/main/myisam.result
index 864939bbda8..477434e2a5c 100644
--- a/mysql-test/main/myisam.result
+++ b/mysql-test/main/myisam.result
@@ -2818,3 +2818,44 @@ drop table t;
 #
 # End of 10.8 tests
 #
+CREATE TABLE t1 (pk INT, a INT, PRIMARY KEY(pk), KEY(a)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1,NULL),(2,NULL),(3,1),(4,5);
+SET @save_optimizer_switch= @@optimizer_switch;
+SET optimizer_switch='index_merge_sort_intersection=on';
+SELECT pk FROM t1 WHERE pk > 2 AND a IS NULL;
+pk
+SET @@optimizer_switch= @save_optimizer_switch;
+drop table t1;
+#
+# MDEV-30104 Server crashes in handler_rowid_filter_check upon ANALYZE TABLE
+#
+CREATE TABLE t1 (a INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1),(2);
+CREATE TABLE t3 (c INT) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (1),(2);
+CREATE TABLE t4 (pk INT, f CHAR(8), PRIMARY KEY(pk), KEY(f)) ENGINE=MyISAM;
+INSERT INTO t4 VALUES (1,'o'),(2,'x');
+ANALYZE TABLE t1, t2, t3, t4 PERSISTENT FOR ALL;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+test.t2	analyze	status	Engine-independent statistics collected
+test.t2	analyze	status	OK
+test.t3	analyze	status	Engine-independent statistics collected
+test.t3	analyze	status	OK
+test.t4	analyze	status	Engine-independent statistics collected
+test.t4	analyze	status	OK
+SELECT * FROM t1 LEFT JOIN (t2 JOIN t3 ON 1) ON 2 IN (SELECT pk FROM t4 WHERE f < 's');
+a	b	c
+1	NULL	NULL
+2	NULL	NULL
+ANALYZE TABLE t4 PERSISTENT FOR ALL;
+Table	Op	Msg_type	Msg_text
+test.t4	analyze	status	Engine-independent statistics collected
+test.t4	analyze	status	Table is already up to date
+DROP TABLE t1, t2, t3, t4;
+#
+# End of 11.0 tests
+#
diff --git a/mysql-test/main/myisam.test b/mysql-test/main/myisam.test
index 53c5f9843cc..1a20f97a54f 100644
--- a/mysql-test/main/myisam.test
+++ b/mysql-test/main/myisam.test
@@ -1916,3 +1916,38 @@ drop table t;
 --echo #
 --echo # End of 10.8 tests
 --echo #
+
+CREATE TABLE t1 (pk INT, a INT, PRIMARY KEY(pk), KEY(a)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1,NULL),(2,NULL),(3,1),(4,5);
+SET @save_optimizer_switch= @@optimizer_switch;
+SET optimizer_switch='index_merge_sort_intersection=on';
+SELECT pk FROM t1 WHERE pk > 2 AND a IS NULL;
+SET @@optimizer_switch= @save_optimizer_switch;
+drop table t1;
+
+--echo #
+--echo # MDEV-30104 Server crashes in handler_rowid_filter_check upon ANALYZE TABLE
+--echo #
+
+CREATE TABLE t1 (a INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2);
+
+CREATE TABLE t2 (b INT) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1),(2); # Optional, fails either way
+
+CREATE TABLE t3 (c INT) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (1),(2); # Optional, fails either way
+
+CREATE TABLE t4 (pk INT, f CHAR(8), PRIMARY KEY(pk), KEY(f)) ENGINE=MyISAM;
+INSERT INTO t4 VALUES (1,'o'),(2,'x');
+
+ANALYZE TABLE t1, t2, t3, t4 PERSISTENT FOR ALL; # Optional, fails either way
+SELECT * FROM t1 LEFT JOIN (t2 JOIN t3 ON 1) ON 2 IN (SELECT pk FROM t4 WHERE f < 's');
+
+ANALYZE TABLE t4 PERSISTENT FOR ALL;
+
+DROP TABLE t1, t2, t3, t4;
+
+--echo #
+--echo # End of 11.0 tests
+--echo #
diff --git a/sql/handler.h b/sql/handler.h
index 924b065057e..209cb527716 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -4647,7 +4647,11 @@ public:
  virtual void cancel_pushed_rowid_filter()
  {
    pushed_rowid_filter= NULL;
-   rowid_filter_is_active= false;
+   if (rowid_filter_is_active)
+   {
+     rowid_filter_is_active= false;
+     rowid_filter_changed();
+   }
  }
 
  virtual void disable_pushed_rowid_filter()