MDEV-21794: Optimizer flag rowid_filter leads to long query

Rowid Filter check is just like Index Condition Pushdown check: before
we check the filter, we must check if we have walked out of the range
we are scanning. (If we did, we should return, and not continue the scan).

Consequences of this:
- Rowid filtering doesn't work for keys that have partially-covered
  blob columns (just like Index Condition Pushdown)
- The rowid filter function has three return values: CHECK_POS (passed)
  CHECK_NEG (filtered out), CHECK_OUT_OF_RANGE.

All of the above is implemented in this patch

1 files changed, 26 insertions, 8 deletions

diff --git a/sql/handler.cc b/sql/handler.cc
index c371da22a87..a7e7e2554cb 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -5970,22 +5970,22 @@ int handler::compare_key2(key_range *range) const
 /**
   ICP callback - to be called by an engine to check the pushed condition
 */
-extern "C" enum icp_result handler_index_cond_check(void* h_arg)
+extern "C" check_result_t handler_index_cond_check(void* h_arg)
 {
   handler *h= (handler*)h_arg;
   THD *thd= h->table->in_use;
-  enum icp_result res;
+  check_result_t res;
 
   enum thd_kill_levels abort_at= h->has_transactions() ?
     THD_ABORT_SOFTLY : THD_ABORT_ASAP;
   if (thd_kill_level(thd) > abort_at)
-    return ICP_ABORTED_BY_USER;
+    return CHECK_ABORTED_BY_USER;
 
   if (h->end_range && h->compare_key2(h->end_range) > 0)
-    return ICP_OUT_OF_RANGE;
+    return CHECK_OUT_OF_RANGE;
   h->increment_statistics(&SSV::ha_icp_attempts);
-  if ((res= h->pushed_idx_cond->val_int()? ICP_MATCH : ICP_NO_MATCH) ==
-      ICP_MATCH)
+  if ((res= h->pushed_idx_cond->val_int()? CHECK_POS : CHECK_NEG) ==
+      CHECK_POS)
     h->increment_statistics(&SSV::ha_icp_match);
   return res;
 }
@@ -5996,12 +5996,30 @@ extern "C" enum icp_result handler_index_cond_check(void* h_arg)
   keys of the rows whose data is to be fetched against the used rowid filter
 */
 
-extern "C" int handler_rowid_filter_check(void *h_arg)
+extern "C"
+check_result_t handler_rowid_filter_check(void *h_arg)
 {
   handler *h= (handler*) h_arg;
   TABLE *tab= h->get_table();
+
+  /*
+    Check for out-of-range and killed conditions only if we haven't done it
+    already in the pushed index condition check
+  */
+  if (!h->pushed_idx_cond)
+  {
+    THD *thd= h->table->in_use;
+    enum thd_kill_levels abort_at= h->has_transactions() ?
+      THD_ABORT_SOFTLY : THD_ABORT_ASAP;
+    if (thd_kill_level(thd) > abort_at)
+      return CHECK_ABORTED_BY_USER;
+
+    if (h->end_range && h->compare_key2(h->end_range) > 0)
+      return CHECK_OUT_OF_RANGE;
+  }
+
   h->position(tab->record[0]);
-  return h->pushed_rowid_filter->check((char *) h->ref);
+  return h->pushed_rowid_filter->check((char*)h->ref)? CHECK_POS: CHECK_NEG;
 }