Fix for LP#697610 ha_index_prev(uchar*): Assertion `inited==INDEX' failed with HANDLER + InnoDB in maria-5.3

mysql-test/suite/handler/innodb.result:
  Added test case
mysql-test/suite/handler/innodb.test:
  Added test case
sql/handler.h:
  Move setting/resetting of active_index to ha_index_init()/ha_index_end() to simplify handler functions index_init()/index_end()
  Fixed that get_index() returns MAX_KEY if index is not inited (this fixed LP#697610)
storage/federated/ha_federated.cc:
  Settting of active_index is not needed anymore
storage/maria/ma_pagecache.c:
  Added error message if we have too little memory for Maria page cache

5 files changed, 37 insertions, 7 deletions

diff --git a/mysql-test/suite/handler/innodb.result b/mysql-test/suite/handler/innodb.result
index 83ce270612d..2cdaa0c4664 100644
--- a/mysql-test/suite/handler/innodb.result
+++ b/mysql-test/suite/handler/innodb.result
@@ -560,3 +560,13 @@ HANDLER t1 READ a NEXT;
 a
 HANDLER t1 CLOSE;
 DROP TABLE t1;
+CREATE TABLE t1 (f1 integer, f2 integer, primary key (f1), key (f2)) engine=innodb;
+INSERT INTO t1 VALUES (1,1),(2,2),(3,3);
+HANDLER t1 OPEN;
+HANDLER t1 READ FIRST WHERE f2 <= 1;
+f1	f2
+1	1
+HANDLER t1 READ `PRIMARY` PREV;
+f1	f2
+3	3
+DROP TABLE t1;
diff --git a/mysql-test/suite/handler/innodb.test b/mysql-test/suite/handler/innodb.test
index f4e4bf7cc3f..6527c4bb8bb 100644
--- a/mysql-test/suite/handler/innodb.test
+++ b/mysql-test/suite/handler/innodb.test
@@ -15,3 +15,14 @@ let $engine_type= InnoDB;
 
 --source init.inc
 --source handler.inc
+
+#
+# LP#697610 ha_index_prev(uchar*): Assertion `inited==INDEX'
+#
+
+CREATE TABLE t1 (f1 integer, f2 integer, primary key (f1), key (f2)) engine=innodb;
+INSERT INTO t1 VALUES (1,1),(2,2),(3,3);
+HANDLER t1 OPEN;
+HANDLER t1 READ FIRST WHERE f2 <= 1;
+HANDLER t1 READ `PRIMARY` PREV;
+DROP TABLE t1;
diff --git a/sql/handler.h b/sql/handler.h
index 52828d4f149..830af71baee 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -1195,14 +1195,18 @@ public:
     DBUG_ENTER("ha_index_init");
     DBUG_ASSERT(inited==NONE);
     if (!(result= index_init(idx, sorted)))
-      inited=INDEX;
+    {
+      inited=       INDEX;
+      active_index= idx;
+    }
     DBUG_RETURN(result);
   }
   int ha_index_end()
   {
     DBUG_ENTER("ha_index_end");
     DBUG_ASSERT(inited==INDEX);
-    inited=NONE;
+    inited=       NONE;
+    active_index= MAX_KEY;
     DBUG_RETURN(index_end());
   }
   /* This is called after index_init() if we need to do a index scan */
@@ -1371,7 +1375,12 @@ public:
     as there may be several calls to this routine.
   */
   virtual void column_bitmaps_signal();
-  uint get_index(void) const { return active_index; }
+  /*
+    We have to check for inited as some engines, like innodb, sets
+    active_index during table scan.
+  */
+  uint get_index(void) const
+  { return inited == INDEX ? active_index : MAX_KEY; }
   virtual int close(void)=0;
 
   /**
@@ -1824,8 +1833,8 @@ private:
   */
 
   virtual int open(const char *name, int mode, uint test_if_locked)=0;
-  virtual int index_init(uint idx, bool sorted) { active_index= idx; return 0; }
-  virtual int index_end() { active_index= MAX_KEY; return 0; }
+  virtual int index_init(uint idx, bool sorted) { return 0; }
+  virtual int index_end() { return 0; }
   /**
     rnd_init() can be called two times without rnd_end() in between
     (it only makes sense if scan=1).
diff --git a/storage/federated/ha_federated.cc b/storage/federated/ha_federated.cc
index 629de4e3ea1..d4cffbed59e 100644
--- a/storage/federated/ha_federated.cc
+++ b/storage/federated/ha_federated.cc
@@ -2453,7 +2453,6 @@ int ha_federated::index_init(uint keynr, bool sorted)
 {
   DBUG_ENTER("ha_federated::index_init");
   DBUG_PRINT("info", ("table: '%s'  key: %u", table->s->table_name.str, keynr));
-  active_index= keynr;
   DBUG_RETURN(0);
 }
 
@@ -2589,7 +2588,6 @@ int ha_federated::index_end(void)
 {
   DBUG_ENTER("ha_federated::index_end");
   free_result();
-  active_index= MAX_KEY;
   DBUG_RETURN(0);
 }
 
diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c
index 220d5eae0bf..1e79b04f7b6 100644
--- a/storage/maria/ma_pagecache.c
+++ b/storage/maria/ma_pagecache.c
@@ -759,6 +759,8 @@ ulong init_pagecache(PAGECACHE *pagecache, size_t use_mem,
   {
     if (blocks < 8)
     {
+      my_message(ENOMEM, "Not enough memory to allocate 8 pagecache pages",
+                 MYF(0));
       my_errno= ENOMEM;
       goto err;
     }