Bug #51357: crash when using handler commands on spatial indexes

Spatial indexes were not checking for out-of-record condition in
the handler next command when the previous command didn't found
rows.

Fixed by making the rtree index to check for end of rows condition
before re-using the key from the previous search.

Fixed another crash if the tree has changed since the last search.
Added a test case for the other error.

3 files changed, 54 insertions, 12 deletions

diff --git a/mysql-test/r/gis-rtree.result b/mysql-test/r/gis-rtree.result
index 68c4a6a13e5..49ccb05c699 100644
--- a/mysql-test/r/gis-rtree.result
+++ b/mysql-test/r/gis-rtree.result
@@ -1526,4 +1526,26 @@ SELECT 1 FROM t1 WHERE a >= GEOMFROMTEXT('LINESTRING(-1 -1, 1 -1, -1 -1, -1 1, 1
 1
 1
 DROP TABLE t1;
+#
+# Bug #51357: crash when using handler commands on spatial indexes
+#
+CREATE TABLE t1(a GEOMETRY NOT NULL,SPATIAL INDEX a(a));
+HANDLER t1 OPEN;
+HANDLER t1 READ a FIRST;
+a
+HANDLER t1 READ a NEXT;
+a
+HANDLER t1 READ a PREV;
+a
+HANDLER t1 READ a LAST;
+a
+HANDLER t1 CLOSE;
+HANDLER t1 OPEN;
+HANDLER t1 READ a FIRST;
+a
+INSERT INTO t1 VALUES (GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
+HANDLER t1 READ a NEXT;
+a
+HANDLER t1 CLOSE;
+DROP TABLE t1;
 End of 5.0 tests.
diff --git a/mysql-test/t/gis-rtree.test b/mysql-test/t/gis-rtree.test
index c325b3bd223..3b341501ab6 100644
--- a/mysql-test/t/gis-rtree.test
+++ b/mysql-test/t/gis-rtree.test
@@ -902,4 +902,26 @@ SELECT 1 FROM t1 WHERE a >= GEOMFROMTEXT('LINESTRING(-1 -1, 1 -1, -1 -1, -1 1, 1
 DROP TABLE t1;
 
 
+--echo #
+--echo # Bug #51357: crash when using handler commands on spatial indexes
+--echo #
+
+CREATE TABLE t1(a GEOMETRY NOT NULL,SPATIAL INDEX a(a));
+HANDLER t1 OPEN;
+HANDLER t1 READ a FIRST;
+HANDLER t1 READ a NEXT;
+HANDLER t1 READ a PREV;
+HANDLER t1 READ a LAST;
+HANDLER t1 CLOSE;
+
+# second crash fixed when the tree has changed since the last search.
+HANDLER t1 OPEN;
+HANDLER t1 READ a FIRST;
+INSERT INTO t1 VALUES (GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
+HANDLER t1 READ a NEXT;
+HANDLER t1 CLOSE;
+
+DROP TABLE t1;
+
+
 --echo End of 5.0 tests.
diff --git a/storage/myisam/rt_index.c b/storage/myisam/rt_index.c
index e094c302f92..31241a83228 100644
--- a/storage/myisam/rt_index.c
+++ b/storage/myisam/rt_index.c
@@ -404,10 +404,16 @@ int rtree_get_first(MI_INFO *info, uint keynr, uint key_length)
 
 int rtree_get_next(MI_INFO *info, uint keynr, uint key_length)
 {
-  my_off_t root;
+  my_off_t root= info->s->state.key_root[keynr];
   MI_KEYDEF *keyinfo = info->s->keyinfo + keynr;
 
-  if (!info->buff_used)
+  if (root == HA_OFFSET_ERROR)
+  {
+    my_errno= HA_ERR_END_OF_FILE;
+    return -1;
+  }
+  
+  if (!info->buff_used && !info->page_changed)
   {
     uint k_len = keyinfo->keylength - info->s->base.rec_reflength;
     /* rt_PAGE_NEXT_KEY(info->int_keypos) */
@@ -428,16 +434,8 @@ int rtree_get_next(MI_INFO *info, uint keynr, uint key_length)
 
     return 0;
   }
-  else
-  {
-    if ((root = info->s->state.key_root[keynr]) == HA_OFFSET_ERROR)
-    {
-      my_errno= HA_ERR_END_OF_FILE;
-      return -1;
-    }
-  
-    return rtree_get_req(info, keyinfo, key_length, root, 0);
-  }
+
+  return rtree_get_req(info, keyinfo, key_length, root, 0);
 }