Bug#17877 - Corrupted spatial index

CHECK TABLE could complain about a fully intact spatial index.
A wrong comparison operator was used for table checking. 
The result was that it checked for non-matching spatial keys. 
This succeeded if at least two different keys were present, 
but failed if only the matching key was present.

I fixed the key comparison.


myisam/mi_check.c:
  Bug#17877 - Corrupted spatial index
  Fixed the comparison operator for checking a spatial index.
  Using MBR_EQUAL | MBR_DATA to compare for equality and
  include the data pointer in the comparison. The latter
  finds the index entry that points to the current record.
  This is necessary for non-unique indexes.
  
  The old operator, SEARCH_SAME, is unknown to the rtree
  search functions and handled like MBR_DISJOINT.
myisam/mi_key.c:
  Bug#17877 - Corrupted spatial index
  Added a missing DBUG_RETURN.
myisam/rt_index.c:
  Bug#17877 - Corrupted spatial index
  Included the data pointer in the copy of the search key.
  This is necessary for searching the index entry that points
  to a specific record if the search_flag contains MBR_DATA.
myisam/rt_mbr.c:
  Bug#17877 - Corrupted spatial index
  Extended the RT_CMP() macro with an assert for an 
  unexpected comparison operator.
mysql-test/r/gis-rtree.result:
  Bug#17877 - Corrupted spatial index
  The test result.
mysql-test/t/gis-rtree.test:
  Bug#17877 - Corrupted spatial index
  The test case.

4 files changed, 14 insertions, 7 deletions

diff --git a/myisam/mi_check.c b/myisam/mi_check.c
index 2395640d5bf..1e62e5e641d 100644
--- a/myisam/mi_check.c
+++ b/myisam/mi_check.c
@@ -1155,12 +1155,13 @@ int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
 	      */
               int search_result= (keyinfo->flag & HA_SPATIAL) ?
                 rtree_find_first(info, key, info->lastkey, key_length,
-                                 SEARCH_SAME) : 
+                                 MBR_EQUAL | MBR_DATA) : 
                 _mi_search(info,keyinfo,info->lastkey,key_length,
                            SEARCH_SAME, info->s->state.key_root[key]);
               if (search_result)
               {
-                mi_check_print_error(param,"Record at: %10s  Can't find key for index: %2d",
+                mi_check_print_error(param,"Record at: %10s  "
+                                     "Can't find key for index: %2d",
                                      llstr(start_recpos,llbuff),key+1);
                 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
                   goto err2;
diff --git a/myisam/mi_key.c b/myisam/mi_key.c
index cb85febd869..eaa854b1a37 100644
--- a/myisam/mi_key.c
+++ b/myisam/mi_key.c
@@ -54,7 +54,7 @@ uint _mi_make_key(register MI_INFO *info, uint keynr, uchar *key,
       TODO: nulls processing
     */
 #ifdef HAVE_SPATIAL
-    return sp_make_key(info,keynr,key,record,filepos);
+    DBUG_RETURN(sp_make_key(info,keynr,key,record,filepos));
 #else
     DBUG_ASSERT(0); /* mi_open should check that this never happens*/
 #endif
diff --git a/myisam/rt_index.c b/myisam/rt_index.c
index 97554dca4e6..1806476dc39 100644
--- a/myisam/rt_index.c
+++ b/myisam/rt_index.c
@@ -183,9 +183,11 @@ int rtree_find_first(MI_INFO *info, uint keynr, uchar *key, uint key_length,
     return -1;
   }
 
-  /* Save searched key */
-  memcpy(info->first_mbr_key, key, keyinfo->keylength -
-	 info->s->base.rec_reflength);
+  /*
+    Save searched key, include data pointer.
+    The data pointer is required if the search_flag contains MBR_DATA.
+  */
+  memcpy(info->first_mbr_key, key, keyinfo->keylength);
   info->last_rkey_length = key_length;
 
   info->rtree_recursion_depth = -1;
diff --git a/myisam/rt_mbr.c b/myisam/rt_mbr.c
index c43daec2f7c..897862c1c9a 100644
--- a/myisam/rt_mbr.c
+++ b/myisam/rt_mbr.c
@@ -52,10 +52,14 @@
     if (EQUAL_CMP(amin, amax, bmin, bmax)) \
       return 1; \
   } \
-  else /* if (nextflag & MBR_DISJOINT) */ \
+  else if (nextflag & MBR_DISJOINT) \
   { \
     if (DISJOINT_CMP(amin, amax, bmin, bmax)) \
       return 1; \
+  }\
+  else /* if unknown comparison operator */ \
+  { \
+    DBUG_ASSERT(0); \
   }
 
 #define RT_CMP_KORR(type, korr_func, len, nextflag) \