Bug#17877 - Corrupted spatial index

CHECK TABLE could complain about a fully intact spatial index.
A wrong comparison operator was used for table checking. 
The result was that it checked for non-matching spatial keys. 
This succeeded if at least two different keys were present, 
but failed if only the matching key was present.

I fixed the key comparison.


myisam/mi_check.c:
  Bug#17877 - Corrupted spatial index
  Fixed the comparison operator for checking a spatial index.
  Using MBR_EQUAL | MBR_DATA to compare for equality and
  include the data pointer in the comparison. The latter
  finds the index entry that points to the current record.
  This is necessary for non-unique indexes.
  
  The old operator, SEARCH_SAME, is unknown to the rtree
  search functions and handled like MBR_DISJOINT.
myisam/mi_key.c:
  Bug#17877 - Corrupted spatial index
  Added a missing DBUG_RETURN.
myisam/rt_index.c:
  Bug#17877 - Corrupted spatial index
  Included the data pointer in the copy of the search key.
  This is necessary for searching the index entry that points
  to a specific record if the search_flag contains MBR_DATA.
myisam/rt_mbr.c:
  Bug#17877 - Corrupted spatial index
  Extended the RT_CMP() macro with an assert for an 
  unexpected comparison operator.
mysql-test/r/gis-rtree.result:
  Bug#17877 - Corrupted spatial index
  The test result.
mysql-test/t/gis-rtree.test:
  Bug#17877 - Corrupted spatial index
  The test case.

2 files changed, 84 insertions, 0 deletions

diff --git a/mysql-test/r/gis-rtree.result b/mysql-test/r/gis-rtree.result
index f479fc41ffb..3fcae8843c0 100644
--- a/mysql-test/r/gis-rtree.result
+++ b/mysql-test/r/gis-rtree.result
@@ -817,3 +817,43 @@ check table t1 extended;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 drop table t1;
+CREATE TABLE t1 (
+c1 geometry NOT NULL default '',
+SPATIAL KEY i1 (c1(32))
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 (c1) VALUES (
+PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+CHECK TABLE t1 EXTENDED;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+DROP TABLE t1;
+CREATE TABLE t1 (
+c1 geometry NOT NULL default '',
+SPATIAL KEY i1 (c1(32))
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 (c1) VALUES (
+PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+INSERT INTO t1 (c1) VALUES (
+PolygonFromText('POLYGON((-65.7402776999 -96.6686111000,
+                            -65.7372222000 -96.5516666000,
+                            -65.8502777000 -96.5461111000,
+                            -65.8527777000 -96.6627777000,
+                            -65.7402776999 -96.6686111000))'));
+INSERT INTO t1 (c1) VALUES (
+PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+CHECK TABLE t1 EXTENDED;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+DROP TABLE t1;
diff --git a/mysql-test/t/gis-rtree.test b/mysql-test/t/gis-rtree.test
index 682f67c61c4..eba53a8a9c5 100644
--- a/mysql-test/t/gis-rtree.test
+++ b/mysql-test/t/gis-rtree.test
@@ -188,4 +188,48 @@ check table t1 extended;
 
 drop table t1;
 
+#
+# Bug#17877 - Corrupted spatial index
+#
+CREATE TABLE t1 (
+  c1 geometry NOT NULL default '',
+  SPATIAL KEY i1 (c1(32))
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 (c1) VALUES (
+  PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+# This showed a missing key.
+CHECK TABLE t1 EXTENDED;
+DROP TABLE t1;
+#
+CREATE TABLE t1 (
+  c1 geometry NOT NULL default '',
+  SPATIAL KEY i1 (c1(32))
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 (c1) VALUES (
+  PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+INSERT INTO t1 (c1) VALUES (
+  PolygonFromText('POLYGON((-65.7402776999 -96.6686111000,
+                            -65.7372222000 -96.5516666000,
+                            -65.8502777000 -96.5461111000,
+                            -65.8527777000 -96.6627777000,
+                            -65.7402776999 -96.6686111000))'));
+# This is the same as the first insert to get a non-unique key.
+INSERT INTO t1 (c1) VALUES (
+  PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+# This showed (and still shows) OK.
+CHECK TABLE t1 EXTENDED;
+DROP TABLE t1;
+
 # End of 4.1 tests