MDEV-12462 SPATIAL index fails to work with CONTAINS.

        Flags are wrongly set for MBR_CONTAINS/MBR_WITHIN functions.

13 files changed, 47 insertions, 31 deletions

diff --git a/mysql-test/r/gis-rt-precise.result b/mysql-test/r/gis-rt-precise.result
index 65583a0ce0d..d3308ed90ba 100644
--- a/mysql-test/r/gis-rt-precise.result
+++ b/mysql-test/r/gis-rt-precise.result
@@ -51,7 +51,7 @@ count(*)
 EXPLAIN SELECT fid, AsText(g) FROM t1 WHERE ST_Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 40))'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	g	g	34	NULL	1	Using where
+1	SIMPLE	t1	range	g	g	34	NULL	4	Using where
 SELECT fid, AsText(g) FROM t1 WHERE ST_Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 40))'));
 fid	AsText(g)
diff --git a/mysql-test/r/gis-rtree.result b/mysql-test/r/gis-rtree.result
index b76d02ef1d7..9373237f1d6 100644
--- a/mysql-test/r/gis-rtree.result
+++ b/mysql-test/r/gis-rtree.result
@@ -301,7 +301,7 @@ count(*)
 EXPLAIN SELECT fid, AsText(g) FROM t2 WHERE Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t2	range	g	g	34	NULL	1	Using where
+1	SIMPLE	t2	range	g	g	34	NULL	4	Using where
 SELECT fid, AsText(g) FROM t2 WHERE Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
 fid	AsText(g)
diff --git a/mysql-test/suite/innodb/r/innodb_gis.result b/mysql-test/suite/innodb/r/innodb_gis.result
index 5a978ea303c..35ea5104acb 100644
--- a/mysql-test/suite/innodb/r/innodb_gis.result
+++ b/mysql-test/suite/innodb/r/innodb_gis.result
@@ -599,3 +599,12 @@ create unique index a on t1(a);
 drop table t1;
 create table t1 (g geometry not null, spatial gk(g)) engine=innodb;
 drop table t1;
+create table t1(id int not null primary key, g1 geometry not null, spatial index(g1));
+insert into t1 values(1, polygonfromtext('POLYGON((0 0, 0 5, 5 5, 5 0, 0 0))'));
+explain select id from t1 where contains(g1, pointfromtext('POINT(1 1)'));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	g1	g1	34	NULL	1	Using where
+select id from t1 where contains(g1, pointfromtext('POINT(1 1)'));
+id
+1
+drop table t1;
diff --git a/mysql-test/suite/innodb/t/innodb_gis.test b/mysql-test/suite/innodb/t/innodb_gis.test
index 45d66d95002..cb04c15f0b6 100644
--- a/mysql-test/suite/innodb/t/innodb_gis.test
+++ b/mysql-test/suite/innodb/t/innodb_gis.test
@@ -11,3 +11,12 @@ SET storage_engine=innodb;
 create table t1 (g geometry not null, spatial gk(g)) engine=innodb;
 drop table t1;
 
+#
+# MDEV-12462 SPATIAL index fails to work with CONTAINS
+#
+
+create table t1(id int not null primary key, g1 geometry not null, spatial index(g1));
+insert into t1 values(1, polygonfromtext('POLYGON((0 0, 0 5, 5 5, 5 0, 0 0))'));
+explain select id from t1 where contains(g1, pointfromtext('POINT(1 1)'));
+select id from t1 where contains(g1, pointfromtext('POINT(1 1)'));
+drop table t1;
diff --git a/mysql-test/suite/innodb_gis/r/0.result b/mysql-test/suite/innodb_gis/r/0.result
index 9f155deef23..90657acaefc 100644
--- a/mysql-test/suite/innodb_gis/r/0.result
+++ b/mysql-test/suite/innodb_gis/r/0.result
@@ -610,7 +610,7 @@ set @g1 = ST_GeomFromText('Polygon((0 0,0 100,100 100,100 0,0 0))');
 set transaction isolation level read uncommitted;
 select count(*) from t1 where ST_Within(t1.c2, @g1);
 count(*)
-0
+1
 disconnect con1;
 connection a;
 commit;
diff --git a/mysql-test/suite/innodb_gis/r/alter_spatial_index.result b/mysql-test/suite/innodb_gis/r/alter_spatial_index.result
index 4a47f0feda8..a6279bdb196 100644
--- a/mysql-test/suite/innodb_gis/r/alter_spatial_index.result
+++ b/mysql-test/suite/innodb_gis/r/alter_spatial_index.result
@@ -66,6 +66,7 @@ c1	ST_Astext(c2)	ST_Astext(c4)
 SET @g1 = ST_GeomFromText('LINESTRING( 300 300,400 400)');
 SELECT c1,ST_Astext(c2),ST_Astext(c4) FROM tab WHERE MBRContains(tab.c4, @g1);
 c1	ST_Astext(c2)	ST_Astext(c4)
+4	POINT(50 50)	POLYGON((300 300,400 400,500 500,300 500,300 400,300 300))
 UPDATE tab SET C2 = ST_GeomFromText('POINT(100 100)')
 WHERE MBRContains(tab.c4, @g1);
 DELETE FROM tab WHERE MBRContains(tab.c4, @g1);
@@ -230,7 +231,6 @@ Table	Op	Msg_type	Msg_text
 test.tab	check	status	OK
 SELECT c1,ST_Astext(c2),ST_Astext(c4) FROM tab ORDER BY c1;
 c1	ST_Astext(c2)	ST_Astext(c4)
-4	POINT(50 50)	POLYGON((300 300,400 400,500 500,300 500,300 400,300 300))
 6	POINT(3 3)	POLYGON((2010 2010,2020 2020,2030 2030,2040 2030,2020 2010,2010 2010))
 7	POINT(60 70)	POLYGON((3010 3010,3020 3020,3030 3030,3040 3030,3020 3010,3010 3010))
 8	POINT(0 0)	POLYGON((3010 3010,3020 3020,3030 3030,3040 3030,3020 3010,3010 3010))
diff --git a/mysql-test/suite/innodb_gis/r/create_spatial_index.result b/mysql-test/suite/innodb_gis/r/create_spatial_index.result
index 1d47fd9688c..71d3092c3b7 100644
--- a/mysql-test/suite/innodb_gis/r/create_spatial_index.result
+++ b/mysql-test/suite/innodb_gis/r/create_spatial_index.result
@@ -76,6 +76,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	tab	range	idx3	idx3	34	NULL	1	Using where; Using filesort
 SELECT c1,ST_Astext(c4) FROM tab WHERE  MBRContains(tab.c4, @g1) ORDER BY c1;
 c1	ST_Astext(c4)
+4	POLYGON((300 300,400 400,500 500,300 500,300 400,300 300))
 EXPLAIN UPDATE tab SET C2 = ST_GeomFromText('POINT(0 0)') 
 WHERE MBRContains(tab.c4, @g1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
@@ -399,6 +400,7 @@ test.tab	check	status	OK
 SET @g1 = ST_GeomFromText('LINESTRING( 300 300,400 400)');
 SELECT c1,ST_Astext(c4) FROM tab WHERE  MBRContains(tab.c4, @g1) ORDER BY c1;
 c1	ST_Astext(c4)
+4	POLYGON((300 300,400 400,500 500,300 500,300 400,300 300))
 DELETE FROM tab WHERE MBRContains(tab.c4, @g1);
 SELECT c1,ST_Astext(c4) FROM tab WHERE  MBRContains(tab.c4, @g1) ORDER BY c1;
 c1	ST_Astext(c4)
@@ -408,7 +410,6 @@ test.tab	check	status	OK
 SET @g1 = ST_GeomFromText('POLYGON((100 200,1010 1010,1020 1020,500 300,300 200,100 300,100 200))');
 SELECT c1,ST_Astext(c4) FROM tab WHERE ST_Crosses(tab.c4, @g1) ORDER BY c1;
 c1	ST_Astext(c4)
-4	POLYGON((300 300,400 400,500 500,300 500,300 400,300 300))
 5	POLYGON((1010 1010,1020 1020,1030 1030,1040 1030,1020 1010,1010 1010))
 DELETE FROM tab WHERE ST_Crosses(tab.c4, @g1);
 SELECT c1,ST_Astext(c4) FROM tab WHERE ST_Crosses(tab.c4, @g1) ORDER BY c1;
@@ -503,6 +504,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	tab	range	idx3	idx3	34	NULL	1	Using where; Using filesort
 SELECT c1,ST_Astext(c4) FROM tab WHERE  MBRContains(tab.c4, @g1) ORDER BY c1;
 c1	ST_Astext(c4)
+4	POLYGON((300 300,400 400,500 500,300 500,300 400,300 300))
 EXPLAIN UPDATE tab SET C2 = ST_GeomFromText('POINT(0 0)') 
 WHERE MBRContains(tab.c4, @g1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
@@ -919,6 +921,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	tab	range	idx3	idx3	34	NULL	1	Using where; Using filesort
 SELECT c1,ST_Astext(c4) FROM tab WHERE  MBRContains(tab.c4, @g1) ORDER BY c1;
 c1	ST_Astext(c4)
+4	POLYGON((300 300,400 400,500 500,300 500,300 400,300 300))
 EXPLAIN UPDATE tab SET C2 = ST_GeomFromText('POINT(0 0)') 
 WHERE MBRContains(tab.c4, @g1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
diff --git a/mysql-test/suite/innodb_gis/r/rt_precise.result b/mysql-test/suite/innodb_gis/r/rt_precise.result
index e52f2ab5aa3..9e2c6adc2d9 100644
--- a/mysql-test/suite/innodb_gis/r/rt_precise.result
+++ b/mysql-test/suite/innodb_gis/r/rt_precise.result
@@ -55,5 +55,8 @@ count(*)
 SELECT fid, ST_AsText(g) FROM t1 WHERE ST_Within(g,
 ST_GeomFromText('Polygon((40 40,60 40,60 60,40 40))')) ORDER BY fid;
 fid	ST_AsText(g)
+45	LINESTRING(51 51,60 60)
+46	LINESTRING(51 41,60 50)
+56	LINESTRING(41 41,50 50)
 DROP TABLE t1;
 End of 5.5 tests.
diff --git a/mysql-test/suite/innodb_gis/r/rtree.result b/mysql-test/suite/innodb_gis/r/rtree.result
index f59949c79e9..285a499b16c 100644
--- a/mysql-test/suite/innodb_gis/r/rtree.result
+++ b/mysql-test/suite/innodb_gis/r/rtree.result
@@ -10,9 +10,12 @@ test.t1	analyze	status	OK
 set @g1 = ST_GeomFromText('Polygon((0 0,0 3,3 3,3 0,0 0))');
 explain select ST_astext(t1.g) from t1 where MBRWithin(t1.g, @g1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	g	g	34	NULL	1	Using where
+1	SIMPLE	t1	ALL	g	NULL	NULL	NULL	5	Using where
 select ST_astext(t1.g) from t1 where MBRWithin(t1.g, @g1);
 ST_astext(t1.g)
+POINT(1 1)
+POINT(1.5 1.5)
+POINT(3 3)
 set @g1 = ST_GeomFromText('Polygon((0 0,0 3,3 3,3 0,0 0))');
 delete from t1 where MBRWithin(t1.g, @g1);
 check table t1;
@@ -20,9 +23,6 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 select ST_astext(t1.g) from t1;
 ST_astext(t1.g)
-POINT(1 1)
-POINT(1.5 1.5)
-POINT(3 3)
 POINT(3.1 3.1)
 POINT(5 5)
 set @g1 = ST_GeomFromText('Polygon((5 5,5 5,5 5,5 5,5 5))');
@@ -32,9 +32,6 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 select ST_astext(t1.g) from t1;
 ST_astext(t1.g)
-POINT(1 1)
-POINT(1.5 1.5)
-POINT(3 3)
 POINT(3.1 3.1)
 POINT(2 2)
 show indexes from t1;
@@ -59,7 +56,6 @@ INSERT INTO t1 VALUES("left3",  ST_GeomFromText('POLYGON (( -3 0, -3 2, -1 2, -1
 SET @p = ST_GeomFromText('POLYGON (( 0 0, 0 2, 2 2, 2 0, 0 0))');
 SELECT name, ST_AsText(square) from t1 where MBRContains(@p, square);
 name	ST_AsText(square)
-small	POLYGON((0 0,0 1,1 1,1 0,0 0))
 SELECT name, ST_AsText(square) from t1 where MBRDisjoint(@p, square);
 name	ST_AsText(square)
 up3	POLYGON((0 3,0 5,2 5,2 3,0 3))
@@ -94,7 +90,6 @@ down2	POLYGON((0 -2,0 0,2 0,2 -2,0 -2))
 left2	POLYGON((-2 0,-2 2,0 2,0 0,-2 0))
 SELECT name, ST_AsText(square) from t1 where MBRWithin(@p, square);
 name	ST_AsText(square)
-big	POLYGON((0 0,0 3,3 3,3 0,0 0))
 SET @vert1   = ST_GeomFromText('POLYGON ((0 -2, 0 2, 0 -2))');
 SET @horiz1  = ST_GeomFromText('POLYGON ((-2 0, 2 0, -2 0))');
 SET @horiz2 = ST_GeomFromText('POLYGON ((-1 0, 3 0, -1 0))');
@@ -157,9 +152,12 @@ test.t1	analyze	status	OK
 set @g1 = ST_GeomFromText('Polygon((0 0,0 3,3 3,3 0,0 0))');
 explain select ST_astext(t1.g) from t1 where MBRWithin(t1.g, @g1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	g	g	34	NULL	1	Using where
+1	SIMPLE	t1	ALL	g	NULL	NULL	NULL	5	Using where
 select ST_astext(t1.g) from t1 where MBRWithin(t1.g, @g1);
 ST_astext(t1.g)
+POINT(1 1)
+POINT(1.5 1.5)
+POINT(3 3)
 set @g1 = ST_GeomFromText('Polygon((0 0,0 3,3 3,3 0,0 0))');
 delete from t1 where MBRWithin(t1.g, @g1);
 check table t1;
@@ -167,9 +165,6 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 select ST_astext(t1.g) from t1;
 ST_astext(t1.g)
-POINT(1 1)
-POINT(1.5 1.5)
-POINT(3 3)
 POINT(3.1 3.1)
 POINT(5 5)
 set @g1 = ST_GeomFromText('Polygon((5 5,5 5,5 5,5 5,5 5))');
@@ -179,9 +174,6 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 select ST_astext(t1.g) from t1;
 ST_astext(t1.g)
-POINT(1 1)
-POINT(1.5 1.5)
-POINT(3 3)
 POINT(3.1 3.1)
 POINT(2 2)
 show indexes from t1;
@@ -225,7 +217,7 @@ SELECT COUNT(*)
 FROM t1
 WHERE ST_CONTAINS(ST_GeomFromText('POLYGON((2 2,4 2, 4 4, 2 4, 2 2))'),way);
 COUNT(*)
-9
+0
 OPTIMIZE TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	note	Table does not support optimize, doing recreate + analyze instead
@@ -234,7 +226,7 @@ SELECT COUNT(*)
 FROM t1
 WHERE ST_CONTAINS(ST_GeomFromText('POLYGON((2 2,4 2, 4 4, 2 4, 2 2))'),way);
 COUNT(*)
-9
+0
 DROP TABLE t1;
 CREATE TABLE t1( i INT, g GEOMETRY NOT NULL, SPATIAL INDEX (g)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(1, LINESTRING(POINT(1,1), POINT(4, 4)));
diff --git a/mysql-test/suite/maria/maria-gis-rtree-dynamic.result b/mysql-test/suite/maria/maria-gis-rtree-dynamic.result
index a0535026722..8bea5edb6fb 100644
--- a/mysql-test/suite/maria/maria-gis-rtree-dynamic.result
+++ b/mysql-test/suite/maria/maria-gis-rtree-dynamic.result
@@ -168,7 +168,7 @@ count(*)
 150
 EXPLAIN SELECT fid, AsText(g) FROM t1 WHERE Within(g, GeomFromText('Polygon((140 140,160 140,160 160,140 160,140 140))'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	ALL	g	NULL	NULL	NULL	150	Using where
+1	SIMPLE	t1	range	g	g	34	NULL	11	Using where
 SELECT fid, AsText(g) FROM t1 WHERE Within(g, GeomFromText('Polygon((140 140,160 140,160 160,140 160,140 140))'));
 fid	AsText(g)
 1	LINESTRING(150 150,150 150)
@@ -302,7 +302,7 @@ count(*)
 EXPLAIN SELECT fid, AsText(g) FROM t2 WHERE Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t2	range	g	g	34	NULL	1	Using where
+1	SIMPLE	t2	range	g	g	34	NULL	4	Using where
 SELECT fid, AsText(g) FROM t2 WHERE Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
 fid	AsText(g)
diff --git a/mysql-test/suite/maria/maria-gis-rtree-trans.result b/mysql-test/suite/maria/maria-gis-rtree-trans.result
index dbcbdeb1661..4a2f97fbd41 100644
--- a/mysql-test/suite/maria/maria-gis-rtree-trans.result
+++ b/mysql-test/suite/maria/maria-gis-rtree-trans.result
@@ -168,7 +168,7 @@ count(*)
 150
 EXPLAIN SELECT fid, AsText(g) FROM t1 WHERE Within(g, GeomFromText('Polygon((140 140,160 140,160 160,140 160,140 140))'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	ALL	g	NULL	NULL	NULL	150	Using where
+1	SIMPLE	t1	range	g	g	34	NULL	11	Using where
 SELECT fid, AsText(g) FROM t1 WHERE Within(g, GeomFromText('Polygon((140 140,160 140,160 160,140 160,140 140))'));
 fid	AsText(g)
 1	LINESTRING(150 150,150 150)
@@ -302,7 +302,7 @@ count(*)
 EXPLAIN SELECT fid, AsText(g) FROM t2 WHERE Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t2	range	g	g	34	NULL	1	Using where
+1	SIMPLE	t2	range	g	g	34	NULL	4	Using where
 SELECT fid, AsText(g) FROM t2 WHERE Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
 fid	AsText(g)
diff --git a/mysql-test/suite/maria/maria-gis-rtree.result b/mysql-test/suite/maria/maria-gis-rtree.result
index 3ecdcc64928..901a7bee397 100644
--- a/mysql-test/suite/maria/maria-gis-rtree.result
+++ b/mysql-test/suite/maria/maria-gis-rtree.result
@@ -168,7 +168,7 @@ count(*)
 150
 EXPLAIN SELECT fid, AsText(g) FROM t1 WHERE Within(g, GeomFromText('Polygon((140 140,160 140,160 160,140 160,140 140))'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	ALL	g	NULL	NULL	NULL	150	Using where
+1	SIMPLE	t1	range	g	g	34	NULL	11	Using where
 SELECT fid, AsText(g) FROM t1 WHERE Within(g, GeomFromText('Polygon((140 140,160 140,160 160,140 160,140 140))'));
 fid	AsText(g)
 1	LINESTRING(150 150,150 150)
@@ -302,7 +302,7 @@ count(*)
 EXPLAIN SELECT fid, AsText(g) FROM t2 WHERE Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t2	range	g	g	34	NULL	1	Using where
+1	SIMPLE	t2	range	g	g	34	NULL	4	Using where
 SELECT fid, AsText(g) FROM t2 WHERE Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
 fid	AsText(g)
diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc
index 7ac2e054e8c..16bad154862 100644
--- a/sql/item_geofunc.cc
+++ b/sql/item_geofunc.cc
@@ -1072,11 +1072,11 @@ Item_func_spatial_rel::get_mm_leaf(RANGE_OPT_PARAM *param,
     tree->max_flag= NO_MAX_RANGE;
     break;
   case SP_WITHIN_FUNC:
-    tree->min_flag= GEOM_FLAG | HA_READ_MBR_WITHIN;// NEAR_MIN;//512;
+    tree->min_flag= GEOM_FLAG | HA_READ_MBR_CONTAIN;// NEAR_MIN;//512;
     tree->max_flag= NO_MAX_RANGE;
     break;
   case SP_CONTAINS_FUNC:
-    tree->min_flag= GEOM_FLAG | HA_READ_MBR_CONTAIN;// NEAR_MIN;//512;
+    tree->min_flag= GEOM_FLAG | HA_READ_MBR_WITHIN;// NEAR_MIN;//512;
     tree->max_flag= NO_MAX_RANGE;
     break;
   case SP_OVERLAPS_FUNC: