Bug #30286 spatial index cause corruption and server crash!

As the result of DOUBLE claculations can be bigger
than DBL_MAX constant we use in code, we shouldn't use this constatn
as a biggest possible value.
Particularly the rtree_pick_key function set 'min_area= DBL_MAX' relying
that any rtree_area_increase result will be less so we return valid
key. Though in rtree_area_increase function we calculate the area
of the rectangle, so the result can be 'inf' if the rectangle is
huge enough, which is bigger than DBL_MAX.

Code of the rtree_pick_key modified so we always return a valid key.


myisam/rt_index.c:
  Bug #30286 spatial index cause corruption and server crash!
  
  always set the best_key with the first key we get, so we always return
  somthing valid.
myisam/rt_mbr.c:
  Bug #30286 spatial index cause corruption and server crash!
  
  function comment extended
mysql-test/r/gis-rtree.result:
  Bug #30286 spatial index cause corruption and server crash!
  test result
mysql-test/t/gis-rtree.test:
  Bug #30286 spatial index cause corruption and server crash!
  test case

2 files changed, 9 insertions, 14 deletions

diff --git a/myisam/rt_index.c b/myisam/rt_index.c
index 238432006a4..f19ecacef63 100644
--- a/myisam/rt_index.c
+++ b/myisam/rt_index.c
@@ -485,15 +485,16 @@ static uchar *rtree_pick_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key,
 			     uint key_length, uchar *page_buf, uint nod_flag)
 {
   double increase;
-  double best_incr = DBL_MAX;
+  double best_incr;
   double area;
   double best_area;
-  uchar *best_key;
+  uchar *best_key= NULL;
   uchar *k = rt_PAGE_FIRST_KEY(page_buf, nod_flag);
   uchar *last = rt_PAGE_END(page_buf);
 
   LINT_INIT(best_area);
   LINT_INIT(best_key);
+  LINT_INIT(best_incr);
 
   for (; k < last; k = rt_PAGE_NEXT_KEY(k, key_length, nod_flag))
   {
@@ -502,22 +503,13 @@ static uchar *rtree_pick_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key,
                                         &area)) == -1.0)
       return NULL;
     /* The following should be safe, even if we compare doubles */
-    if (increase < best_incr)
+    if (!best_key || increase < best_incr ||
+        ((increase == best_incr) && (area < best_area)))
     {
       best_key = k;
       best_area = area;
       best_incr = increase;
     }
-    else
-    {
-      /* The following should be safe, even if we compare doubles */
-      if ((increase == best_incr) && (area < best_area))
-      {
-        best_key = k;
-        best_area = area;
-        best_incr = increase;
-      }
-    }
   }
   return best_key;
 }
diff --git a/myisam/rt_mbr.c b/myisam/rt_mbr.c
index 897862c1c9a..31eaac0ae70 100644
--- a/myisam/rt_mbr.c
+++ b/myisam/rt_mbr.c
@@ -525,7 +525,10 @@ double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar* a, uchar* b,
 }
 
 /*
-Calculates MBR_AREA(a+b) - MBR_AREA(a)
+  Calculates MBR_AREA(a+b) - MBR_AREA(a)
+  Note: when 'a' and 'b' objects are far from each other,
+  the area increase can be really big, so this function
+  can return 'inf' as a result.
 */
 double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b, 
                           uint key_length, double *ab_area)