Fix for bug #5532 (error 22 inserting GIS data)

This bug is the result of weird error happening with mi_float8get and double
arythmetic. I described that in 'Bug that looks potentially dangerous' email


myisam/rt_mbr.c:
  *ab_area was changed with the local variable just to make the function
  working without errors
mysql-test/r/gis.result:
  appropriate test result
mysql-test/t/gis.test:
  Test case

3 files changed, 186 insertions, 4 deletions

diff --git a/myisam/rt_mbr.c b/myisam/rt_mbr.c
index 7b556979904..c7fde674729 100644
--- a/myisam/rt_mbr.c
+++ b/myisam/rt_mbr.c
@@ -505,7 +505,7 @@ double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar* a, uchar* b,
    amax = korr_func(a+len); \
    bmax = korr_func(b+len); \
    a_area *= (((double)amax) - ((double)amin)); \
-   *ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
+   loc_ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
 }
 
 #define RT_AREA_INC_GET(type, get_func, len)\
@@ -516,7 +516,7 @@ double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar* a, uchar* b,
    get_func(amax, a+len); \
    get_func(bmax, b+len); \
    a_area *= (((double)amax) - ((double)amin)); \
-   *ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
+   loc_ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
 }
 
 /*
@@ -526,6 +526,7 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
                           uint key_length, double *ab_area)
 {
   double a_area= 1.0;
+  double loc_ab_area= 1.0;
   
   *ab_area= 1.0;
   for (; (int)key_length > 0; keyseg += 2)
@@ -575,7 +576,7 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
       RT_AREA_INC_GET(double, mi_float8get, 8);
       break;
     case HA_KEYTYPE_END:
-      return *ab_area - a_area;
+      goto safe_end;
     default:
       return -1;
     }
@@ -584,7 +585,9 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
     a+= keyseg_length;
     b+= keyseg_length;
   }
-  return *ab_area - a_area;
+safe_end:
+  *ab_area= loc_ab_area;
+  return loc_ab_area - a_area;
 }
 
 #define RT_PERIM_INC_KORR(type, korr_func, len) \
diff --git a/mysql-test/r/gis.result b/mysql-test/r/gis.result
index 9f5dd286cf9..65a4c7df342 100644
--- a/mysql-test/r/gis.result
+++ b/mysql-test/r/gis.result
@@ -485,3 +485,92 @@ MBRContains(GeomFromText('Polygon((0 0, 0 7, 7 7, 7 0, 0 0))'), a);
 AsText(a)
 POINT(1 1)
 drop table t1;
+CREATE TABLE t1 (Coordinates POINT NOT NULL, SPATIAL INDEX(Coordinates));
+INSERT INTO t1 VALUES(GeomFromText('POINT(383293632 1754448)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(564952612 157516260)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(903994614 180726515)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(98128178 141127631)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(862547902 799334546)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(341989013 850270906)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(803302376 93039099)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(857439153 817431356)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(319757546 343162742)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(826341972 717484432)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(305066789 201736238)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(626068992 616241497)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(55789424 755830108)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(802874458 312435220)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(153795660 551723671)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(242207428 537089292)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(553478119 807160039)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(694605552 457472733)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(987886554 792733729)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(598600363 850434457)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(592068275 940589376)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(700705362 395370650)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(33628474 558144514)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(212802006 353386020)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(901307256 39143977)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(70870451 206374045)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(240880214 696939443)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(822615542 296669638)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(452769551 625489999)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(609104858 606565210)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(177213669 851312285)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(143654501 730691787)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(658472325 838260052)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(188164520 646358878)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(630993781 786764883)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(496793334 223062055)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(727354258 197498696)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(618432704 760982731)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(755643210 831234710)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(114368751 656950466)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(870378686 185239202)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(863324511 111258900)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(882178645 685940052)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(407928538 334948195)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(311430051 17033395)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(941513405 488643719)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(868345680 85167906)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(219335507 526818004)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(923427958 407500026)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(173176882 554421738)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(194264908 669970217)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(777483793 921619165)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(867468912 395916497)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(682601897 623112122)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(227151206 796970647)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(280062588 97529892)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(982209849 143387099)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(208788792 864388493)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(829327151 616717329)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(199336688 140757201)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(633750724 140850093)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(629400920 502096404)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(226017998 848736426)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(28914408 149445955)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(256236452 202091290)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(703867693 450501360)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(872061506 481351486)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(372120524 739530418)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(877267982 54722420)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(362642540 104419188)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(851693067 642705127)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(201949080 833902916)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(786092225 410737872)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(698291409 615419376)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(27455201 897628096)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(756176576 661205925)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(38478189 385577496)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(163302328 264496186)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(234313922 192216735)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(413942141 490550373)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(394308025 117809834)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(941051732 266369530)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(599161319 313172256)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(5899948 476429301)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(367894677 368542487)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(580848489 219587743)'));
+INSERT INTO t1 VALUES(GeomFromText('POINT(11247614 782797569)'));
+drop table t1;
diff --git a/mysql-test/t/gis.test b/mysql-test/t/gis.test
index e35b9996a44..e44f09c1d8d 100644
--- a/mysql-test/t/gis.test
+++ b/mysql-test/t/gis.test
@@ -190,3 +190,93 @@ select AsText(a) from t1 where
   and
   MBRContains(GeomFromText('Polygon((0 0, 0 7, 7 7, 7 0, 0 0))'), a);
 drop table t1;
+
+CREATE TABLE t1 (Coordinates POINT NOT NULL, SPATIAL INDEX(Coordinates)); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(383293632 1754448)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(564952612 157516260)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(903994614 180726515)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(98128178 141127631)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(862547902 799334546)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(341989013 850270906)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(803302376 93039099)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(857439153 817431356)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(319757546 343162742)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(826341972 717484432)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(305066789 201736238)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(626068992 616241497)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(55789424 755830108)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(802874458 312435220)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(153795660 551723671)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(242207428 537089292)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(553478119 807160039)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(694605552 457472733)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(987886554 792733729)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(598600363 850434457)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(592068275 940589376)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(700705362 395370650)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(33628474 558144514)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(212802006 353386020)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(901307256 39143977)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(70870451 206374045)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(240880214 696939443)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(822615542 296669638)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(452769551 625489999)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(609104858 606565210)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(177213669 851312285)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(143654501 730691787)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(658472325 838260052)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(188164520 646358878)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(630993781 786764883)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(496793334 223062055)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(727354258 197498696)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(618432704 760982731)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(755643210 831234710)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(114368751 656950466)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(870378686 185239202)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(863324511 111258900)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(882178645 685940052)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(407928538 334948195)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(311430051 17033395)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(941513405 488643719)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(868345680 85167906)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(219335507 526818004)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(923427958 407500026)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(173176882 554421738)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(194264908 669970217)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(777483793 921619165)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(867468912 395916497)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(682601897 623112122)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(227151206 796970647)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(280062588 97529892)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(982209849 143387099)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(208788792 864388493)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(829327151 616717329)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(199336688 140757201)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(633750724 140850093)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(629400920 502096404)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(226017998 848736426)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(28914408 149445955)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(256236452 202091290)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(703867693 450501360)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(872061506 481351486)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(372120524 739530418)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(877267982 54722420)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(362642540 104419188)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(851693067 642705127)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(201949080 833902916)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(786092225 410737872)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(698291409 615419376)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(27455201 897628096)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(756176576 661205925)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(38478189 385577496)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(163302328 264496186)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(234313922 192216735)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(413942141 490550373)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(394308025 117809834)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(941051732 266369530)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(599161319 313172256)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(5899948 476429301)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(367894677 368542487)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(580848489 219587743)')); 
+INSERT INTO t1 VALUES(GeomFromText('POINT(11247614 782797569)')); 
+drop table t1;