wl 1562 (To improve RTree indexes)

some changes to make code nicer


include/myisampack.h:
  mi_sint1korr and similar things added to do conversion in an
  uniform way
myisam/rt_index.c:
  'if' simplified
myisam/rt_mbr.c:
  some fixes to make code nicer and smaller
myisam/rt_test.c:
  some modifications to extend test
mysql-test/r/gis-rtree.result:
  result became slightly different because of changes made

5 files changed, 256 insertions, 270 deletions

diff --git a/include/myisampack.h b/include/myisampack.h
index 95793e2aaeb..06c94fea75f 100644
--- a/include/myisampack.h
+++ b/include/myisampack.h
@@ -21,6 +21,10 @@
   better compression
 */
 
+/* these two are for uniformity */
+#define mi_sint1korr(A) (int8)(*A)
+#define mi_uint1korr(A) (uint8)(*A)
+
 #define mi_sint2korr(A) (int16) (((int16) ((uchar) (A)[1])) +\
 				 ((int16) ((int16) (A)[0]) << 8))
 #define mi_sint3korr(A) ((int32) ((((uchar) (A)[0]) & 128) ? \
@@ -75,6 +79,9 @@
 				    (((uint32) ((uchar) (A)[0])) << 24))) <<\
 				    32))
 
+/* This one is for uniformity */
+#define mi_int1store(T,A) *((uchar*)(T))= (uchar) (A)
+
 #define mi_int2store(T,A)  { uint def_temp= (uint) (A) ;\
 			     *((uchar*) ((T)+1))= (uchar)(def_temp); \
 			     *((uchar*) ((T)+0))= (uchar)(def_temp >> 8); }
diff --git a/myisam/rt_index.c b/myisam/rt_index.c
index 824cb7a396f..8d8a5412c7b 100644
--- a/myisam/rt_index.c
+++ b/myisam/rt_index.c
@@ -462,21 +462,13 @@ static uchar *rtree_pick_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key,
     if ((increase = rtree_perimeter_increase(keyinfo->seg, k, key, key_length,
 					     &perimeter)) == -1)
       return NULL;
-    if (increase < best_incr)
+    if ((increase < best_incr)||
+	(increase == best_incr && perimeter < best_perimeter))
     {
       best_key = k;
       best_perimeter= perimeter;
       best_incr = increase;
     }
-    else
-    {
-      if ((increase == best_incr) && (perimeter < best_perimeter))
-      {
-        best_key = k;
-        best_perimeter= perimeter;
-        best_incr = increase;
-      }
-    }
   }
   return best_key;
 }
diff --git a/myisam/rt_mbr.c b/myisam/rt_mbr.c
index da427e4b67a..1f036eff196 100644
--- a/myisam/rt_mbr.c
+++ b/myisam/rt_mbr.c
@@ -24,7 +24,7 @@
 #define CONTAIN_CMP(amin, amax, bmin, bmax) ((bmin > amin)  || (bmax <  amax))
 #define WITHIN_CMP(amin, amax, bmin, bmax) ((amin > bmin)  || (amax <  bmax))
 #define DISJOINT_CMP(amin, amax, bmin, bmax) ((amin <= bmax) && (bmin <= amax))
-#define EQUAL_CMP(amix, amax, bmin, bmax) ((amix != bmin) || (amax != bmax))
+#define EQUAL_CMP(amin, amax, bmin, bmax) ((amin != bmin) || (amax != bmax))
 
 #define FCMP(A, B) ((int)(A) - (int)(B))
 #define p_inc(A, B, X)  {A += X; B += X;}
@@ -61,12 +61,9 @@
   type amin, amax, bmin, bmax; \
   amin = korr_func(a); \
   bmin = korr_func(b); \
-  p_inc(a, b, len); \
-  amax = korr_func(a); \
-  bmax = korr_func(b); \
+  amax = korr_func(a+len); \
+  bmax = korr_func(b+len); \
   RT_CMP(nextflag); \
-  p_inc(a, b, len); \
-  break; \
 }
 
 #define RT_CMP_GET(type, get_func, len, nextflag) \
@@ -74,12 +71,9 @@
   type amin, amax, bmin, bmax; \
   get_func(amin, a); \
   get_func(bmin, b); \
-  p_inc(a, b, len); \
-  get_func(amax, a); \
-  get_func(bmax, b); \
+  get_func(amax, a+len); \
+  get_func(bmax, b+len); \
   RT_CMP(nextflag); \
-  p_inc(a, b, len); \
-  break; \
 }
 
 /*
@@ -98,54 +92,54 @@ int rtree_key_cmp(HA_KEYSEG *keyseg, uchar *b, uchar *a, uint key_length,
 {
   for (; (int) key_length > 0; keyseg += 2 )
   {
-    key_length -= keyseg->length * 2;
-
     switch ((enum ha_base_keytype) keyseg->type) {
-    case HA_KEYTYPE_TEXT:
+    case HA_KEYTYPE_INT8:
+      RT_CMP_KORR(int8, mi_sint1korr, 1, nextflag);
+      break;
     case HA_KEYTYPE_BINARY:
-    case HA_KEYTYPE_VARTEXT:
-    case HA_KEYTYPE_VARBINARY:
-    case HA_KEYTYPE_NUM:
-    default:
-      return 1;
+      RT_CMP_KORR(uint8, mi_uint1korr, 1, nextflag);
       break;
-    case HA_KEYTYPE_INT8:
-      {
-        int amin,amax,bmin,bmax;
-        amin = (int)*((signed char *)a);
-        bmin = (int)*((signed char *)b);
-        p_inc(a, b, 1);
-        amax = (int)*((signed char *)a);
-        bmax = (int)*((signed char *)b);
-        RT_CMP(nextflag);
-        p_inc(a, b, 1);
-        break;
-      }
     case HA_KEYTYPE_SHORT_INT:
       RT_CMP_KORR(int16, mi_sint2korr, 2, nextflag);
+      break;
     case HA_KEYTYPE_USHORT_INT:
       RT_CMP_KORR(uint16, mi_uint2korr, 2, nextflag);
+      break;
     case HA_KEYTYPE_INT24:
       RT_CMP_KORR(int32, mi_sint3korr, 3, nextflag);
+      break;
     case HA_KEYTYPE_UINT24:
       RT_CMP_KORR(uint32, mi_uint3korr, 3, nextflag);
+      break;
     case HA_KEYTYPE_LONG_INT:
       RT_CMP_KORR(int32, mi_sint4korr, 4, nextflag);
+      break;
     case HA_KEYTYPE_ULONG_INT:
       RT_CMP_KORR(uint32, mi_uint4korr, 4, nextflag);
+      break;
 #ifdef HAVE_LONG_LONG
     case HA_KEYTYPE_LONGLONG:
       RT_CMP_KORR(longlong, mi_sint8korr, 8, nextflag)
+      break;
     case HA_KEYTYPE_ULONGLONG:
       RT_CMP_KORR(ulonglong, mi_uint8korr, 8, nextflag)
+      break;
 #endif
     case HA_KEYTYPE_FLOAT:
       RT_CMP_GET(float, mi_float4get, 4, nextflag);
+      break;
     case HA_KEYTYPE_DOUBLE:
       RT_CMP_GET(double, mi_float8get, 8, nextflag);
+      break;
     case HA_KEYTYPE_END:
       goto end;
+    default:
+      return 1;
     }
+    uint32 keyseg_length= keyseg->length * 2;
+    key_length-= keyseg_length;
+    a+= keyseg_length;
+    b+= keyseg_length;
   }
 
 end:
@@ -165,22 +159,16 @@ end:
 { \
   type amin, amax; \
   amin = korr_func(a); \
-  a += len; \
-  amax = korr_func(a); \
-  a += len; \
+  amax = korr_func(a+len); \
   res *= (cast(amax) - cast(amin)); \
-  break; \
 }
 
 #define RT_VOL_GET(type, get_func, len, cast) \
 { \
   type amin, amax; \
   get_func(amin, a); \
-  a += len; \
-  get_func(amax, a); \
-  a += len; \
+  get_func(amax, a+len); \
   res *= (cast(amax) - cast(amin)); \
-  break; \
 }
 
 /*
@@ -191,53 +179,54 @@ double rtree_rect_volume(HA_KEYSEG *keyseg, uchar *a, uint key_length)
   double res = 1;
   for (; (int)key_length > 0; keyseg += 2)
   {
-    key_length -= keyseg->length * 2;
-    
     switch ((enum ha_base_keytype) keyseg->type) {
-    case HA_KEYTYPE_TEXT:
+    case HA_KEYTYPE_INT8:
+      RT_VOL_KORR(int8, mi_sint1korr, 1, (double));
+      break;
     case HA_KEYTYPE_BINARY:
-    case HA_KEYTYPE_VARTEXT:
-    case HA_KEYTYPE_VARBINARY:
-    case HA_KEYTYPE_NUM:
-    default:
-      return 1;
+      RT_VOL_KORR(uint8, mi_uint1korr, 1, (double));
       break;
-    case HA_KEYTYPE_INT8:
-      {
-        int amin, amax;
-        amin = (int)*((signed char *)a);
-        a += 1;
-        amax = (int)*((signed char *)a);
-        a += 1;
-        res *= ((double)amax - (double)amin);
-        break;
-      }
     case HA_KEYTYPE_SHORT_INT:
       RT_VOL_KORR(int16, mi_sint2korr, 2, (double));
+      break;
     case HA_KEYTYPE_USHORT_INT:
       RT_VOL_KORR(uint16, mi_uint2korr, 2, (double));
+      break;
     case HA_KEYTYPE_INT24:
       RT_VOL_KORR(int32, mi_sint3korr, 3, (double));
+      break;
     case HA_KEYTYPE_UINT24:
       RT_VOL_KORR(uint32, mi_uint3korr, 3, (double));
+      break;
     case HA_KEYTYPE_LONG_INT:
       RT_VOL_KORR(int32, mi_sint4korr, 4, (double));
+      break;
     case HA_KEYTYPE_ULONG_INT:
       RT_VOL_KORR(uint32, mi_uint4korr, 4, (double));
+      break;
 #ifdef HAVE_LONG_LONG
     case HA_KEYTYPE_LONGLONG:
       RT_VOL_KORR(longlong, mi_sint8korr, 8, (double));
+      break;
     case HA_KEYTYPE_ULONGLONG:
       RT_VOL_KORR(longlong, mi_sint8korr, 8, ulonglong2double);
+      break;
 #endif
     case HA_KEYTYPE_FLOAT:
       RT_VOL_GET(float, mi_float4get, 4, (double));
+      break;
     case HA_KEYTYPE_DOUBLE:
       RT_VOL_GET(double, mi_float8get, 8, (double));
+      break;
     case HA_KEYTYPE_END:
       key_length = 0;
       break;
+    default:
+      return -1;
     }
+    uint32 keyseg_length= keyseg->length * 2;
+    key_length-= keyseg_length;
+    a+= keyseg_length;
   }
   return res;
 }
@@ -246,24 +235,18 @@ double rtree_rect_volume(HA_KEYSEG *keyseg, uchar *a, uint key_length)
 { \
   type amin, amax; \
   amin = korr_func(a); \
-  a += len; \
-  amax = korr_func(a); \
-  a += len; \
+  amax = korr_func(a+len); \
   *res++ = cast(amin); \
   *res++ = cast(amax); \
-  break; \
 }
 
 #define RT_D_MBR_GET(type, get_func, len, cast) \
 { \
   type amin, amax; \
   get_func(amin, a); \
-  a += len; \
-  get_func(amax, a); \
-  a += len; \
+  get_func(amax, a+len); \
   *res++ = cast(amin); \
   *res++ = cast(amax); \
-  break; \
 }
 
 /*
@@ -273,54 +256,54 @@ int rtree_d_mbr(HA_KEYSEG *keyseg, uchar *a, uint key_length, double *res)
 {
   for (; (int)key_length > 0; keyseg += 2)
   {
-    key_length -= keyseg->length * 2;
-    
     switch ((enum ha_base_keytype) keyseg->type) {
-    case HA_KEYTYPE_TEXT:
+    case HA_KEYTYPE_INT8:
+      RT_D_MBR_KORR(int8, mi_sint1korr, 1, (double));
+      break;
     case HA_KEYTYPE_BINARY:
-    case HA_KEYTYPE_VARTEXT:
-    case HA_KEYTYPE_VARBINARY:
-    case HA_KEYTYPE_NUM:
-    default:
-      return 1;
+      RT_D_MBR_KORR(uint8, mi_uint1korr, 1, (double));
       break;
-    case HA_KEYTYPE_INT8:
-      {
-        int amin, amax;
-        amin = (int)*((signed char *)a);
-        a += 1;
-        amax = (int)*((signed char *)a);
-        a += 1;
-        *res++ = (double)amin;
-        *res++ = (double)amax;
-        break;
-      }
     case HA_KEYTYPE_SHORT_INT:
       RT_D_MBR_KORR(int16, mi_sint2korr, 2, (double));
+      break;
     case HA_KEYTYPE_USHORT_INT:
       RT_D_MBR_KORR(uint16, mi_uint2korr, 2, (double));
+      break;
     case HA_KEYTYPE_INT24:
       RT_D_MBR_KORR(int32, mi_sint3korr, 3, (double));
+      break;
     case HA_KEYTYPE_UINT24:
       RT_D_MBR_KORR(uint32, mi_uint3korr, 3, (double));
+      break;
     case HA_KEYTYPE_LONG_INT:
       RT_D_MBR_KORR(int32, mi_sint4korr, 4, (double));
+      break;
     case HA_KEYTYPE_ULONG_INT:
       RT_D_MBR_KORR(uint32, mi_uint4korr, 4, (double));
+      break;
 #ifdef HAVE_LONG_LONG
     case HA_KEYTYPE_LONGLONG:
       RT_D_MBR_KORR(longlong, mi_sint8korr, 8, (double));
+      break;
     case HA_KEYTYPE_ULONGLONG:
       RT_D_MBR_KORR(longlong, mi_sint8korr, 8, ulonglong2double);
+      break;
 #endif
     case HA_KEYTYPE_FLOAT:
       RT_D_MBR_GET(float, mi_float4get, 4, (double));
+      break;
     case HA_KEYTYPE_DOUBLE:
       RT_D_MBR_GET(double, mi_float8get, 8, (double));
+      break;
     case HA_KEYTYPE_END:
       key_length = 0;
       break;
+    default:
+      return 1;
     }
+    uint32 keyseg_length= keyseg->length * 2;
+    key_length-= keyseg_length;
+    a+= keyseg_length;
   }
   return 0;
 }
@@ -330,17 +313,12 @@ int rtree_d_mbr(HA_KEYSEG *keyseg, uchar *a, uint key_length, double *res)
   type amin, amax, bmin, bmax; \
   amin = korr_func(a); \
   bmin = korr_func(b); \
-  p_inc(a, b, len); \
-  amax = korr_func(a); \
-  bmax = korr_func(b); \
-  p_inc(a, b, len); \
+  amax = korr_func(a+len); \
+  bmax = korr_func(b+len); \
   amin = min(amin, bmin); \
   amax = max(amax, bmax); \
   store_func(c, amin); \
-  c += len; \
-  store_func(c, amax); \
-  c += len; \
-  break; \
+  store_func(c+len, amax); \
 }
 
 #define RT_COMB_GET(type, get_func, store_func, len) \
@@ -348,17 +326,12 @@ int rtree_d_mbr(HA_KEYSEG *keyseg, uchar *a, uint key_length, double *res)
   type amin, amax, bmin, bmax; \
   get_func(amin, a); \
   get_func(bmin, b); \
-  p_inc(a, b, len); \
-  get_func(amax, a); \
-  get_func(bmax, b); \
-  p_inc(a, b, len); \
+  get_func(amax, a+len); \
+  get_func(bmax, b+len); \
   amin = min(amin, bmin); \
   amax = max(amax, bmax); \
   store_func(c, amin); \
-  c += len; \
-  store_func(c, amax); \
-  c += len; \
-  break; \
+  store_func(c+len, amax); \
 }
 
 /*
@@ -370,62 +343,57 @@ int rtree_d_mbr(HA_KEYSEG *keyseg, uchar *a, uint key_length, double *res)
 int rtree_combine_rect(HA_KEYSEG *keyseg, uchar* a, uchar* b, uchar* c, 
 		       uint key_length)
 {
-
   for ( ; (int) key_length > 0 ; keyseg += 2)
   {
-    key_length -= keyseg->length * 2;
-    
     switch ((enum ha_base_keytype) keyseg->type) {
-    case HA_KEYTYPE_TEXT:
+    case HA_KEYTYPE_INT8:
+      RT_COMB_KORR(int8, mi_sint1korr, mi_int1store, 1);
+      break;
     case HA_KEYTYPE_BINARY:
-    case HA_KEYTYPE_VARTEXT:
-    case HA_KEYTYPE_VARBINARY:
-    case HA_KEYTYPE_NUM:
-    default:
-      return 1;
+      RT_COMB_KORR(uint8, mi_uint1korr, mi_int1store, 1);
       break;
-    case HA_KEYTYPE_INT8:
-      {
-        int amin, amax, bmin, bmax;
-        amin = (int)*((signed char *)a);
-        bmin = (int)*((signed char *)b);
-        p_inc(a, b, 1);
-        amax = (int)*((signed char *)a);
-        bmax = (int)*((signed char *)b);
-        p_inc(a, b, 1);
-        amin = min(amin, bmin);
-        amax = max(amax, bmax);
-        *((signed char*)c) = amin;
-        c += 1;
-        *((signed char*)c) = amax;
-        c += 1;
-        break;
-      }
     case HA_KEYTYPE_SHORT_INT:
       RT_COMB_KORR(int16, mi_sint2korr, mi_int2store, 2);
+      break;
     case HA_KEYTYPE_USHORT_INT:
       RT_COMB_KORR(uint16, mi_uint2korr, mi_int2store, 2);
+      break;
     case HA_KEYTYPE_INT24:
       RT_COMB_KORR(int32, mi_sint3korr, mi_int3store, 3);
+      break;
     case HA_KEYTYPE_UINT24:
       RT_COMB_KORR(uint32, mi_uint3korr, mi_int3store, 3);
+      break;
     case HA_KEYTYPE_LONG_INT:
       RT_COMB_KORR(int32, mi_sint4korr, mi_int4store, 4);
+      break;
     case HA_KEYTYPE_ULONG_INT:
       RT_COMB_KORR(uint32, mi_uint4korr, mi_int4store, 4);
+      break;
 #ifdef HAVE_LONG_LONG
     case HA_KEYTYPE_LONGLONG:
       RT_COMB_KORR(longlong, mi_sint8korr, mi_int8store, 8);
+      break;
     case HA_KEYTYPE_ULONGLONG:
       RT_COMB_KORR(ulonglong, mi_uint8korr, mi_int8store, 8);
+      break;
 #endif
     case HA_KEYTYPE_FLOAT:
       RT_COMB_GET(float, mi_float4get, mi_float4store, 4);
+      break;
     case HA_KEYTYPE_DOUBLE:
       RT_COMB_GET(double, mi_float8get, mi_float8store, 8);
+      break;
     case HA_KEYTYPE_END:
       return 0;
+    default:
+      return 1;
     }
+    uint32 keyseg_length= keyseg->length * 2;
+    key_length-= keyseg_length;
+    a+= keyseg_length;
+    b+= keyseg_length;
+    c+= keyseg_length;
   }
   return 0;
 }
@@ -435,16 +403,13 @@ int rtree_combine_rect(HA_KEYSEG *keyseg, uchar* a, uchar* b, uchar* c,
   type amin, amax, bmin, bmax; \
   amin = korr_func(a); \
   bmin = korr_func(b); \
-  p_inc(a, b, len); \
-  amax = korr_func(a); \
-  bmax = korr_func(b); \
-  p_inc(a, b, len); \
+  amax = korr_func(a+len); \
+  bmax = korr_func(b+len); \
   amin = max(amin, bmin); \
   amax = min(amax, bmax); \
   if (amin >= amax) \
     return 0; \
   res *= amax - amin; \
-  break; \
 }
 
 #define RT_OVL_AREA_GET(type, get_func, len) \
@@ -452,16 +417,13 @@ int rtree_combine_rect(HA_KEYSEG *keyseg, uchar* a, uchar* b, uchar* c,
   type amin, amax, bmin, bmax; \
   get_func(amin, a); \
   get_func(bmin, b); \
-  p_inc(a, b, len); \
-  get_func(amax, a); \
-  get_func(bmax, b); \
-  p_inc(a, b, len); \
+  get_func(amax, a+len); \
+  get_func(bmax, b+len); \
   amin = max(amin, bmin); \
   amax = min(amax, bmax); \
   if (amin >= amax)  \
     return 0; \
   res *= amax - amin; \
-  break; \
 }
 
 /*
@@ -473,58 +435,54 @@ double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar* a, uchar* b,
   double res = 1;
   for (; (int) key_length > 0 ; keyseg += 2)
   {
-    key_length -= keyseg->length * 2;
-    
     switch ((enum ha_base_keytype) keyseg->type) {
-    case HA_KEYTYPE_TEXT:
+    case HA_KEYTYPE_INT8:
+      RT_OVL_AREA_KORR(int8, mi_sint1korr, 1);
+      break;
     case HA_KEYTYPE_BINARY:
-    case HA_KEYTYPE_VARTEXT:
-    case HA_KEYTYPE_VARBINARY:
-    case HA_KEYTYPE_NUM:
-    default:
-      return -1;
+      RT_OVL_AREA_KORR(uint8, mi_uint1korr, 1);
       break;
-    case HA_KEYTYPE_INT8:
-      {
-        int amin, amax, bmin, bmax;
-        amin = (int)*((signed char *)a);
-        bmin = (int)*((signed char *)b);
-        p_inc(a, b, 1);
-        amax = (int)*((signed char *)a);
-        bmax = (int)*((signed char *)b);
-        p_inc(a, b, 1);
-        amin = max(amin, bmin);
-        amax = min(amax, bmax);
-        if (amin >= amax) 
-          return 0;
-        res *= amax - amin;
-        break;
-      }
     case HA_KEYTYPE_SHORT_INT:
       RT_OVL_AREA_KORR(int16, mi_sint2korr, 2);
+      break;
     case HA_KEYTYPE_USHORT_INT:
       RT_OVL_AREA_KORR(uint16, mi_uint2korr, 2);
+      break;
     case HA_KEYTYPE_INT24:
       RT_OVL_AREA_KORR(int32, mi_sint3korr, 3);
+      break;
     case HA_KEYTYPE_UINT24:
       RT_OVL_AREA_KORR(uint32, mi_uint3korr, 3);
+      break;
     case HA_KEYTYPE_LONG_INT:
       RT_OVL_AREA_KORR(int32, mi_sint4korr, 4);
+      break;
     case HA_KEYTYPE_ULONG_INT:
       RT_OVL_AREA_KORR(uint32, mi_uint4korr, 4);
+      break;
 #ifdef HAVE_LONG_LONG
     case HA_KEYTYPE_LONGLONG:
       RT_OVL_AREA_KORR(longlong, mi_sint8korr, 8);
+      break;
     case HA_KEYTYPE_ULONGLONG:
       RT_OVL_AREA_KORR(longlong, mi_sint8korr, 8);
+      break;
 #endif
     case HA_KEYTYPE_FLOAT:
       RT_OVL_AREA_GET(float, mi_float4get, 4);
+      break;
     case HA_KEYTYPE_DOUBLE:
       RT_OVL_AREA_GET(double, mi_float8get, 8);
+      break;
     case HA_KEYTYPE_END:
       return res;
+    default:
+      return -1;
     }
+    uint32 keyseg_length= keyseg->length * 2;
+    key_length-= keyseg_length;
+    a+= keyseg_length;
+    b+= keyseg_length;
   }
   return res;
 }
@@ -534,13 +492,10 @@ double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar* a, uchar* b,
    type amin, amax, bmin, bmax; \
    amin = korr_func(a); \
    bmin = korr_func(b); \
-   p_inc(a, b, len); \
-   amax = korr_func(a); \
-   bmax = korr_func(b); \
-   p_inc(a, b, len); \
+   amax = korr_func(a+len); \
+   bmax = korr_func(b+len); \
    a_area *= (((double)amax) - ((double)amin)); \
    *ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
-   break; \
 }
 
 #define RT_AREA_INC_GET(type, get_func, len)\
@@ -548,13 +503,10 @@ double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar* a, uchar* b,
    type amin, amax, bmin, bmax; \
    get_func(amin, a); \
    get_func(bmin, b); \
-   p_inc(a, b, len); \
-   get_func(amax, a); \
-   get_func(bmax, b); \
-   p_inc(a, b, len); \
+   get_func(amax, a+len); \
+   get_func(bmax, b+len); \
    a_area *= (((double)amax) - ((double)amin)); \
    *ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
-   break; \
 }
 
 /*
@@ -568,8 +520,6 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
   *ab_area= 1.0;
   for (; (int)key_length > 0; keyseg += 2)
   {
-    key_length -= keyseg->length * 2;
-    
     /* Handle NULL part */
     if (keyseg->null_bit)
     {
@@ -577,52 +527,53 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
     }
 
     switch ((enum ha_base_keytype) keyseg->type) {
-    case HA_KEYTYPE_TEXT:
+    case HA_KEYTYPE_INT8:
+      RT_AREA_INC_KORR(int8, mi_sint1korr, 1);
+      break;
     case HA_KEYTYPE_BINARY:
-    case HA_KEYTYPE_VARTEXT:
-    case HA_KEYTYPE_VARBINARY:
-    case HA_KEYTYPE_NUM:
-    default:
-      return 1;
+      RT_AREA_INC_KORR(uint8, mi_uint1korr, 1);
       break;
-    case HA_KEYTYPE_INT8:
-      {
-        int amin, amax, bmin, bmax;
-        amin = (int)*((signed char *)a);
-        bmin = (int)*((signed char *)b);
-        p_inc(a, b, 1);
-        amax = (int)*((signed char *)a);
-        bmax = (int)*((signed char *)b);
-        p_inc(a, b, 1);
-        a_area *= (((double)amax) - ((double)amin));
-        *ab_area *= ((double)max(amax, bmax) - (double)min(amin, bmin));
-        break;
-      }
     case HA_KEYTYPE_SHORT_INT:
       RT_AREA_INC_KORR(int16, mi_sint2korr, 2);
+      break;
     case HA_KEYTYPE_USHORT_INT:
       RT_AREA_INC_KORR(uint16, mi_uint2korr, 2);
+      break;
     case HA_KEYTYPE_INT24:
       RT_AREA_INC_KORR(int32, mi_sint3korr, 3);
+      break;
     case HA_KEYTYPE_UINT24:
       RT_AREA_INC_KORR(int32, mi_uint3korr, 3);
+      break;
     case HA_KEYTYPE_LONG_INT:
       RT_AREA_INC_KORR(int32, mi_sint4korr, 4);
+      break;
     case HA_KEYTYPE_ULONG_INT:
       RT_AREA_INC_KORR(uint32, mi_uint4korr, 4);
+      break;
 #ifdef HAVE_LONG_LONG
     case HA_KEYTYPE_LONGLONG:
       RT_AREA_INC_KORR(longlong, mi_sint8korr, 8);
+      break;
     case HA_KEYTYPE_ULONGLONG:
       RT_AREA_INC_KORR(longlong, mi_sint8korr, 8);
+      break;
 #endif
     case HA_KEYTYPE_FLOAT:
       RT_AREA_INC_GET(float, mi_float4get, 4);
+      break;
     case HA_KEYTYPE_DOUBLE:
       RT_AREA_INC_GET(double, mi_float8get, 8);
+      break;
     case HA_KEYTYPE_END:
       return *ab_area - a_area;
+    default:
+      return -1;
     }
+    uint32 keyseg_length= keyseg->length * 2;
+    key_length-= keyseg_length;
+    a+= keyseg_length;
+    b+= keyseg_length;
   }
   return *ab_area - a_area;
 }
@@ -632,13 +583,10 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
    type amin, amax, bmin, bmax; \
    amin = korr_func(a); \
    bmin = korr_func(b); \
-   p_inc(a, b, len); \
-   amax = korr_func(a); \
-   bmax = korr_func(b); \
-   p_inc(a, b, len); \
+   amax = korr_func(a+len); \
+   bmax = korr_func(b+len); \
    a_perim+= (((double)amax) - ((double)amin)); \
    *ab_perim+= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
-   break; \
 }
 
 #define RT_PERIM_INC_GET(type, get_func, len)\
@@ -646,13 +594,10 @@ double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
    type amin, amax, bmin, bmax; \
    get_func(amin, a); \
    get_func(bmin, b); \
-   p_inc(a, b, len); \
-   get_func(amax, a); \
-   get_func(bmax, b); \
-   p_inc(a, b, len); \
+   get_func(amax, a+len); \
+   get_func(bmax, b+len); \
    a_perim+= (((double)amax) - ((double)amin)); \
    *ab_perim+= ((double)max(amax, bmax) - (double)min(amin, bmin)); \
-   break; \
 }
 
 /*
@@ -666,8 +611,6 @@ double rtree_perimeter_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
   *ab_perim= 0.0;
   for (; (int)key_length > 0; keyseg += 2)
   {
-    key_length -= keyseg->length * 2;
-    
     /* Handle NULL part */
     if (keyseg->null_bit)
     {
@@ -675,52 +618,53 @@ double rtree_perimeter_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
     }
 
     switch ((enum ha_base_keytype) keyseg->type) {
-    case HA_KEYTYPE_TEXT:
+    case HA_KEYTYPE_INT8:
+      RT_PERIM_INC_KORR(int8, mi_sint1korr, 1);
+      break;
     case HA_KEYTYPE_BINARY:
-    case HA_KEYTYPE_VARTEXT:
-    case HA_KEYTYPE_VARBINARY:
-    case HA_KEYTYPE_NUM:
-    default:
-      return 1;
+      RT_PERIM_INC_KORR(uint8, mi_uint1korr, 1);
       break;
-    case HA_KEYTYPE_INT8:
-      {
-        int amin, amax, bmin, bmax;
-        amin = (int)*((signed char *)a);
-        bmin = (int)*((signed char *)b);
-        p_inc(a, b, 1);
-        amax = (int)*((signed char *)a);
-        bmax = (int)*((signed char *)b);
-        p_inc(a, b, 1);
-        a_perim+= (((double)amax) - ((double)amin));
-        *ab_perim+= ((double)max(amax, bmax) - (double)min(amin, bmin));
-        break;
-      }
     case HA_KEYTYPE_SHORT_INT:
       RT_PERIM_INC_KORR(int16, mi_sint2korr, 2);
+      break;
     case HA_KEYTYPE_USHORT_INT:
       RT_PERIM_INC_KORR(uint16, mi_uint2korr, 2);
+      break;
     case HA_KEYTYPE_INT24:
       RT_PERIM_INC_KORR(int32, mi_sint3korr, 3);
+      break;
     case HA_KEYTYPE_UINT24:
       RT_PERIM_INC_KORR(int32, mi_uint3korr, 3);
+      break;
     case HA_KEYTYPE_LONG_INT:
       RT_PERIM_INC_KORR(int32, mi_sint4korr, 4);
+      break;
     case HA_KEYTYPE_ULONG_INT:
       RT_PERIM_INC_KORR(uint32, mi_uint4korr, 4);
+      break;
 #ifdef HAVE_LONG_LONG
     case HA_KEYTYPE_LONGLONG:
       RT_PERIM_INC_KORR(longlong, mi_sint8korr, 8);
+      break;
     case HA_KEYTYPE_ULONGLONG:
       RT_PERIM_INC_KORR(longlong, mi_sint8korr, 8);
+      break;
 #endif
     case HA_KEYTYPE_FLOAT:
       RT_PERIM_INC_GET(float, mi_float4get, 4);
+      break;
     case HA_KEYTYPE_DOUBLE:
       RT_PERIM_INC_GET(double, mi_float8get, 8);
+      break;
     case HA_KEYTYPE_END:
       return *ab_perim - a_perim;
+    default:
+      return -1;
     }
+    uint32 keyseg_length= keyseg->length * 2;
+    key_length-= keyseg_length;
+    a+= keyseg_length;
+    b+= keyseg_length;
   }
   return *ab_perim - a_perim;
 }
@@ -746,7 +690,6 @@ double rtree_perimeter_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
   store_func(c, amax); \
   c += len; \
   inc += 2 * len; \
-  break; \
 }
 
 #define RT_PAGE_MBR_GET(type, get_func, store_func, len) \
@@ -769,7 +712,6 @@ double rtree_perimeter_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b,
   store_func(c, amax); \
   c += len; \
   inc += 2 * len; \
-  break; \
 }
 
 /*
@@ -797,61 +739,48 @@ int rtree_page_mbr(MI_INFO *info, HA_KEYSEG *keyseg, uchar *page_buf,
     k = rt_PAGE_FIRST_KEY(page_buf, nod_flag);
 
     switch ((enum ha_base_keytype) keyseg->type) {
-    case HA_KEYTYPE_TEXT:
+    case HA_KEYTYPE_INT8:
+      RT_PAGE_MBR_KORR(int8, mi_sint1korr, mi_int1store, 1);
+      break;
     case HA_KEYTYPE_BINARY:
-    case HA_KEYTYPE_VARTEXT:
-    case HA_KEYTYPE_VARBINARY:
-    case HA_KEYTYPE_NUM:
-    default:
-      return 1;
+      RT_PAGE_MBR_KORR(uint8, mi_uint1korr, mi_int1store, 1);
       break;
-    case HA_KEYTYPE_INT8:
-      {
-        int amin, amax, bmin, bmax;
-        amin = (int)*((signed char *)(k + inc));
-        amax = (int)*((signed char *)(k + inc + 1));
-        k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag);
-        for (; k < last; k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag))
-        {
-          bmin = (int)*((signed char *)(k + inc));
-          bmax = (int)*((signed char *)(k + inc + 1));
-
-          if (amin > bmin)
-            amin = bmin;
-          if (amax < bmax)
-            amax = bmax;
-        }
-        *((signed char*)c) = amin;
-        c += 1;
-        *((signed char*)c) = amax;
-        c += 1;
-        inc += 1 * 2;
-        break;
-      }
     case HA_KEYTYPE_SHORT_INT:
       RT_PAGE_MBR_KORR(int16, mi_sint2korr, mi_int2store, 2);
+      break;
     case HA_KEYTYPE_USHORT_INT:
       RT_PAGE_MBR_KORR(uint16, mi_uint2korr, mi_int2store, 2);
+      break;
     case HA_KEYTYPE_INT24:
       RT_PAGE_MBR_KORR(int32, mi_sint3korr, mi_int3store, 3);
+      break;
     case HA_KEYTYPE_UINT24:
       RT_PAGE_MBR_KORR(uint32, mi_uint3korr, mi_int3store, 3);
+      break;
     case HA_KEYTYPE_LONG_INT:
       RT_PAGE_MBR_KORR(int32, mi_sint4korr, mi_int4store, 4);
+      break;
     case HA_KEYTYPE_ULONG_INT:
       RT_PAGE_MBR_KORR(uint32, mi_uint4korr, mi_int4store, 4);
+      break;
 #ifdef HAVE_LONG_LONG
     case HA_KEYTYPE_LONGLONG:
       RT_PAGE_MBR_KORR(longlong, mi_sint8korr, mi_int8store, 8);
+      break;
     case HA_KEYTYPE_ULONGLONG:
       RT_PAGE_MBR_KORR(ulonglong, mi_uint8korr, mi_int8store, 8);
+      break;
 #endif
     case HA_KEYTYPE_FLOAT:
       RT_PAGE_MBR_GET(float, mi_float4get, mi_float4store, 4);
+      break;
     case HA_KEYTYPE_DOUBLE:
       RT_PAGE_MBR_GET(double, mi_float8get, mi_float8store, 8);
+      break;
     case HA_KEYTYPE_END:
       return 0;
+    default:
+      return 1;
     }
   }
   return 0;
diff --git a/myisam/rt_test.c b/myisam/rt_test.c
index bbeb8fce2d1..db4231aa0f5 100644
--- a/myisam/rt_test.c
+++ b/myisam/rt_test.c
@@ -31,6 +31,51 @@ static void create_record1(char *record,uint rownr);
 static void print_record(char * record,my_off_t offs,const char * tail);
 static  int run_test(const char *filename);
 
+static double rt_data[]=
+{
+  /*1*/  0,10,0,10,
+  /*2*/  5,15,0,10,
+  /*3*/  0,10,5,15,
+  /*4*/  10,20,10,20,
+  /*5*/  0,10,0,10,
+  /*6*/  5,15,0,10,
+  /*7*/  0,10,5,15,
+  /*8*/  10,20,10,20,
+  /*9*/  0,10,0,10,
+  /*10*/  5,15,0,10,
+  /*11*/  0,10,5,15,
+  /*12*/  10,20,10,20,
+  /*13*/  0,10,0,10,
+  /*14*/  5,15,0,10,
+  /*15*/  0,10,5,15,
+  /*16*/  10,20,10,20,
+  /*17*/  5,15,0,10,
+  /*18*/  0,10,5,15,
+  /*19*/  10,20,10,20,
+  /*20*/  0,10,0,10,
+
+  /*1*/  100,110,0,10,
+  /*2*/  105,115,0,10,
+  /*3*/  100,110,5,15,
+  /*4*/  110,120,10,20,
+  /*5*/  100,110,0,10,
+  /*6*/  105,115,0,10,
+  /*7*/  100,110,5,15,
+  /*8*/  110,120,10,20,
+  /*9*/  100,110,0,10,
+  /*10*/  105,115,0,10,
+  /*11*/  100,110,5,15,
+  /*12*/  110,120,10,20,
+  /*13*/  100,110,0,10,
+  /*14*/  105,115,0,10,
+  /*15*/  100,110,5,15,
+  /*16*/  110,120,10,20,
+  /*17*/  105,115,0,10,
+  /*18*/  100,110,5,15,
+  /*19*/  110,120,10,20,
+  /*20*/  100,110,0,10,
+  -1
+};
 
 int main(int argc __attribute__((unused)),char *argv[] __attribute__((unused)))
 {
@@ -55,7 +100,7 @@ int run_test(const char *filename)
   int key_type=HA_KEYTYPE_DOUBLE;
   int key_length=8;
   int null_fields=0;
-  int nrecords=300;
+  int nrecords=sizeof(rt_data)/(sizeof(double)*4);/* 3000;*/
   int rec_length=0;
   int uniques=0;
   int i;
@@ -399,7 +444,7 @@ static void create_record1(char *record,uint rownr)
 }
 
 
-static void create_record(char *record,uint rownr)
+static void create_record0(char *record,uint rownr)
 {
    int i;
    char * pos;
@@ -419,3 +464,16 @@ static void create_record(char *record,uint rownr)
       pos+=sizeof(c);
    }
 }
+
+static void create_record(char *record,uint rownr)
+{
+   int i;
+   char *pos;
+   double *data= rt_data+rownr*4;
+   record[0]=0x01; /* DEL marker */
+   for ( pos=record+1, i=0; i<ndims*2; i++)
+   {
+      float8store(pos,data[i]);
+      pos+=8;
+   }
+}
diff --git a/mysql-test/r/gis-rtree.result b/mysql-test/r/gis-rtree.result
index b66ef6d6a31..3fb1a5dd31c 100644
--- a/mysql-test/r/gis-rtree.result
+++ b/mysql-test/r/gis-rtree.result
@@ -167,12 +167,10 @@ count(*)
 150
 EXPLAIN SELECT fid, AsText(g) FROM t1 WHERE Within(g, GeomFromText('Polygon((140 140,160 140,160 160,140 160,140 140))'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	g	g	32	NULL	4	Using where
+1	SIMPLE	t1	range	g	g	32	NULL	7	Using where
 SELECT fid, AsText(g) FROM t1 WHERE Within(g, GeomFromText('Polygon((140 140,160 140,160 160,140 160,140 140))'));
 fid	AsText(g)
 1	LINESTRING(150 150,150 150)
-11	LINESTRING(140 140,160 160)
-2	LINESTRING(149 149,151 151)
 3	LINESTRING(148 148,152 152)
 4	LINESTRING(147 147,153 153)
 5	LINESTRING(146 146,154 154)
@@ -181,6 +179,8 @@ fid	AsText(g)
 8	LINESTRING(143 143,157 157)
 9	LINESTRING(142 142,158 158)
 10	LINESTRING(141 141,159 159)
+11	LINESTRING(140 140,160 160)
+2	LINESTRING(149 149,151 151)
 DROP TABLE t1;
 CREATE TABLE t2 (
 fid INT NOT NULL AUTO_INCREMENT PRIMARY KEY, 
@@ -305,10 +305,10 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 SELECT fid, AsText(g) FROM t2 WHERE Within(g, 
 GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
 fid	AsText(g)
+46	LINESTRING(51 41,60 50)
+56	LINESTRING(41 41,50 50)
 45	LINESTRING(51 51,60 60)
 55	LINESTRING(41 51,50 60)
-56	LINESTRING(41 41,50 50)
-46	LINESTRING(51 41,60 50)
 DELETE FROM t2 WHERE Within(g, Envelope(GeometryFromWKB(LineString(Point(10 * 10 - 9, 10 * 10 - 9), Point(10 * 10, 10 * 10)))));
 SELECT count(*) FROM t2;
 count(*)