fix for byte ordering bug in HEAP rb-tree

2 files changed, 65 insertions, 0 deletions

diff --git a/heap/hp_create.c b/heap/hp_create.c
index 6c38d54cb12..40b8202d94f 100644
--- a/heap/hp_create.c
+++ b/heap/hp_create.c
@@ -55,6 +55,22 @@ int heap_create(const char *name, uint keys, HP_KEYDEF *keydef,
 	  if (keyinfo->algorithm == HA_KEY_ALG_BTREE)
 	    keyinfo->rb_tree.size_of_element++;
 	}
+	switch (keyinfo->seg[j].type) {
+	case HA_KEYTYPE_SHORT_INT:
+	case HA_KEYTYPE_LONG_INT:
+	case HA_KEYTYPE_FLOAT:
+	case HA_KEYTYPE_DOUBLE:
+	case HA_KEYTYPE_USHORT_INT:
+	case HA_KEYTYPE_ULONG_INT:
+	case HA_KEYTYPE_LONGLONG:
+	case HA_KEYTYPE_ULONGLONG:
+	case HA_KEYTYPE_INT24:
+	case HA_KEYTYPE_UINT24:
+	case HA_KEYTYPE_INT8:
+	  keyinfo->seg[j].flag|= HA_SWAP_KEY;
+	default:
+	  break;
+	}
       }
       keyinfo->length= length;
       length+= keyinfo->rb_tree.size_of_element + 
diff --git a/heap/hp_hash.c b/heap/hp_hash.c
index 946659621fe..cd70d2ab532 100644
--- a/heap/hp_hash.c
+++ b/heap/hp_hash.c
@@ -443,6 +443,43 @@ uint hp_rb_make_key(HP_KEYDEF *keydef, byte *key,
       if (!(*key++= 1 - test(rec[seg->null_pos] & seg->null_bit)))
         continue;
     }
+    if (seg->flag & HA_SWAP_KEY)
+    {
+      uint length= seg->length;
+      byte *pos= (byte*) rec + seg->start;
+      
+#ifdef HAVE_ISNAN
+      if (seg->type == HA_KEYTYPE_FLOAT)
+      {
+	float nr;
+	float4get(nr, pos);
+	if (isnan(nr))
+	{
+	  /* Replace NAN with zero */
+ 	  bzero(key, length);
+	  key+= length;
+	  continue;
+	}
+      }
+      else if (seg->type == HA_KEYTYPE_DOUBLE)
+      {
+	double nr;
+	float8get(nr, pos);
+	if (isnan(nr))
+	{
+ 	  bzero(key, length);
+	  key+= length;
+	  continue;
+	}
+      }
+#endif
+      pos+= length;
+      while (length--)
+      {
+	*key++= *--pos;
+      }
+      continue;
+    }
     memcpy(key, rec + seg->start, (size_t) seg->length);
     key+= seg->length;
   }
@@ -467,6 +504,18 @@ uint hp_rb_pack_key(HP_KEYDEF *keydef, uchar *key, const uchar *old, uint k_len)
         continue;
       }
     }
+    if (seg->flag & HA_SWAP_KEY)
+    {
+      uint length= seg->length;
+      byte *pos= (byte*) old + length;
+      
+      k_len-= length;
+      while (length--)
+      {
+	*key++= *--pos;
+      }
+      continue;
+    }
     memcpy((byte*) key, old, seg->length);
     key+= seg->length;
     k_len-= seg->length;