Bug#9112 - Merge table with composite index producing invalid results with some queries

The problem was an ab-use of last_rkey_length.
Formerly we saved the packed key length (of the search key)
in this element. But in certain cases it got replaced by 
the (packed) result key length.
Now we use a new element of MI_INFO to save the packed key 
length of the search key.

4 files changed, 21 insertions, 7 deletions

diff --git a/myisam/mi_dbug.c b/myisam/mi_dbug.c
index fe5b36fd304..8e7777e92a5 100644
--- a/myisam/mi_dbug.c
+++ b/myisam/mi_dbug.c
@@ -40,12 +40,12 @@ void _mi_print_key(FILE *stream, register MI_KEYSEG *keyseg,
     end= key+ keyseg->length;
     if (keyseg->flag & HA_NULL_PART)
     {
-      if (!*key)
+      /* A NULL value is encoded by a 1-byte flag. Zero means NULL. */
+      if (! *(key++))
       {
 	fprintf(stream,"NULL");
 	continue;
       }
-      key++;
     }
 
     switch (keyseg->type) {
diff --git a/myisam/mi_rkey.c b/myisam/mi_rkey.c
index 1bb478efd3d..70122288d6c 100644
--- a/myisam/mi_rkey.c
+++ b/myisam/mi_rkey.c
@@ -31,8 +31,8 @@ int mi_rkey(MI_INFO *info, byte *buf, int inx, const byte *key, uint key_len,
   MI_KEYSEG *last_used_keyseg;
   uint pack_key_length, use_key_length, nextflag;
   DBUG_ENTER("mi_rkey");
-  DBUG_PRINT("enter",("base: %lx  inx: %d  search_flag: %d",
-		      info,inx,search_flag));
+  DBUG_PRINT("enter", ("base: %p  buf: %p  inx: %d  search_flag: %d",
+                       info, buf, inx, search_flag));
 
   if ((inx = _mi_check_index(info,inx)) < 0)
     DBUG_RETURN(my_errno);
@@ -44,9 +44,12 @@ int mi_rkey(MI_INFO *info, byte *buf, int inx, const byte *key, uint key_len,
   {
     if (key_len == 0)
       key_len=USE_WHOLE_KEY;
+    /* Save the packed key for later use in the second buffer of lastkey. */
     key_buff=info->lastkey+info->s->base.max_key_length;
     pack_key_length=_mi_pack_key(info, (uint) inx, key_buff, (uchar*) key,
 				 key_len, &last_used_keyseg);
+    /* Save packed_key_length for use by the MERGE engine. */
+    info->pack_key_length= pack_key_length;
     DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE, keyinfo->seg,
 				     key_buff, pack_key_length););
   }
diff --git a/myisam/mi_search.c b/myisam/mi_search.c
index cfc1c5cc3c0..c0be1a427df 100644
--- a/myisam/mi_search.c
+++ b/myisam/mi_search.c
@@ -1237,11 +1237,21 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
   reg1 MI_KEYSEG *keyseg;
   uchar *start_key,*page,*page_end,*from,*from_end;
   uint length,tmp;
+  DBUG_ENTER("_mi_get_binary_pack_key");
 
   page= *page_pos;
   page_end=page+MI_MAX_KEY_BUFF+1;
   start_key=key;
 
+  /*
+    Keys are compressed the following way:
+
+    prefix length    Packed length of prefix for the prev key. (1 or 3 bytes)
+    for each key segment:
+      [is null]        Null indicator if can be null (1 byte, zero means null)
+      [length]         Packed length if varlength (1 or 3 bytes)
+    pointer          Reference to the data file (last_keyseg->length).
+  */
   get_key_length(length,page);
   if (length)
   {
@@ -1251,7 +1261,7 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
                           length, keyinfo->maxlength, *page_pos));
       DBUG_DUMP("key",(char*) *page_pos,16);
       my_errno=HA_ERR_CRASHED;
-      return 0;                                 /* Wrong key */
+      DBUG_RETURN(0);                                 /* Wrong key */
     }
     from=key;  from_end=key+length;
   }
@@ -1312,12 +1322,12 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
     {
       DBUG_PRINT("error",("Error when unpacking key"));
       my_errno=HA_ERR_CRASHED;
-      return 0;                                 /* Error */
+      DBUG_RETURN(0);                                 /* Error */
     }
     memcpy((byte*) key,(byte*) from,(size_t) length);
     *page_pos= from+length;
   }
-  return((uint) (key-start_key)+keyseg->length);
+  DBUG_RETURN((uint) (key-start_key)+keyseg->length);
 }
 
 
diff --git a/myisam/myisamdef.h b/myisam/myisamdef.h
index 51a2dd3a2b3..11d7a5ea21f 100644
--- a/myisam/myisamdef.h
+++ b/myisam/myisamdef.h
@@ -255,6 +255,7 @@ struct st_myisam_info {
   uint	lastkey_length;			/* Length of key in lastkey */
   uint	last_rkey_length;		/* Last length in mi_rkey() */
   uint  save_lastkey_length;
+  uint  pack_key_length;                /* For MYISAMMRG */
   int	errkey;				/* Got last error on this key */
   int   lock_type;			/* How database was locked */
   int   tmp_lock_type;			/* When locked by readinfo */