Bug#12565 - ERROR 1034 when running simple UPDATE or DELETE on large MyISAM table

Changed end-space comparison so that the key is not used past
its end. This is due to the new end-space behaviour in 4.1.
See also bug 6151 and 9188.


mysql-test/r/key.result:
  Bug#12565 - ERROR 1034 when running simple UPDATE or DELETE on large MyISAM table
  The test result.
mysql-test/t/key.test:
  Bug#12565 - ERROR 1034 when running simple UPDATE or DELETE on large MyISAM table
  The test case.

1 files changed, 23 insertions, 9 deletions

diff --git a/myisam/mi_search.c b/myisam/mi_search.c
index 390e32b679d..6ed245d9715 100644
--- a/myisam/mi_search.c
+++ b/myisam/mi_search.c
@@ -316,19 +316,21 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
   get_key_pack_length(kseg_len,length_pack,kseg);
   key_len_skip=length_pack+kseg_len;
   key_len_left=(int) key_len- (int) key_len_skip;
+  /* If key_len is 0, then lenght_pack is 1, then key_len_left is -1. */
   cmplen=(key_len_left>=0) ? kseg_len : key_len-length_pack;
   DBUG_PRINT("info",("key: '%.*s'",kseg_len,kseg));
 
   /*
     Keys are compressed the following way:
 
-    If the max length of first key segment <= 127 characters the prefix is
+    If the max length of first key segment <= 127 bytes the prefix is
     1 byte else it's 2 byte
 
-    prefix         The high bit is set if this is a prefix for the prev key
-    length         Packed length if the previous was a prefix byte
-    [length]       Length character of data
-    next-key-seg   Next key segments
+    (prefix) length  The high bit is set if this is a prefix for the prev key.
+    [suffix length]  Packed length of suffix if the previous was a prefix.
+    (suffix) data    Key data bytes (past the common prefix or whole segment).
+    [next-key-seg]   Next key segments (([packed length], data), ...)
+    pointer          Reference to the data file (last_keyseg->length).
   */
 
   matched=0;  /* how many char's from prefix were alredy matched */
@@ -349,16 +351,23 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
 
     if (packed)
     {
-      if (suffix_len == 0)                      /* Same key */
+      if (suffix_len == 0)
+      {
+        /* == 0x80 or 0x8000, same key, prefix length == old key length. */
         prefix_len=len;
+      }
       else
       {
+        /* > 0x80 or 0x8000, this is prefix lgt, packed suffix lgt follows. */
         prefix_len=suffix_len;
         get_key_length(suffix_len,vseg);
       }
     }
     else
+    {
+      /* Not packed. No prefix used from last key. */
       prefix_len=0;
+    }
 
     len=prefix_len+suffix_len;
     seg_len_pack=get_pack_length(len);
@@ -414,7 +423,12 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
       uint  left;
       uchar *k=kseg+prefix_len;
 
-      left=(len>cmplen) ? cmplen-prefix_len : suffix_len;
+      /*
+        If prefix_len > cmplen then we are in the end-space comparison
+        phase. Do not try to acces the key any more ==> left= 0.
+      */
+      left= ((len <= cmplen) ? suffix_len :
+             ((prefix_len < cmplen) ? cmplen - prefix_len : 0));
 
       matched=prefix_len+left;
 
@@ -451,7 +465,7 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
 	    my_flag= -1;
 	  else
 	  {
-	    /* We have to compare k and vseg as if they where space extended */
+	    /* We have to compare k and vseg as if they were space extended */
 	    uchar *end= k+ (cmplen - len);
 	    for ( ; k < end && *k == ' '; k++) ;
 	    if (k == end)
@@ -470,7 +484,7 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
 	  if ((nextflag & SEARCH_PREFIX) && key_len_left == 0)
 	    goto fix_flag;
 
-	  /* We have to compare k and vseg as if they where space extended */
+	  /* We have to compare k and vseg as if they were space extended */
 	  for (end=vseg + (len-cmplen) ;
 	       vseg < end && *vseg == (uchar) ' ';
 	       vseg++, matched++) ;