MDEV-5817 MySQL BUG#11825482: Broken key length calculation for btree index

just as in 5.6 fix - copy the correct null-handling code from MyISAM

1 files changed, 10 insertions, 0 deletions

diff --git a/storage/heap/hp_hash.c b/storage/heap/hp_hash.c
index f96cf866138..993d3377b73 100644
--- a/storage/heap/hp_hash.c
+++ b/storage/heap/hp_hash.c
@@ -875,9 +875,19 @@ uint hp_rb_pack_key(HP_KEYDEF *keydef, uchar *key, const uchar *old,
     keypart_map>>= 1;
     if (seg->null_bit)
     {
+      /* Convert NULL from MySQL representation into HEAP's. */
       if (!(*key++= (char) 1 - *old++))
+      {
+        /*
+          Skip length part of a variable length field.
+          Length of key-part used with heap_rkey() always 2.
+          See also hp_hashnr().
+        */
+        if (seg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART))
+          old+= 2;
         continue;
       }
+    }
     if (seg->flag & HA_SWAP_KEY)
     {
       uint length= seg->length;