Fix for bug#38484 DELETE causes crash or index corruption

There is no test cases as it's not trivial to do a test case for this.
The new code for Maria is however executed by the test case in the bug report.

mysql-test/mysql-test-run.pl:
  Avoid warnings when running with --extern
storage/maria/ha_maria.cc:
  Disable not complete assert until Sanja can push a proper fix
storage/maria/ma_delete.c:
  Fix that pageflag for page is calculated based on original values
storage/maria/ma_search.c:
  Ensure that prev_length structure variable is properly reset when not used
storage/myisam/mi_search.c:
  Ensure that prev_length structure variable is properly reset when not used

3 files changed, 10 insertions, 5 deletions

diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc
index 3b3e06163a3..0f763e3fafe 100644
--- a/storage/maria/ha_maria.cc
+++ b/storage/maria/ha_maria.cc
@@ -2363,7 +2363,9 @@ int ha_maria::external_lock(THD *thd, int lock_type)
             This is a bit excessive, ACID requires this only if there are some
             changes to commit (rollback shouldn't be tested).
           */
+#ifdef WAITING_FOR_PATCH_FROM_SANJA
           DBUG_ASSERT(!thd->main_da.is_sent);
+#endif
           /* autocommit ? rollback a transaction */
 #ifdef MARIA_CANNOT_ROLLBACK
           if (ma_commit(trn))
diff --git a/storage/maria/ma_delete.c b/storage/maria/ma_delete.c
index 1d6bb565e02..92f27938d0c 100644
--- a/storage/maria/ma_delete.c
+++ b/storage/maria/ma_delete.c
@@ -780,6 +780,7 @@ static int underflow(register MARIA_HA *info, MARIA_KEYDEF *keyinfo,
       first_key)
   {
     size_t tmp_length;
+    uint next_page_flag;
     /* Use page right of anc-page */
     DBUG_PRINT("test",("use right page"));
 
@@ -805,6 +806,7 @@ static int underflow(register MARIA_HA *info, MARIA_KEYDEF *keyinfo,
                            DFLT_INIT_HITS, buff, 0, &next_page_link))
       goto err;
     next_buff_length= _ma_get_page_used(share, buff);
+    next_page_flag=   _ma_get_keypage_flag(share,buff);
     DBUG_DUMP("next", buff, next_buff_length);
 
     /* find keys to make a big key-page */
@@ -829,7 +831,7 @@ static int underflow(register MARIA_HA *info, MARIA_KEYDEF *keyinfo,
     _ma_store_page_used(share, buff, buff_length);
 
     /* Set page flag from combination of both key pages and parting key */
-    page_flag= (_ma_get_keypage_flag(share, buff) |
+    page_flag= (next_page_flag |
                 _ma_get_keypage_flag(share, leaf_buff));
     if (anc_key.flag & (SEARCH_USER_KEY_HAS_TRANSID |
                         SEARCH_PAGE_KEY_HAS_TRANSID))
diff --git a/storage/maria/ma_search.c b/storage/maria/ma_search.c
index 8b0c0426f1b..cf7ec5f9724 100644
--- a/storage/maria/ma_search.c
+++ b/storage/maria/ma_search.c
@@ -2194,9 +2194,10 @@ int _ma_calc_bin_pack_key_length(const MARIA_KEY *int_key,
 
     if (next_length > ref_length)
     {
-      /* We put a key with different case between two keys with the same prefix
-         Extend next key to have same prefix as
-         this key */
+      /*
+        We put a key with different case between two keys with the same prefix
+        Extend next key to have same prefix as this key
+      */
       s_temp->n_ref_length= ref_length;
       s_temp->prev_length=  next_length-ref_length;
       s_temp->prev_key+=    ref_length;
@@ -2206,13 +2207,13 @@ int _ma_calc_bin_pack_key_length(const MARIA_KEY *int_key,
     }
     /* Check how many characters are identical to next key */
     key= s_temp->key+next_length;
+    s_temp->prev_length= 0;
     while (*key++ == *next_key++) ;
     if ((ref_length= (uint) (key - s_temp->key)-1) == next_length)
     {
       s_temp->next_key_pos=0;
       return (s_temp->move_length= length);  /* Can't pack next key */
     }
-    s_temp->prev_length=0;
     s_temp->n_ref_length=ref_length;
     return s_temp->move_length= (int) (length-(ref_length - next_length) -
                                        next_length_pack +