MDEV-21749: page_cur_insert_rec_low(): Assertion rdm - rd + bd <= insert_buf + rec_size failed.

This bug was introduced in
commit 7ae21b18a6b73bbc3bf1ff448faf60c29ac1d386
(the main commit of MDEV-12353).

page_cur_insert_rec_low(): Before entering the comparison loop, make sure
that the range does not exceed c_end already at the start of the loop.
The loop is only comparing for pointer equality, and that condition
cdm == c_end would never hold if the end was already exceeded in
the beginning. Also, skip the comparison altogether if we could find
at most 2 equal bytes.

PageBulk::insertPage(): Apply a similar change. It seems that this
code was correct, because the loop checks for cdm < c_end.

2 files changed, 35 insertions, 29 deletions

diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc
index d892c429a1e..4d73a6a4ac2 100644
--- a/storage/innobase/btr/btr0bulk.cc
+++ b/storage/innobase/btr/btr0bulk.cc
@@ -275,20 +275,23 @@ no_data:
           goto no_data;
 
       /* Try to copy any data bytes of the preceding record. */
-      const byte *cdm= cd;
-      const byte *rdm= rd;
-      for (; cdm < c_end && *rdm == *cdm; cdm++, rdm++)
-      ut_ad(rdm - rd + bd <= insert_rec_end);
-      size_t len= static_cast<size_t>(rdm - rd);
-      ut_ad(!memcmp(rd, cd, len));
-      if (len > 2)
+      if (c_end - cd > 2)
       {
-        m_mtr.memcpy<mtr_t::FORCED>(*m_block, b, r, m_cur_rec - c);
-        memcpy(bd, cd, len);
-        m_mtr.memmove(*m_block, page_offset(bd), page_offset(cd), len);
-        c= cdm;
-        b= rdm - rd + bd;
-        r= rdm;
+        const byte *cdm= cd;
+        const byte *rdm= rd;
+        for (; cdm < c_end && *rdm == *cdm; cdm++, rdm++)
+        ut_ad(rdm - rd + bd <= insert_rec_end);
+        size_t len= static_cast<size_t>(rdm - rd);
+        ut_ad(!memcmp(rd, cd, len));
+        if (len > 2)
+        {
+          m_mtr.memcpy<mtr_t::FORCED>(*m_block, b, r, m_cur_rec - c);
+          memcpy(bd, cd, len);
+          m_mtr.memmove(*m_block, page_offset(bd), page_offset(cd), len);
+          c= cdm;
+          b= rdm - rd + bd;
+          r= rdm;
+        }
       }
     }
 
diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc
index 45cf5886302..dd20f4bca0f 100644
--- a/storage/innobase/page/page0cur.cc
+++ b/storage/innobase/page/page0cur.cc
@@ -1342,23 +1342,26 @@ no_data:
 
         /* Try to copy any data bytes of the preceding record. */
         const byte * const cd= cur->rec + (rd - rec);
-        const byte *cdm= cd;
-        const byte *rdm= rd;
-        while (*rdm++ == *cdm++)
-          if (cdm == c_end)
-            break;
-        cdm--, rdm--;
-        ut_ad(rdm - rd + bd <= insert_buf + rec_size);
-        size_t len= static_cast<size_t>(rdm - rd);
-        ut_ad(!memcmp(rd, cd, len));
-        if (len > 2)
+        if (c_end - cd > 2)
         {
-          mtr->memcpy<mtr_t::FORCED>(*block, b, r, cur->rec - c);
-          memcpy(bd, cd, len);
-          mtr->memmove(*block, page_offset(bd), page_offset(cd), len);
-          c= cdm;
-          b= rdm - rd + bd;
-          r= rdm;
+          const byte *cdm= cd;
+          const byte *rdm= rd;
+          while (*rdm++ == *cdm++)
+            if (cdm == c_end)
+              break;
+          cdm--, rdm--;
+          ut_ad(rdm - rd + bd <= insert_buf + rec_size);
+          size_t len= static_cast<size_t>(rdm - rd);
+          ut_ad(!memcmp(rd, cd, len));
+          if (len > 2)
+          {
+            mtr->memcpy<mtr_t::FORCED>(*block, b, r, cur->rec - c);
+            memcpy(bd, cd, len);
+            mtr->memmove(*block, page_offset(bd), page_offset(cd), len);
+            c= cdm;
+            b= rdm - rd + bd;
+            r= rdm;
+          }
         }
       }
     }