Import wiredtiger: 6958f7386fd688cd05dec0daeb83da40d9965144 from branch mongodb-master

ref: 027ed1577f..6958f7386f
for: 6.1.0-rc0

WT-9257 Fix column store search near ignoring update list (#8102)

5 files changed, 230 insertions, 7 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 959c19196d3..61a86ba2eda 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "027ed1577f97f48f84c4ada8dde10aca9e9b0cd0"
+    "commit": "6958f7386fd688cd05dec0daeb83da40d9965144"
 }
diff --git a/src/third_party/wiredtiger/src/btree/col_srch.c b/src/third_party/wiredtiger/src/btree/col_srch.c
index 06190986888..ea1e25f0549 100644
--- a/src/third_party/wiredtiger/src/btree/col_srch.c
+++ b/src/third_party/wiredtiger/src/btree/col_srch.c
@@ -279,6 +279,10 @@ leaf_only:
     return (0);
 
 past_end:
+    /* We don't always set these below, add a catch-all. */
+    cbt->ins_head = NULL;
+    cbt->ins = NULL;
+
     /*
      * A record past the end of the page's standard information. Check the append list; by
      * definition, any record on the append list is closer than the last record on the page, so it's
@@ -286,11 +290,39 @@ past_end:
      * because column-store files are dense, but in this case the caller searched past the end of
      * the table.
      */
-    cbt->ins_head = WT_COL_APPEND(page);
-    if ((cbt->ins = __col_insert_search(cbt->ins_head, cbt->ins_stack, cbt->next_stack, recno)) ==
-      NULL)
+    ins_head = WT_COL_APPEND(page);
+    ins = __col_insert_search(ins_head, cbt->ins_stack, cbt->next_stack, recno);
+    if (ins == NULL) {
+        /*
+         * There is nothing on the append list, so search the insert list. (The append list would
+         * have been closer to the search record).
+         */
+        if (cbt->recno != WT_RECNO_OOB) {
+            if (page->type == WT_PAGE_COL_FIX)
+                ins_head = WT_COL_UPDATE_SINGLE(page);
+            else {
+                ins_head = WT_COL_UPDATE_SLOT(page, cbt->slot);
+
+                /*
+                 * Set this, otherwise the code in cursor_valid will assume there's no on-disk value
+                 * underneath ins_head.
+                 */
+                F_SET(cbt, WT_CBT_VAR_ONPAGE_MATCH);
+            }
+
+            ins = WT_SKIP_LAST(ins_head);
+            if (ins != NULL && cbt->recno == WT_INSERT_RECNO(ins)) {
+                cbt->ins_head = ins_head;
+                cbt->ins = ins;
+            }
+        }
+
         cbt->compare = -1;
-    else {
+    } else {
+        WT_ASSERT(session, page->type == WT_PAGE_COL_FIX || !F_ISSET(cbt, WT_CBT_VAR_ONPAGE_MATCH));
+
+        cbt->ins_head = ins_head;
+        cbt->ins = ins;
         cbt->recno = WT_INSERT_RECNO(cbt->ins);
         if (recno == cbt->recno)
             cbt->compare = 0;
diff --git a/src/third_party/wiredtiger/src/include/column_inline.h b/src/third_party/wiredtiger/src/include/column_inline.h
index c09a1321b23..f63657c85c5 100644
--- a/src/third_party/wiredtiger/src/include/column_inline.h
+++ b/src/third_party/wiredtiger/src/include/column_inline.h
@@ -238,7 +238,7 @@ __col_var_last_recno(WT_REF *ref)
      * records, our callers must handle that explicitly, if they care.
      */
     if (!WT_COL_VAR_REPEAT_SET(page))
-        return (page->entries == 0 ? 0 : ref->ref_recno + (page->entries - 1));
+        return (page->entries == 0 ? WT_RECNO_OOB : ref->ref_recno + (page->entries - 1));
 
     repeat = &page->pg_var_repeats[page->pg_var_nrepeats - 1];
     return ((repeat->recno + repeat->rle) - 1 + (page->entries - (repeat->indx + 1)));
@@ -259,7 +259,7 @@ __col_fix_last_recno(WT_REF *ref)
      * If there's an append list, there may be more records on the page. This function ignores those
      * records, our callers must handle that explicitly, if they care.
      */
-    return (page->entries == 0 ? 0 : ref->ref_recno + (page->entries - 1));
+    return (page->entries == 0 ? WT_RECNO_OOB : ref->ref_recno + (page->entries - 1));
 }
 
 /*
diff --git a/src/third_party/wiredtiger/test/suite/test_search_near05.py b/src/third_party/wiredtiger/test/suite/test_search_near05.py
new file mode 100644
index 00000000000..f54485817c1
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_search_near05.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-present MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_search_near05.py
+#       Search_near with a key past the end.
+
+import wttest
+from wtscenario import make_scenarios
+
+class test_search_near05(wttest.WiredTigerTestCase):
+    uri = 'file:test_search_near05'
+
+    key_format_values = [
+        ('fix', dict(key_format='r', value_format='8t')),
+        ('var', dict(key_format='r', value_format='I')),
+        ('row', dict(key_format='Q', value_format='I')),
+    ]
+
+    ops = [
+        ('update', dict(delete=False)),
+        ('delete', dict(delete=True)),
+    ]
+
+    scenarios = make_scenarios(key_format_values, ops)
+
+    def evict(self, value):
+        evict_cursor = self.session.open_cursor(self.uri, None, "debug=(release_evict)")
+        self.session.begin_transaction()
+        for i in range(1, 1001):
+            v = evict_cursor[i]
+            self.assertEqual(v, value)
+            self.assertEqual(evict_cursor.reset(), 0)
+        self.session.rollback_transaction()
+
+    def test_implicit_record_cursor_insert_next(self):
+        self.session.create(self.uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format))
+        cursor = self.session.open_cursor(self.uri)
+        value1 = 1
+        value2 = 2
+        for i in range(1, 1001):
+            cursor[i] = value1
+
+        # Do a checkpoint to write everything to the disk image
+        self.session.checkpoint()
+        # Evict the data
+        self.evict(value1)
+
+        # Update or delete the last key
+        if self.delete:
+            self.session.begin_transaction()
+            cursor.set_key(1000)
+            cursor.remove()
+            self.session.commit_transaction()
+        else:
+            cursor[1000] = value2
+
+        self.session.begin_transaction()
+        cursor.set_key(1100)
+        cursor.search_near()
+
+        if self.delete:
+            if self.value_format == "8t":
+                self.assertEqual(cursor.get_key(), 1000)
+                self.assertEqual(cursor.get_value(), 0)
+            else:
+                self.assertEqual(cursor.get_key(), 999)
+                self.assertEqual(cursor.get_value(), value1)
+        else:
+            self.assertEqual(cursor.get_key(), 1000)
+            self.assertEqual(cursor.get_value(), value2)
+
+if __name__ == '__main__':
+    wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_search_near06.py b/src/third_party/wiredtiger/test/suite/test_search_near06.py
new file mode 100644
index 00000000000..81f9f853c76
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_search_near06.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-present MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_search_near06.py
+#       Search_near with a key past the end much like search_near_05, but
+#       this time use timestamps to ensure the update isn't visible.
+
+import wttest
+from wtscenario import make_scenarios
+
+class test_search_near06(wttest.WiredTigerTestCase):
+    uri = 'file:test_search_near06'
+
+    key_format_values = [
+        ('fix', dict(key_format='r', value_format='8t')),
+        ('var', dict(key_format='r', value_format='I')),
+        ('row', dict(key_format='Q', value_format='I')),
+    ]
+
+    ops = [
+        ('update', dict(delete=False)),
+        ('delete', dict(delete=True)),
+    ]
+
+    scenarios = make_scenarios(key_format_values, ops)
+
+    def evict(self, value):
+        evict_cursor = self.session.open_cursor(self.uri, None, "debug=(release_evict)")
+        self.session.begin_transaction()
+        for i in range(1, 1001):
+            v = evict_cursor[i]
+            self.assertEqual(v, value)
+            self.assertEqual(evict_cursor.reset(), 0)
+        self.session.rollback_transaction()
+
+    def test_implicit_record_cursor_insert_next(self):
+        self.session.create(self.uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format))
+        cursor = self.session.open_cursor(self.uri)
+        value1 = 1
+        value2 = 2
+        for i in range(1, 1001):
+            cursor[i] = value1
+
+        # Do a checkpoint to write everything to the disk image
+        self.session.checkpoint()
+        # Evict the data
+        self.evict(value1)
+
+        # Update or delete the last key
+        if self.delete:
+            self.session.begin_transaction()
+            cursor.set_key(1000)
+            cursor.remove()
+            self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10))
+        else:
+            self.session.begin_transaction()
+            cursor[1000] = value2
+            self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10))
+
+        self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5))
+        cursor.set_key(1100)
+        cursor.search_near()
+
+        self.assertEqual(cursor.get_key(), 1000)
+        self.assertEqual(cursor.get_value(), value1)
+
+if __name__ == '__main__':
+    wttest.run()