Merge branch 'develop' into wt-2426-checkpoint-locking

5 files changed, 214 insertions, 60 deletions

diff --git a/test/format/backup.c b/test/format/backup.c
index 56657940514..2b1463bd0e3 100644
--- a/test/format/backup.c
+++ b/test/format/backup.c
@@ -67,6 +67,13 @@ copy_file(const char *name)
 	    "cp %s/%s %s/%s", g.home, name, g.home_backup, name);
 	testutil_checkfmt(system(cmd), "backup copy: %s", cmd);
 	free(cmd);
+
+	len = strlen(g.home) + strlen(g.home_backup2) + strlen(name) * 2 + 20;
+	cmd = dmalloc(len);
+	(void)snprintf(cmd, len,
+	    "cp %s/%s %s/%s", g.home, name, g.home_backup2, name);
+	testutil_checkfmt(system(cmd), "backup copy: %s", cmd);
+	free(cmd);
 }
 
 /*
diff --git a/test/format/format.h b/test/format/format.h
index c54fd061736..a129c5395fd 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -109,6 +109,7 @@ typedef struct {
 
 	char *home;				/* Home directory */
 	char *home_backup;			/* Hot-backup directory */
+	char *home_backup2;			/* Saved Hot-backup directory */
 	char *home_backup_init;			/* Initialize backup command */
 	char *home_bdb;				/* BDB directory */
 	char *home_config;			/* Run CONFIG file path */
diff --git a/test/format/util.c b/test/format/util.c
index 347b2ea1db3..2e4c869366c 100644
--- a/test/format/util.c
+++ b/test/format/util.c
@@ -310,6 +310,10 @@ path_setup(const char *home)
 	g.home_backup = dmalloc(len);
 	snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP");
 
+	len = strlen(g.home) + strlen("BACKUP2") + 2;
+	g.home_backup2 = dmalloc(len);
+	snprintf(g.home_backup2, len, "%s/%s", g.home, "BACKUP2");
+
 	/* BDB directory. */
 	len = strlen(g.home) + strlen("bdb") + 2;
 	g.home_bdb = dmalloc(len);
@@ -340,13 +344,15 @@ path_setup(const char *home)
 	/* Backup directory initialize command, remove and re-create it. */
 #undef	CMD
 #ifdef _WIN32
-#define	CMD	"del /s /q >:nul && mkdir %s"
+#define	CMD	"del /s /q >:nul && mkdir %s %s"
 #else
-#define	CMD	"rm -rf %s && mkdir %s"
+#define	CMD	"rm -rf %s %s && mkdir %s %s"
 #endif
-	len = strlen(g.home_backup) * 2 + strlen(CMD) + 1;
+	len = strlen(g.home_backup) * 2 +
+	    strlen(g.home_backup2) * 2 + strlen(CMD) + 1;
 	g.home_backup_init = dmalloc(len);
-	snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup);
+	snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup2,
+	    g.home_backup, g.home_backup2);
 
 	/*
 	 * Salvage command, save the interesting files so we can replay the
diff --git a/test/suite/test_bug008.py b/test/suite/test_bug008.py
index 8f0526d9cef..0243887e258 100644
--- a/test/suite/test_bug008.py
+++ b/test/suite/test_bug008.py
@@ -33,65 +33,208 @@ import wiredtiger, wttest
 from helper import simple_populate, key_populate, value_populate
 from wtscenario import check_scenarios
 
-# Tests for invisible updates.
+# Test search/search-near operations, including invisible values and keys
+# past the end of the table.
 class test_bug008(wttest.WiredTigerTestCase):
+    uri = 'file:test_bug008'                # This is a btree layer test.
     scenarios = check_scenarios([
-        ('fix', dict(fmt='key_format=r,value_format=8t', empty=1)),
-        ('row', dict(fmt='key_format=S', empty=0)),
-        ('var', dict(fmt='key_format=r', empty=0))
+        ('fix', dict(fmt='key_format=r,value_format=8t', empty=1, colvar=0)),
+        ('row', dict(fmt='key_format=S', empty=0, colvar=0)),
+        ('var', dict(fmt='key_format=r', empty=0, colvar=1))
     ])
 
+    # Verify cursor search and search-near operations in an empty table.
+    def test_search_empty(self):
+        # Create the object and open a cursor.
+        self.session.create(self.uri, self.fmt)
+        cursor = self.session.open_cursor(self.uri, None)
+
+        # Search for a record past the end of the table, which should fail.
+        cursor.set_key(key_populate(cursor, 100))
+        self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
+
+        # Search-near for a record past the end of the table, which should fail.
+        cursor.set_key(key_populate(cursor, 100))
+        self.assertEqual(cursor.search_near(), wiredtiger.WT_NOTFOUND)
+
+    # Verify cursor search and search-near operations at and past the end of
+    # a file, with a set of on-page visible records.
+    def test_search_eot(self):
+        # Populate the tree and reopen the connection, forcing it to disk
+        # and moving the records to an on-page format.
+        simple_populate(self, self.uri, self.fmt, 100)
+        self.reopen_conn()
+
+        # Open a cursor.
+        cursor = self.session.open_cursor(self.uri, None)
+
+        # Search for a record at the end of the table, which should succeed.
+        cursor.set_key(key_populate(cursor, 100))
+        self.assertEqual(cursor.search(), 0)
+        self.assertEqual(cursor.get_key(), key_populate(cursor, 100))
+        self.assertEqual(cursor.get_value(), value_populate(cursor, 100))
+
+        # Search-near for a record at the end of the table, which should
+        # succeed, returning the last record.
+        cursor.set_key(key_populate(cursor, 100))
+        self.assertEqual(cursor.search_near(), 0)
+        self.assertEqual(cursor.get_key(), key_populate(cursor, 100))
+        self.assertEqual(cursor.get_value(), value_populate(cursor, 100))
+
+        # Search for a record past the end of the table, which should fail.
+        cursor.set_key(key_populate(cursor, 200))
+        self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
+
+        # Search-near for a record past the end of the table, which should
+        # succeed, returning the last record.
+        cursor.set_key(key_populate(cursor, 200))
+        self.assertEqual(cursor.search_near(), -1)
+        self.assertEqual(cursor.get_key(), key_populate(cursor, 100))
+        self.assertEqual(cursor.get_value(), value_populate(cursor, 100))
+
+    # Verify cursor search-near operations before and after a set of
+    # column-store duplicates.
+    def test_search_duplicate(self):
+        if self.colvar == 0:
+                return
+
+        # Populate the tree.
+        simple_populate(self, self.uri, self.fmt, 105)
+
+        # Set up deleted records before and after a set of duplicate records,
+        # and make sure search/search-near returns the correct record.
+        cursor = self.session.open_cursor(self.uri, None)
+        for i in range(20, 100):
+            cursor[key_populate(cursor, i)] = '=== IDENTICAL VALUE ==='
+        for i in range(15, 25):
+            cursor.set_key(key_populate(cursor, i))
+            self.assertEqual(cursor.remove(), 0)
+        for i in range(95, 106):
+            cursor.set_key(key_populate(cursor, i))
+            self.assertEqual(cursor.remove(), 0)
+        cursor.close()
+
+        # Reopen the connection, forcing it to disk and moving the records to
+        # an on-page format.
+        self.reopen_conn()
+
+        # Open a cursor.
+        cursor = self.session.open_cursor(self.uri, None)
+
+        # Search-near for a record in the deleted set before the duplicate set,
+        # which should succeed, returning the first record in the duplicate set.
+        cursor.set_key(key_populate(cursor, 18))
+        self.assertEqual(cursor.search_near(), 1)
+        self.assertEqual(cursor.get_key(), key_populate(cursor, 25))
+
+        # Search-near for a record in the deleted set after the duplicate set,
+        # which should succeed, returning the last record in the duplicate set.
+        cursor.set_key(key_populate(cursor, 98))
+        self.assertEqual(cursor.search_near(), -1)
+        self.assertEqual(cursor.get_key(), key_populate(cursor, 94))
+
     # Verify cursor search and search-near operations on a file with a set of
     # on-page visible records, and a set of insert-list invisible records.
     def test_search_invisible_one(self):
-        uri = 'file:test_bug008'                # This is a btree layer test.
+        # Populate the tree.
+        simple_populate(self, self.uri, self.fmt, 100)
 
-        # Populate the tree and reopen the connection, forcing it to disk
-        # and moving the records to an on-page format.
-        simple_populate(self, uri, self.fmt, 100)
+        # Delete a range of records.
+        for i in range(5, 10):
+            cursor = self.session.open_cursor(self.uri, None)
+            cursor.set_key(key_populate(cursor, i))
+            self.assertEqual(cursor.remove(), 0)
+
+        # Reopen the connection, forcing it to disk and moving the records to
+        # an on-page format.
         self.reopen_conn()
 
-        # Begin a transaction, and add some additional records.
+        # Add updates to the existing records (in both the deleted an undeleted
+        # range), as well as some new records after the end. Put the updates in
+        # a separate transaction so they're invisible to another cursor.
         self.session.begin_transaction()
-        cursor = self.session.open_cursor(uri, None)
+        cursor = self.session.open_cursor(self.uri, None)
+        for i in range(5, 10):
+            cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000)
+        for i in range(30, 40):
+            cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000)
         for i in range(100, 140):
-            cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+            cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000)
 
         # Open a separate session and cursor.
         s = self.conn.open_session()
-        cursor = s.open_cursor(uri, None)
+        cursor = s.open_cursor(self.uri, None)
 
-        # Search for an invisible record.
-        cursor.set_key(key_populate(cursor, 130))
-        if self.empty:
-            # Invisible updates to fixed-length column-store objects are
-            # invisible to the reader, but the fact that they exist past
-            # the end of the initial records causes the instantiation of
-            # empty records: confirm successful return of an empty row.
-            cursor.search()
-            self.assertEqual(cursor.get_key(), 130)
-            self.assertEqual(cursor.get_value(), 0)
-        else:
-            # Otherwise, we should not find any matching records.
-            self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
+        # Search for an existing record in the deleted range, should not find
+        # it.
+        for i in range(5, 10):
+            cursor.set_key(key_populate(cursor, i))
+            if self.empty:
+                # Fixed-length column-store rows always exist.
+                self.assertEqual(cursor.search(), 0)
+                self.assertEqual(cursor.get_key(), i)
+                self.assertEqual(cursor.get_value(), 0)
+            else:
+                self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
 
-        # Search-near for an invisible record, which should succeed, returning
-        # the last visible record.
-        cursor.set_key(key_populate(cursor, 130))
-        cursor.search_near()
-        if self.empty:
-            # Invisible updates to fixed-length column-store objects are
-            # invisible to the reader, but the fact that they exist past
-            # the end of the initial records causes the instantiation of
-            # empty records: confirm successful return of an empty row.
-            cursor.search()
-            self.assertEqual(cursor.get_key(), 130)
-            self.assertEqual(cursor.get_value(), 0)
-        else:
-            # Otherwise, we should find the closest record for which we can see
-            # the value.
-            self.assertEqual(cursor.get_key(), key_populate(cursor, 100))
-            self.assertEqual(cursor.get_value(), value_populate(cursor, 100))
+        # Search for an existing record in the updated range, should see the
+        # original value.
+        for i in range(30, 40):
+            cursor.set_key(key_populate(cursor, i))
+            self.assertEqual(cursor.search(), 0)
+            self.assertEqual(cursor.get_key(), key_populate(cursor, i))
+
+        # Search for a added record, should not find it.
+        for i in range(120, 130):
+            cursor.set_key(key_populate(cursor, i))
+            if self.empty:
+                # Invisible updates to fixed-length column-store objects are
+                # invisible to the reader, but the fact that they exist past
+                # the end of the initial records causes the instantiation of
+                # empty records: confirm successful return of an empty row.
+                self.assertEqual(cursor.search(), 0)
+                self.assertEqual(cursor.get_key(), i)
+                self.assertEqual(cursor.get_value(), 0)
+            else:
+                # Otherwise, we should not find any matching records.
+                self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
+
+        # Search-near for an existing record in the deleted range, should find
+        # the next largest record. (This depends on the implementation behavior
+        # which currently includes a bias to prefix search.)
+        for i in range(5, 10):
+            cursor.set_key(key_populate(cursor, i))
+            if self.empty:
+                # Fixed-length column-store rows always exist.
+                self.assertEqual(cursor.search_near(), 0)
+                self.assertEqual(cursor.get_key(), i)
+                self.assertEqual(cursor.get_value(), 0)
+            else:
+                self.assertEqual(cursor.search_near(), 1)
+                self.assertEqual(cursor.get_key(), key_populate(cursor, 10))
+
+        # Search-near for an existing record in the updated range, should see
+        # the original value.
+        for i in range(30, 40):
+            cursor.set_key(key_populate(cursor, i))
+            self.assertEqual(cursor.search_near(), 0)
+            self.assertEqual(cursor.get_key(), key_populate(cursor, i))
+
+        # Search-near for an added record, should find the previous largest
+        # record.
+        for i in range(120, 130):
+            cursor.set_key(key_populate(cursor, i))
+            if self.empty:
+                # Invisible updates to fixed-length column-store objects are
+                # invisible to the reader, but the fact that they exist past
+                # the end of the initial records causes the instantiation of
+                # empty records: confirm successful return of an empty row.
+                self.assertEqual(cursor.search_near(), 0)
+                self.assertEqual(cursor.get_key(), i)
+                self.assertEqual(cursor.get_value(), 0)
+            else:
+                self.assertEqual(cursor.search_near(), -1)
+                self.assertEqual(cursor.get_key(), key_populate(cursor, 100))
 
     # Verify cursor search and search-near operations on a file with a set of
     # on-page visible records, a set of insert-list visible records, and a set
@@ -101,28 +244,26 @@ class test_bug008(wttest.WiredTigerTestCase):
     # fallback happens, whether the correct position is in the page slots or
     # the insert list.)
     def test_search_invisible_two(self):
-        uri = 'file:test_bug008'                # This is a btree layer test.
-
         # Populate the tree and reopen the connection, forcing it to disk
         # and moving the records to an on-page format.
-        simple_populate(self, uri, self.fmt, 100)
+        simple_populate(self, self.uri, self.fmt, 100)
         self.reopen_conn()
 
         # Add some additional visible records.
-        cursor = self.session.open_cursor(uri, None)
+        cursor = self.session.open_cursor(self.uri, None)
         for i in range(100, 120):
             cursor[key_populate(cursor, i)] = value_populate(cursor, i)
         cursor.close()
 
         # Begin a transaction, and add some additional records.
         self.session.begin_transaction()
-        cursor = self.session.open_cursor(uri, None)
+        cursor = self.session.open_cursor(self.uri, None)
         for i in range(120, 140):
             cursor[key_populate(cursor, i)] = value_populate(cursor, i)
 
         # Open a separate session and cursor.
         s = self.conn.open_session()
-        cursor = s.open_cursor(uri, None)
+        cursor = s.open_cursor(self.uri, None)
 
         # Search for an invisible record.
         cursor.set_key(key_populate(cursor, 130))
diff --git a/test/suite/test_drop.py b/test/suite/test_drop.py
index 5663b85d661..52ea7251ab5 100644
--- a/test/suite/test_drop.py
+++ b/test/suite/test_drop.py
@@ -41,12 +41,11 @@ class test_drop(wttest.WiredTigerTestCase):
     scenarios = check_scenarios([
         ('file', dict(uri='file:')),
         ('table', dict(uri='table:')),
-        #Not yet: drop failing with an open cursor needs handle locking
-        #('table-lsm', dict(uri='table:', extra_config=',type=lsm')),
+        ('table-lsm', dict(uri='table:', extra_config=',type=lsm')),
     ])
 
     # Populate an object, remove it and confirm it no longer exists.
-    def drop(self, populate, with_cursor, close_session, drop_index):
+    def drop(self, populate, with_cursor, reopen, drop_index):
         uri = self.uri + self.name
         populate(self, uri, 'key_format=S' + self.extra_config, 10)
 
@@ -57,7 +56,7 @@ class test_drop(wttest.WiredTigerTestCase):
                 lambda: self.session.drop(uri, None))
             cursor.close()
 
-        if close_session:
+        if reopen:
             self.reopen_conn()
 
         if drop_index:
@@ -73,17 +72,17 @@ class test_drop(wttest.WiredTigerTestCase):
         # Try all combinations except dropping the index, the simple
         # case has no indices.
         for with_cursor in [False, True]:
-            for close_session in [False, True]:
-                self.drop(simple_populate, with_cursor, close_session, False)
+            for reopen in [False, True]:
+                self.drop(simple_populate, with_cursor, reopen, False)
 
         # A complex, multi-file table object.
         # Try all test combinations.
         if self.uri == "table:":
             for with_cursor in [False, True]:
-                for close_session in [False, True]:
+                for reopen in [False, True]:
                     for drop_index in [False, True]:
                         self.drop(complex_populate, with_cursor,
-                                  close_session, drop_index)
+                                  reopen, drop_index)
 
     # Test drop of a non-existent object: force succeeds, without force fails.
     def test_drop_dne(self):