Import wiredtiger: f301aed47ca31b7f54fecf28d49199c313f86bc2 from branch mongodb-5.0

ref: 4f64671423..f301aed47c
for: 4.9.0

WT-6981       Add randomness to Python test suite runs
WT-7071       Add column store support to test_hs16
WT-7091       Restrict usage of LSM to only operate in conjunction with compatible incremental backup mechanism

9 files changed, 94 insertions, 18 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 15265cb36d4..1b00f016cb6 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-5.0",
-    "commit": "4f64671423a710f83dc6fca55326b081bec8cff6"
+    "commit": "f301aed47ca31b7f54fecf28d49199c313f86bc2"
 }
diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c
index 6a03b7ea5b3..274786c2866 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_backup.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c
@@ -611,6 +611,11 @@ __backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[
               session, EINVAL, "Incremental primary cursor must have a known source identifier");
         F_SET(cb, WT_CURBACKUP_INCR);
     }
+
+    /* Return an error if block-based incremental backup is performed with open LSM trees. */
+    if (incremental_config && !TAILQ_EMPTY(&conn->lsmqh))
+        WT_ERR_MSG(session, ENOTSUP, "LSM does not work with block-based incremental backup");
+
 err:
     if (ret != 0 && cb->incr_src != NULL) {
         F_CLR(cb->incr_src, WT_BLKINCR_INUSE);
diff --git a/src/third_party/wiredtiger/src/docs/backup.dox b/src/third_party/wiredtiger/src/docs/backup.dox
index 82979f16423..265253d8959 100644
--- a/src/third_party/wiredtiger/src/docs/backup.dox
+++ b/src/third_party/wiredtiger/src/docs/backup.dox
@@ -171,6 +171,9 @@ database directory has not been opened and recovery run.  Once recovery
 has run in a backup directory, you can no longer back up to that
 database directory.
 
+Block-based incremental backup does not work with LSM trees. An error
+will be returned in that case.
+
 An example of opening the backup data source for block-based incremental backup:
 
 @snippet ex_all.c incremental block backup
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index 8d1ec39f61c..c4c79f5e39d 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -1447,6 +1447,15 @@ tasks:
           dependent_task: compile-linux-no-ftruncate
       - func: "unit test"
 
+  # Run the tests that uses suite_random with a random starting seed
+  - name: unit-test-random-seed
+    depends_on:
+    - name: compile
+    commands:
+      - func: "fetch artifacts"
+      - func: "unit test"
+        vars:
+          unit_test_args: -v 2 -R cursor13 join02 join07 schema03 timestamp22
   # Break out Python unit tests into multiple buckets/tasks.  We have a fixed number of buckets,
   # and we use the -b option of the test/suite/run.py script to split up the tests.
 
@@ -2453,6 +2462,7 @@ buildvariants:
     - name: configure-combinations
     - name: checkpoint-filetypes-test
     - name: unit-test-long
+    - name: unit-test-random-seed
     - name: spinlock-gcc-test
     - name: spinlock-pthread-adaptive-test
     - name: compile-wtperf
diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c
index 46b475b219d..13d91b793e3 100644
--- a/src/third_party/wiredtiger/test/format/config.c
+++ b/src/third_party/wiredtiger/test/format/config.c
@@ -784,11 +784,25 @@ config_lsm_reset(void)
         config_single("transaction.timestamps=off", false);
     }
 
-    /* LSM may not work with backups, turn off backups if lsm is configured. */
+    /*
+     * LSM does not work with block-based incremental backup, change the incremental backup
+     * mechanism if block based in configured.
+     */
     if (g.c_backups) {
-        if (config_is_perm("backup"))
-            testutil_die(EINVAL, "LSM is incompatible with backup configurations");
-        config_single("backup=off", false);
+        if (config_is_perm("backup.incremental") && g.c_backup_incr_flag == INCREMENTAL_BLOCK)
+            testutil_die(EINVAL, "LSM does not work with backup.incremental=block configuration.");
+
+        if (g.c_backup_incr_flag == INCREMENTAL_BLOCK)
+            switch (mmrand(NULL, 1, 2)) {
+            case 1:
+                /* 50% */
+                config_single("backup.incremental=off", false);
+                break;
+            case 2:
+                /* 50% */
+                config_single("backup.incremental=log", false);
+                break;
+            }
     }
 }
 
diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py
index 924cd61cc23..7d342a7a471 100755
--- a/src/third_party/wiredtiger/test/suite/run.py
+++ b/src/third_party/wiredtiger/test/suite/run.py
@@ -124,6 +124,9 @@ Options:\n\
   -t      | --timestamp          name WT_TEST according to timestamp\n\
   -v N    | --verbose N          set verboseness to N (0<=N<=3, default=1)\n\
   -i      | --ignore-stdout      dont fail on unexpected stdout or stderr\n\
+  -R      | --randomseed         run with random seeds for generates random numbers\n\
+  -S      | --seed               run with two seeds that generates random numbers, \n\
+                                 format "seed1.seed2", seed1 or seed2 can\'t be zero\n\
 \n\
 Tests:\n\
   may be a file name in test/suite: (e.g. test_base01.py)\n\
@@ -303,6 +306,7 @@ if __name__ == '__main__':
     parallel = 0
     random_sample = 0
     batchtotal = batchnum = 0
+    seed = seedw = seedz = 0
     configfile = None
     configwrite = False
     dirarg = None
@@ -415,6 +419,20 @@ if __name__ == '__main__':
                 configfile = args.pop(0)
                 configwrite = True
                 continue
+            if option == '-randomseed' or option == 'R':
+                seedw = random.randint(1, 0xffffffff)
+                seedz = random.randint(1, 0xffffffff)
+                continue
+            if option == '-seed' or option == 'S':
+                if seed != 0 or len(args) == 0:
+                    usage()
+                    sys.exit(2)
+                seed = args.pop(0)
+                [seedw, seedz] = seed.split('.')
+                if seedw == 0 or seedz == 0:
+                    usage()
+                    sys.exit(2)
+                continue
             print('unknown arg: ' + arg)
             usage()
             sys.exit(2)
@@ -501,7 +519,7 @@ if __name__ == '__main__':
     # That way, verbose printing can be done at the class definition level.
     wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, lldbSub,
                                           verbose, wt_builddir, dirarg,
-                                          longtest, ignoreStdout)
+                                          longtest, ignoreStdout, seedw, seedz)
 
     # Without any tests listed as arguments, do discovery
     if len(testargs) == 0:
diff --git a/src/third_party/wiredtiger/test/suite/suite_random.py b/src/third_party/wiredtiger/test/suite/suite_random.py
index f329dd8cabc..b4da5c8b92f 100644
--- a/src/third_party/wiredtiger/test/suite/suite_random.py
+++ b/src/third_party/wiredtiger/test/suite/suite_random.py
@@ -28,6 +28,8 @@
 
 # suite_random.py
 #    A quick and predictable pseudo random number generator.
+import wttest
+
 class suite_random:
     """
     Generate random 32 bit integers that are predictable,
@@ -37,6 +39,7 @@ class suite_random:
     """
     def __init__(self, *args):
         arglen = len(args)
+        seedw, seedz = wttest.getseed()
         if arglen == 1:
             self.seedw = int(args[0]) & 0xffffffff
             self.seedz = int(args[0]) & 0xffffffff
@@ -44,8 +47,8 @@ class suite_random:
             self.seedw = int(args[0]) & 0xffffffff
             self.seedz = int(args[1]) & 0xffffffff
         else:
-            self.seedw = 521288629
-            self.seedz = 362436069
+            self.seedw = int(seedw) & 0xffffffff
+            self.seedz = int(seedz) & 0xffffffff
 
     def rand32(self):
         """
@@ -54,8 +57,9 @@ class suite_random:
         w = self.seedw
         z = self.seedz
         if w == 0 or z == 0:
-            self.seedw = 521288629
-            self.seedz = 362436069
+            seedw, seedz = wttest.getRandomSeed()
+            self.seedw = int(seedw) & 0xffffffff
+            self.seedz = int(seedz) & 0xffffffff
 
         self.seedz = (36969 * (z & 65535) + (z >> 16)) & 0xffffffff
         self.seedw = (18000 * (w & 65535) + (w >> 16)) & 0xffffffff
diff --git a/src/third_party/wiredtiger/test/suite/test_hs16.py b/src/third_party/wiredtiger/test/suite/test_hs16.py
index 1e5da4a19a1..b8b658127e9 100644
--- a/src/third_party/wiredtiger/test/suite/test_hs16.py
+++ b/src/third_party/wiredtiger/test/suite/test_hs16.py
@@ -27,6 +27,7 @@
 # OTHER DEALINGS IN THE SOFTWARE.
 
 import time, wiredtiger, wttest
+from wtscenario import make_scenarios
 
 def timestamp_str(t):
     return '%x' % t
@@ -36,36 +37,47 @@ def timestamp_str(t):
 class test_hs16(wttest.WiredTigerTestCase):
     conn_config = 'cache_size=5MB'
     session_config = 'isolation=snapshot'
+    key_format_values = (
+        ('column', dict(key_format='r')),
+        ('string', dict(key_format='S'))
+    )
+    scenarios = make_scenarios(key_format_values)
+
+    def create_key(self,i):
+        if self.key_format == 'S':
+            return str(i)
+        return i
 
     def test_hs16(self):
         uri = 'table:test_hs16'
-        self.session.create(uri, 'key_format=S,value_format=S')
+        create_params = 'key_format={}, value_format=S'.format(self.key_format)
+        self.session.create(uri, create_params)
         cursor = self.session.open_cursor(uri)
 
         # Insert an update without timestamp
         self.session.begin_transaction()
-        cursor[str(0)] = 'a'
+        cursor[self.create_key(1)] = 'a'
         self.session.commit_transaction()
 
         # Update an update at timestamp 1
         self.session.begin_transaction()
-        cursor[str(0)] = 'b'
+        cursor[self.create_key(1)] = 'b'
         self.session.commit_transaction('commit_timestamp=' + timestamp_str(1))
 
         # Open anther session to make the next update without timestamp non-globally visible
         session2 = self.setUpSessionOpen(self.conn)
         cursor2 = session2.open_cursor(uri)
         session2.begin_transaction()
-        cursor[str(1)] = 'a'
+        cursor[self.create_key(2)] = 'a'
 
         # Update an update without timestamp
         self.session.begin_transaction()
-        cursor[str(0)] = 'c'
+        cursor[self.create_key(1)] = 'c'
         self.session.commit_transaction()
 
         # Update an update at timestamp 2
         self.session.begin_transaction()
-        cursor[str(0)] = 'd'
+        cursor[self.create_key(1)] = 'd'
         self.session.commit_transaction('commit_timestamp=' + timestamp_str(2))
 
         # Do a checkpoint, it should not panic
diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py
index 0d11bfb193b..15b328ad1bb 100755
--- a/src/third_party/wiredtiger/test/suite/wttest.py
+++ b/src/third_party/wiredtiger/test/suite/wttest.py
@@ -198,7 +198,7 @@ class WiredTigerTestCase(unittest.TestCase):
     @staticmethod
     def globalSetup(preserveFiles = False, useTimestamp = False,
                     gdbSub = False, lldbSub = False, verbose = 1, builddir = None, dirarg = None,
-                    longtest = False, ignoreStdout = False):
+                    longtest = False, ignoreStdout = False, seedw = 0, seedz = 0):
         WiredTigerTestCase._preserveFiles = preserveFiles
         d = 'WT_TEST' if dirarg == None else dirarg
         if useTimestamp:
@@ -221,6 +221,11 @@ class WiredTigerTestCase(unittest.TestCase):
         WiredTigerTestCase._concurrent = False
         WiredTigerTestCase._globalSetup = True
         WiredTigerTestCase._ttyDescriptor = None
+        WiredTigerTestCase._seeds = [521288629, 362436069]
+        WiredTigerTestCase._randomseed = False
+        if seedw != 0 and seedz != 0:
+            WiredTigerTestCase._randomseed = True
+            WiredTigerTestCase._seeds = [seedw, seedz]
 
     def fdSetUp(self):
         self.captureout = CapturedFd('stdout.txt', 'standard output')
@@ -470,7 +475,6 @@ class WiredTigerTestCase(unittest.TestCase):
             except:
                 pass
         self._connections = []
-
         try:
             self.fdTearDown()
             self.captureout.check(self)
@@ -757,6 +761,9 @@ def longtest(description):
 def islongtest():
     return WiredTigerTestCase._longtest
 
+def getseed():
+    return WiredTigerTestCase._seeds
+
 def runsuite(suite, parallel):
     suite_to_run = suite
     if parallel > 1:
@@ -766,6 +773,9 @@ def runsuite(suite, parallel):
         WiredTigerTestCase._concurrent = True
         suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel))
     try:
+        if WiredTigerTestCase._randomseed:
+            WiredTigerTestCase.prout("Starting test suite with seedw={0} and seedz={1}. Rerun this test with -seed {0}.{1} to get the same randomness"
+                .format(str(WiredTigerTestCase._seeds[0]), str(WiredTigerTestCase._seeds[1])))
         return unittest.TextTestRunner(
             verbosity=WiredTigerTestCase._verbose).run(suite_to_run)
     except BaseException as e: