Import wiredtiger: 4880ad2a4a0ddb42b3ea9fabf054760872780606 from branch mongodb-4.6

ref: 42e4868b7b..4880ad2a4a
for: 4.5.1

WT-6505       Add debugging for missing file failure
WT-6613       Add python test for early_load flag

5 files changed, 63 insertions, 13 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 84b893446d0..45bf0228d0b 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-4.6",
-    "commit": "42e4868b7b40ec5a49eec6a0e6cc1bf7eff2cad0"
+    "commit": "4880ad2a4a0ddb42b3ea9fabf054760872780606"
 }
diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/main.c b/src/third_party/wiredtiger/test/csuite/random_directio/main.c
index 6a9967a4f3d..5e21751781d 100644
--- a/src/third_party/wiredtiger/test/csuite/random_directio/main.c
+++ b/src/third_party/wiredtiger/test/csuite/random_directio/main.c
@@ -153,7 +153,8 @@ static const char *const uri_rev = "table:rev";
  * that has schema operations happens again at id 200, assuming frequency
  * set to 100. So it is a good test of schema operations 'in flight'.
  */
-#define SCHEMA_OP_FREQUENCY 100
+#define SCHEMA_FREQUENCY_DEFAULT 100
+static uint64_t schema_frequency;
 
 #define TEST_STREQ(expect, got, message)                                 \
     do {                                                                 \
@@ -203,6 +204,8 @@ usage(void)
     fprintf(stderr, "usage: %s [options]\n", progname);
     fprintf(stderr, "options:\n");
     fprintf(stderr, "  %-20s%s\n", "-d data_size", "approximate size of keys and values [1000]");
+    fprintf(stderr, "  %-20s%s\n", "-f schema frequency",
+      "restart schema sequence every frequency period [100]");
     fprintf(stderr, "  %-20s%s\n", "-h home", "WiredTiger home directory [WT_TEST.directio]");
     fprintf(
       stderr, "  %-20s%s\n", "-i interval", "interval timeout between copy/recover cycles [3]");
@@ -243,7 +246,7 @@ usage(void)
 static bool
 has_schema_operation(uint64_t id, uint32_t offset)
 {
-    return (id >= offset && (id - offset) % SCHEMA_OP_FREQUENCY < 10);
+    return (id >= offset && (id - offset) % schema_frequency < 10);
 }
 
 /*
@@ -355,7 +358,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o
         /*
         fprintf(stderr, "CREATE: %s\n", uri1);
         */
+        testutil_check(session->log_printf(session, "CREATE: %s", uri1));
         testutil_check(session->create(session, uri1, "key_format=S,value_format=S"));
+        testutil_check(session->log_printf(session, "CREATE: DONE %s", uri1));
         break;
     case 1:
         /* Insert a value into the table. */
@@ -366,7 +371,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o
         testutil_check(session->open_cursor(session, uri1, NULL, NULL, &cursor));
         cursor->set_key(cursor, uri1);
         cursor->set_value(cursor, uri1);
+        testutil_check(session->log_printf(session, "INSERT: %s", uri1));
         testutil_check(cursor->insert(cursor));
+        testutil_check(session->log_printf(session, "INSERT: DONE %s", uri1));
         testutil_check(cursor->close(cursor));
         break;
     case 2:
@@ -378,7 +385,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o
             /*
             fprintf(stderr, "RENAME: %s->%s\n", uri1, uri2);
             */
+            testutil_check(session->log_printf(session, "RENAME: %s->%s", uri1, uri2));
             ret = session->rename(session, uri1, uri2, NULL);
+            testutil_check(session->log_printf(session, "RENAME: DONE %s->%s", uri1, uri2));
         }
         break;
     case 3:
@@ -391,7 +400,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o
         /*
         fprintf(stderr, "UPDATE: %s\n", uri2);
         */
+        testutil_check(session->log_printf(session, "UPDATE: %s", uri2));
         testutil_check(cursor->update(cursor));
+        testutil_check(session->log_printf(session, "UPDATE: DONE %s", uri2));
         testutil_check(cursor->close(cursor));
         break;
     case 4:
@@ -402,7 +413,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o
             /*
             fprintf(stderr, "DROP: %s\n", uri1);
             */
+            testutil_check(session->log_printf(session, "DROP: %s", uri1));
             ret = session->drop(session, uri1, NULL);
+            testutil_check(session->log_printf(session, "DROP: DONE %s", uri1));
         }
     }
     /*
@@ -585,7 +598,8 @@ fill_db(uint32_t nth, uint32_t datasize, const char *method, uint32_t flags)
     testutil_check(wiredtiger_open(".", NULL, envconf, &conn));
 
     datasize += 1; /* Add an extra byte for string termination */
-    printf("Create %" PRIu32 " writer threads\n", nth);
+    printf(
+      "Create %" PRIu32 " writer threads. Schema frequency %" PRIu64 "\n", nth, schema_frequency);
     for (i = 0; i < nth; ++i) {
         td[i].conn = conn;
         td[i].data = dcalloc(datasize, 1);
@@ -756,7 +770,7 @@ check_schema(WT_SESSION *session, uint64_t lastid, uint32_t threadid, uint32_t f
  *     Make a copy of the database and verify its contents.
  */
 static bool
-check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags)
+check_db(uint32_t nth, uint32_t datasize, pid_t pid, bool directio, uint32_t flags)
 {
     WT_CONNECTION *conn;
     WT_CURSOR *cursor, *meta, *rev;
@@ -765,7 +779,8 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags)
     uint64_t gotid, id;
     uint64_t *lastid;
     uint32_t gotth, kvsize, th, threadmap;
-    char checkdir[4096], savedir[4096];
+    int status;
+    char checkdir[4096], dbgdir[4096], savedir[4096];
     char *gotkey, *gotvalue, *keybuf, *p;
     char **large_arr;
 
@@ -778,6 +793,7 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags)
         large_buf(large_arr[th], LARGE_WRITE_SIZE, th, true);
     }
     testutil_check(__wt_snprintf(checkdir, sizeof(checkdir), "%s.CHECK", home));
+    testutil_check(__wt_snprintf(dbgdir, sizeof(savedir), "%s.DEBUG", home));
     testutil_check(__wt_snprintf(savedir, sizeof(savedir), "%s.SAVE", home));
 
     /*
@@ -788,10 +804,30 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags)
       "Copy database home directory using direct I/O to run recovery,\n"
       "along with a saved 'pre-recovery' copy.\n");
     copy_directory(home, checkdir, directio);
+    /* Copy the original home directory explicitly without direct I/O. */
+    copy_directory(home, dbgdir, false);
     copy_directory(checkdir, savedir, false);
 
     printf("Open database, run recovery and verify content\n");
-    testutil_check(wiredtiger_open(checkdir, NULL, ENV_CONFIG_REC, &conn));
+    ret = wiredtiger_open(checkdir, NULL, ENV_CONFIG_REC, &conn);
+    /* If this fails, abort the child process before we die so we can see what it was doing. */
+    if (ret != 0) {
+        if (pid != 0) {
+            /*
+             * The child is stopped, it won't process an abort until it is continued. First signal
+             * the abort, then signal continue so that the child process will process the abort and
+             * dump core.
+             */
+            printf("Send abort to child process ID %d\n", (int)pid);
+            if (kill(pid, SIGABRT) != 0)
+                testutil_die(errno, "kill");
+            if (kill(pid, SIGCONT) != 0)
+                testutil_die(errno, "kill");
+            if (waitpid(pid, &status, 0) == -1)
+                testutil_die(errno, "waitpid");
+        }
+        testutil_check(ret);
+    }
     testutil_check(conn->open_session(conn, NULL, NULL, &session));
     testutil_check(session->open_cursor(session, uri_main, NULL, NULL, &cursor));
     testutil_check(session->open_cursor(session, uri_rev, NULL, NULL, &rev));
@@ -1025,6 +1061,7 @@ main(int argc, char *argv[])
     working_dir = "WT_TEST.random-directio";
     method = "none";
     pid = 0;
+    schema_frequency = SCHEMA_FREQUENCY_DEFAULT;
     memset(args, 0, sizeof(args));
 
     if (!has_direct_io()) {
@@ -1038,7 +1075,7 @@ main(int argc, char *argv[])
           __wt_snprintf_len_set(p, sizeof(args) - (size_t)(p - args), &size, " %s", argv[i]));
         p += size;
     }
-    while ((ch = __wt_getopt(progname, argc, argv, "d:h:i:m:n:pS:T:t:v")) != EOF)
+    while ((ch = __wt_getopt(progname, argc, argv, "d:f:h:i:m:n:pS:T:t:v")) != EOF)
         switch (ch) {
         case 'd':
             datasize = (uint32_t)atoi(__wt_optarg);
@@ -1047,6 +1084,9 @@ main(int argc, char *argv[])
                 return (EXIT_FAILURE);
             }
             break;
+        case 'f':
+            schema_frequency = (uint64_t)atoi(__wt_optarg);
+            break;
         case 'h':
             working_dir = __wt_optarg;
             break;
@@ -1189,7 +1229,7 @@ main(int argc, char *argv[])
                 testutil_die(errno, "kill");
             printf("Check DB\n");
             fflush(stdout);
-            if (!check_db(nth, datasize, true, flags))
+            if (!check_db(nth, datasize, pid, true, flags))
                 return (EXIT_FAILURE);
             if (kill(pid, SIGCONT) != 0)
                 testutil_die(errno, "kill");
@@ -1204,7 +1244,7 @@ main(int argc, char *argv[])
         if (waitpid(pid, &status, 0) == -1)
             testutil_die(errno, "waitpid");
     }
-    if (verify_only && !check_db(nth, datasize, false, flags)) {
+    if (verify_only && !check_db(nth, datasize, 0, false, flags)) {
         printf("FAIL\n");
         return (EXIT_FAILURE);
     }
diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh
index 815b8a21c9f..7d5b96d9b57 100755
--- a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh
+++ b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh
@@ -23,7 +23,7 @@ for threads in $TEST_THREADS; do
     for method in $TEST_METHODS; do
         RUN_TEST="$RUN_TEST_CMD -t 5 -m $method"
         $RUN_TEST -T $threads                              || exit 1
-        $RUN_TEST -T $threads -S create,drop,verbose       || exit 1
+        $RUN_TEST -f 20 -T $threads -S create,drop,verbose       || exit 1
 
         # Here are successively tougher schema tests that do not yet
         # reliably pass.  'verbose' can be added to any.
diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt01.py b/src/third_party/wiredtiger/test/suite/test_encrypt01.py
index 92032a765fe..09031c67a28 100644
--- a/src/third_party/wiredtiger/test/suite/test_encrypt01.py
+++ b/src/third_party/wiredtiger/test/suite/test_encrypt01.py
@@ -61,13 +61,20 @@ class test_encrypt01(wttest.WiredTigerTestCase):
         ('none-snappy', dict(log_compress=None, block_compress='snappy')),
         ('snappy-lz4', dict(log_compress='snappy', block_compress='lz4')),
     ]
-    scenarios = make_scenarios(types, encrypt, compress)
+    loadExt = [
+        ('earlyLoadTrue', dict(earlyLoad=True)),
+        ('earlyLoadFalse', dict(earlyLoad=False)),
+    ]
+
+    scenarios = make_scenarios(types, encrypt, compress, loadExt)
 
     nrecords = 5000
     bigvalue = "abcdefghij" * 1001    # len(bigvalue) = 10010
 
     def conn_extensions(self, extlist):
         extlist.skip_if_missing = True
+        if self.earlyLoad == True:
+            extlist.early_load_ext = True
         extlist.extension('encryptors', self.sys_encrypt)
         extlist.extension('encryptors', self.file_encrypt)
         extlist.extension('compressors', self.block_compress)
diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py
index f88f26b5da7..a0a86731f1c 100755
--- a/src/third_party/wiredtiger/test/suite/wttest.py
+++ b/src/third_party/wiredtiger/test/suite/wttest.py
@@ -263,8 +263,11 @@ class WiredTigerTestCase(unittest.TestCase):
         result = ''
         extfiles = {}
         skipIfMissing = False
+        earlyLoading = ''
         if hasattr(exts, 'skip_if_missing'):
             skipIfMissing = exts.skip_if_missing
+        if hasattr(exts, 'early_load_ext') and exts.early_load_ext == True:
+            earlyLoading = '=(early_load=true)'
         for ext in exts:
             extconf = ''
             if '=' in ext:
@@ -302,7 +305,7 @@ class WiredTigerTestCase(unittest.TestCase):
             else:
                 extfiles[ext] = complete
         if len(extfiles) != 0:
-            result = ',extensions=[' + ','.join(list(extfiles.values())) + ']'
+            result = ',extensions=[' + ','.join(list(extfiles.values())) + earlyLoading + ']'
         return result
 
     # Can be overridden, but first consider setting self.conn_config