From cbd21efb0254b018e971113e2516ef307ee6fa38 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Mon, 7 Sep 2020 16:55:33 +1000 Subject: Import wiredtiger: 4880ad2a4a0ddb42b3ea9fabf054760872780606 from branch mongodb-4.6 ref: 42e4868b7b..4880ad2a4a for: 4.5.1 WT-6505 Add debugging for missing file failure WT-6613 Add python test for early_load flag --- src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/test/csuite/random_directio/main.c | 58 ++++++++++++++++++---- .../test/csuite/random_directio/smoke.sh | 2 +- .../wiredtiger/test/suite/test_encrypt01.py | 9 +++- src/third_party/wiredtiger/test/suite/wttest.py | 5 +- 5 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 84b893446d0..45bf0228d0b 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.6", - "commit": "42e4868b7b40ec5a49eec6a0e6cc1bf7eff2cad0" + "commit": "4880ad2a4a0ddb42b3ea9fabf054760872780606" } diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/main.c b/src/third_party/wiredtiger/test/csuite/random_directio/main.c index 6a9967a4f3d..5e21751781d 100644 --- a/src/third_party/wiredtiger/test/csuite/random_directio/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_directio/main.c @@ -153,7 +153,8 @@ static const char *const uri_rev = "table:rev"; * that has schema operations happens again at id 200, assuming frequency * set to 100. So it is a good test of schema operations 'in flight'. */ -#define SCHEMA_OP_FREQUENCY 100 +#define SCHEMA_FREQUENCY_DEFAULT 100 +static uint64_t schema_frequency; #define TEST_STREQ(expect, got, message) \ do { \ @@ -203,6 +204,8 @@ usage(void) fprintf(stderr, "usage: %s [options]\n", progname); fprintf(stderr, "options:\n"); fprintf(stderr, " %-20s%s\n", "-d data_size", "approximate size of keys and values [1000]"); + fprintf(stderr, " %-20s%s\n", "-f schema frequency", + "restart schema sequence every frequency period [100]"); fprintf(stderr, " %-20s%s\n", "-h home", "WiredTiger home directory [WT_TEST.directio]"); fprintf( stderr, " %-20s%s\n", "-i interval", "interval timeout between copy/recover cycles [3]"); @@ -243,7 +246,7 @@ usage(void) static bool has_schema_operation(uint64_t id, uint32_t offset) { - return (id >= offset && (id - offset) % SCHEMA_OP_FREQUENCY < 10); + return (id >= offset && (id - offset) % schema_frequency < 10); } /* @@ -355,7 +358,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o /* fprintf(stderr, "CREATE: %s\n", uri1); */ + testutil_check(session->log_printf(session, "CREATE: %s", uri1)); testutil_check(session->create(session, uri1, "key_format=S,value_format=S")); + testutil_check(session->log_printf(session, "CREATE: DONE %s", uri1)); break; case 1: /* Insert a value into the table. */ @@ -366,7 +371,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o testutil_check(session->open_cursor(session, uri1, NULL, NULL, &cursor)); cursor->set_key(cursor, uri1); cursor->set_value(cursor, uri1); + testutil_check(session->log_printf(session, "INSERT: %s", uri1)); testutil_check(cursor->insert(cursor)); + testutil_check(session->log_printf(session, "INSERT: DONE %s", uri1)); testutil_check(cursor->close(cursor)); break; case 2: @@ -378,7 +385,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o /* fprintf(stderr, "RENAME: %s->%s\n", uri1, uri2); */ + testutil_check(session->log_printf(session, "RENAME: %s->%s", uri1, uri2)); ret = session->rename(session, uri1, uri2, NULL); + testutil_check(session->log_printf(session, "RENAME: DONE %s->%s", uri1, uri2)); } break; case 3: @@ -391,7 +400,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o /* fprintf(stderr, "UPDATE: %s\n", uri2); */ + testutil_check(session->log_printf(session, "UPDATE: %s", uri2)); testutil_check(cursor->update(cursor)); + testutil_check(session->log_printf(session, "UPDATE: DONE %s", uri2)); testutil_check(cursor->close(cursor)); break; case 4: @@ -402,7 +413,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o /* fprintf(stderr, "DROP: %s\n", uri1); */ + testutil_check(session->log_printf(session, "DROP: %s", uri1)); ret = session->drop(session, uri1, NULL); + testutil_check(session->log_printf(session, "DROP: DONE %s", uri1)); } } /* @@ -585,7 +598,8 @@ fill_db(uint32_t nth, uint32_t datasize, const char *method, uint32_t flags) testutil_check(wiredtiger_open(".", NULL, envconf, &conn)); datasize += 1; /* Add an extra byte for string termination */ - printf("Create %" PRIu32 " writer threads\n", nth); + printf( + "Create %" PRIu32 " writer threads. Schema frequency %" PRIu64 "\n", nth, schema_frequency); for (i = 0; i < nth; ++i) { td[i].conn = conn; td[i].data = dcalloc(datasize, 1); @@ -756,7 +770,7 @@ check_schema(WT_SESSION *session, uint64_t lastid, uint32_t threadid, uint32_t f * Make a copy of the database and verify its contents. */ static bool -check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) +check_db(uint32_t nth, uint32_t datasize, pid_t pid, bool directio, uint32_t flags) { WT_CONNECTION *conn; WT_CURSOR *cursor, *meta, *rev; @@ -765,7 +779,8 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) uint64_t gotid, id; uint64_t *lastid; uint32_t gotth, kvsize, th, threadmap; - char checkdir[4096], savedir[4096]; + int status; + char checkdir[4096], dbgdir[4096], savedir[4096]; char *gotkey, *gotvalue, *keybuf, *p; char **large_arr; @@ -778,6 +793,7 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) large_buf(large_arr[th], LARGE_WRITE_SIZE, th, true); } testutil_check(__wt_snprintf(checkdir, sizeof(checkdir), "%s.CHECK", home)); + testutil_check(__wt_snprintf(dbgdir, sizeof(savedir), "%s.DEBUG", home)); testutil_check(__wt_snprintf(savedir, sizeof(savedir), "%s.SAVE", home)); /* @@ -788,10 +804,30 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) "Copy database home directory using direct I/O to run recovery,\n" "along with a saved 'pre-recovery' copy.\n"); copy_directory(home, checkdir, directio); + /* Copy the original home directory explicitly without direct I/O. */ + copy_directory(home, dbgdir, false); copy_directory(checkdir, savedir, false); printf("Open database, run recovery and verify content\n"); - testutil_check(wiredtiger_open(checkdir, NULL, ENV_CONFIG_REC, &conn)); + ret = wiredtiger_open(checkdir, NULL, ENV_CONFIG_REC, &conn); + /* If this fails, abort the child process before we die so we can see what it was doing. */ + if (ret != 0) { + if (pid != 0) { + /* + * The child is stopped, it won't process an abort until it is continued. First signal + * the abort, then signal continue so that the child process will process the abort and + * dump core. + */ + printf("Send abort to child process ID %d\n", (int)pid); + if (kill(pid, SIGABRT) != 0) + testutil_die(errno, "kill"); + if (kill(pid, SIGCONT) != 0) + testutil_die(errno, "kill"); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); + } + testutil_check(ret); + } testutil_check(conn->open_session(conn, NULL, NULL, &session)); testutil_check(session->open_cursor(session, uri_main, NULL, NULL, &cursor)); testutil_check(session->open_cursor(session, uri_rev, NULL, NULL, &rev)); @@ -1025,6 +1061,7 @@ main(int argc, char *argv[]) working_dir = "WT_TEST.random-directio"; method = "none"; pid = 0; + schema_frequency = SCHEMA_FREQUENCY_DEFAULT; memset(args, 0, sizeof(args)); if (!has_direct_io()) { @@ -1038,7 +1075,7 @@ main(int argc, char *argv[]) __wt_snprintf_len_set(p, sizeof(args) - (size_t)(p - args), &size, " %s", argv[i])); p += size; } - while ((ch = __wt_getopt(progname, argc, argv, "d:h:i:m:n:pS:T:t:v")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "d:f:h:i:m:n:pS:T:t:v")) != EOF) switch (ch) { case 'd': datasize = (uint32_t)atoi(__wt_optarg); @@ -1047,6 +1084,9 @@ main(int argc, char *argv[]) return (EXIT_FAILURE); } break; + case 'f': + schema_frequency = (uint64_t)atoi(__wt_optarg); + break; case 'h': working_dir = __wt_optarg; break; @@ -1189,7 +1229,7 @@ main(int argc, char *argv[]) testutil_die(errno, "kill"); printf("Check DB\n"); fflush(stdout); - if (!check_db(nth, datasize, true, flags)) + if (!check_db(nth, datasize, pid, true, flags)) return (EXIT_FAILURE); if (kill(pid, SIGCONT) != 0) testutil_die(errno, "kill"); @@ -1204,7 +1244,7 @@ main(int argc, char *argv[]) if (waitpid(pid, &status, 0) == -1) testutil_die(errno, "waitpid"); } - if (verify_only && !check_db(nth, datasize, false, flags)) { + if (verify_only && !check_db(nth, datasize, 0, false, flags)) { printf("FAIL\n"); return (EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh index 815b8a21c9f..7d5b96d9b57 100755 --- a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh @@ -23,7 +23,7 @@ for threads in $TEST_THREADS; do for method in $TEST_METHODS; do RUN_TEST="$RUN_TEST_CMD -t 5 -m $method" $RUN_TEST -T $threads || exit 1 - $RUN_TEST -T $threads -S create,drop,verbose || exit 1 + $RUN_TEST -f 20 -T $threads -S create,drop,verbose || exit 1 # Here are successively tougher schema tests that do not yet # reliably pass. 'verbose' can be added to any. diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt01.py b/src/third_party/wiredtiger/test/suite/test_encrypt01.py index 92032a765fe..09031c67a28 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt01.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt01.py @@ -61,13 +61,20 @@ class test_encrypt01(wttest.WiredTigerTestCase): ('none-snappy', dict(log_compress=None, block_compress='snappy')), ('snappy-lz4', dict(log_compress='snappy', block_compress='lz4')), ] - scenarios = make_scenarios(types, encrypt, compress) + loadExt = [ + ('earlyLoadTrue', dict(earlyLoad=True)), + ('earlyLoadFalse', dict(earlyLoad=False)), + ] + + scenarios = make_scenarios(types, encrypt, compress, loadExt) nrecords = 5000 bigvalue = "abcdefghij" * 1001 # len(bigvalue) = 10010 def conn_extensions(self, extlist): extlist.skip_if_missing = True + if self.earlyLoad == True: + extlist.early_load_ext = True extlist.extension('encryptors', self.sys_encrypt) extlist.extension('encryptors', self.file_encrypt) extlist.extension('compressors', self.block_compress) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index f88f26b5da7..a0a86731f1c 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -263,8 +263,11 @@ class WiredTigerTestCase(unittest.TestCase): result = '' extfiles = {} skipIfMissing = False + earlyLoading = '' if hasattr(exts, 'skip_if_missing'): skipIfMissing = exts.skip_if_missing + if hasattr(exts, 'early_load_ext') and exts.early_load_ext == True: + earlyLoading = '=(early_load=true)' for ext in exts: extconf = '' if '=' in ext: @@ -302,7 +305,7 @@ class WiredTigerTestCase(unittest.TestCase): else: extfiles[ext] = complete if len(extfiles) != 0: - result = ',extensions=[' + ','.join(list(extfiles.values())) + ']' + result = ',extensions=[' + ','.join(list(extfiles.values())) + earlyLoading + ']' return result # Can be overridden, but first consider setting self.conn_config -- cgit v1.2.1