From f07694654f3733ff664cdb8b48cee37486095c55 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Thu, 17 Jan 2019 14:36:04 +1100 Subject: Import wiredtiger: 21838832317b7c38b1a3e929283170515b08bb79 from branch mongodb-4.2 ref: fa402fef19..2183883231 for: 4.1.8 WT-4452 Fix bugs in the salvage test for big endian WT-4503 Pre-allocating log file takes long time on PPC machine WT-4524 wt2853_perf failures in valgrind WT-4525 wt4333_handle_locks failures in valgrind --- src/third_party/wiredtiger/dist/s_string.ok | 1 + src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/block/block_open.c | 7 +- src/third_party/wiredtiger/src/block/block_read.c | 12 +- src/third_party/wiredtiger/src/block/block_write.c | 2 +- .../wiredtiger/test/csuite/wt2853_perf/main.c | 4 + .../test/csuite/wt4333_handle_locks/main.c | 6 + .../wiredtiger/test/salvage/Makefile.am | 2 +- src/third_party/wiredtiger/test/salvage/salvage.c | 137 ++++++++++++--------- .../wiredtiger/test/suite/test_bug019.py | 4 +- 10 files changed, 101 insertions(+), 76 deletions(-) diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index e6892f63c72..f5381727e9a 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1385,6 +1385,7 @@ xff xxxx xxxxx xxxxxx +zSeries zalloc zf zfree diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index efadc6294c9..b836b282e8a 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "fa402fef1974b30ef61770caff58b0eabda97781", + "commit": "21838832317b7c38b1a3e929283170515b08bb79", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index 0d60e751c29..68f7224ff76 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -372,11 +372,8 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block) __wt_verbose(session, WT_VERB_BLOCK, "%s: magic %" PRIu32 - ", major/minor: %" PRIu32 "/%" PRIu32 - ", checksum %#" PRIx32, - block->name, desc->magic, - desc->majorv, desc->minorv, - desc->checksum); + ", major/minor: %" PRIu32 "/%" PRIu32, + block->name, desc->magic, desc->majorv, desc->minorv); err: __wt_scr_free(session, &buf); return (ret); diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 83c1ba17d6d..977fb165b84 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -120,7 +120,7 @@ __wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_DECL_RET; size_t chunk, i, nchunks; -#define WT_CORRUPT_FMT "{%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "}" +#define WT_CORRUPT_FMT "{%" PRIuMAX ", %" PRIu32 ", %#" PRIx32 "}" if (buf->size == 0) { __wt_errx(session, WT_CORRUPT_FMT ": empty buffer, no dump available", @@ -226,7 +226,7 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t page_checksum; __wt_verbose(session, WT_VERB_READ, - "off %" PRIuMAX ", size %" PRIu32 ", checksum %" PRIu32, + "off %" PRIuMAX ", size %" PRIu32 ", checksum %#" PRIx32, (uintmax_t)offset, size, checksum); WT_STAT_CONN_INCR(session, block_read); @@ -276,8 +276,8 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, __wt_errx(session, "%s: read checksum error for %" PRIu32 "B block at " "offset %" PRIuMAX ": calculated block checksum " - "of %" PRIu32 " doesn't match expected checksum " - "of %" PRIu32, + "of %#" PRIx32 " doesn't match expected checksum " + "of %#" PRIx32, block->name, size, (uintmax_t)offset, page_checksum, checksum); } else @@ -285,8 +285,8 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, __wt_errx(session, "%s: read checksum error for %" PRIu32 "B block at " "offset %" PRIuMAX ": block header checksum " - "of %" PRIu32 " doesn't match expected checksum " - "of %" PRIu32, + "of %#" PRIx32 " doesn't match expected checksum " + "of %#" PRIx32, block->name, size, (uintmax_t)offset, swap.checksum, checksum); diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index 51baa55b383..4de128494d0 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -374,7 +374,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, session, block_byte_write_checkpoint, align_size); __wt_verbose(session, WT_VERB_WRITE, - "off %" PRIuMAX ", size %" PRIuMAX ", checksum %" PRIu32, + "off %" PRIuMAX ", size %" PRIuMAX ", checksum %#" PRIx32, (uintmax_t)offset, (uintmax_t)align_size, checksum); *offsetp = offset; diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index 75510fcdbda..6172e417c46 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -82,6 +82,10 @@ main(int argc, char *argv[]) int i, nfail; const char *tablename; + /* Bypass this test for valgrind. */ + if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) + return (EXIT_SUCCESS); + opts = &_opts; sharedopts = &_sharedopts; memset(opts, 0, sizeof(*opts)); diff --git a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c index 2c8140f8f26..5c8b965ceb5 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c @@ -338,6 +338,12 @@ main(int argc, char *argv[]) u_int i, n; int ch; + /* + * Bypass this test for valgrind. It has a fairly low thread limit. + */ + if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) + return (EXIT_SUCCESS); + (void)testutil_set_progname(argv); __wt_random_init_seed(NULL, &rnd); diff --git a/src/third_party/wiredtiger/test/salvage/Makefile.am b/src/third_party/wiredtiger/test/salvage/Makefile.am index b9ba9c35d72..b1540e639ba 100644 --- a/src/third_party/wiredtiger/test/salvage/Makefile.am +++ b/src/third_party/wiredtiger/test/salvage/Makefile.am @@ -14,4 +14,4 @@ TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WiredTiger* __slvg* core.* *.core + rm -rf core.* *.core WT_TEST diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index 9e491c499ce..d3429788bfc 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -30,21 +30,17 @@ #include -#define DUMP "__slvg.dump" /* Dump file */ -#define LOAD "__slvg.load" /* Build file */ -#define RSLT "__slvg.result" /* Result file */ -#define SLVG "__slvg.slvg" /* Salvage file */ +#define HOME "WT_TEST" +#define DUMP "WT_TEST/__slvg.dump" /* Dump file */ +#define LOAD "WT_TEST/__slvg.load" /* Build file */ +#define LOAD_URI "file:__slvg.load" /* Build URI */ +#define RSLT "WT_TEST/__slvg.result" /* Result file */ +#define SLVG "WT_TEST/__slvg.slvg" /* Salvage file */ +#define SLVG_URI "file:__slvg.slvg" /* Salvage URI */ #define PSIZE (2 * 1024) #define OSIZE (PSIZE / 20) -#define CHECK(x) do { \ - if (!(x)) { \ - fprintf(stderr, "%s: failed\n", #x); \ - assert(0); \ - } \ -} while (0) - void build(int, int, int); void copy(u_int, u_int); void empty(int); @@ -162,8 +158,9 @@ run(int r) printf("\t%s: run %d\n", __wt_page_type_string(page_type), r); - CHECK(system("rm -f WiredTiger* __slvg.*") == 0); - CHECK((res_fp = fopen(RSLT, "w")) != NULL); + testutil_make_work_dir(HOME); + + testutil_assert((res_fp = fopen(RSLT, "w")) != NULL); /* * Each run builds the LOAD file, and then appends the first page of @@ -435,7 +432,7 @@ run(int r) exit(EXIT_FAILURE); } - CHECK(fclose(res_fp) == 0); + testutil_assert(fclose(res_fp) == 0); process(); @@ -446,6 +443,8 @@ run(int r) "check failed, salvage results were incorrect\n"); exit(EXIT_FAILURE); } + + testutil_clean_work_dir(HOME); } /* @@ -478,10 +477,10 @@ build(int ikey, int ivalue, int cnt) * Disable logging: we're modifying files directly, we don't want to * run recovery. */ - CHECK(wiredtiger_open( - NULL, NULL, "create,log=(enabled=false)", &conn) == 0); - CHECK(conn->open_session(conn, NULL, NULL, &session) == 0); - CHECK(session->drop(session, "file:" LOAD, "force") == 0); + testutil_check(wiredtiger_open( + HOME, NULL, "create,log=(enabled=false)", &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->drop(session, LOAD_URI, "force")); switch (page_type) { case WT_PAGE_COL_FIX: @@ -511,9 +510,9 @@ build(int ikey, int ivalue, int cnt) default: assert(0); } - CHECK(session->create(session, "file:" LOAD, config) == 0); - CHECK(session->open_cursor( - session, "file:" LOAD, NULL, "bulk,append", &cursor) == 0); + testutil_check(session->create(session, LOAD_URI, config)); + testutil_check(session->open_cursor( + session, LOAD_URI, NULL, "bulk,append", &cursor)); for (; cnt > 0; --cnt, ++ikey, ++ivalue) { switch (page_type) { /* Build the key. */ case WT_PAGE_COL_FIX: @@ -540,7 +539,7 @@ build(int ikey, int ivalue, int cnt) value.size = 20; cursor->set_value(cursor, &value); } - CHECK(cursor->insert(cursor) == 0); + testutil_check(cursor->insert(cursor)); } /* @@ -550,10 +549,10 @@ build(int ikey, int ivalue, int cnt) */ new_slvg = !file_exists(SLVG); if (new_slvg) { - CHECK(session->drop(session, "file:" SLVG, "force") == 0); - CHECK(session->create(session, "file:" SLVG, config) == 0); + testutil_check(session->drop(session, SLVG_URI, "force")); + testutil_check(session->create(session, SLVG_URI, config)); } - CHECK(conn->close(conn, 0) == 0); + testutil_check(conn->close(conn, 0)); if (new_slvg) (void)remove(SLVG); } @@ -568,9 +567,11 @@ copy(u_int gen, u_int recno) FILE *ifp, *ofp; WT_BLOCK_HEADER *blk; WT_PAGE_HEADER *dsk; + uint64_t recno64; + uint32_t cksum32, gen32; char buf[PSIZE]; - CHECK((ifp = fopen(LOAD, "r")) != NULL); + testutil_assert((ifp = fopen(LOAD, "r")) != NULL); /* * If the salvage file doesn't exist, then we're creating it: @@ -578,31 +579,47 @@ copy(u_int gen, u_int recno) * Otherwise, we are appending to an existing file. */ if (file_exists(SLVG)) - CHECK((ofp = fopen(SLVG, "a")) != NULL); + testutil_assert((ofp = fopen(SLVG, "a")) != NULL); else { - CHECK((ofp = fopen(SLVG, "w")) != NULL); - CHECK(fread(buf, 1, PSIZE, ifp) == PSIZE); - CHECK(fwrite(buf, 1, PSIZE, ofp) == PSIZE); + testutil_assert((ofp = fopen(SLVG, "w")) != NULL); + testutil_assert(fread(buf, 1, PSIZE, ifp) == PSIZE); + testutil_assert(fwrite(buf, 1, PSIZE, ofp) == PSIZE); } /* * If there's data, copy/update the first formatted page. */ if (gen != 0) { - CHECK(fseek(ifp, (long)PSIZE, SEEK_SET) == 0); - CHECK(fread(buf, 1, PSIZE, ifp) == PSIZE); + testutil_assert(fseek(ifp, (long)PSIZE, SEEK_SET) == 0); + testutil_assert(fread(buf, 1, PSIZE, ifp) == PSIZE); + + /* + * Page headers are written in little-endian format, swap before + * calculating the checksum on big-endian hardware. Checksums + * always returned in little-endian format, no swap is required. + */ + gen32 = gen; + recno64 = recno; +#ifdef WORDS_BIGENDIAN + gen32 = __wt_bswap32(gen32); + recno64 = __wt_bswap64(recno64); +#endif dsk = (void *)buf; if (page_type != WT_PAGE_ROW_LEAF) - dsk->recno = recno; - dsk->write_gen = gen; + dsk->recno = recno64; + dsk->write_gen = gen32; blk = WT_BLOCK_HEADER_REF(buf); blk->checksum = 0; - blk->checksum = __wt_checksum(dsk, PSIZE); - CHECK(fwrite(buf, 1, PSIZE, ofp) == PSIZE); + cksum32 = __wt_checksum(dsk, PSIZE); +#ifdef WORDS_BIGENDIAN + cksum32 = __wt_bswap32(cksum32); +#endif + blk->checksum = cksum32; + testutil_assert(fwrite(buf, 1, PSIZE, ofp) == PSIZE); } - CHECK(fclose(ifp) == 0); - CHECK(fclose(ofp) == 0); + testutil_assert(fclose(ifp) == 0); + testutil_assert(fclose(ofp) == 0); } /* @@ -627,35 +644,35 @@ process(void) progname)); strcat(config, "log=(enabled=false),"); - CHECK(wiredtiger_open(NULL, NULL, config, &conn) == 0); - CHECK(conn->open_session(conn, NULL, NULL, &session) == 0); - CHECK(session->salvage(session, "file:" SLVG, 0) == 0); - CHECK(conn->close(conn, 0) == 0); + testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->salvage(session, SLVG_URI, 0)); + testutil_check(conn->close(conn, 0)); /* Verify. */ - CHECK(wiredtiger_open(NULL, NULL, config, &conn) == 0); - CHECK(conn->open_session(conn, NULL, NULL, &session) == 0); - CHECK(session->verify(session, "file:" SLVG, 0) == 0); - CHECK(conn->close(conn, 0) == 0); + testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->verify(session, SLVG_URI, 0)); + testutil_check(conn->close(conn, 0)); /* Dump. */ - CHECK((fp = fopen(DUMP, "w")) != NULL); - CHECK(wiredtiger_open(NULL, NULL, config, &conn) == 0); - CHECK(conn->open_session(conn, NULL, NULL, &session) == 0); - CHECK(session->open_cursor( - session, "file:" SLVG, NULL, "dump=print", &cursor) == 0); + testutil_assert((fp = fopen(DUMP, "w")) != NULL); + testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor( + session, SLVG_URI, NULL, "dump=print", &cursor)); while (cursor->next(cursor) == 0) { if (page_type == WT_PAGE_ROW_LEAF) { - CHECK(cursor->get_key(cursor, &key) == 0); - CHECK(fputs(key, fp) >= 0); - CHECK(fputc('\n', fp) >= 0); + testutil_check(cursor->get_key(cursor, &key)); + testutil_assert(fputs(key, fp) >= 0); + testutil_assert(fputc('\n', fp) >= 0); } - CHECK(cursor->get_value(cursor, &value) == 0); - CHECK(fputs(value, fp) >= 0); - CHECK(fputc('\n', fp) >= 0); + testutil_check(cursor->get_value(cursor, &value)); + testutil_assert(fputs(value, fp) >= 0); + testutil_assert(fputc('\n', fp) >= 0); } - CHECK(conn->close(conn, 0) == 0); - CHECK(fclose(fp) == 0); + testutil_check(conn->close(conn, 0)); + testutil_assert(fclose(fp) == 0); } /* @@ -669,7 +686,7 @@ empty(int cnt) if (page_type == WT_PAGE_COL_FIX) for (i = 0; i < cnt; ++i) - CHECK(fputs("\\00\n", res_fp)); + testutil_assert(fputs("\\00\n", res_fp)); } /* diff --git a/src/third_party/wiredtiger/test/suite/test_bug019.py b/src/third_party/wiredtiger/test/suite/test_bug019.py index a386bbdabeb..7f878f70874 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug019.py +++ b/src/third_party/wiredtiger/test/suite/test_bug019.py @@ -61,9 +61,9 @@ class test_bug019(wttest.WiredTigerTestCase): c.close() # Wait for a log file to be pre-allocated. Avoid timing problems, but - # assert a file is created within 30 seconds. + # assert a file is created within 90 seconds. def prepfiles(self): - for i in range(1,30): + for i in range(1,90): f = fnmatch.filter(os.listdir('.'), "*Prep*") if f: return f -- cgit v1.2.1