From e4be1b7d5e23018957c0d0301a0f62ee651c7082 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Mon, 15 Jun 2020 17:30:26 +1000 Subject: Import wiredtiger: 930bbacc3761a10483875585dbd4ecb58271d57e from branch mongodb-4.4 ref: f650b1124b..930bbacc37 for: 4.4.0-rc10 WT-6175 tcmalloc fragmentation is worse in 4.4 with durable history WT-6344 Clean-up timestamped updates to cater for globally visible full updates WT-6408 test/format bulk load can set an incorrect row count WT-6413 Remove globally visible check in __wt_checkpoint_close WT-6414 Block running rebalance with timestamp set in test format WT-6418 Account for aborted updates when doing the first scan of updates before inserting them to the history store WT-6419 Make sure we dump core on Evergreen PPC machines --- src/third_party/wiredtiger/test/evergreen.yml | 5 ++- src/third_party/wiredtiger/test/format/bulk.c | 45 +++++++++++++------------ src/third_party/wiredtiger/test/format/config.c | 6 ++++ src/third_party/wiredtiger/test/format/smoke.sh | 3 +- 4 files changed, 33 insertions(+), 26 deletions(-) (limited to 'src/third_party/wiredtiger/test') diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index b3b16040e9d..9bf45be39dd 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -171,8 +171,8 @@ functions: script: | set -o errexit set -o verbose + ${format_test_setting|} for i in $(seq ${times|1}); do - ${format_test_setting|} ${test_env_vars|} ./format.sh ${smp_command|} ${format_test_script_args|} 2>&1 done "many dbs test": @@ -2204,8 +2204,6 @@ tasks: - func: "compile wiredtiger with builtins" - func: "format test script" vars: - # Make sure we dump core on failure - format_test_setting: ulimit -c unlimited #run for 2 hours ( 2 * 60 = 120 minutes), use default config format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 120 @@ -2536,6 +2534,7 @@ buildvariants: run_on: - ubuntu1804-power8-test expansions: + format_test_setting: ulimit -c unlimited smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make test_env_vars: diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index d11d0060664..89e0406c3cb 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -80,7 +80,7 @@ wts_load(void) WT_DECL_RET; WT_ITEM key, value; WT_SESSION *session; - uint32_t keyno; + uint32_t committed_keyno, keyno, v; bool is_bulk; conn = g.wts_conn; @@ -112,19 +112,7 @@ wts_load(void) if (g.c_txn_timestamps) bulk_begin_transaction(session); - for (keyno = 0; ++keyno <= g.c_rows;) { - /* Do some checking every 10K operations. */ - if (keyno % 10000 == 0) { - /* Report on progress. */ - track("bulk load", keyno, NULL); - - /* Restart the enclosing transaction so we don't overflow the cache. */ - if (g.c_txn_timestamps) { - bulk_commit_transaction(session); - bulk_begin_transaction(session); - } - } - + for (committed_keyno = keyno = 0; ++keyno <= g.c_rows;) { key_gen(&key, keyno); val_gen(NULL, &value, keyno); @@ -176,21 +164,34 @@ wts_load(void) g.c_delete_pct += g.c_insert_pct - 5; g.c_insert_pct = 5; } - g.c_delete_pct += g.c_write_pct / 2; - g.c_write_pct = g.c_write_pct / 2; + v = g.c_write_pct / 2; + g.c_delete_pct += v; + g.c_write_pct -= v; break; } + + /* Restart the enclosing transaction every 5K operations so we don't overflow the cache. */ + if (keyno % 5000 == 0) { + /* Report on progress. */ + track("bulk load", keyno, NULL); + + if (g.c_txn_timestamps) { + bulk_commit_transaction(session); + committed_keyno = keyno; + bulk_begin_transaction(session); + } + } } /* - * We may have exited the loop early, reset our counters to match our insert count. If the count - * changed, rewrite the CONFIG file so reopens aren't surprised. + * Ideally, the insert loop runs until the number of rows plus one, in which case row counts are + * correct. If the loop exited early, reset the counters and rewrite the CONFIG file (so reopens + * aren't surprised). */ - --keyno; - if (g.rows != keyno) { - g.rows = keyno; - g.c_rows = (uint32_t)keyno; + if (keyno != g.c_rows + 1) { + g.rows = committed_keyno; + g.c_rows = (uint32_t)committed_keyno; config_print(false); } diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index e2f933526bf..1db9cc5854c 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -930,6 +930,12 @@ config_transaction(void) if (g.c_txn_freq != 100 && config_is_perm("transaction.frequency")) testutil_die(EINVAL, "timestamps require transaction frequency set to 100"); } + /* FIXME-WT-6410: temporarily disable rebalance with timestamps. */ + if (g.c_txn_timestamps && g.c_rebalance) { + if (config_is_perm("ops.rebalance")) + testutil_die(EINVAL, "rebalance cannot run with timestamps"); + config_single("ops.rebalance=off", false); + } if (g.c_isolation_flag == ISOLATION_SNAPSHOT && config_is_perm("transaction.isolation")) { if (!g.c_txn_timestamps && config_is_perm("transaction.timestamps")) testutil_die(EINVAL, "snapshot isolation requires timestamps"); diff --git a/src/third_party/wiredtiger/test/format/smoke.sh b/src/third_party/wiredtiger/test/format/smoke.sh index 067ef8e2589..82162874d9e 100755 --- a/src/third_party/wiredtiger/test/format/smoke.sh +++ b/src/third_party/wiredtiger/test/format/smoke.sh @@ -18,4 +18,5 @@ args="$args runs.threads=4 " $TEST_WRAPPER ./t $args runs.type=row # Force a rebalance to occur with statistics logging to test the utility -$TEST_WRAPPER ./t $args runs.type=row statistics.server=1 ops.rebalance=1 +# FIXME-WT-6410: temporarily disable running rebalance with timestamps +# $TEST_WRAPPER ./t $args runs.type=row statistics.server=1 ops.rebalance=1 -- cgit v1.2.1