diff options
| -rw-r--r-- | src/third_party/wiredtiger/import.data | 2 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/btree/bt_debug.c | 26 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/btree/bt_misc.c | 8 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/conn/conn_open.c | 9 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/history/hs.c | 29 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/include/extern.h | 3 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/include/session.h | 4 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/meta/meta_ckpt.c | 2 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/txn/txn.c | 45 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/txn/txn_ckpt.c | 16 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/src/txn/txn_recover.c | 10 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c | 33 | ||||
| -rwxr-xr-x | src/third_party/wiredtiger/test/evergreen.yml | 11 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/bulk.c | 2 |
14 files changed, 155 insertions, 45 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 7381ba0bcfe..e62088151eb 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "154719f3ff736872f67f2b3e9f4d8b396d2d8adf" + "commit": "cd0eca9e685e9ac2446e8e6d5b398b366a7ba420" } diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index f5a8cf0019e..99956204900 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -146,6 +146,10 @@ __debug_item_value(WT_DBG *ds, const char *tag, const void *data_arg, size_t siz if (size == 0) return (ds->f(ds, "\t%s%s{}\n", tag == NULL ? "" : tag, tag == NULL ? "" : " ")); + if (session->dump_raw) + return (ds->f(ds, "\t%s%s{%s}\n", tag == NULL ? "" : tag, tag == NULL ? "" : " ", + __wt_buf_set_printable(session, data_arg, size, ds->t1))); + /* * If the format is 'S', it's a string and our version of it may not yet be nul-terminated. */ @@ -955,8 +959,6 @@ __wt_debug_cursor_page(void *cursor_arg, const char *ofile) cbt = cursor_arg; session = CUR2S(cursor_arg); - WT_RET(__wt_debug_cursor_tree_hs(cursor_arg, "/tmp/ohs")); - WT_WITH_BTREE(session, CUR2BT(cbt), ret = __wt_debug_page(session, NULL, cbt->ref, ofile)); return (ret); } @@ -1022,13 +1024,17 @@ __debug_page(WT_DBG *ds, WT_REF *ref, uint32_t flags) session = ds->session; WT_RET(__wt_scr_alloc(session, 100, &ds->key)); - /* Set up history store support. */ + /* + * Set up history store support, opening a history store cursor on demand. Ignore errors if that + * doesn't work, we may be running in-memory. + */ if (!WT_IS_HS(S2BT(session))) { - WT_RET(__wt_scr_alloc(session, 0, &ds->hs_key)); - WT_RET(__wt_scr_alloc(session, 0, &ds->hs_value)); - if (session->hs_cursor == NULL) { - WT_RET(__wt_hs_cursor(session, &ds->session_flags, &ds->is_owner)); + if (session->hs_cursor == NULL && + __wt_hs_cursor(session, &ds->session_flags, &ds->is_owner) == 0) ds->hs_is_local = true; + if (session->hs_cursor != NULL) { + WT_RET(__wt_scr_alloc(session, 0, &ds->hs_key)); + WT_RET(__wt_scr_alloc(session, 0, &ds->hs_value)); } } @@ -1364,7 +1370,7 @@ __debug_page_row_leaf(WT_DBG *ds, WT_PAGE *page) if ((upd = WT_ROW_UPDATE(page, rip)) != NULL) WT_RET(__debug_update(ds, upd, false)); - if (!WT_IS_HS(S2BT(session))) + if (!WT_IS_HS(S2BT(session)) && session->hs_cursor != NULL) WT_RET(__debug_hs_key(ds)); if ((insert = WT_ROW_INSERT(page, rip)) != NULL) @@ -1390,7 +1396,7 @@ __debug_col_skip(WT_DBG *ds, WT_INSERT_HEAD *head, const char *tag, bool hexbyte WT_RET(ds->f(ds, "\t%s %" PRIu64 "\n", tag, WT_INSERT_RECNO(ins))); WT_RET(__debug_update(ds, ins->upd, hexbyte)); - if (!WT_IS_HS(S2BT(session))) { + if (!WT_IS_HS(S2BT(session)) && session->hs_cursor != NULL) { p = ds->key->mem; WT_RET(__wt_vpack_uint(&p, 0, WT_INSERT_RECNO(ins))); ds->key->size = WT_PTRDIFF(p, ds->key->mem); @@ -1416,7 +1422,7 @@ __debug_row_skip(WT_DBG *ds, WT_INSERT_HEAD *head) WT_RET(__debug_item_key(ds, "insert", WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins))); WT_RET(__debug_update(ds, ins->upd, false)); - if (!WT_IS_HS(S2BT(session))) { + if (!WT_IS_HS(S2BT(session)) && session->hs_cursor != NULL) { WT_RET(__wt_buf_set(session, ds->key, WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins))); WT_RET(__debug_hs_key(ds)); } diff --git a/src/third_party/wiredtiger/src/btree/bt_misc.c b/src/third_party/wiredtiger/src/btree/bt_misc.c index 072b66800f8..536ac075b46 100644 --- a/src/third_party/wiredtiger/src/btree/bt_misc.c +++ b/src/third_party/wiredtiger/src/btree/bt_misc.c @@ -88,6 +88,12 @@ __wt_key_string( WT_SESSION_IMPL *session, const void *data_arg, size_t size, const char *key_format, WT_ITEM *buf) { WT_ITEM tmp; + +#ifdef HAVE_DIAGNOSTIC + if (session->dump_raw) + return (__wt_buf_set_printable(session, data_arg, size, buf)); +#endif + /* * If the format is 'S', it's a string and our version of it may not yet be nul-terminated. */ @@ -101,7 +107,7 @@ __wt_key_string( size = sizeof(WT_ERR_STRING); } } - return __wt_buf_set_printable_format(session, data_arg, size, key_format, buf); + return (__wt_buf_set_printable_format(session, data_arg, size, key_format, buf)); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 01577317a32..68f370fd829 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -218,8 +218,13 @@ __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_meta_track_init(session)); /* - * Create the history store file. This will only actually create it on upgrade or when creating - * a new database. + * Drop the lookaside file if it still exists. + */ + WT_RET(__wt_hs_cleanup_las(session)); + + /* + * Create the history store file. This will only actually create it on a clean upgrade or when + * creating a new database. */ WT_RET(__wt_hs_create(session, cfg)); diff --git a/src/third_party/wiredtiger/src/history/hs.c b/src/third_party/wiredtiger/src/history/hs.c index 06ac36fc2ac..b295ac730b4 100644 --- a/src/third_party/wiredtiger/src/history/hs.c +++ b/src/third_party/wiredtiger/src/history/hs.c @@ -144,11 +144,11 @@ err: } /* - * __wt_hs_create -- - * Initialize the database's history store. + * __wt_hs_cleanup_las -- + * Drop the lookaside file if it exists. */ int -__wt_hs_create(WT_SESSION_IMPL *session, const char **cfg) +__wt_hs_cleanup_las(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -156,16 +156,33 @@ __wt_hs_create(WT_SESSION_IMPL *session, const char **cfg) conn = S2C(session); - /* Read-only and in-memory configurations don't need the history store table. */ + /* Read-only and in-memory configurations won't drop the lookaside. */ if (F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_READONLY)) return (0); /* The LAS table may exist on upgrade. Discard it. */ WT_WITH_SCHEMA_LOCK( session, ret = __wt_schema_drop(session, "file:WiredTigerLAS.wt", drop_cfg)); - WT_RET(ret); - /* Re-create the table. */ + return (ret); +} + +/* + * __wt_hs_create -- + * Initialize the database's history store. + */ +int +__wt_hs_create(WT_SESSION_IMPL *session, const char **cfg) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + /* Read-only and in-memory configurations don't need the history store table. */ + if (F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_READONLY)) + return (0); + + /* Create the table. */ WT_RET(__wt_session_create(session, WT_HS_URI, WT_HS_CONFIG)); WT_RET(__wt_hs_config(session, cfg)); diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index a8b80658f05..9ada3107728 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -750,6 +750,8 @@ extern int __wt_history_store_verify(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_history_store_verify_one(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_hs_cleanup_las(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_config(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_create(WT_SESSION_IMPL *session, const char **cfg) @@ -1754,6 +1756,7 @@ extern void __wt_thread_group_start_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool is_locked); extern void __wt_thread_group_stop_one(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group); extern void __wt_timestamp_to_hex_string(wt_timestamp_t ts, char *hex_timestamp); +extern void __wt_txn_bump_snapshot(WT_SESSION_IMPL *session); extern void __wt_txn_clear_durable_timestamp(WT_SESSION_IMPL *session); extern void __wt_txn_clear_read_timestamp(WT_SESSION_IMPL *session); extern void __wt_txn_clear_timestamp_queues(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 84d7c14c914..5da9919e15a 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -160,6 +160,10 @@ struct __wt_session_impl { /* Sessions have an associated statistics bucket based on its ID. */ u_int stat_bucket; /* Statistics bucket offset */ +#ifdef HAVE_DIAGNOSTIC + uint8_t dump_raw; /* Configure debugging page dump */ +#endif + /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_SESSION_BACKUP_CURSOR 0x00000001u #define WT_SESSION_BACKUP_DUP 0x00000002u diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index d435e37d80d..97474d992ec 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -342,7 +342,7 @@ __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT * __wt_encrypt_size(session, kencryptor, a->size, &encrypt_size); WT_ERR(__wt_buf_grow(session, b, encrypt_size)); WT_ERR(__wt_encrypt(session, kencryptor, 0, a, b)); - WT_ERR(__wt_buf_grow(session, a, b->size * 2)); + WT_ERR(__wt_buf_grow(session, a, b->size * 2 + 1)); __wt_fill_hex(b->mem, b->size, a->mem, a->memsize, &a->size); metadata = a->data; diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index fe92141aa98..11aac26e2b5 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -141,11 +141,11 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) } /* - * __wt_txn_get_snapshot -- - * Allocate a snapshot. + * __txn_get_snapshot_int -- + * Allocate a snapshot, optionally update our shared txn ids. */ -void -__wt_txn_get_snapshot(WT_SESSION_IMPL *session) +static void +__txn_get_snapshot_int(WT_SESSION_IMPL *session, bool publish) { WT_CONNECTION_IMPL *conn; WT_TXN *txn; @@ -177,16 +177,21 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) /* * Include the checkpoint transaction, if one is running: we should ignore any uncommitted * changes the checkpoint has written to the metadata. We don't have to keep the checkpoint's - * changes pinned so don't including it in the published pinned ID. + * changes pinned so don't go including it in the published pinned ID. + * + * We can assume that if a function calls without intention to publish then it is the special + * case of checkpoint calling it twice. In which case do not include the checkpoint id. */ if ((id = txn_global->checkpoint_txn_shared.id) != WT_TXN_NONE) { - txn->snapshot[n++] = id; - txn_shared->metadata_pinned = id; + if (txn->id != id) + txn->snapshot[n++] = id; + if (publish) + txn_shared->metadata_pinned = id; } /* For pure read-only workloads, avoid scanning. */ if (prev_oldest_id == current_id) { - txn_shared->pinned_id = current_id; + pinned_id = current_id; /* Check that the oldest ID has not moved in the meantime. */ WT_ASSERT(session, prev_oldest_id == txn_global->oldest_id); goto done; @@ -240,14 +245,34 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) */ WT_ASSERT(session, WT_TXNID_LE(prev_oldest_id, pinned_id)); WT_ASSERT(session, prev_oldest_id == txn_global->oldest_id); - txn_shared->pinned_id = pinned_id; - done: + if (publish) + txn_shared->pinned_id = pinned_id; __wt_readunlock(session, &txn_global->rwlock); __txn_sort_snapshot(session, n, current_id); } /* + * __wt_txn_get_snapshot -- + * Common case, allocate a snapshot and update our shared ids. + */ +void +__wt_txn_get_snapshot(WT_SESSION_IMPL *session) +{ + __txn_get_snapshot_int(session, true); +} + +/* + * __wt_txn_bump_snapshot -- + * Uncommon case, allocate a snapshot but skip updating our shared ids. + */ +void +__wt_txn_bump_snapshot(WT_SESSION_IMPL *session) +{ + __txn_get_snapshot_int(session, false); +} + +/* * __txn_oldest_scan -- * Sweep the running transactions to calculate the oldest ID required. */ diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 78c04c792cd..f8914a33dac 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -530,6 +530,7 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, bool *trackingp, const char *cfg[ WT_TXN *txn; WT_TXN_GLOBAL *txn_global; WT_TXN_SHARED *txn_shared; + uint64_t original_snap_min; const char *txn_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_begin_transaction), "isolation=snapshot", NULL}; bool use_timestamp; @@ -550,7 +551,9 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, bool *trackingp, const char *cfg[ */ WT_STAT_CONN_SET(session, txn_checkpoint_prep_running, 1); __wt_epoch(session, &conn->ckpt_prep_start); + WT_RET(__wt_txn_begin(session, txn_cfg)); + original_snap_min = session->txn->snap_min; WT_DIAGNOSTIC_YIELD; @@ -628,6 +631,19 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, bool *trackingp, const char *cfg[ __wt_writeunlock(session, &txn_global->rwlock); + /* + * Refresh our snapshot here without publishing our shared ids to the world, doing so prevents + * us from racing with the stable timestamp moving ahead of current snapshot. i.e. if the stable + * timestamp moves after we begin the checkpoint transaction but before we set the checkpoint + * timestamp we can end up missing updates in our checkpoint. + */ + __wt_txn_bump_snapshot(session); + + /* Assert that our snapshot min didn't somehow move backwards. */ + WT_ASSERT(session, session->txn->snap_min >= original_snap_min); + /* Flag as unused for non diagnostic builds. */ + WT_UNUSED(original_snap_min); + if (use_timestamp) __wt_verbose_timestamp( session, txn_global->checkpoint_timestamp, "Checkpoint requested at stable timestamp"); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 0779bad0839..2a6d7d12e38 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -732,6 +732,16 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[]) goto done; } + if (!hs_exists) { + __wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS, "%s", + "Creating the history store before applying log records. Likely recovering after an" + "unclean shutdown on an earlier version"); + /* + * Create the history store as we might need it while applying log records in recovery. + */ + WT_ERR(__wt_hs_create(session, cfg)); + } + /* * Recovery can touch more data than fits in cache, so it relies on regular eviction to manage * paging. Start eviction threads for recovery without history store cursors. diff --git a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c index aaa1b49f961..4e1393b0449 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c @@ -351,7 +351,15 @@ wt_open_corrupt(const char *sfx) testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%s", home, sfx)); else testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home)); - ret = wiredtiger_open(buf, &event_handler, NULL, &conn); + + /* + * Opening the database may cause a panic and core dump. Change dir to database directory so the + * core will be left someplace we will clean up. + */ + if (chdir(buf) != 0) + testutil_die(errno, "Child chdir: %s", home); + + ret = wiredtiger_open(NULL, &event_handler, NULL, &conn); /* * Not all out of sync combinations lead to corruption. We keep the previous checkpoint in the * file so some combinations of future or old turtle files and metadata files will succeed. @@ -496,7 +504,7 @@ main(int argc, char *argv[]) printf("corrupt metadata\n"); corrupt_file(WT_METAFILE, CORRUPT); testutil_check(__wt_snprintf( - buf, sizeof(buf), "cp -p %s/WiredTiger.wt ./%s.SAVE/WiredTiger.wt.CORRUPT", home, home)); + buf, sizeof(buf), "cp -p %s/WiredTiger.wt ./%s.%s/WiredTiger.wt.CORRUPT", home, home, SAVE)); printf("copy: %s\n", buf); if ((ret = system(buf)) < 0) testutil_die(ret, "system: %s", buf); @@ -508,26 +516,23 @@ main(int argc, char *argv[]) printf("corrupt turtle\n"); corrupt_file(WT_METADATA_TURTLE, WT_METAFILE_URI); testutil_check(__wt_snprintf(buf, sizeof(buf), - "cp -p %s/WiredTiger.turtle ./%s.SAVE/WiredTiger.turtle.CORRUPT", home, home)); + "cp -p %s/WiredTiger.turtle ./%s.%s/WiredTiger.turtle.CORRUPT", home, home, SAVE)); printf("copy: %s\n", buf); if ((ret = system(buf)) < 0) testutil_die(ret, "system: %s", buf); run_all_verification(NULL, &table_data[0]); - /* - * We need to set up the string before we clean up the structure. Then after the clean up we - * will run this command. - */ - testutil_check(__wt_snprintf(buf, sizeof(buf), "rm -rf core* %s*", home)); - testutil_cleanup(opts); - - /* - * We've created a lot of extra directories and possibly some core files from child process - * aborts. Manually clean them up. - */ + /* Remove saved copy of the original database directory. */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "rm -rf %s.%s", home, SAVE)); printf("cleanup and remove: %s\n", buf); if ((ret = system(buf)) < 0) testutil_die(ret, "system: %s", buf); + /* + * Cleanup from test. This will delete the database directory along with the core files left + * there by our children. + */ + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index f0354b089c1..b3b16040e9d 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -2091,6 +2091,17 @@ tasks: ./time_shift_test.sh /usr/local/lib/faketime/libfaketimeMT.so.1 0-1 2>&1 + - name: format-stress-pull-request-test + tags: ["pull_request"] + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "format test script" + vars: + smp_command: -j $(grep -c ^processor /proc/cpuinfo) + # run for 10 minutes. + format_test_script_args: -t 10 rows=10000 ops=50000 + - name: format-wtperf-test commands: - func: "get project" diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index 02e4cd404bd..d11d0060664 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -178,6 +178,8 @@ wts_load(void) } g.c_delete_pct += g.c_write_pct / 2; g.c_write_pct = g.c_write_pct / 2; + + break; } } |