diff options
80 files changed, 1120 insertions, 553 deletions
diff --git a/src/third_party/wiredtiger/NEWS b/src/third_party/wiredtiger/NEWS index 7bf3b0e7edb..1e821835386 100644 --- a/src/third_party/wiredtiger/NEWS +++ b/src/third_party/wiredtiger/NEWS @@ -1,6 +1,14 @@ Ticket reference tags refer to tickets in the MongoDB JIRA tracking system: https://jira.mongodb.org +WiredTiger release 3.1.0, 2018-07-12 +------------------------------------ + +See the upgrading documentation for details of API and behavior changes. + +See JIRA changelog for a full listing: +https://jira.mongodb.org/projects/WT/versions/19708 + WiredTiger release 3.0.0, 2018-01-08 ------------------------------------ diff --git a/src/third_party/wiredtiger/README b/src/third_party/wiredtiger/README index f21ff213a7c..fd8757621bf 100644 --- a/src/third_party/wiredtiger/README +++ b/src/third_party/wiredtiger/README @@ -1,6 +1,6 @@ -WiredTiger 3.1.0: (April 23, 2018) +WiredTiger 3.1.1: (July 12, 2018) -This is version 3.1.0 of WiredTiger. +This is version 3.1.1 of WiredTiger. WiredTiger release packages and documentation can be found at: @@ -8,7 +8,7 @@ WiredTiger release packages and documentation can be found at: The documentation for this specific release can be found at: - http://source.wiredtiger.com/3.1.0/index.html + http://source.wiredtiger.com/3.1.1/index.html The WiredTiger source code can be found at: diff --git a/src/third_party/wiredtiger/RELEASE_INFO b/src/third_party/wiredtiger/RELEASE_INFO index ee25ecd6c56..2014ba3ee74 100644 --- a/src/third_party/wiredtiger/RELEASE_INFO +++ b/src/third_party/wiredtiger/RELEASE_INFO @@ -1,6 +1,6 @@ WIREDTIGER_VERSION_MAJOR=3 WIREDTIGER_VERSION_MINOR=1 -WIREDTIGER_VERSION_PATCH=0 +WIREDTIGER_VERSION_PATCH=1 WIREDTIGER_VERSION="$WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH" WIREDTIGER_RELEASE_DATE=`date "+%B %e, %Y"` diff --git a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 index 7d0df5d65ac..9d07958bad9 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 @@ -53,19 +53,6 @@ AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_ZSTD], [test "$wt_cv_with_builtin_extension_zstd" = "yes"]) AC_MSG_RESULT($with_builtins) -AH_TEMPLATE( - HAVE_CRC32_HARDWARE, [Define to 1 to configure CRC32 hardware support.]) -AC_MSG_CHECKING(if --enable-crc32-hardware option specified) -AC_ARG_ENABLE(crc32-hardware, - AS_HELP_STRING([--enable-crc32-hardware], - [Enable CRC32 hardware support.]), r=$enableval, r=yes) -case "$r" in -no) wt_cv_enable_crc32_hardware=no;; -*) AC_DEFINE(HAVE_CRC32_HARDWARE) - wt_cv_enable_crc32_hardware=yes;; -esac -AC_MSG_RESULT($wt_cv_enable_crc32_hardware) - AH_TEMPLATE(HAVE_DIAGNOSTIC, [Define to 1 for diagnostic tests.]) AC_MSG_CHECKING(if --enable-diagnostic option specified) AC_ARG_ENABLE(diagnostic, diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 index 3f87f7f6507..8b39a5d09d6 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 @@ -2,8 +2,8 @@ dnl build by dist/s_version VERSION_MAJOR=3 VERSION_MINOR=1 -VERSION_PATCH=0 -VERSION_STRING='"WiredTiger 3.1.0: (April 23, 2018)"' +VERSION_PATCH=1 +VERSION_STRING='"WiredTiger 3.1.1: (July 12, 2018)"' AC_SUBST(VERSION_MAJOR) AC_SUBST(VERSION_MINOR) diff --git a/src/third_party/wiredtiger/build_win/wiredtiger.def b/src/third_party/wiredtiger/build_win/wiredtiger.def index 3ee9f6b6a9d..79fa84a11e0 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger.def +++ b/src/third_party/wiredtiger/build_win/wiredtiger.def @@ -1,8 +1,8 @@ LIBRARY WIREDTIGER EXPORTS - wiredtiger_checksum_crc32c wiredtiger_config_parser_open wiredtiger_config_validate + wiredtiger_crc32c_func wiredtiger_open wiredtiger_pack_close wiredtiger_pack_int diff --git a/src/third_party/wiredtiger/build_win/wiredtiger_config.h b/src/third_party/wiredtiger/build_win/wiredtiger_config.h index 55431f59fae..bb4cc7848f8 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger_config.h +++ b/src/third_party/wiredtiger/build_win/wiredtiger_config.h @@ -25,9 +25,6 @@ /* Define to 1 if you have the `clock_gettime' function. */ /* #undef HAVE_CLOCK_GETTIME */ -/* Define to 1 to enable CRC32 hardware support. */ -/* #undef HAVE_CRC32_HARDWARE */ - /* Define to 1 for diagnostic tests. */ /* #undef HAVE_DIAGNOSTIC */ diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index d29e9655fb3..e96a147fc70 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -590,8 +590,7 @@ connection_runtime_config = [ type='list', undoc=True, choices=[ 'checkpoint_slow', 'lookaside_sweep_race', 'split_1', 'split_2', - 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8', - 'split_9']), + 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8']), Config('verbose', '', r''' enable messages for various events. Options are given as a list, such as <code>"verbose=[evictserver,read]"</code>''', @@ -601,6 +600,7 @@ connection_runtime_config = [ 'checkpoint', 'checkpoint_progress', 'compact', + 'error_returns', 'evict', 'evict_stuck', 'evictserver', @@ -709,7 +709,7 @@ wiredtiger_open_log_configuration = [ Config('recover', 'on', r''' run recovery or error if recovery needs to run after an unclean shutdown''', - choices=['error','on']) + choices=['error', 'on', 'salvage']) ]), ] diff --git a/src/third_party/wiredtiger/dist/s_export.list b/src/third_party/wiredtiger/dist/s_export.list index 72ce553ac9b..e49fa113d96 100644 --- a/src/third_party/wiredtiger/dist/s_export.list +++ b/src/third_party/wiredtiger/dist/s_export.list @@ -1,7 +1,7 @@ # List of OK external symbols. -wiredtiger_checksum_crc32c wiredtiger_config_parser_open wiredtiger_config_validate +wiredtiger_crc32c_func wiredtiger_open wiredtiger_pack_close wiredtiger_pack_int diff --git a/src/third_party/wiredtiger/dist/s_funcs.list b/src/third_party/wiredtiger/dist/s_funcs.list index eed29e91fc1..95c568a19ff 100644 --- a/src/third_party/wiredtiger/dist/s_funcs.list +++ b/src/third_party/wiredtiger/dist/s_funcs.list @@ -33,7 +33,6 @@ __wt_stat_join_aggregate __wt_stat_join_clear_all __wt_stream_set_no_buffer __wt_try_readlock -wiredtiger_checksum_crc32c wiredtiger_config_parser_open wiredtiger_config_validate wiredtiger_pack_int diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 64b9758877e..215f1c36e8b 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -207,8 +207,8 @@ LLLLLLL LOGREC LOGSCAN LOOKASIDE +LRSVv LRU -LRVv LSB LSM LSN diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index 9c5f6711da0..f97b910620f 100755 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -55,6 +55,8 @@ func_ok() -e '/int __wt_block_compact_start$/d' \ -e '/int __wt_block_manager_size$/d' \ -e '/int __wt_block_write_size$/d' \ + -e '/int __wt_buf_catfmt$/d' \ + -e '/int __wt_buf_fmt$/d' \ -e '/int __wt_curjoin_joined$/d' \ -e '/int __wt_cursor_close$/d' \ -e '/int __wt_cursor_noop$/d' \ @@ -65,6 +67,7 @@ func_ok() -e '/int __wt_once$/d' \ -e '/int __wt_posix_directory_list_free$/d' \ -e '/int __wt_session_breakpoint$/d' \ + -e '/int __wt_set_return_func$/d' \ -e '/int __wt_spin_init$/d' \ -e '/int __wt_spin_trylock$/d' \ -e '/int __wt_stat_connection_desc$/d' \ diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index 139f39fe673..190c2c421d3 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -1356,8 +1356,9 @@ main(int argc, char *argv[]) const char *buffer = "some string"; size_t len = strlen(buffer); /*! [Checksum a buffer] */ - uint32_t crc32c; - crc32c = wiredtiger_checksum_crc32c(buffer, len); + uint32_t crc32c, (*func)(const void *, size_t); + func = wiredtiger_crc32c_func(); + crc32c = func(buffer, len); /*! [Checksum a buffer] */ (void)crc32c; } diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index ea08c6eb2d1..3392241798d 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "333460997573558235254cb7e84dad765bb844a7", + "commit": "1be1b793becdfb5f4b2ae950449aa3710ca320ce", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c index db755db198a..cb3cd8986f3 100644 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ b/src/third_party/wiredtiger/src/async/async_api.c @@ -145,7 +145,7 @@ retry: */ if (op == NULL || op->state != WT_ASYNCOP_FREE) { WT_STAT_CONN_INCR(session, async_full); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* * Set the state of this op handle as READY for the user to use. diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 16e25c1fe25..d18e3887ca5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -53,7 +53,7 @@ static int __debug_ref(WT_DBG *, WT_REF *); static int __debug_row_skip(WT_DBG *, WT_INSERT_HEAD *); static int __debug_tree(WT_SESSION_IMPL *, WT_REF *, const char *, uint32_t); static int __debug_update(WT_DBG *, WT_UPDATE *, bool); -static int __dmsg_wrapup(WT_DBG *); +static int __debug_wrapup(WT_DBG *); /* * __wt_debug_set_verbose -- @@ -270,7 +270,7 @@ __debug_config(WT_SESSION_IMPL *session, WT_DBG *ds, const char *ofile) ds->f = __dmsg_event; } else { if ((ds->fp = fopen(ofile, "w")) == NULL) - return (EIO); + return (__wt_set_return(session, EIO)); __wt_stream_set_line_buffer(ds->fp); ds->f = __dmsg_file; } @@ -282,12 +282,13 @@ __debug_config(WT_SESSION_IMPL *session, WT_DBG *ds, const char *ofile) } /* - * __dmsg_wrapup -- + * __debug_wrapup -- * Flush any remaining output, release resources. */ static int -__dmsg_wrapup(WT_DBG *ds) +__debug_wrapup(WT_DBG *ds) { + WT_DECL_RET; WT_ITEM *msg; WT_SESSION_IMPL *session; @@ -303,7 +304,7 @@ __dmsg_wrapup(WT_DBG *ds) */ if (msg != NULL) { if (msg->size != 0) - WT_RET(__wt_msg(session, "%s", (char *)msg->mem)); + ret = __wt_msg(session, "%s", (char *)msg->mem); __wt_scr_free(session, &ds->msg); } @@ -311,7 +312,7 @@ __dmsg_wrapup(WT_DBG *ds) if (ds->fp != NULL) (void)fclose(ds->fp); - return (0); + return (ret); } /* @@ -435,59 +436,61 @@ __wt_debug_disk( WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, const char *ofile) { WT_DBG *ds, _ds; + WT_DECL_RET; ds = &_ds; WT_RET(__debug_config(session, ds, ofile)); - WT_RET(ds->f(ds, "%s page", __wt_page_type_string(dsk->type))); + WT_ERR(ds->f(ds, "%s page", __wt_page_type_string(dsk->type))); switch (dsk->type) { case WT_PAGE_BLOCK_MANAGER: break; case WT_PAGE_COL_FIX: case WT_PAGE_COL_INT: case WT_PAGE_COL_VAR: - WT_RET(ds->f(ds, ", recno %" PRIu64, dsk->recno)); + WT_ERR(ds->f(ds, ", recno %" PRIu64, dsk->recno)); /* FALLTHROUGH */ case WT_PAGE_ROW_INT: case WT_PAGE_ROW_LEAF: - WT_RET(ds->f(ds, ", entries %" PRIu32, dsk->u.entries)); + WT_ERR(ds->f(ds, ", entries %" PRIu32, dsk->u.entries)); break; case WT_PAGE_OVFL: - WT_RET(ds->f(ds, ", datalen %" PRIu32, dsk->u.datalen)); + WT_ERR(ds->f(ds, ", datalen %" PRIu32, dsk->u.datalen)); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE_ERR(session); } if (F_ISSET(dsk, WT_PAGE_COMPRESSED)) - WT_RET(ds->f(ds, ", compressed")); + WT_ERR(ds->f(ds, ", compressed")); if (F_ISSET(dsk, WT_PAGE_ENCRYPTED)) - WT_RET(ds->f(ds, ", encrypted")); + WT_ERR(ds->f(ds, ", encrypted")); if (F_ISSET(dsk, WT_PAGE_EMPTY_V_ALL)) - WT_RET(ds->f(ds, ", empty-all")); + WT_ERR(ds->f(ds, ", empty-all")); if (F_ISSET(dsk, WT_PAGE_EMPTY_V_NONE)) - WT_RET(ds->f(ds, ", empty-none")); + WT_ERR(ds->f(ds, ", empty-none")); if (F_ISSET(dsk, WT_PAGE_LAS_UPDATE)) - WT_RET(ds->f(ds, ", LAS-update")); + WT_ERR(ds->f(ds, ", LAS-update")); - WT_RET(ds->f(ds, ", generation %" PRIu64 "\n", dsk->write_gen)); + WT_ERR(ds->f(ds, ", generation %" PRIu64 "\n", dsk->write_gen)); switch (dsk->type) { case WT_PAGE_BLOCK_MANAGER: break; case WT_PAGE_COL_FIX: - WT_RET(__debug_dsk_col_fix(ds, dsk)); + WT_ERR(__debug_dsk_col_fix(ds, dsk)); break; case WT_PAGE_COL_INT: case WT_PAGE_COL_VAR: case WT_PAGE_ROW_INT: case WT_PAGE_ROW_LEAF: - WT_RET(__debug_dsk_cell(ds, dsk)); + WT_ERR(__debug_dsk_cell(ds, dsk)); break; default: break; } - return (__dmsg_wrapup(ds)); +err: WT_TRET(__debug_wrapup(ds)); + return (ret); } /* @@ -620,9 +623,9 @@ __wt_debug_tree_shape( WT_WITH_PAGE_INDEX(session, ret = __debug_tree_shape_worker(ds, page, 1)); - WT_RET(ret); - return (__dmsg_wrapup(ds)); + WT_TRET(__debug_wrapup(ds)); + return (ret); } /* AUTOMATIC FLAG VALUE GENERATION START */ @@ -705,7 +708,7 @@ __wt_debug_page( WT_WITH_BTREE(session, btree, ret = __debug_page(ds, ref, WT_DEBUG_TREE_LEAF)); - WT_TRET(__dmsg_wrapup(ds)); + WT_TRET(__debug_wrapup(ds)); return (ret); } @@ -744,7 +747,7 @@ __debug_tree( ret = __debug_page(ds, ref, flags); - WT_TRET(__dmsg_wrapup(ds)); + WT_TRET(__debug_wrapup(ds)); return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c index d31f76f629c..0d49adc19ca 100644 --- a/src/third_party/wiredtiger/src/btree/bt_discard.c +++ b/src/third_party/wiredtiger/src/btree/bt_discard.c @@ -32,29 +32,14 @@ __wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref) */ WT_ASSERT(session, S2BT(session)->evict_ref != ref); -#ifdef HAVE_DIAGNOSTIC - { - WT_HAZARD *hp; - int i; /* * Make sure no other thread has a hazard pointer on the page we are * about to discard. This is complicated by the fact that readers * publish their hazard pointer before re-checking the page state, so * our check can race with readers without indicating a real problem. - * Wait for up to a second for hazard pointers to be cleared. + * If we find a hazard pointer, wait for it to be cleared. */ - for (hp = NULL, i = 0; i < 100; i++) { - if ((hp = __wt_hazard_check(session, ref)) == NULL) - break; - __wt_sleep(0, 10000); - } - if (hp != NULL) - __wt_errx(session, - "discarded page has hazard pointer: (%p: %s, line %d)", - (void *)hp->ref, hp->file, hp->line); - WT_ASSERT(session, hp == NULL); - } -#endif + WT_ASSERT(session, __wt_hazard_check_assert(session, ref, true)); __wt_page_out(session, &ref->page); } @@ -263,6 +248,9 @@ __wt_free_ref( if (ref == NULL) return; + /* Assert there are no hazard pointers. */ + WT_ASSERT(session, __wt_hazard_check_assert(session, ref, false)); + /* * Optionally free the referenced pages. (The path to free referenced * page is used for error cleanup, no instantiated and then discarded diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 9160ff1dd21..613a95f321b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -915,10 +915,11 @@ __wt_btree_immediately_durable(WT_SESSION_IMPL *session) /* * This is used to determine whether timestamp updates should - * be rolled back for this btree. It's likely that the particular - * test required here will change when rollback to stable is - * supported with in-memory configurations. + * be rolled back for this btree. With in-memory, the logging + * setting on tables is still important and when enabled they + * should be considered "durable". */ - return (FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ENABLED) && + return ((FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ENABLED) || + (F_ISSET(S2C(session), WT_CONN_IN_MEMORY))) && !F_ISSET(btree, WT_BTREE_NO_LOGGING)); } diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 54f4eaa8f52..411c7ed7cfe 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -2350,7 +2350,7 @@ __slvg_ovfl_ref(WT_SESSION_IMPL *session, WT_TRACK *trk, bool multi_panic) { if (F_ISSET(trk, WT_TRACK_OVFL_REFD)) { if (!multi_panic) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); WT_PANIC_RET(session, EINVAL, "overflow record unexpectedly referenced multiple times " "during leaf page merge"); diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index a98de6c6c9f..2a69ef84540 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -449,7 +449,7 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) children = pindex->entries / btree->split_deepen_per_child; if (children < 10) { if (pindex->entries < 100) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); children = 10; } chunk = pindex->entries / children; @@ -872,6 +872,8 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, /* Free the backing block and address. */ WT_TRET(__wt_ref_block_free(session, next_ref)); + WT_ASSERT(session, + __wt_hazard_check_assert(session, next_ref, false)); WT_TRET(__split_safe_free( session, split_gen, exclusive, next_ref, sizeof(WT_REF))); parent_decr += sizeof(WT_REF); @@ -915,7 +917,7 @@ err: __wt_scr_free(session, &scr); * being deleted, but don't be noisy, there's nothing wrong. */ if (empty_parent) - ret = EBUSY; + ret = __wt_set_return(session, EBUSY); break; case WT_ERR_PANIC: __wt_err(session, ret, "fatal error during parent page split"); @@ -982,7 +984,7 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) children = pindex->entries / btree->split_deepen_per_child; if (children < 10) { if (pindex->entries < 100) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); children = 10; } chunk = pindex->entries / children; @@ -1214,7 +1216,7 @@ __split_internal_lock( * the parent, give up to avoid that deadlock. */ if (!trylock && !__wt_btree_can_evict_dirty(session)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * Get a page-level lock on the parent to single-thread splits into the diff --git a/src/third_party/wiredtiger/src/btree/col_srch.c b/src/third_party/wiredtiger/src/btree/col_srch.c index 123b640cdf4..4b7b3d3d727 100644 --- a/src/third_party/wiredtiger/src/btree/col_srch.c +++ b/src/third_party/wiredtiger/src/btree/col_srch.c @@ -180,7 +180,7 @@ descend: /* } /* Encourage races. */ - __wt_timing_stress(session, WT_TIMING_STRESS_SPLIT_9); + WT_DIAGNOSTIC_YIELD; /* * Swap the current page for the child page. If the page splits diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index a3f05a2700f..38921471d74 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -432,7 +432,7 @@ append: if (__wt_split_descent_race( } descend: /* Encourage races. */ - __wt_timing_stress(session, WT_TIMING_STRESS_SPLIT_9); + WT_DIAGNOSTIC_YIELD; /* * Swap the current page for the child page. If the page splits diff --git a/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c b/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c index 01740dcd953..f09c15be450 100644 --- a/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c +++ b/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c @@ -26,9 +26,10 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "wt_internal.h" +#include <inttypes.h> +#include <stddef.h> -#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) +#if defined(__linux__) #include <asm/hwcap.h> #include <sys/auxv.h> @@ -84,23 +85,28 @@ __wt_checksum_hw(const void *chunk, size_t len) } #endif +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +#if defined(__GNUC__) +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) + __attribute__((visibility("default"))); +#else +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t); +#endif + /* - * __wt_checksum_init -- - * WiredTiger: detect CRC hardware and set the checksum function. + * wiredtiger_crc32c_func -- + * WiredTiger: detect CRC hardware and return the checksum function. */ -void -__wt_checksum_init(void) - WT_GCC_FUNC_ATTRIBUTE((cold)) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) { -#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) +#if defined(__linux__) unsigned long caps = getauxval(AT_HWCAP); if (caps & HWCAP_CRC32) - __wt_process.checksum = __wt_checksum_hw; - else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_hw); + return (__wt_checksum_sw); #else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_sw); #endif } diff --git a/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c b/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c index 8626fa42136..cb427ea7a67 100644 --- a/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c +++ b/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c @@ -1,5 +1,6 @@ #if defined(__powerpc64__) -#include "wt_internal.h" +#include <inttypes.h> +#include <stddef.h> #define CRC_TABLE #include "crc32_constants.h" @@ -80,17 +81,23 @@ __wt_checksum_hw(const void *chunk, size_t len) return (crc32_vpmsum(0, chunk, len)); } +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +#if defined(__GNUC__) +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) + __attribute__((visibility("default"))); +#else +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t); +#endif + /* - * __wt_checksum_init -- - * WiredTiger: detect CRC hardware and set the checksum function. + * wiredtiger_crc32c_func -- + * WiredTiger: detect CRC hardware and return the checksum function. */ -void -__wt_checksum_init(void) - WT_GCC_FUNC_ATTRIBUTE((cold)) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) { -#if defined(HAVE_CRC32_HARDWARE) - __wt_process.checksum = __wt_checksum_hw; +#if defined(__powerpc64__) + return (__wt_checksum_hw); #else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_sw); #endif } diff --git a/src/third_party/wiredtiger/src/checksum/software/checksum.c b/src/third_party/wiredtiger/src/checksum/software/checksum.c index 1228c9a0ce1..4d93f8bf1ea 100644 --- a/src/third_party/wiredtiger/src/checksum/software/checksum.c +++ b/src/third_party/wiredtiger/src/checksum/software/checksum.c @@ -38,7 +38,8 @@ * little endian. */ -#include "wt_internal.h" +#include <inttypes.h> +#include <stddef.h> /* * The CRC slicing tables. @@ -1095,13 +1096,14 @@ static const uint32_t g_crc_slicing[8][256] = { #endif }; +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); + /* * __wt_checksum_sw -- * Return a checksum for a chunk of memory, computed in software. */ uint32_t __wt_checksum_sw(const void *chunk, size_t len) - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { uint32_t crc, next; size_t nqwords; diff --git a/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c b/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c index 73199018a7d..2ff237f883e 100644 --- a/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c +++ b/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c @@ -26,9 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "wt_internal.h" +#include <inttypes.h> +#include <stddef.h> -#if defined(HAVE_CRC32_HARDWARE) #if (defined(__amd64) || defined(__x86_64)) /* * __wt_checksum_hw -- @@ -116,17 +116,21 @@ __wt_checksum_hw(const void *chunk, size_t len) return (~crc); } #endif -#endif /* HAVE_CRC32_HARDWARE */ + +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +#if defined(__GNUC__) +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) + __attribute__((visibility("default"))); +#else +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t); +#endif /* - * __wt_checksum_init -- - * WiredTiger: detect CRC hardware and set the checksum function. + * wiredtiger_crc32c_func -- + * WiredTiger: detect CRC hardware and return the checksum function. */ -void -__wt_checksum_init(void) - WT_GCC_FUNC_ATTRIBUTE((cold)) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) { -#if defined(HAVE_CRC32_HARDWARE) #if (defined(__amd64) || defined(__x86_64)) unsigned int eax, ebx, ecx, edx; @@ -137,9 +141,8 @@ __wt_checksum_init(void) #define CPUID_ECX_HAS_SSE42 (1 << 20) if (ecx & CPUID_ECX_HAS_SSE42) - __wt_process.checksum = __wt_checksum_hw; - else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_hw); + return (__wt_checksum_sw); #elif defined(_M_AMD64) int cpuInfo[4]; @@ -148,14 +151,10 @@ __wt_checksum_init(void) #define CPUID_ECX_HAS_SSE42 (1 << 20) if (cpuInfo[2] & CPUID_ECX_HAS_SSE42) - __wt_process.checksum = __wt_checksum_hw; - else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_hw); + return (__wt_checksum_sw); + #else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_sw); #endif - -#else /* !HAVE_CRC32_HARDWARE */ - __wt_process.checksum = __wt_checksum_sw; -#endif /* HAVE_CRC32_HARDWARE */ } diff --git a/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c b/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c index ec7adb02cba..72117d7509d 100644 --- a/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c +++ b/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c @@ -6,12 +6,13 @@ * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> * */ -#include "wt_internal.h" #include <sys/types.h> #include <endian.h> +#include <inttypes.h> +#include <stddef.h> -#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) +#if defined(__linux__) #include <sys/auxv.h> /* RHEL 7 has kernel support, but does not define this constant in the lib c headers. */ @@ -92,23 +93,29 @@ __wt_checksum_hw(const void *chunk, size_t len) #endif +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +#if defined(__GNUC__) +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) + __attribute__((visibility("default"))); +#else +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t); +#endif + /* - * __wt_checksum_init -- - * WiredTiger: detect CRC hardware and set the checksum function. + * wiredtiger_crc32c_func -- + * WiredTiger: detect CRC hardware and return the checksum function. */ -void -__wt_checksum_init(void) - WT_GCC_FUNC_ATTRIBUTE((cold)) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) { -#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) +#if defined(__linux__) unsigned long caps = getauxval(AT_HWCAP); if (caps & HWCAP_S390_VX) - __wt_process.checksum = __wt_checksum_hw; + return (__wt_checksum_hw); else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_sw); #else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_sw); #endif } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 0945d768ce2..abbea9b1bb9 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -190,17 +190,18 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -774,7 +775,7 @@ static const WT_CONFIG_CHECK { "path", "string", NULL, NULL, NULL, 0 }, { "prealloc", "boolean", NULL, NULL, NULL, 0 }, { "recover", "string", - NULL, "choices=[\"error\",\"on\"]", + NULL, "choices=[\"error\",\"on\",\"salvage\"]", NULL, 0 }, { "zero_fill", "boolean", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } @@ -881,7 +882,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "transaction_sync", "category", NULL, NULL, @@ -890,13 +891,14 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "use_environment_priv", "boolean", NULL, NULL, NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -985,7 +987,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "transaction_sync", "category", NULL, NULL, @@ -994,13 +996,14 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "use_environment_priv", "boolean", NULL, NULL, NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -1086,20 +1089,21 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -1185,20 +1189,21 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 589560acc88..47ab622a7f4 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1832,6 +1832,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "checkpoint", WT_VERB_CHECKPOINT }, { "checkpoint_progress",WT_VERB_CHECKPOINT_PROGRESS }, { "compact", WT_VERB_COMPACT }, + { "error_returns", WT_VERB_ERROR_RETURNS }, { "evict", WT_VERB_EVICT }, { "evict_stuck", WT_VERB_EVICT_STUCK }, { "evictserver", WT_VERB_EVICTSERVER }, @@ -2025,7 +2026,6 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) { "split_6", WT_TIMING_STRESS_SPLIT_6 }, { "split_7", WT_TIMING_STRESS_SPLIT_7 }, { "split_8", WT_TIMING_STRESS_SPLIT_8 }, - { "split_9", WT_TIMING_STRESS_SPLIT_9 }, { NULL, 0 } }; WT_CONFIG_ITEM cval, sval; @@ -2750,15 +2750,3 @@ err: /* Discard the scratch buffers. */ return (ret); } - -/* - * wiredtiger_checksum_crc32c -- - * CRC32C checksum function entry point. - */ -uint32_t -wiredtiger_checksum_crc32c(const void *buffer, size_t len) -{ - if (__wt_process.checksum == NULL) - __wt_checksum_init(); - return (__wt_process.checksum(buffer, len)); -} diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 7c24f3c126f..7013ade0f27 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -47,7 +47,7 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session) if ((ret = __wt_metadata_search(session, dhandle->name, &metaconf)) != 0) { if (ret == WT_NOTFOUND) - ret = ENOENT; + ret = __wt_set_return(session, ENOENT); WT_RET(ret); } @@ -703,7 +703,7 @@ __conn_dhandle_remove(WT_SESSION_IMPL *session, bool final) /* Check if the handle was reacquired by a session while we waited. */ if (!final && (dhandle->session_inuse != 0 || dhandle->session_ref != 0)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); WT_CONN_DHANDLE_REMOVE(conn, dhandle, bucket); return (0); diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index d8eb095d6d2..34570fdd0be 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -299,6 +299,13 @@ __logmgr_config( session, cfg, "log.recover", 0, &cval)); if (WT_STRING_MATCH("error", cval.str, cval.len)) FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR); + else if (WT_STRING_MATCH("salvage", cval.str, cval.len)) { + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_RET_MSG(session, EINVAL, + "Readonly configuration incompatible with " + "log=(recover=salvage)"); + FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_SALVAGE); + } } WT_RET(__wt_config_gets(session, cfg, "log.zero_fill", &cval)); diff --git a/src/third_party/wiredtiger/src/docs/config-strings.dox b/src/third_party/wiredtiger/src/docs/config-strings.dox index a583573214f..d6291d5b4ba 100644 --- a/src/third_party/wiredtiger/src/docs/config-strings.dox +++ b/src/third_party/wiredtiger/src/docs/config-strings.dox @@ -44,7 +44,7 @@ columns in a table, values are nested using parentheses. For example: All types of parentheses are treated equivalently by the parser. -When an integer values is expected, the value may have multiplier characters +When an integer value is expected, the value may have multiplier characters appended, as follows: <table> diff --git a/src/third_party/wiredtiger/src/docs/top/main.dox b/src/third_party/wiredtiger/src/docs/top/main.dox index e4de22ff042..d802443a9d8 100644 --- a/src/third_party/wiredtiger/src/docs/top/main.dox +++ b/src/third_party/wiredtiger/src/docs/top/main.dox @@ -6,12 +6,12 @@ WiredTiger is an high performance, scalable, production quality, NoSQL, @section releases Releases <table> -@row{<b>WiredTiger 3.0.0</b> (current), +@row{<b>WiredTiger 3.1.0</b> (current), + <a href="releases/wiredtiger-3.1.0.tar.bz2"><b>[Release package]</b></a>, + <a href="3.1.0/index.html"><b>[Documentation]</b></a>} +@row{<b>WiredTiger 3.0.0</b> (previous), <a href="releases/wiredtiger-3.0.0.tar.bz2"><b>[Release package]</b></a>, <a href="3.0.0/index.html"><b>[Documentation]</b></a>} -@row{<b>WiredTiger 2.9.3</b> (previous), - <a href="releases/wiredtiger-2.9.3.tar.bz2"><b>[Release package]</b></a>, - <a href="2.9.3/index.html"><b>[Documentation]</b></a>} @row{<b>Development branch</b>, <a href="https://github.com/wiredtiger/wiredtiger"><b>[Source code]</b></a>, <a href="develop/index.html"><b>[Documentation]</b></a>} diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index 2e4990e8a33..7e89d23230f 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,5 +1,61 @@ /*! @page upgrading Upgrading WiredTiger applications +@section version_310 Upgrading to Version 3.1.0 +<dl> + +<dt>WiredTiger on-disk log file format change</dt> +<dd> +The WiredTiger on-disk file format for write-ahead log files has changed +as the log file version number was incremented. See +<a href=https://jira.mongodb.org/browse/WT-4029>WT-4029</a> for details. +</dd> + +<dt>::wiredtiger_open compatibility configuration changes</dt> +<dd> +The compatibility setting now takes additional options that can define +the minimum or maximum required version of existing data files. See +<a href=https://jira.mongodb.org/browse/WT-4056>WT-4056</a> and +<a href=https://jira.mongodb.org/browse/WT-4098>WT-4098</a> for details. +</dd> +<dt>::wiredtiger_open cache configuration changes</dt> +<dd> +The cache configuration options \c eviction_checkpoint_target, \c +eviction_dirty_target, \c eviction_dirty_trigger, \c eviction_target and \c +eviction_trigger have changed. The options can now take an absolute size. It would +be a percentage of the cache size if the value is within the range of 0 to 100 +or an absolute size when greater than 100. This API change is compatible with +existing usage. See <a href=https://jira.mongodb.org/browse/WT-3632>WT-3632</a> +for details. +</dd> + +<dt>Changed transaction semantics around schema operations</dt> +<dd> +WiredTiger does not offer fully transactional create and drop operations. +We have made some changes to how create and drop are implemented +if done within the scope of an explicit transaction. If an application +is relying on particular visibility/atomicity guarantees around table +create or drop, care should be taken when upgrading. See +<a href=https://jira.mongodb.org/browse/WT-3964>WT-3964</a> for details. +</dd> + +<dt>On-disk format change for metadata</dt> +<dd> +There was a change to the content stored in the WiredTiger owned metadata +files, which means metadata created or updated by this version of WiredTiger +is not compatible with earlier versions. See +<a href=https://jira.mongodb.org/browse/WT-3905>WT-3905</a> for details. +</dd> + +<dt>Implement a per-session cursor cache</dt> +<dd> +WiredTiger now holds a cache of recently closed cursors in each +session. This improves performance for applications that open and +close cursors frequently, but increases memory overhead. The cache +is enabled by default, but can be disabled. See +<a href=https://jira.mongodb.org/browse/WT-1228>WT-1228</a> for details. +</dd> + +</dl><hr> @section version_300 Upgrading to Version 3.0.0 <dl> @@ -28,16 +84,6 @@ The performance visualization tool \c wtstats has been removed and is no longer supported. </dd> -<dt>::wiredtiger_open cache configuration changes</dt> -<dd> -The cache configuration options \c eviction_checkpoint_target, \c -eviction_dirty_target, \c eviction_dirty_trigger, \c eviction_target and \c -eviction_trigger have changed. The options can now take absolute size. It would -be a percentage of the cache size if the value is within the range of 0 to 100 -or an absolute size when greater than 100. This API change is compatible with -existing usage. -</dd> - </dl><hr> @section version_292 Upgrading to Version 2.9.2 <dl> diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 05397843fc7..13b5dfc4c8d 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -451,7 +451,7 @@ __evict_server(WT_SESSION_IMPL *session, bool *did_work) "Cache stuck for too long, giving up"); WT_RET(__wt_verbose_dump_txn(session)); WT_RET(__wt_verbose_dump_cache(session)); - return (ETIMEDOUT); + return (__wt_set_return(session, ETIMEDOUT)); #else if (WT_VERBOSE_ISSET(session, WT_VERB_EVICT_STUCK)) { WT_RET(__wt_verbose_dump_txn(session)); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 0daccdf5b1c..71e084773d0 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -42,7 +42,7 @@ __evict_exclusive(WT_SESSION_IMPL *session, WT_REF *ref) WT_STAT_DATA_INCR(session, cache_eviction_hazard); WT_STAT_CONN_INCR(session, cache_eviction_hazard); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* @@ -453,7 +453,7 @@ __evict_child_check(WT_SESSION_IMPL *session, WT_REF *parent) * page. */ if (__wt_page_del_active(session, child, true)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); break; case WT_REF_LOOKASIDE: /* @@ -461,10 +461,10 @@ __evict_child_check(WT_SESSION_IMPL *session, WT_REF *parent) * can be ignored. */ if (__wt_page_las_active(session, child)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); break; default: - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } } WT_INTL_FOREACH_END; @@ -528,7 +528,7 @@ __evict_review( * should be uncommon - we don't add clean pages to the queue. */ if (F_ISSET(conn, WT_CONN_IN_MEMORY) && !modified && !closing) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* Check if the page can be evicted. */ if (!closing) { @@ -541,7 +541,7 @@ __evict_review( session, WT_TXN_OLDEST_STRICT)); if (!__wt_page_can_evict(session, ref, inmem_splitp)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * Check for an append-only workload needing an in-memory @@ -563,7 +563,7 @@ __evict_review( * eviction that writes to lookaside), give up. */ if (F_ISSET(session, WT_SESSION_NO_RECONCILE)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If the page is dirty, reconcile it to decide if we can evict it. @@ -636,7 +636,7 @@ __evict_review( if (WT_SESSION_IS_CHECKPOINT(session) && !__wt_page_is_modified(page) && !__wt_txn_visible_all(session, page->modify->rec_max_txn, WT_TIMESTAMP_NULL(&page->modify->rec_max_timestamp))) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If reconciliation fails but reports it might succeed if we use the @@ -663,7 +663,7 @@ __evict_review( */ if (WT_SESSION_IS_CHECKPOINT(session) && page->modify->rec_result == WT_PM_REC_MULTIBLOCK) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * Success: assert the page is clean or reconciliation was configured diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 7813f1299fd..ed3480a40d0 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1357,13 +1357,12 @@ __wt_page_evict_retry(WT_SESSION_IMPL *session, WT_PAGE *page) return (true); #ifdef HAVE_TIMESTAMPS - if (!__wt_timestamp_iszero(&mod->last_eviction_timestamp)) { - __wt_txn_pinned_timestamp(session, &pinned_ts); - if (__wt_timestamp_cmp( - &mod->last_eviction_timestamp, - &txn_global->pinned_timestamp) != 0) - return (true); - } + if (__wt_timestamp_iszero(&mod->last_eviction_timestamp)) + return (true); + + __wt_txn_pinned_timestamp(session, &pinned_ts); + if (__wt_timestamp_cmp(&pinned_ts, &mod->last_eviction_timestamp) > 0) + return (true); #endif return (false); diff --git a/src/third_party/wiredtiger/src/include/cache.i b/src/third_party/wiredtiger/src/include/cache.i index 7f12949e162..2e3700f6287 100644 --- a/src/third_party/wiredtiger/src/include/cache.i +++ b/src/third_party/wiredtiger/src/include/cache.i @@ -220,9 +220,9 @@ __wt_cache_update_lookaside_score( global_score = cache->evict_lookaside_score; if (score > global_score && global_score < 100) - __wt_atomic_addi32(&cache->evict_lookaside_score, 1); + (void)__wt_atomic_addi32(&cache->evict_lookaside_score, 1); else if (score < global_score && global_score > 0) - __wt_atomic_subi32(&cache->evict_lookaside_score, 1); + (void)__wt_atomic_subi32(&cache->evict_lookaside_score, 1); } /* diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 22459b0072c..19bb21ce2f1 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -322,7 +322,8 @@ struct __wt_connection_impl { #define WT_CONN_LOG_RECOVER_DIRTY 0x020u /* Recovering unclean */ #define WT_CONN_LOG_RECOVER_DONE 0x040u /* Recovery completed */ #define WT_CONN_LOG_RECOVER_ERR 0x080u /* Error if recovery required */ -#define WT_CONN_LOG_ZERO_FILL 0x100u /* Manually zero files */ +#define WT_CONN_LOG_RECOVER_SALVAGE 0x100u /* Salvage log files */ +#define WT_CONN_LOG_ZERO_FILL 0x200u /* Manually zero files */ /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t log_flags; /* Global logging configuration */ WT_CONDVAR *log_cond; /* Log server wait mutex */ @@ -413,34 +414,35 @@ struct __wt_connection_impl { #define WT_VERB_CHECKPOINT 0x000000004u #define WT_VERB_CHECKPOINT_PROGRESS 0x000000008u #define WT_VERB_COMPACT 0x000000010u -#define WT_VERB_EVICT 0x000000020u -#define WT_VERB_EVICTSERVER 0x000000040u -#define WT_VERB_EVICT_STUCK 0x000000080u -#define WT_VERB_FILEOPS 0x000000100u -#define WT_VERB_HANDLEOPS 0x000000200u -#define WT_VERB_LOG 0x000000400u -#define WT_VERB_LOOKASIDE 0x000000800u -#define WT_VERB_LOOKASIDE_ACTIVITY 0x000001000u -#define WT_VERB_LSM 0x000002000u -#define WT_VERB_LSM_MANAGER 0x000004000u -#define WT_VERB_METADATA 0x000008000u -#define WT_VERB_MUTEX 0x000010000u -#define WT_VERB_OVERFLOW 0x000020000u -#define WT_VERB_READ 0x000040000u -#define WT_VERB_REBALANCE 0x000080000u -#define WT_VERB_RECONCILE 0x000100000u -#define WT_VERB_RECOVERY 0x000200000u -#define WT_VERB_RECOVERY_PROGRESS 0x000400000u -#define WT_VERB_SALVAGE 0x000800000u -#define WT_VERB_SHARED_CACHE 0x001000000u -#define WT_VERB_SPLIT 0x002000000u -#define WT_VERB_TEMPORARY 0x004000000u -#define WT_VERB_THREAD_GROUP 0x008000000u -#define WT_VERB_TIMESTAMP 0x010000000u -#define WT_VERB_TRANSACTION 0x020000000u -#define WT_VERB_VERIFY 0x040000000u -#define WT_VERB_VERSION 0x080000000u -#define WT_VERB_WRITE 0x100000000u +#define WT_VERB_ERROR_RETURNS 0x000000020u +#define WT_VERB_EVICT 0x000000040u +#define WT_VERB_EVICTSERVER 0x000000080u +#define WT_VERB_EVICT_STUCK 0x000000100u +#define WT_VERB_FILEOPS 0x000000200u +#define WT_VERB_HANDLEOPS 0x000000400u +#define WT_VERB_LOG 0x000000800u +#define WT_VERB_LOOKASIDE 0x000001000u +#define WT_VERB_LOOKASIDE_ACTIVITY 0x000002000u +#define WT_VERB_LSM 0x000004000u +#define WT_VERB_LSM_MANAGER 0x000008000u +#define WT_VERB_METADATA 0x000010000u +#define WT_VERB_MUTEX 0x000020000u +#define WT_VERB_OVERFLOW 0x000040000u +#define WT_VERB_READ 0x000080000u +#define WT_VERB_REBALANCE 0x000100000u +#define WT_VERB_RECONCILE 0x000200000u +#define WT_VERB_RECOVERY 0x000400000u +#define WT_VERB_RECOVERY_PROGRESS 0x000800000u +#define WT_VERB_SALVAGE 0x001000000u +#define WT_VERB_SHARED_CACHE 0x002000000u +#define WT_VERB_SPLIT 0x004000000u +#define WT_VERB_TEMPORARY 0x008000000u +#define WT_VERB_THREAD_GROUP 0x010000000u +#define WT_VERB_TIMESTAMP 0x020000000u +#define WT_VERB_TRANSACTION 0x040000000u +#define WT_VERB_VERIFY 0x080000000u +#define WT_VERB_VERSION 0x100000000u +#define WT_VERB_WRITE 0x200000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t verbose; @@ -459,7 +461,6 @@ struct __wt_connection_impl { #define WT_TIMING_STRESS_SPLIT_6 0x080u #define WT_TIMING_STRESS_SPLIT_7 0x100u #define WT_TIMING_STRESS_SPLIT_8 0x200u -#define WT_TIMING_STRESS_SPLIT_9 0x400u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t timing_stress_flags; diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index f94c3d7b880..c85f67e6f0b 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -19,6 +19,13 @@ #define WT_DIAGNOSTIC_YIELD #endif +#define __wt_err(session, error, ...) \ + __wt_err_func(session, error, __func__, __LINE__, __VA_ARGS__) +#define __wt_errx(session, ...) \ + __wt_errx_func(session, __func__, __LINE__, __VA_ARGS__) +#define __wt_set_return(session, error) \ + __wt_set_return_func(session, __func__, __LINE__, error) + /* Set "ret" and branch-to-err-label tests. */ #define WT_ERR(a) do { \ if ((ret = (a)) != 0) \ @@ -125,8 +132,10 @@ */ #ifdef HAVE_DIAGNOSTIC #define WT_ASSERT(session, exp) do { \ - if (!(exp)) \ - __wt_assert(session, 0, __func__, __LINE__, "%s", #exp);\ + if (!(exp)) { \ + __wt_errx(session, "%s", #exp); \ + __wt_abort(session); \ + } \ } while (0) #else #define WT_ASSERT(session, exp) \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 24e16adefd1..6df9da759c3 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -215,8 +215,7 @@ extern int __wt_las_cursor_position(WT_CURSOR *cursor, uint64_t pageid) WT_GCC_F extern int __wt_las_remove_block(WT_SESSION_IMPL *session, uint64_t pageid, bool lock_wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_save_dropped(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_sweep(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern uint32_t __wt_checksum_sw(const void *chunk, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_checksum_init(void) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)); +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); extern void __wt_config_initn(WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str, size_t len); extern void __wt_config_init(WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str); extern void __wt_config_subinit(WT_SESSION_IMPL *session, WT_CONFIG *conf, WT_CONFIG_ITEM *item); @@ -549,7 +548,7 @@ extern int __wt_nfilename(WT_SESSION_IMPL *session, const char *name, size_t nam extern int __wt_filename_construct(WT_SESSION_IMPL *session, const char *path, const char *file_prefix, uintmax_t id_1, uint32_t id_2, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -676,23 +675,15 @@ extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_ extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep); extern void __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler); -extern void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))); +extern void __wt_err_func(WT_SESSION_IMPL *session, int error, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_errx_func(WT_SESSION_IMPL *session, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_set_return_func(WT_SESSION_IMPL *session, const char*func, int line, int err) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_err_printf(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)); extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_msg_printf(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_ext_strerror(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, int error); extern int __wt_progress(WT_SESSION_IMPL *session, const char *s, uint64_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void -__wt_assert(WT_SESSION_IMPL *session, - int error, const char *file_name, int line_number, const char *fmt, ...) - WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) - WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) -#ifdef HAVE_DIAGNOSTIC - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) -#endif - WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_illegal_value_func(WT_SESSION_IMPL *session, const char *tag, const char *file, int line) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_inmem_unsupported_op(WT_SESSION_IMPL *session, const char *tag) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -724,6 +715,7 @@ extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_ extern void __wt_hazard_close(WT_SESSION_IMPL *session); extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref); extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref); +extern bool __wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_fill_hex(const uint8_t *src, size_t src_max, uint8_t *dest, size_t dest_max, size_t *lenp); extern int __wt_raw_to_hex(WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_raw_to_esc_hex(WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 7060e6cea23..c56dea0a074 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -338,3 +338,50 @@ union __wt_rand_state { continue; \ } #define WT_TAILQ_SAFE_REMOVE_END } + +/* + * WT_VA_ARGS_BUF_FORMAT -- + * Format into a scratch buffer, extending it as necessary. This is a + * macro because we need to repeatedly call va_start/va_end and there's no + * way to do that inside a function call. + */ +#define WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, concatenate) do { \ + size_t __len, __space; \ + va_list __ap; \ + int __ret_xx; /* __ret already used by WT_RET */ \ + char *__p; \ + \ + /* \ + * This macro is used to both initialize and concatenate into a \ + * buffer. If not concatenating, clear the size so we don't use \ + * any existing contents. \ + */ \ + if (!concatenate) \ + (buf)->size = 0; \ + for (;;) { \ + WT_ASSERT(session, (buf)->memsize >= (buf)->size); \ + __p = (char *)((uint8_t *)(buf)->mem + (buf)->size); \ + __space = (buf)->memsize - (buf)->size; \ + \ + /* Format into the buffer. */ \ + va_start(__ap, fmt); \ + __ret_xx = __wt_vsnprintf_len_set( \ + __p, __space, &__len, fmt, __ap); \ + va_end(__ap); \ + WT_RET(__ret_xx); \ + \ + /* Check if there was enough space. */ \ + if (__len < __space) { \ + (buf)->data = (buf)->mem; \ + (buf)->size += __len; \ + break; \ + } \ + \ + /* \ + * If not, double the size of the buffer: we're dealing \ + * with strings, we don't expect the size to get huge. \ + */ \ + WT_RET(__wt_buf_extend( \ + session, buf, (buf)->size + __len + 1)); \ + } \ +} while (0) diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index 2c380e95ade..5c9f95bc08a 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -259,15 +259,26 @@ __wt_spin_backoff(uint64_t *yield_count, uint64_t *sleep_usecs) static inline void __wt_timing_stress(WT_SESSION_IMPL *session, u_int flag) { - WT_CONNECTION_IMPL *conn; - uint64_t sleep_usecs; + uint64_t i; - conn = S2C(session); - - /* Only sleep when the specified configuration flag is set. */ - if (!FLD_ISSET(conn->timing_stress_flags, flag)) + /* Optionally only sleep when a specified configuration flag is set. */ + if (flag != 0 && !FLD_ISSET(S2C(session)->timing_stress_flags, flag)) return; - sleep_usecs = __wt_random(&session->rnd) % WT_TIMING_STRESS_MAX_DELAY; - __wt_sleep(0, sleep_usecs); + /* + * We need a fast way to choose a sleep time. We want to sleep a short + * period most of the time, but occasionally wait longer. Divide the + * maximum period of time into 10 buckets (where bucket 0 doesn't sleep + * at all), and roll dice, advancing to the next bucket 50% of the time. + * That means we'll hit the maximum roughly every 1K calls. + */ + for (i = 0;;) + if (__wt_random(&session->rnd) & 0x1 || ++i > 9) + break; + + if (i == 0) + __wt_yield(); + else + /* The default maximum delay is 1/10th of a second. */ + __wt_sleep(0, i * (WT_TIMING_STRESS_MAX_DELAY / 10)); } diff --git a/src/third_party/wiredtiger/src/include/os_fhandle.i b/src/third_party/wiredtiger/src/include/os_fhandle.i index 7c09a83132c..78d01abca4b 100644 --- a/src/third_party/wiredtiger/src/include/os_fhandle.i +++ b/src/third_party/wiredtiger/src/include/os_fhandle.i @@ -72,7 +72,7 @@ __wt_fextend(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset) if (handle->fh_extend != NULL) return (handle->fh_extend( handle, (WT_SESSION *)session, offset)); - return (ENOTSUP); + return (__wt_set_return(session, ENOTSUP)); } /* @@ -157,7 +157,7 @@ __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset) if (handle->fh_truncate != NULL) return (handle->fh_truncate( handle, (WT_SESSION *)session, offset)); - return (ENOTSUP); + return (__wt_set_return(session, ENOTSUP)); } /* diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index a4ba834d5ef..2dcf65319b8 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2334,15 +2334,15 @@ struct __wt_connection { * given as a list\, such as * <code>"verbose=[evictserver\,read]"</code>., a list\, with values * chosen from the following options: \c "api"\, \c "block"\, \c - * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c "evict"\, - * \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, - * \c "log"\, \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c - * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c - * "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, \c - * "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, - * \c "temporary"\, \c "thread_group"\, \c "timestamp"\, \c - * "transaction"\, \c "verify"\, \c "version"\, \c "write"; default - * empty.} + * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c + * "error_returns"\, \c "evict"\, \c "evict_stuck"\, \c "evictserver"\, + * \c "fileops"\, \c "handleops"\, \c "log"\, \c "lookaside"\, \c + * "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, + * \c "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c + * "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c "salvage"\, + * \c "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, + * \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c + * "write"; default empty.} * @configend * @errors */ @@ -2902,9 +2902,10 @@ struct __wt_connection { * flag; default \c true.} * @config{ recover, run recovery * or error if recovery needs to run after an unclean shutdown., a string\, - * chosen from the following options: \c "error"\, \c "on"; default \c on.} - * @config{ zero_fill, manually write zeroes into log - * files., a boolean flag; default \c false.} + * chosen from the following options: \c "error"\, \c "on"\, \c "salvage"; + * default \c on.} + * @config{ zero_fill, manually write + * zeroes into log files., a boolean flag; default \c false.} * @config{ ),,} * @config{lsm_manager = (, configure database wide options for LSM tree * management. The LSM manager is started automatically the first time an LSM @@ -3017,14 +3018,14 @@ struct __wt_connection { * @config{verbose, enable messages for various events. Options are given as a * list\, such as <code>"verbose=[evictserver\,read]"</code>., a list\, with * values chosen from the following options: \c "api"\, \c "block"\, \c - * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c "evict"\, \c - * "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, - * \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c - * "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c - * "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c - * "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c - * "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; - * default empty.} + * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c "error_returns"\, + * \c "evict"\, \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c + * "handleops"\, \c "log"\, \c "lookaside"\, \c "lookaside_activity"\, \c + * "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c + * "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, \c + * "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c + * "temporary"\, \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c + * "verify"\, \c "version"\, \c "write"; default empty.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to * files. Ignored on non-Windows systems. Options are given as a list\, such * as <code>"write_through=[data]"</code>. Configuring \c write_through requires @@ -3549,18 +3550,17 @@ struct __wt_config_parser { */ /*! - * Return a buffer's CRC32C checksum. + * Return a pointer to a function that calculates a CRC32C checksum. * * The WiredTiger library CRC32C checksum function uses hardware support where * available, else it falls back to a software implementation. * * @snippet ex_all.c Checksum a buffer * - * @param buffer a pointer to a buffer - * @param len the number of valid bytes in the buffer - * @returns the buffer's CRC32C checksum + * @returns a pointer to a function that takes a buffer and length and returns + * the CRC32C checksum */ -uint32_t wiredtiger_checksum_crc32c(const void *buffer, size_t len) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! @} */ diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 6ce26e03c5d..243af64cb18 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -10,6 +10,7 @@ static int __log_newfile(WT_SESSION_IMPL *, bool, bool *); static int __log_openfile(WT_SESSION_IMPL *, uint32_t, uint32_t, WT_FH **); +static int __log_truncate(WT_SESSION_IMPL *, WT_LSN *, bool, bool); static int __log_write_internal( WT_SESSION_IMPL *, WT_ITEM *, WT_LSN *, uint32_t); @@ -926,7 +927,7 @@ err: __wt_scr_free(session, &buf); */ static int __log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, - WT_LSN *lsnp, uint16_t *versionp) + WT_LSN *lsnp, uint16_t *versionp, bool *need_salvagep) { WT_CONNECTION_IMPL *conn; WT_DECL_ITEM(buf); @@ -937,11 +938,15 @@ __log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, WT_LOG_RECORD *logrec; uint32_t allocsize, rectype; const uint8_t *end, *p; + bool need_salvage, salvage_mode; conn = S2C(session); + fh = NULL; log = conn->log; + need_salvage = false; WT_RET(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__log_openfile(session, id, 0, &fh)); + salvage_mode = (need_salvagep != NULL && + FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_SALVAGE)); if (log == NULL) allocsize = WT_LOG_ALIGN; @@ -953,17 +958,30 @@ __log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, memset(buf->mem, 0, allocsize); /* + * Any operation that fails from here on out indicates corruption + * that could be salvaged. + */ + need_salvage = true; + + /* * Read in the log file header and verify it. */ + WT_ERR(__log_openfile(session, id, 0, &fh)); WT_ERR(__wt_read(session, fh, 0, allocsize, buf->mem)); logrec = (WT_LOG_RECORD *)buf->mem; __wt_log_record_byteswap(logrec); desc = (WT_LOG_DESC *)logrec->record; __wt_log_desc_byteswap(desc); - if (desc->log_magic != WT_LOG_MAGIC) - WT_PANIC_RET(session, WT_ERROR, - "log file %s corrupted: Bad magic number %" PRIu32, - fh->name, desc->log_magic); + if (desc->log_magic != WT_LOG_MAGIC) { + if (salvage_mode) + WT_ERR_MSG(session, WT_ERROR, + "log file %s corrupted: Bad magic number %" PRIu32, + fh->name, desc->log_magic); + else + WT_PANIC_RET(session, WT_ERROR, + "log file %s corrupted: Bad magic number %" PRIu32, + fh->name, desc->log_magic); + } /* * We cannot read future log file formats. */ @@ -1043,7 +1061,14 @@ err: __wt_scr_free(session, &buf); */ if (fhp != NULL && ret == 0) *fhp = fh; - else + else if (ret != 0 && need_salvage && salvage_mode) { + /* Let the caller know this file must be salvaged. */ + ret = 0; + WT_TRET(__wt_close(session, &fh)); + if (fhp != NULL) + *fhp = NULL; + *need_salvagep = true; + } else WT_TRET(__wt_close(session, &fh)); return (ret); @@ -1131,7 +1156,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) WT_STAT_CONN_INCR(session, log_close_yields); __wt_log_wrlsn(session, NULL); if (++yield_cnt > 10000) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); __wt_yield(); } /* @@ -1200,7 +1225,8 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) * we must pass in a local file handle. Otherwise there is a wide * window where another thread could see a NULL log file handle. */ - WT_RET(__log_open_verify(session, log->fileid, &log_fh, NULL, NULL)); + WT_RET(__log_open_verify(session, log->fileid, &log_fh, NULL, NULL, + NULL)); /* * Write the LSN at the end of the last record in the previous log file * as the first record in this log file. @@ -1412,25 +1438,31 @@ __log_truncate_file(WT_SESSION_IMPL *session, WT_FH *log_fh, wt_off_t offset) * it will truncate between the given LSN and the trunc_lsn. That is, * since we pre-allocate log files, it will free that space and allow the * log to be traversed. We use the trunc_lsn because logging has already - * opened the new/next log file before recovery ran. This function assumes - * we are in recovery or other dedicated time and not during live running. + * opened the new/next log file before recovery ran. If salvage_mode is + * set, we verify headers of log files visited and recreate them if they + * are damaged. This function assumes we are in recovery or other + * dedicated time and not during live running. */ static int -__log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) +__log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log, + bool salvage_mode) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_FH *log_fh; WT_LOG *log; - uint32_t lognum; + uint32_t lognum, salvage_first, salvage_last; u_int i, logcount; char **logfiles; + bool need_salvage, opened; conn = S2C(session); log = conn->log; log_fh = NULL; logcount = 0; logfiles = NULL; + salvage_first = salvage_last = 0; + need_salvage = false; /* * Truncate the log file to the given LSN. @@ -1446,6 +1478,10 @@ __log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) WT_ERR(__wt_fsync(session, log_fh, true)); WT_ERR(__wt_close(session, &log_fh)); + if (salvage_mode) + WT_ERR(__wt_msg(session, + "salvage: log file %" PRIu32 " truncated", lsn->l.file)); + /* * If we just want to truncate the current log, return and skip * looking for intervening logs. @@ -1456,7 +1492,32 @@ __log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) for (i = 0; i < logcount; i++) { WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); if (lognum > lsn->l.file && lognum < log->trunc_lsn.l.file) { - WT_ERR(__log_openfile(session, lognum, 0, &log_fh)); + opened = false; + if (salvage_mode) { + /* + * When salvaging, we verify that the + * header of the log file is valid. + * If not, create a new, empty one. + */ + need_salvage = false; + WT_ERR(__log_open_verify(session, lognum, + &log_fh, NULL, NULL, &need_salvage)); + if (need_salvage) { + WT_ASSERT(session, log_fh == NULL); + WT_ERR(__wt_log_remove(session, + WT_LOG_FILENAME, lognum)); + WT_ERR(__wt_log_allocfile(session, + lognum, WT_LOG_FILENAME)); + } else + opened = true; + + if (salvage_first == 0) + salvage_first = lognum; + salvage_last = lognum; + } + if (!opened) + WT_ERR(__log_openfile(session, lognum, 0, + &log_fh)); /* * If there are intervening files pre-allocated, * truncate them to the end of the log file header. @@ -1469,6 +1530,17 @@ __log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) } err: WT_TRET(__wt_close(session, &log_fh)); WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); + if (salvage_first != 0) { + if (salvage_last > salvage_first) + WT_TRET(__wt_msg(session, + "salvage: log files %" PRIu32 "-%" PRIu32 + " truncated at beginning", salvage_first, + salvage_last)); + else + WT_TRET(__wt_msg(session, + "salvage: log file %" PRIu32 + " truncated at beginning", salvage_first)); + } return (ret); } @@ -1566,13 +1638,12 @@ __wt_log_open(WT_SESSION_IMPL *session) uint16_t version; u_int i, logcount; char **logfiles; + bool need_salvage; conn = S2C(session); log = conn->log; logfiles = NULL; logcount = 0; - lastlog = 0; - firstlog = UINT32_MAX; /* * Open up a file handle to the log directory if we haven't. @@ -1587,9 +1658,14 @@ __wt_log_open(WT_SESSION_IMPL *session) if (!F_ISSET(conn, WT_CONN_READONLY)) WT_ERR(__log_prealloc_remove(session)); +again: /* * Now look at the log files and set our LSNs. */ + lastlog = 0; + firstlog = UINT32_MAX; + need_salvage = false; + WT_ERR(__log_get_files(session, WT_LOG_FILENAME, &logfiles, &logcount)); for (i = 0; i < logcount; i++) { WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); @@ -1610,8 +1686,23 @@ __wt_log_open(WT_SESSION_IMPL *session) * we create a new log file so that we can detect an unsupported * version before modifying the file space. */ - WT_ERR(__log_open_verify(session, - lastlog, NULL, NULL, &version)); + WT_ERR(__log_open_verify(session, lastlog, NULL, NULL, + &version, &need_salvage)); + + /* + * If we were asked to salvage and the last log file was + * indeed corrupt, remove it and try all over again. + */ + if (need_salvage) { + WT_ERR(__wt_log_remove( + session, WT_LOG_FILENAME, lastlog)); + WT_ERR(__wt_msg(session, + "salvage: log file %" PRIu32 " removed", lastlog)); + WT_ERR(__wt_fs_directory_list_free(session, &logfiles, + logcount)); + logfiles = NULL; + goto again; + } } /* @@ -1641,7 +1732,7 @@ __wt_log_open(WT_SESSION_IMPL *session) * have to close the file. */ WT_ERR(__log_open_verify(session, - lognum, NULL, NULL, &version)); + lognum, NULL, NULL, &version, NULL)); /* * If we find any log file at the wrong version * set the flag and we're done. @@ -1957,7 +2048,7 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, u_int i, logcount; int firstrecord; char **logfiles; - bool eol, partial_record; + bool eol, need_salvage, partial_record; conn = S2C(session); log = conn->log; @@ -1966,6 +2057,7 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, logfiles = NULL; eol = false; firstrecord = 1; + need_salvage = false; /* * If the caller did not give us a callback function there is nothing @@ -2064,8 +2156,9 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, if (!WT_IS_INIT_LSN(lsnp)) start_lsn = *lsnp; } - WT_ERR(__log_open_verify(session, - start_lsn.l.file, &log_fh, &prev_lsn, NULL)); + WT_ERR(__log_open_verify(session, start_lsn.l.file, &log_fh, &prev_lsn, + NULL, &need_salvage)); + WT_ERR_TEST(need_salvage, WT_ERROR); WT_ERR(__wt_filesize(session, log_fh, &log_size)); rd_lsn = start_lsn; if (LF_ISSET(WT_LOGSCAN_RECOVER)) @@ -2103,7 +2196,8 @@ advance: __wt_verbose(session, WT_VERB_LOG, "Truncate end of log %" PRIu32 "/%" PRIu32, rd_lsn.l.file, rd_lsn.l.offset); - WT_ERR(__log_truncate(session, &rd_lsn, true)); + WT_ERR(__log_truncate(session, &rd_lsn, true, + false)); } /* * If we had a partial record, we'll want to break @@ -2128,7 +2222,9 @@ advance: " through %" PRIu32, rd_lsn.l.file, end_lsn.l.file); WT_ERR(__log_open_verify(session, - rd_lsn.l.file, &log_fh, &prev_lsn, &version)); + rd_lsn.l.file, &log_fh, &prev_lsn, &version, + &need_salvage)); + WT_ERR_TEST(need_salvage, WT_ERROR); /* * Opening the log file reads with verify sets up the * previous LSN from the first record. This detects @@ -2161,10 +2257,15 @@ advance: } /* * Read the minimum allocation size a record could be. + * Conditionally set the need_salvage flag so that if the + * read fails, we know this is an situation we can salvage. */ WT_ASSERT(session, buf->memsize >= allocsize); + need_salvage = FLD_ISSET(conn->log_flags, + WT_CONN_LOG_RECOVER_SALVAGE); WT_ERR(__wt_read(session, log_fh, rd_lsn.l.offset, (size_t)allocsize, buf->mem)); + need_salvage = false; /* * See if we need to read more than the allocation size. We * expect that we rarely will have to read more. Most log @@ -2215,7 +2316,9 @@ advance: WT_STAT_CONN_INCR(session, log_scan_rereads); } /* - * We read in the record, verify checksum. + * We read in the record, verify checksum. A failed checksum + * does not imply corruption, it may be the result of a + * partial write. * * Handle little- and big-endian objects. Objects are written * in little-endian format: save the header checksum, and @@ -2291,11 +2394,22 @@ advance: __wt_verbose(session, WT_VERB_LOG, "End of recovery truncate end of log %" PRIu32 "/%" PRIu32, rd_lsn.l.file, rd_lsn.l.offset); - WT_ERR(__log_truncate(session, &rd_lsn, false)); + WT_ERR(__log_truncate(session, &rd_lsn, false, false)); } err: WT_STAT_CONN_INCR(session, log_scans); /* + * If we are salvaging and failed a salvageable operation, then + * truncate the log at the fail point. + */ + if (ret != 0 && need_salvage) { + WT_TRET(__wt_close(session, &log_fh)); + log_fh = NULL; + WT_TRET(__log_truncate(session, &rd_lsn, false, true)); + ret = 0; + } + + /* * If the first attempt to read a log record results in * an error recovery is likely going to fail. Try to provide * a helpful failure message. diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index 8deda5e242f..c75181d0687 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -119,7 +119,7 @@ retry: * decide if retrying is necessary or not. */ if (forced && WT_LOG_SLOT_INPROGRESS(old_state)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If someone else is switching out this slot we lost. Nothing to * do but return. Return WT_NOTFOUND anytime the given slot was diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 16b28a1aecc..03feafe3c8c 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -419,7 +419,8 @@ __lsm_tree_find(WT_SESSION_IMPL *session, */ if (!__wt_atomic_cas_ptr( &lsm_tree->excl_session, NULL, session)) - return (EBUSY); + return (__wt_set_return( + session, EBUSY)); /* * Drain the work queue before checking for @@ -431,7 +432,8 @@ __lsm_tree_find(WT_SESSION_IMPL *session, if (lsm_tree->refcnt != 1) { __wt_lsm_tree_release( session, lsm_tree); - return (EBUSY); + return (__wt_set_return( + session, EBUSY)); } } else { (void)__wt_atomic_add32(&lsm_tree->refcnt, 1); @@ -445,7 +447,8 @@ __lsm_tree_find(WT_SESSION_IMPL *session, lsm_tree->refcnt > 0); __wt_lsm_tree_release( session, lsm_tree); - return (EBUSY); + return (__wt_set_return( + session, EBUSY)); } } diff --git a/src/third_party/wiredtiger/src/os_common/os_abort.c b/src/third_party/wiredtiger/src/os_common/os_abort.c index 85dcc741855..54cae3e61aa 100644 --- a/src/third_party/wiredtiger/src/os_common/os_abort.c +++ b/src/third_party/wiredtiger/src/os_common/os_abort.c @@ -15,6 +15,7 @@ void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_ATTRIBUTE((noreturn)) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { #ifdef HAVE_ATTACH u_int i; diff --git a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c index cb7a05e05d9..182f89cc5e1 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c +++ b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c @@ -63,7 +63,7 @@ __im_handle_remove(WT_SESSION_IMPL *session, if (im_fh->ref != 0) { __wt_err(session, EBUSY, "%s: file-remove", im_fh->iface.name); if (!force) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } bucket = im_fh->name_hash % WT_HASH_ARRAY_SIZE; @@ -272,7 +272,7 @@ __im_fs_size(WT_FILE_SYSTEM *file_system, /* Search for the handle, then get its size. */ if ((im_fh = __im_handle_search(file_system, name)) == NULL) - ret = ENOENT; + ret = __wt_set_return(session, ENOENT); else *sizep = (wt_off_t)im_fh->buf.size; diff --git a/src/third_party/wiredtiger/src/os_posix/os_dir.c b/src/third_party/wiredtiger/src/os_posix/os_dir.c index 2c2cb084a91..54614d67649 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_dir.c +++ b/src/third_party/wiredtiger/src/os_posix/os_dir.c @@ -41,11 +41,12 @@ __directory_list_worker(WT_FILE_SYSTEM *file_system, * but various static analysis programs remain unconvinced, check both. */ WT_SYSCALL_RETRY(((dirp = opendir(directory)) == NULL ? -1 : 0), ret); - if (dirp == NULL && ret == 0) - ret = EINVAL; - if (ret != 0) + if (dirp == NULL || ret != 0) { + if (ret == 0) + ret = EINVAL; WT_RET_MSG(session, ret, "%s: directory-list: opendir", directory); + } for (count = 0; (dp = readdir(dirp)) != NULL;) { /* diff --git a/src/third_party/wiredtiger/src/os_posix/os_fallocate.c b/src/third_party/wiredtiger/src/os_posix/os_fallocate.c index 6f6e6a6bdc2..cde6adfe780 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fallocate.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fallocate.c @@ -33,9 +33,9 @@ __posix_std_fallocate( return (ret); #else WT_UNUSED(file_handle); - WT_UNUSED(wt_session); WT_UNUSED(offset); - return (ENOTSUP); + + return (__wt_set_return((WT_SESSION_IMPL *)wt_session, ENOTSUP)); #endif } @@ -66,9 +66,9 @@ __posix_sys_fallocate( return (ret); #else WT_UNUSED(file_handle); - WT_UNUSED(wt_session); WT_UNUSED(offset); - return (ENOTSUP); + + return (__wt_set_return((WT_SESSION_IMPL *)wt_session, ENOTSUP)); #endif } @@ -92,9 +92,9 @@ __posix_posix_fallocate( return (ret); #else WT_UNUSED(file_handle); - WT_UNUSED(wt_session); WT_UNUSED(offset); - return (ENOTSUP); + + return (__wt_set_return((WT_SESSION_IMPL *)wt_session, ENOTSUP)); #endif } diff --git a/src/third_party/wiredtiger/src/os_posix/os_fs.c b/src/third_party/wiredtiger/src/os_posix/os_fs.c index 0af67ad38c5..42e6e411440 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fs.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fs.c @@ -338,7 +338,7 @@ __posix_file_advise(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, */ if (ret == EINVAL) { file_handle->fh_advise = NULL; - return (ENOTSUP); + return (__wt_set_return(session, ENOTSUP)); } WT_RET_MSG(session, ret, diff --git a/src/third_party/wiredtiger/src/os_posix/os_map.c b/src/third_party/wiredtiger/src/os_posix/os_map.c index b9ec284e124..f04a966c468 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_map.c +++ b/src/third_party/wiredtiger/src/os_posix/os_map.c @@ -33,7 +33,7 @@ __wt_posix_map(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, * mmap(2) of files with direct I/O to the same files. */ if (pfh->direct_io) - return (ENOTSUP); + return (__wt_set_return(session, ENOTSUP)); /* * There's no locking here to prevent the underlying file from changing diff --git a/src/third_party/wiredtiger/src/os_win/os_fs.c b/src/third_party/wiredtiger/src/os_win/os_fs.c index 66f4de87299..c75e8365116 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fs.c +++ b/src/third_party/wiredtiger/src/os_win/os_fs.c @@ -393,7 +393,7 @@ __win_file_set_end( if (SetEndOfFile(win_fh->filehandle_secondary) == FALSE) { if (GetLastError() == ERROR_USER_MAPPED_FILE) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); windows_error = __wt_getlasterror(); ret = __wt_map_windows_error(windows_error); __wt_err(session, ret, diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 2ec28e31201..7faf19734a5 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -420,7 +420,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, if (LF_ISSET(WT_REC_EVICT) && !__wt_page_can_evict(session, ref, NULL)) { WT_PAGE_UNLOCK(session, page); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* Initialize the reconciliation structure for each new run. */ @@ -442,8 +442,9 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, if (LF_ISSET(WT_REC_EVICT)) { mod->last_eviction_id = oldest_id; #ifdef HAVE_TIMESTAMPS - __wt_txn_pinned_timestamp( - session, &mod->last_eviction_timestamp); + if (S2C(session)->txn_global.has_pinned_timestamp) + __wt_txn_pinned_timestamp( + session, &mod->last_eviction_timestamp); #endif mod->last_evict_pass_gen = S2C(session)->cache->evict_pass_gen; } @@ -651,7 +652,7 @@ __rec_write_check_complete( return (0); *lookaside_retryp = true; - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* @@ -1397,7 +1398,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, if (F_ISSET(r, WT_REC_UPDATE_RESTORE) && *updp != NULL && uncommitted) { r->leave_dirty = true; - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } if (upd->type == WT_UPDATE_BIRTHMARK) @@ -1517,9 +1518,9 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, * the WT_REC_LOOKASIDE flag. */ if (!F_ISSET(r, WT_REC_LOOKASIDE | WT_REC_UPDATE_RESTORE)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); if (uncommitted && !F_ISSET(r, WT_REC_UPDATE_RESTORE)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); WT_ASSERT(session, r->max_txn != WT_TXN_NONE); @@ -1704,7 +1705,7 @@ __rec_child_deleted(WT_SESSION_IMPL *session, * the original page and which should see the deleted page). */ if (F_ISSET(r, WT_REC_EVICT)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If there are deleted child pages we can't discard immediately, keep @@ -1789,7 +1790,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, */ WT_ASSERT(session, !F_ISSET(r, WT_REC_EVICT)); if (F_ISSET(r, WT_REC_EVICT)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If called during checkpoint, the child is being @@ -1817,7 +1818,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, if (F_ISSET(r, WT_REC_EVICT) && __wt_page_las_active(session, ref)) { WT_ASSERT(session, false); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* @@ -1842,7 +1843,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, */ WT_ASSERT(session, !F_ISSET(r, WT_REC_EVICT)); if (F_ISSET(r, WT_REC_EVICT)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If called during checkpoint, acquire a hazard pointer @@ -1877,7 +1878,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, */ WT_ASSERT(session, !F_ISSET(r, WT_REC_EVICT)); if (F_ISSET(r, WT_REC_EVICT)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); goto done; case WT_REF_SPLIT: @@ -1894,7 +1895,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, * for checkpoint. */ WT_ASSERT(session, WT_REF_SPLIT != WT_REF_SPLIT); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); WT_ILLEGAL_VALUE(session); } @@ -3704,7 +3705,7 @@ __rec_split_write(WT_SESSION_IMPL *session, WT_RECONCILE *r, * allocate a zero-length array. */ if (r->page->type != WT_PAGE_ROW_LEAF && chunk->entries == 0) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); if (F_ISSET(r, WT_REC_LOOKASIDE)) { r->cache_write_lookaside = true; diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index e880cb415c8..aba708f0e0b 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -53,7 +53,7 @@ err: __wt_free(session, config); * there was no metadata entry. */ if (ret == WT_NOTFOUND) - ret = ENOENT; + ret = __wt_set_return(session, ENOENT); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 0f8806dd462..990f2636bf9 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -28,7 +28,7 @@ __wt_schema_get_table_uri(WT_SESSION_IMPL *session, WT_ERR(__wt_session_get_dhandle(session, uri, NULL, NULL, flags)); table = (WT_TABLE *)session->dhandle; if (!ok_incomplete && !table->cg_complete) { - ret = EINVAL; + ret = __wt_set_return(session, EINVAL); WT_TRET(__wt_session_release_dhandle(session)); WT_ERR_MSG(session, ret, "'%s' cannot be used " "until all column groups are created", diff --git a/src/third_party/wiredtiger/src/schema/schema_plan.c b/src/third_party/wiredtiger/src/schema/schema_plan.c index abe76013e12..7fb2a391784 100644 --- a/src/third_party/wiredtiger/src/schema/schema_plan.c +++ b/src/third_party/wiredtiger/src/schema/schema_plan.c @@ -298,7 +298,7 @@ __find_column_format(WT_SESSION_IMPL *session, WT_TABLE *table, if (k.len == colname->len && strncmp(colname->str, k.str, k.len) == 0) { if (value_only && inkey) - return (EINVAL); + return (__wt_set_return(session, EINVAL)); return (0); } } diff --git a/src/third_party/wiredtiger/src/schema/schema_util.c b/src/third_party/wiredtiger/src/schema/schema_util.c index ceec6db6cb5..a281ec3fe12 100644 --- a/src/third_party/wiredtiger/src/schema/schema_util.c +++ b/src/third_party/wiredtiger/src/schema/schema_util.c @@ -39,7 +39,7 @@ __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) } for (i = 0; backup_list[i] != NULL; ++i) { if (strcmp(backup_list[i], name) == 0) { - ret = EBUSY; + ret = __wt_set_return(session, EBUSY); break; } } diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index caa775686cf..30399cafd22 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -140,7 +140,7 @@ __wt_session_lock_dhandle( if (!LF_ISSET(WT_DHANDLE_LOCK_ONLY) && (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || (btree != NULL && F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)))) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); ++dhandle->excl_ref; return (0); } @@ -167,7 +167,7 @@ __wt_session_lock_dhandle( * give up. */ if (btree != NULL && F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If the handle is open, get a read lock and recheck. diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 0569d0545e6..6010b310ba1 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -174,7 +174,7 @@ __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler) */ static int __eventv(WT_SESSION_IMPL *session, bool msg_event, int error, - const char *file_name, int line_number, const char *fmt, va_list ap) + const char *func_name, int line_number, const char *fmt, va_list ap) WT_GCC_FUNC_ATTRIBUTE((cold)) { struct timespec ts; @@ -231,8 +231,8 @@ __eventv(WT_SESSION_IMPL *session, bool msg_event, int error, WT_ERROR_APPEND(p, remain, ", %s", prefix); WT_ERROR_APPEND(p, remain, ": "); - if (file_name != NULL) - WT_ERROR_APPEND(p, remain, "%s, %d: ", file_name, line_number); + if (func_name != NULL) + WT_ERROR_APPEND(p, remain, "%s, %d: ", func_name, line_number); WT_ERROR_APPEND_AP(p, remain, fmt, ap); @@ -309,13 +309,14 @@ err: if (fprintf(stderr, } /* - * __wt_err -- + * __wt_err_func -- * Report an error. */ void -__wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) +__wt_err_func(WT_SESSION_IMPL *session, + int error, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) - WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { va_list ap; @@ -325,18 +326,21 @@ __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) * an error value to return. */ va_start(ap, fmt); - WT_IGNORE_RET(__eventv(session, false, error, NULL, 0, fmt, ap)); + WT_IGNORE_RET(__eventv(session, + false, error, func_name, line_number, fmt, ap)); va_end(ap); } /* - * __wt_errx -- + * __wt_errx_func -- * Report an error with no error code. */ void -__wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) +__wt_errx_func(WT_SESSION_IMPL *session, + const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) - WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { va_list ap; @@ -345,11 +349,25 @@ __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) * an error value to return. */ va_start(ap, fmt); - WT_IGNORE_RET(__eventv(session, false, 0, NULL, 0, fmt, ap)); + WT_IGNORE_RET(__eventv(session, + false, 0, func_name, line_number, fmt, ap)); va_end(ap); } /* + * __wt_set_return_func -- + * Conditionally log the source of an error code and return the error. + */ +int +__wt_set_return_func( + WT_SESSION_IMPL *session, const char* func, int line, int err) +{ + __wt_verbose(session, + WT_VERB_ERROR_RETURNS, "%s: %d Error: %d", func, line, err); + return (err); +} + +/* * __wt_ext_err_printf -- * Extension API call to print to the error stream. */ @@ -388,44 +406,28 @@ __wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) } /* - * info_msg -- + * __wt_msg -- * Informational message. */ -static int -info_msg(WT_SESSION_IMPL *session, const char *fmt, va_list ap) +int +__wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((cold)) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) { + WT_DECL_ITEM(buf); + WT_DECL_RET; WT_EVENT_HANDLER *handler; WT_SESSION *wt_session; - /* - * !!! - * SECURITY: - * Buffer placed at the end of the stack in case snprintf overflows. - */ - char s[2048]; + WT_RET(__wt_scr_alloc(session, 0, &buf)); - WT_RET(__wt_vsnprintf(s, sizeof(s), fmt, ap)); + WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, false); wt_session = (WT_SESSION *)session; handler = session->event_handler; - return (handler->handle_message(handler, wt_session, s)); -} + ret = handler->handle_message(handler, wt_session, buf->data); -/* - * __wt_msg -- - * Informational message. - */ -int -__wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) - WT_GCC_FUNC_ATTRIBUTE((cold)) - WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) -{ - WT_DECL_RET; - va_list ap; - - va_start(ap, fmt); - ret = info_msg(session, fmt, ap); - va_end(ap); + __wt_scr_free(session, &buf); return (ret); } @@ -439,16 +441,24 @@ __wt_ext_msg_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) { + WT_DECL_ITEM(buf); WT_DECL_RET; + WT_EVENT_HANDLER *handler; WT_SESSION_IMPL *session; - va_list ap; if ((session = (WT_SESSION_IMPL *)wt_session) == NULL) session = ((WT_CONNECTION_IMPL *)wt_api->conn)->default_session; - va_start(ap, fmt); - ret = info_msg(session, fmt, ap); - va_end(ap); + WT_RET(__wt_scr_alloc(session, 0, &buf)); + + WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, false); + + wt_session = (WT_SESSION *)session; + handler = session->event_handler; + ret = handler->handle_message(handler, wt_session, buf->data); + + __wt_scr_free(session, &buf); + return (ret); } @@ -487,34 +497,6 @@ __wt_progress(WT_SESSION_IMPL *session, const char *s, uint64_t v) } /* - * __wt_assert -- - * Assert and other unexpected failures, includes file/line information - * for debugging. - */ -void -__wt_assert(WT_SESSION_IMPL *session, - int error, const char *file_name, int line_number, const char *fmt, ...) - WT_GCC_FUNC_ATTRIBUTE((cold)) - WT_GCC_FUNC_ATTRIBUTE((format (printf, 5, 6))) -#ifdef HAVE_DIAGNOSTIC - WT_GCC_FUNC_ATTRIBUTE((noreturn)) -#endif - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) -{ - va_list ap; - - va_start(ap, fmt); - WT_IGNORE_RET(__eventv( - session, false, error, file_name, line_number, fmt, ap)); - va_end(ap); - -#ifdef HAVE_DIAGNOSTIC - __wt_abort(session); /* Drop core if testing. */ - /* NOTREACHED */ -#endif -} - -/* * __wt_panic -- * A standard error message when we panic. */ diff --git a/src/third_party/wiredtiger/src/support/global.c b/src/third_party/wiredtiger/src/support/global.c index d1271e0d427..f71f91a4daa 100644 --- a/src/third_party/wiredtiger/src/support/global.c +++ b/src/third_party/wiredtiger/src/support/global.c @@ -117,7 +117,7 @@ __wt_global_once(void) return; } - __wt_checksum_init(); + __wt_process.checksum = wiredtiger_crc32c_func(); __global_calibrate_ticks(); TAILQ_INIT(&__wt_process.connqh); diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index 815c876f444..4116638e31c 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -401,6 +401,29 @@ __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref) #ifdef HAVE_DIAGNOSTIC /* + * __wt_hazard_check_assert -- + * Assert there's no hazard pointer to the page. + */ +bool +__wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor) +{ + WT_HAZARD *hp; + int i; + + for (i = 0;;) { + if ((hp = __wt_hazard_check(session, ref)) == NULL) + return (true); + if (!waitfor || ++i > 100) + break; + __wt_sleep(0, 10000); + } + __wt_errx(session, + "hazard pointer reference to discarded object: (%p: %s, line %d)", + (void *)hp->ref, hp->file, hp->line); + return (false); +} + +/* * __hazard_dump -- * Display the list of hazard pointers. */ diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c index fd66a1a40bb..959405dee50 100644 --- a/src/third_party/wiredtiger/src/support/mtx_rw.c +++ b/src/third_party/wiredtiger/src/support/mtx_rw.c @@ -137,7 +137,7 @@ __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) /* This read lock can only be granted if there are no active writers. */ if (old.u.s.current != old.u.s.next) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * The replacement lock value is a result of adding an active reader. @@ -146,7 +146,7 @@ __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) */ new.u.v = old.u.v; if (++new.u.s.readers_active == 0) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* We rely on this atomic operation to provide a barrier. */ return (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY); @@ -331,7 +331,7 @@ __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) */ old.u.v = l->u.v; if (old.u.s.current != old.u.s.next || old.u.s.readers_active != 0) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * We've checked above that there is no writer active (since diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index 2ead79a1c1c..72e312da325 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -71,30 +71,9 @@ __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { - WT_DECL_RET; - size_t len; - va_list ap; - - for (;;) { - va_start(ap, fmt); - ret = __wt_vsnprintf_len_set( - buf->mem, buf->memsize, &len, fmt, ap); - va_end(ap); - WT_RET(ret); - - /* Check if there was enough space. */ - if (len < buf->memsize) { - buf->data = buf->mem; - buf->size = len; - return (0); - } + WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, false); - /* - * If not, double the size of the buffer: we're dealing with - * strings, and we don't expect these numbers to get huge. - */ - WT_RET(__wt_buf_extend(session, buf, len + 1)); - } + return (0); } /* @@ -106,11 +85,6 @@ __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { - WT_DECL_RET; - size_t len, space; - char *p; - va_list ap; - /* * If we're appending data to an existing buffer, any data field should * point into the allocated memory. (It wouldn't be insane to copy any @@ -119,27 +93,9 @@ __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) */ WT_ASSERT(session, buf->data == NULL || WT_DATA_IN_ITEM(buf)); - for (;;) { - va_start(ap, fmt); - p = (char *)((uint8_t *)buf->mem + buf->size); - WT_ASSERT(session, buf->memsize >= buf->size); - space = buf->memsize - buf->size; - ret = __wt_vsnprintf_len_set(p, space, &len, fmt, ap); - va_end(ap); - WT_RET(ret); - - /* Check if there was enough space. */ - if (len < space) { - buf->size += len; - return (0); - } + WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, true); - /* - * If not, double the size of the buffer: we're dealing with - * strings, and we don't expect these numbers to get huge. - */ - WT_RET(__wt_buf_extend(session, buf, buf->size + len + 1)); - } + return (0); } /* @@ -400,16 +356,19 @@ __wt_scr_discard(WT_SESSION_IMPL *session) if (*bufp == NULL) continue; if (F_ISSET(*bufp, WT_ITEM_INUSE)) +#ifdef HAVE_DIAGNOSTIC __wt_errx(session, "scratch buffer allocated and never discarded" -#ifdef HAVE_DIAGNOSTIC ": %s: %d", session-> scratch_track[bufp - session->scratch].file, session-> scratch_track[bufp - session->scratch].line -#endif ); +#else + __wt_errx(session, + "scratch buffer allocated and never discarded"); +#endif __wt_buf_free(session, *bufp); __wt_free(session, *bufp); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index ad8351923a0..59d9fa2e7c1 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1896,7 +1896,7 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) if (btree->modified && !bulk && S2C(session)->txn_global.has_stable_timestamp && !__wt_btree_immediately_durable(session)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * Turn on metadata tracking if: diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index e01db53fda9..ba00198adbe 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -466,7 +466,8 @@ __wt_txn_rollback_to_stable(WT_SESSION_IMPL *session, const char *cfg[]) * trees in cache populates a list that is used to check which * lookaside records should be removed. */ - WT_ERR(__txn_rollback_to_stable_lookaside_fixup(session)); + if (!F_ISSET(conn, WT_CONN_IN_MEMORY)) + WT_ERR(__txn_rollback_to_stable_lookaside_fixup(session)); err: F_CLR(conn, WT_CONN_EVICTION_NO_LOOKASIDE); __wt_free(session, conn->stable_rollback_bitstring); diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 2d08c4c5274..7042736e2a2 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -11,7 +11,7 @@ const char *home = "."; /* Home directory */ const char *progname; /* Program name */ /* Global arguments */ -const char *usage_prefix = "[-LRVv] [-C config] [-E secretkey] [-h home]"; +const char *usage_prefix = "[-LRSVv] [-C config] [-E secretkey] [-h home]"; bool verbose = false; /* Verbose flag */ static const char *command; /* Command name */ @@ -19,6 +19,7 @@ static const char *command; /* Command name */ #define REC_ERROR "log=(recover=error)" #define REC_LOGOFF "log=(enabled=false)" #define REC_RECOVER "log=(recover=on)" +#define REC_SALVAGE "log=(recover=salvage)" static void usage(void) @@ -70,7 +71,7 @@ main(int argc, char *argv[]) int ch, major_v, minor_v, tret, (*func)(WT_SESSION *, int, char *[]); const char *cmd_config, *config, *p1, *p2, *p3, *rec_config; char *p, *secretkey; - bool logoff, needconn, recover; + bool logoff, needconn, recover, salvage; conn = NULL; p = NULL; @@ -105,9 +106,9 @@ main(int argc, char *argv[]) * needed, the user can specify -R to run recovery. */ rec_config = REC_ERROR; - logoff = recover = false; + logoff = recover = salvage = false; /* Check for standard options. */ - while ((ch = __wt_getopt(progname, argc, argv, "C:E:h:LRVv")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "C:E:h:LRSVv")) != EOF) switch (ch) { case 'C': /* wiredtiger_open config */ cmd_config = __wt_optarg; @@ -131,6 +132,10 @@ main(int argc, char *argv[]) rec_config = REC_RECOVER; recover = true; break; + case 'S': /* salvage */ + rec_config = REC_SALVAGE; + salvage = true; + break; case 'V': /* version */ printf("%s\n", wiredtiger_version(NULL, NULL, NULL)); goto done; @@ -142,8 +147,9 @@ main(int argc, char *argv[]) usage(); goto err; } - if (logoff && recover) { - fprintf(stderr, "Only one of -L and -R is allowed.\n"); + if ((logoff && recover) || (logoff && salvage) || + (recover && salvage)) { + fprintf(stderr, "Only one of -L, -R, and -S is allowed.\n"); goto err; } argc -= __wt_optind; diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 8a1781eae45..11cabf4e9db 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -210,7 +210,7 @@ thread_ckpt_run(void *arg) session, "use_timestamp=true")); testutil_check(td->conn->query_timestamp( td->conn, buf, "get=last_checkpoint")); - sscanf(buf, "%" SCNx64, &stable); + testutil_assert(sscanf(buf, "%" SCNx64, &stable) == 1); printf("Checkpoint %d complete at stable %" PRIu64 ".\n", i, stable); fflush(stdout); @@ -728,7 +728,7 @@ main(int argc, char *argv[]) if (use_ts) { testutil_check( conn->query_timestamp(conn, buf, "get=recovery")); - sscanf(buf, "%" SCNx64, &stable_val); + testutil_assert(sscanf(buf, "%" SCNx64, &stable_val) == 1); printf("Got stable_val %" PRIu64 "\n", stable_val); } diff --git a/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c b/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c index 9a786e5d222..fff7b55bbf9 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c @@ -44,65 +44,51 @@ static void run(void) { size_t len; - uint32_t crc32c; + uint32_t crc32c, (*func)(const void *, size_t); uint8_t *data; /* Allocate aligned memory for the data. */ data = dcalloc(100, sizeof(uint8_t)); + /* Get a pointer to the CRC32C function. */ + func = wiredtiger_crc32c_func(); + /* * Some simple known checksums. */ len = 1; - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0x527d5351, len, "nul x1"); len = 2; - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0xf16177d2, len, "nul x2"); len = 3; - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0x6064a37a, len, "nul x3"); len = 4; - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0x48674bc7, len, "nul x4"); len = strlen("123456789"); memcpy(data, "123456789", len); - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0xe3069283, len, "known string #1"); len = strlen("The quick brown fox jumps over the lazy dog"); memcpy(data, "The quick brown fox jumps over the lazy dog", len); - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0x22620404, len, "known string #2"); free(data); } int -main(int argc, char *argv[]) +main(void) { - TEST_OPTS *opts, _opts; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - /* - * The external API should work before the library configures itself, - * run before and after calling wiredtiger_open(). - */ - run(); - - testutil_check( - wiredtiger_open(opts->home, NULL, "create", &opts->conn)); - run(); - testutil_cleanup(opts); return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index c398c1a96b2..70e8165e97d 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -370,10 +370,6 @@ static CONFIG c[] = { "stress splits (#8)", /* 2% */ C_BOOL, 2, 0, 0, &g.c_timing_stress_split_8, NULL }, - { "timing_stress_split_9", - "stress splits (#9)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_9, NULL }, - { "transaction_timestamps", /* 10% */ "enable transaction timestamp support", C_BOOL, 10, 0, 0, &g.c_txn_timestamps, NULL }, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 1406d2b3fb5..83b75f70f26 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -222,7 +222,6 @@ typedef struct { uint32_t c_timing_stress_split_6; uint32_t c_timing_stress_split_7; uint32_t c_timing_stress_split_8; - uint32_t c_timing_stress_split_9; uint32_t c_truncate; uint32_t c_txn_freq; uint32_t c_txn_timestamps; diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 8040142aa19..6452f74384d 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -264,8 +264,6 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) CONFIG_APPEND(p, ",split_7"); if (g.c_timing_stress_split_8) CONFIG_APPEND(p, ",split_8"); - if (g.c_timing_stress_split_9) - CONFIG_APPEND(p, ",split_9"); CONFIG_APPEND(p, "]"); /* Extensions. */ diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index 9c8a90d37b9..895a31620d9 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -670,7 +670,7 @@ empty(int cnt) if (page_type == WT_PAGE_COL_FIX) for (i = 0; i < cnt; ++i) - fputs("\\00\n", res_fp); + CHECK(fputs("\\00\n", res_fp)); } /* diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 6b668ad3e07..74fa259c3c4 100644 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -238,7 +238,7 @@ if __name__ == '__main__': tests = unittest.TestSuite() # Turn numbers and ranges into test module names - preserve = timestamp = debug = dryRun = gdbSub = longtest = False + preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = False parallel = 0 configfile = None configwrite = False @@ -269,6 +269,9 @@ if __name__ == '__main__': if option == '-gdb' or option == 'g': gdbSub = True continue + if option == '-lldb': + lldbSub = True + continue if option == '-help' or option == 'h': usage() sys.exit(0) @@ -323,7 +326,7 @@ if __name__ == '__main__': # All global variables should be set before any test classes are loaded. # That way, verbose printing can be done at the class definition level. - wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, + wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, lldbSub, verbose, wt_builddir, dirarg, longtest) diff --git a/src/third_party/wiredtiger/test/suite/suite_subprocess.py b/src/third_party/wiredtiger/test/suite/suite_subprocess.py index c2e9d99f691..4b0f6823e06 100644..100755 --- a/src/third_party/wiredtiger/test/suite/suite_subprocess.py +++ b/src/third_party/wiredtiger/test/suite/suite_subprocess.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, subprocess, sys +import os, re, subprocess, sys from run import wt_builddir from wttest import WiredTigerTestCase @@ -79,23 +79,51 @@ class suite_subprocess: print '********************************' self.fail('ERROR found in output file: ' + filename) + # If the string is of the form '/.../', then return just the embedded + # pattern, otherwise, return None + def convert_to_pattern(self, s): + if len(s) >= 2 and s[0] == '/' and s[-1] == '/': + return s[1:-1] + else: + return None + def check_file_content(self, filename, expect): with open(filename, 'r') as f: got = f.read(len(expect) + 100) self.assertEqual(got, expect, filename + ': does not contain expected:\n\'' + expect + '\', but contains:\n\'' + got + '\'.') - def check_file_contains(self, filename, expect): + def check_file_contains_one_of(self, filename, expectlist): """ Check that the file contains the expected string in the first 100K bytes """ maxbytes = 1024*100 with open(filename, 'r') as f: got = f.read(maxbytes) - if not (expect in got): + found = False + for expect in expectlist: + pat = self.convert_to_pattern(expect) + if pat == None: + if expect in got: + found = True + break + else: + if re.search(pat, got): + found = True + break + if not found: + if len(expectlist) == 1: + expect = '\'' + expectlist[0] + '\'' + else: + expect = str(expectlist) + gotstr = '\'' + \ + (got if len(got) < 1000 else (got[0:1000] + '...')) + '\'' if len(got) >= maxbytes: - self.fail(filename + ': does not contain expected \'' + expect + '\', or output is too large') + self.fail(filename + ': does not contain expected ' + expect + ', or output is too large, got ' + gotstr) else: - self.fail(filename + ': does not contain expected \'' + expect + '\'') + self.fail(filename + ': does not contain expected ' + expect + ', got ' + gotstr) + + def check_file_contains(self, filename, expect): + self.check_file_contains_one_of(filename, [expect]) def check_empty_file(self, filename): """ @@ -165,6 +193,8 @@ class suite_subprocess: procargs = [ wtexe ] if self._gdbSubprocess: procargs = [ "gdb", "--args" ] + procargs + elif self._lldbSubprocess: + procargs = [ "lldb", "--" ] + procargs procargs.extend(args) if self._gdbSubprocess: infilepart = "" @@ -177,6 +207,17 @@ class suite_subprocess: ">" + wtoutname + " 2>" + wterrname print "*********************************************" returncode = subprocess.call(procargs) + elif self._lldbSubprocess: + infilepart = "" + if infilename != None: + infilepart = "<" + infilename + " " + print str(procargs) + print "*********************************************" + print "**** Run 'wt' via: run " + \ + " ".join(procargs[3:]) + infilepart + \ + ">" + wtoutname + " 2>" + wterrname + print "*********************************************" + returncode = subprocess.call(procargs) elif infilename: with open(infilename, "r") as wtin: returncode = subprocess.call( diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py new file mode 100755 index 00000000000..79b040a4cec --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_txn19.py +# Transactions: test recovery with corrupted log files +# + +import fnmatch, os, shutil, time +from wtscenario import make_scenarios +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# This test uses an artificially small log file limit, and creates +# large records so two fit into a log file. This allows us to test +# both the case when corruption happens at the beginning of a log file +# (an even number of records have been created), and when corruption +# happens in the middle of a log file (with an odd number of records). + +def corrupt(fname, truncate, offset, writeit): + with open(fname, 'r+') as log: + if truncate: + log.truncate() + if offset: + if offset < 0: # Negative offset means seek to the end + log.seek(0, 2) + else: + log.seek(offset) + if writeit: + log.write(writeit) + +class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): + base_config = 'log=(archive=false,enabled,file_max=100K),' + \ + 'transaction_sync=(enabled,method=none)' + conn_config = base_config + corruption_type = [ + ('removal', dict(kind='removal', f=lambda fname: + os.remove(fname))), + ('truncate', dict(kind='truncate', f=lambda fname: + corrupt(fname, True, 0, None))), + ('zero-begin', dict(kind='zero', f=lambda fname: + corrupt(fname, False, 0, '\0' * 4096))), + ('zero-trunc', dict(kind='zero', f=lambda fname: + corrupt(fname, True, 0, '\0' * 4096))), + ('zero-end', dict(kind='zero-end', f=lambda fname: + corrupt(fname, False, -1, '\0' * 4096))), + ('garbage-begin', dict(kind='garbage-begin', f=lambda fname: + corrupt(fname, False, 0, 'Bad!' * 1024))), + ('garbage-middle', dict(kind='garbage-middle', f=lambda fname: + corrupt(fname, False, 1024 * 25, 'Bad!' * 1024))), + ('garbage-end', dict(kind='garbage-end', f=lambda fname: + corrupt(fname, False, -1, 'Bad!' * 1024))), + ] + # The list comprehension below expands each entry in the integer tuple + # list to a scenario. For example, (3, 4, 2) expands to: + # ('corrupt=[3,4],checkpoint=2', dict(corruptpos=3, corruptpos2=4, chkpt=2)) + # + # Each number corresponds to a log file, so for this example, we have + # corruption in log file 3 (using the style of corruption from + # corruption_type), there is a second corruption in log file 4, + # and there is a checkpoint in log file 2. A value of 0 means no + # corruption or checkpoint. + corruption_pos = [ + ('corrupt=[' + str(x) + ',' + str(y) + '],checkpoint=' + str(z), + dict(corruptpos=x,corruptpos2=y,chkpt=z)) for (x,y,z) in ( + (0, 0, 0), (0, 0, 2), (6, 0, 0), (6, 0, 3), (3, 0, 0), + (3, 0, 2), (3, 4, 2), (3, 5, 2), (3, 0, 4))] + nrecords = [('nrecords=10', dict(nrecords=10)), + ('nrecords=11', dict(nrecords=11))] + + # This function prunes out unnecessary or problematic test cases + # from the list of scenarios. + def includeFunc(name, dictarg): + kind = dictarg['kind'] + corruptpos = dictarg['corruptpos'] + chkpt = dictarg['chkpt'] + + # corruptpos == 0 indicates there is no corruption. + # (i.e. corrupt log file 0, which doesn't exist) + # We do want to test the case of no corruption, but we don't + # need to try it against every type of corruption, only one. + if corruptpos == 0: + return kind == 'removal' + + # NOTE: + # The removal or truncation of a middle log file (not first or last) + # that would be used in recovery is not currently handled gracefully. + if (kind == 'removal' or kind == 'truncate') and \ + corruptpos != 6 and corruptpos > chkpt: + return False + + # All the other cases are valid + return True + + scenarios = make_scenarios( + corruption_type, corruption_pos, nrecords, + include=includeFunc, prune=20, prunelong=1000) + + uri = 'table:test_txn19' + create_params = 'key_format=i,value_format=S' + + # Return the log file number that contains the given record + # number. In this test, two records fit into each log file, and + # after each even record is written, a new log file is created + # (having no records initially). The last log file is this + # (nrecords/2 + 1), given that we start with log 1. + def record_to_logfile(self, recordnum): + return recordnum / 2 + 1 + + # Returns the first record number in a log file. + def logfile_to_record(self, logfile): + return (logfile - 1) * 2 + + # Return true if the log file is corrupted. + # If not corrupted, the log file will produce no errors, + # and all the records originally written should be recovered. + def corrupted(self): + # Corruptpos == 0 means to do no corruption in any log file + if self.corruptpos == 0: + return False + + # Adding zeroes to the end of a log file is indistinguishable + # from having a log file that is preallocated that has not been + # totally filled. One might argue that if this does not occur + # in the final log file, it could/should have been truncated. + # At any rate, we consider this particular corruption to be benign. + if self.kind == 'zero-end': + return False + + # NOTE: + # If garbage is added to an otherwise valid log file, the invalid + # portion is effectively 'skipped over'. + if self.kind == 'garbage-end': + return False + return True + + def show_logs(self, homedir, msg): + loglist = [] + for i in range(0, 10): + basename = 'WiredTigerLog.000000000' + str(i) + fullname = homedir + os.sep + basename + if os.path.isfile(fullname): + loglist.append(i) + if os.stat(fullname).st_size == 0: + self.tty('LOGS ' + msg + ': ' + str(i) + ' is empty') + self.tty('LOGS ' + msg + ': ' + str(loglist)) + + def copy_for_crash_restart(self, olddir, newdir): + ''' Simulate a crash from olddir and restart in newdir. ''' + # with the connection still open, copy files to new directory + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + # Skip lock file on Windows since it is locked + if os.path.isfile(fullname) and \ + "WiredTiger.lock" not in fullname and \ + "Tmplog" not in fullname and \ + "Preplog" not in fullname: + shutil.copy(fullname, newdir) + + # Generate a value that is a bit over half the size of the log file. + def valuegen(self, i): + return str(i) + 'A' * (1024 * 60) # ~60K + + # Insert a list of keys + def inserts(self, keylist): + c = self.session.open_cursor(self.uri) + for i in keylist: + if self.chkpt > 0 and self.logfile_to_record(self.chkpt) == i: + self.session.checkpoint() + c[i] = self.valuegen(i) + c.close() + + def checks(self, expectlist): + c = self.session.open_cursor(self.uri, None, None) + gotlist = [] + for key, value in c: + gotlist.append(key) + self.assertEqual(self.valuegen(key), value) + self.assertEqual(expectlist, gotlist) + c.close() + + def log_number_to_file_name(self, homedir, n): + self.assertLess(n, 10) # assuming 1 digit + return homedir + os.sep + 'WiredTigerLog.000000000' + str(n) + + def corrupt_log(self, homedir): + if not self.corrupted(): + return + self.f(self.log_number_to_file_name(homedir, self.corruptpos)) + + # Corrupt a second log file if needed + if self.corruptpos2 != 0: + self.f(self.log_number_to_file_name(homedir, self.corruptpos2)) + + # Return true iff the log has been damaged in a way that + # is not detected as a corruption. + # This includes: + # - removal of the last log + # - corruption in the middle of a log file. + def log_corrupt_but_valid(self): + # NOTE: + # Currently, corruption in the middle of a log file is not yet + # detected. + if self.kind == 'garbage-middle': + return True + if self.corruptpos == self.record_to_logfile(self.nrecords): + if self.kind == 'removal': + return True + return False + + # For this test, at least, salvage identifies and fixes all + # recovery failures. + def expect_salvage_messages(self): + return self.expect_recovery_failure() + + def expect_recovery_failure(self): + return self.corrupted() and \ + self.corruptpos >= self.chkpt and \ + not self.log_corrupt_but_valid() + + def recovered_records(self): + if not self.corrupted() or self.chkpt > self.corruptpos: + return self.nrecords + return self.logfile_to_record(self.corruptpos) + + def test_corrupt_log(self): + ''' Corrupt the log and restart with different kinds of recovery ''' + + # This test creates some data, then simulates a crash with corruption. + # Then does a restart with recovery, then starts again with salvage, + # and finally starts again with recovery (adding new records). + + self.session.create(self.uri, self.create_params) + self.inserts([x for x in range(0, self.nrecords)]) + newdir = "RESTART" + self.copy_for_crash_restart(self.home, newdir) + self.close_conn() + #self.show_logs(newdir, 'before corruption') + self.corrupt_log(newdir) + #self.show_logs(newdir, 'after corruption') + salvage_config = self.base_config + ',log=(recover=salvage)' + errfile = 'list.err' + outfile = 'list.out' + expect_fail = self.expect_recovery_failure() + + # In cases of corruption, we cannot always call wiredtiger_open + # directly, because there may be a panic, and abort() is called + # in diagnostic mode which terminates the Python interpreter. + # + # Running any wt command externally to Python allows + # us to observe the failure or success safely. + # Use -R to force recover=on, which is the default for + # wiredtiger_open, (wt utilities normally have recover=error) + self.runWt(['-h', newdir, '-C', self.base_config, '-R', 'list'], + errfilename=errfile, outfilename=outfile, failure=expect_fail, + closeconn=False) + + if expect_fail: + self.check_file_contains_one_of(errfile, + ['/log file.*corrupted/', 'WT_ERROR: non-specific WiredTiger error']) + else: + self.check_empty_file(errfile) + self.check_file_contains(outfile, self.uri) + + found_records = self.recovered_records() + expect = [x for x in range(0, found_records)] + + # If we are salvaging, expect an informational message + if self.expect_salvage_messages(): + errpat = '.*' + # Possible messages: + # salvage: log files x-y truncated at beginning + # salvage: log file x truncated at beginning + # salvage: log file x truncated + # salvage: log file x removed + outpat = 'salvage: log file' + else: + errpat = '^$' + outpat = '^$' + with self.expectedStdoutPattern(outpat): + with self.expectedStderrPattern(errpat): + self.conn = self.wiredtiger_open(newdir, salvage_config) + self.session = self.setUpSessionOpen(self.conn) + self.checks(expect) + + # Insert a couple more and simulate another crash. + newdir2 = "RESTART2" + self.inserts([self.nrecords, self.nrecords + 1]) + expect.extend([self.nrecords, self.nrecords + 1]) + self.copy_for_crash_restart(newdir, newdir2) + self.checks(expect) + self.reopen_conn(newdir) + self.checks(expect) + self.reopen_conn(newdir2, self.conn_config) + self.checks(expect) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wtscenario.py b/src/third_party/wiredtiger/test/suite/wtscenario.py index 76824e428df..d953b7f627f 100644 --- a/src/third_party/wiredtiger/test/suite/wtscenario.py +++ b/src/third_party/wiredtiger/test/suite/wtscenario.py @@ -68,6 +68,9 @@ def make_scenarios(*args, **kwargs): """ The standard way to create scenarios for WT tests. Scenarios can be combined by listing them all as arguments. + If some scenario combinations should not be included, + a include= argument function may be listed, which given a name and + dictionary argument, returns True if the scenario should be included. A final prune= and/or prunelong= argument may be given that forces the list of entries in the scenario to be pruned. The result is a (combined) scenario that has been checked @@ -76,14 +79,19 @@ def make_scenarios(*args, **kwargs): scenes = multiply_scenarios('.', *args) pruneval = None prunelong = None + includefunc = None for key in kwargs: if key == 'prune': pruneval = kwargs[key] elif key == 'prunelong': prunelong = kwargs[key] + elif key == 'include': + includefunc = kwargs[key] else: raise AssertionError( 'make_scenarios: unexpected named arg: ' + key) + if includefunc: + scenes = [(name, d) for (name, d) in scenes if includefunc(name, d)] if pruneval != None or prunelong != None: pruneval = pruneval if pruneval != None else -1 prunelong = prunelong if prunelong != None else -1 diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 8c0915bc9c7..1a2fddce031 100644 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -101,7 +101,7 @@ class CapturedFd(object): ' unexpected ' + self.desc + ', contains:\n"' + contents + '"') testcase.fail('unexpected ' + self.desc + ', contains: "' + - shortenWithEllipsis(contents,100) + '"') + contents + '"') self.expectpos = filesize def checkAdditional(self, testcase, expect): @@ -180,7 +180,7 @@ class WiredTigerTestCase(unittest.TestCase): @staticmethod def globalSetup(preserveFiles = False, useTimestamp = False, - gdbSub = False, verbose = 1, builddir = None, dirarg = None, + gdbSub = False, lldbSub = False, verbose = 1, builddir = None, dirarg = None, longtest = False): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg @@ -194,6 +194,7 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._origcwd = os.getcwd() WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 0) # unbuffered WiredTigerTestCase._gdbSubprocess = gdbSub + WiredTigerTestCase._lldbSubprocess = lldbSub WiredTigerTestCase._longtest = longtest WiredTigerTestCase._verbose = verbose WiredTigerTestCase._dupout = os.dup(sys.stdout.fileno()) @@ -339,20 +340,25 @@ class WiredTigerTestCase(unittest.TestCase): self.conn.close(config) self.conn = None - def open_conn(self, directory="."): + def open_conn(self, directory=".", config=None): """ Open the connection if already closed. """ if self.conn == None: + if config != None: + self._old_config = self.conn_config + self.conn_config = config self.conn = self.setUpConnectionOpen(directory) + if config != None: + self.conn_config = self._old_config self.session = self.setUpSessionOpen(self.conn) - def reopen_conn(self, directory="."): + def reopen_conn(self, directory=".", config=None): """ Reopen the connection. """ self.close_conn() - self.open_conn(directory) + self.open_conn(directory, config) def setUp(self): if not hasattr(self.__class__, 'wt_ntests'): |