diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2020-09-09 11:37:44 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-09-09 01:52:32 +0000 |
| commit | 0f0eb111c9dbd5dd4ca86529aee81bad59c6579b (patch) | |
| tree | 94f3efa909755fcc8b381bf68214ebb9bb326964 /src | |
| parent | 103b545528e7cf05537bd68bda4bc8235bf6fefa (diff) | |
| download | mongo-0f0eb111c9dbd5dd4ca86529aee81bad59c6579b.tar.gz | |
Import wiredtiger: 6a7db4f96fe828fdb1b9a31d83460c0573a2c3b1 from branch mongodb-4.4
ref: 579966149b..6a7db4f96f
for: 4.4.2
WT-5144 Use wt_clock instead of wt_epoch in perf programs
WT-5585 Remove cache_overflow config option
WT-5693 Enable test_wt4105_large_doc_small_upd
WT-5940 Migrate to Clang Format 10
WT-6000 Enhance incremental backup testing in format to support restart
WT-6006 Revert test binaries to 10 in checkpoint-stress-test
WT-6027 Fix docs spelling errors and warnings
WT-6322 Split full compatibility tests into smaller groups
WT-6390 Extend compact02 timeout from 8 => 10 minutes
WT-6404 Add timing stress that delays checkpoint after it calls __wt_txn_begin
WT-6451 Do not evict clean metadata pages if needed for historic reads
WT-6458 read row-store leaf pages with prepared updates in a single pass
WT-6463 History store operations should honor cache size
WT-6471 Avoid the error message for non-existent clang-format binary
WT-6472 Update timestamp_abort test cache configuration
WT-6478 Cursor cache statistics not getting incremented
WT-6505 Add debugging for missing file failure
WT-6507 Exit cache eviction worker after our operation has timed out
WT-6526 Fix assertion failure when opening DB in readonly mode after unclean shutdown
WT-6532 Consider update structure overhead in split length calculation
WT-6542 Add an assert to ensure we are not unintentionally returning empty values
WT-6544 Onpage value not appended to the tombstone restored from the data or history store
WT-6556 Fix internal sessions to use internal session close function than public API to avoid memory leak
WT-6559 Use the session id from the new session to determine statistics bucket
WT-6560 Fix usage of global salvage in WT utility
WT-6561 Provide MongoDB configuration in the wt utility usage output
WT-6569 Squash the prepared updates into a single update before writing it to data store
WT-6570 RTS to remove the left over updates in the history store without stop timestamp
WT-6571 Lseek cannot use error_sys_check because it does not return an int
WT-6577 History store dump outputs confusing time window
WT-6578 Prevent reconciliation from looking past the on-disk value
WT-6581 Fix class name in test_hs15
WT-6585 Panic if updates that are older than the updates in history store are inserted to history store
WT-6586 Tombstone inserted to history store should also be flagged as WT_UPDATE_HS
WT-6589 Fix disabled cursor cache python tests
WT-6591 Stop checkpoint thread before closing connection in Python tests
WT-6593 Retry conflicting operations in test_rollback_to_stable10
WT-6596 Increase cache for timestamp abort test and separate key spaces for all abort tests
WT-6598 Add new API allowing changing dhandle hash bucket size
WT-6602 Allow operation timeout ms to be passed to commit and rollback
WT-6604 Fix typo in the comment descibing WT_CELL structure
WT-6610 Fix incremental backup checkpoint parsing to handle upgrades
WT-6611 Revert enhancement allowing rename and incremental backup
WT-6613 Add python test for early_load flag
WT-6615 Initialize last_upd where it is actually used
WT-6619 Eliminate possibility of infinite loop in test_cursor13.py
WT-6623 Set the connection level file id in recovery file scan
WT-6625 Remove outdated TODO
WT-6635 Disable mix and column filetype test
WT-6640 Coverity: Failure to restore saved dhandle
WT-6641 Coverity: Unused value
Diffstat (limited to 'src')
240 files changed, 5579 insertions, 3612 deletions
diff --git a/src/third_party/wiredtiger/.clang-format b/src/third_party/wiredtiger/.clang-format index 79eb168c0c0..51cc1257cbb 100644 --- a/src/third_party/wiredtiger/.clang-format +++ b/src/third_party/wiredtiger/.clang-format @@ -3,24 +3,29 @@ Language: Cpp # BasedOnStyle: LLVM AccessModifierOffset: 0 AlignAfterOpenBracket: DontAlign +AlignConsecutiveMacros: false AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false -AlignEscapedNewlinesLeft: true +AlignEscapedNewlines: Left AlignOperands: false AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true AllowAllParametersOfDeclarationOnNextLine: true -AllowShortBlocksOnASingleLine: false +AllowShortBlocksOnASingleLine: Never AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: Empty -AllowShortIfStatementsOnASingleLine: false +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never AllowShortLoopsOnASingleLine: false AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: AllDefinitions AlwaysBreakBeforeMultilineStrings: true -AlwaysBreakTemplateDeclarations: false +AlwaysBreakTemplateDeclarations: MultiLine BinPackArguments: true BinPackParameters: true BraceWrapping: + AfterCaseLabel: false AfterClass: false AfterControlStatement: false AfterEnum: false @@ -29,22 +34,34 @@ BraceWrapping: AfterObjCDeclaration: false AfterStruct: false AfterUnion: false + AfterExternBlock: false BeforeCatch: false BeforeElse: false IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true BreakBeforeBinaryOperators: None BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: false BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true ColumnLimit: 100 CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false ConstructorInitializerAllOnOneLineOrOnePerLine: false ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 2 Cpp11BracedListStyle: true +DeriveLineEnding: false DerivePointerAlignment: false DisableFormat: false ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true ForEachMacros: - Q_FOREACH - BOOST_FOREACH @@ -62,44 +79,71 @@ ForEachMacros: - WT_MODIFY_FOREACH_REVERSE - WT_ROW_FOREACH - WT_SKIP_FOREACH +IncludeBlocks: Preserve IncludeCategories: - Regex: '^"(llvm|llvm-c|clang|clang-c)/' Priority: 2 - - Regex: '^(<|"(gtest|isl|json)/)' + SortPriority: 0 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' Priority: 3 + SortPriority: 0 - Regex: '.*' Priority: 1 + SortPriority: 0 +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' IndentCaseLabels: false +IndentGotoLabels: false +IndentPPDirectives: None IndentWidth: 4 IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: true MacroBlockBegin: '^WT_PACKED_STRUCT_BEGIN$' MacroBlockEnd: '^WT_PACKED_STRUCT_END$' MaxEmptyLinesToKeep: 1 NamespaceIndentation: None +ObjCBinPackProtocolList: Auto ObjCBlockIndentWidth: 2 ObjCSpaceAfterProperty: false ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 PenaltyBreakBeforeFirstCallParameter: 19 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 60 PointerAlignment: Right ReflowComments: true SortIncludes: false +SortUsingDeclarations: true SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 1 SpacesInAngles: false +SpacesInConditionalStatement: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false -Standard: Cpp11 +SpaceBeforeSquareBrackets: false +Standard: Latest +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION TabWidth: 8 +UseCRLF: false UseTab: Never ... diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx index 739b50ebb27..e8ee0298007 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx +++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx @@ -841,9 +841,9 @@ int ThreadRunner::op_run(Operation *op) { (track->ops % _workload->options.sample_rate == 0); VERBOSE(*this, "OP " << op->_optype << " " << op->_table._uri.c_str() << ", recno=" << recno); - timespec start; + uint64_t start; if (measure_latency) - workgen_epoch(&start); + workgen_clock(&start); // Whether or not we are measuring latency, we track how many operations // are in progress, or that complete. @@ -929,15 +929,15 @@ int ThreadRunner::op_run(Operation *op) { } if (measure_latency) { - timespec stop; - workgen_epoch(&stop); - track->complete_with_latency(ts_us(stop - start)); + uint64_t stop; + workgen_clock(&stop); + track->complete_with_latency(ns_to_us(stop - start)); } else if (track != NULL) track->complete(); if (op->_group != NULL) { uint64_t endtime = 0; - timespec now; + uint64_t now; if (op->_timed != 0.0) endtime = _op_time_us + secs_us(op->_timed); @@ -951,8 +951,8 @@ int ThreadRunner::op_run(Operation *op) { i != op->_group->end(); i++) WT_ERR(op_run(&*i)); } - workgen_epoch(&now); - } while (!_stop && ts_us(now) < endtime); + workgen_clock(&now); + } while (!_stop && ns_to_us(now) < endtime); if (op->_timed != 0.0) _op_time_us = endtime; @@ -1487,14 +1487,13 @@ void SleepOperationInternal::parse_config(const std::string &config) int SleepOperationInternal::run(ThreadRunner *runner, WT_SESSION *session) { uint64_t endtime; - timespec now; - uint64_t now_us; + uint64_t now, now_us; (void)runner; /* not used */ (void)session; /* not used */ - workgen_epoch(&now); - now_us = ts_us(now); + workgen_clock(&now); + now_us = ns_to_us(now); if (runner->_thread->options.synchronized) endtime = runner->_op_time_us + secs_us(_sleepvalue); else @@ -1509,8 +1508,8 @@ int SleepOperationInternal::run(ThreadRunner *runner, WT_SESSION *session) else usleep(sleep_us); - workgen_epoch(&now); - now_us = ts_us(now); + workgen_clock(&now); + now_us = ns_to_us(now); } return (0); } diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_func.c b/src/third_party/wiredtiger/bench/workgen/workgen_func.c index 14fa5f64d94..88ca4c597d1 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen_func.c +++ b/src/third_party/wiredtiger/bench/workgen/workgen_func.c @@ -50,6 +50,12 @@ workgen_atomic_add64(uint64_t *vp, uint64_t v) } void +workgen_clock(uint64_t *clockp) +{ + *clockp = __wt_clock(NULL); +} + +void workgen_epoch(struct timespec *tsp) { __wt_epoch(NULL, tsp); diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_func.h b/src/third_party/wiredtiger/bench/workgen/workgen_func.h index 7d303d0e4e9..9132bcc9358 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen_func.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen_func.h @@ -29,6 +29,7 @@ struct workgen_random_state; extern uint32_t workgen_atomic_add32(uint32_t *vp, uint32_t v); extern uint64_t workgen_atomic_add64(uint64_t *vp, uint64_t v); +extern void workgen_clock(uint64_t *tsp); extern void workgen_epoch(struct timespec *tsp); extern uint32_t workgen_random(struct workgen_random_state volatile *rnd_state); extern int workgen_random_alloc(WT_SESSION *session, struct workgen_random_state **rnd_state); diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_int.h b/src/third_party/wiredtiger/bench/workgen/workgen_int.h index d5ed99c8c53..ae1d21d1343 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen_int.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen_int.h @@ -51,10 +51,10 @@ struct WorkgenTimeStamp { WorkgenTimeStamp() {} static uint64_t get_timestamp_lag(double seconds) { - timespec start_time; - workgen_epoch(&start_time); + uint64_t start_time; + workgen_clock(&start_time); - return (ts_us(start_time) - secs_us(seconds)); + return (ns_to_us(start_time) - secs_us(seconds)); } static void sleep(double seconds) { @@ -62,10 +62,9 @@ struct WorkgenTimeStamp { } static uint64_t get_timestamp() { - timespec start_time; - workgen_epoch(&start_time); - - return (ts_us(start_time)); + uint64_t start_time; + workgen_clock(&start_time); + return (ns_to_us(start_time)); } }; diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index 9e589e2a55f..7e3c947e466 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -200,10 +200,8 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) /* Move to the next workload slot. */ if (wtperf->workload_cnt == WORKLOAD_MAX) { - fprintf(stderr, - "too many workloads configured, only %d workloads " - "supported\n", - WORKLOAD_MAX); + fprintf( + stderr, "too many workloads configured, only %d workloads supported\n", WORKLOAD_MAX); return (EINVAL); } workp = &wtperf->workload[wtperf->workload_cnt++]; @@ -394,10 +392,7 @@ config_opt(WTPERF *wtperf, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) break; } if (desc == NULL) { - fprintf(stderr, - "wtperf: Error: " - "unknown option \'%.*s\'\n", - (int)k->len, k->str); + fprintf(stderr, "wtperf: Error: unknown option \'%.*s\'\n", (int)k->len, k->str); fprintf(stderr, "Options:\n"); for (i = 0; i < WT_ELEMENTS(config_opts_desc); i++) fprintf(stderr, "\t%s\n", config_opts_desc[i].name); @@ -407,26 +402,20 @@ config_opt(WTPERF *wtperf, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) switch (desc->type) { case BOOL_TYPE: if (v->type != WT_CONFIG_ITEM_BOOL) { - fprintf(stderr, - "wtperf: Error: " - "bad bool value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); + fprintf(stderr, "wtperf: Error: bad bool value for \'%.*s=%.*s\'\n", (int)k->len, + k->str, (int)v->len, v->str); return (EINVAL); } *(int *)valueloc = (int)v->val; break; case INT_TYPE: if (v->type != WT_CONFIG_ITEM_NUM) { - fprintf(stderr, - "wtperf: Error: " - "bad int value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); + fprintf(stderr, "wtperf: Error: bad int value for \'%.*s=%.*s\'\n", (int)k->len, k->str, + (int)v->len, v->str); return (EINVAL); } if (v->val > INT_MAX) { - fprintf(stderr, - "wtperf: Error: " - "int value out of range for \'%.*s=%.*s\'\n", + fprintf(stderr, "wtperf: Error: int value out of range for \'%.*s=%.*s\'\n", (int)k->len, k->str, (int)v->len, v->str); return (EINVAL); } @@ -434,16 +423,12 @@ config_opt(WTPERF *wtperf, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) break; case UINT32_TYPE: if (v->type != WT_CONFIG_ITEM_NUM) { - fprintf(stderr, - "wtperf: Error: " - "bad uint32 value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); + fprintf(stderr, "wtperf: Error: bad uint32 value for \'%.*s=%.*s\'\n", (int)k->len, + k->str, (int)v->len, v->str); return (EINVAL); } if (v->val < 0 || v->val > UINT_MAX) { - fprintf(stderr, - "wtperf: Error: " - "uint32 value out of range for \'%.*s=%.*s\'\n", + fprintf(stderr, "wtperf: Error: uint32 value out of range for \'%.*s=%.*s\'\n", (int)k->len, k->str, (int)v->len, v->str); return (EINVAL); } @@ -454,10 +439,8 @@ config_opt(WTPERF *wtperf, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) * Configuration parsing uses string/ID to distinguish between quoted and unquoted values. */ if (v->type != WT_CONFIG_ITEM_STRING && v->type != WT_CONFIG_ITEM_ID) { - fprintf(stderr, - "wtperf: Error: " - "bad string value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); + fprintf(stderr, "wtperf: Error: bad string value for \'%.*s=%.*s\'\n", (int)k->len, + k->str, (int)v->len, v->str); return (EINVAL); } strp = (char **)valueloc; @@ -486,10 +469,8 @@ config_opt(WTPERF *wtperf, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) return (config_threads(wtperf, v->str, v->len)); if (v->type != WT_CONFIG_ITEM_STRING && v->type != WT_CONFIG_ITEM_ID) { - fprintf(stderr, - "wtperf: Error: " - "bad string value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); + fprintf(stderr, "wtperf: Error: bad string value for \'%.*s=%.*s\'\n", (int)k->len, + k->str, (int)v->len, v->str); return (EINVAL); } strp = (char **)valueloc; @@ -740,8 +721,8 @@ config_sanity(WTPERF *wtperf) /* Various intervals should be less than the run-time. */ if (opts->run_time > 0 && ((opts->checkpoint_threads != 0 && opts->checkpoint_interval > opts->run_time) || - opts->report_interval > opts->run_time || opts->sample_interval > opts->run_time || - opts->scan_interval > opts->run_time)) { + opts->report_interval > opts->run_time || opts->sample_interval > opts->run_time || + opts->scan_interval > opts->run_time)) { fprintf(stderr, "interval value longer than the run-time\n"); return (EINVAL); } @@ -768,8 +749,9 @@ config_sanity(WTPERF *wtperf) /* If we have separate tables for scanning, we need a separate count. */ if ((opts->scan_icount > 0 && opts->scan_table_count == 0) || (opts->scan_icount == 0 && opts->scan_table_count > 0)) { - fprintf(stderr, "scan_icount %" PRIu32 " and scan_table_count %" PRIu32 - " must both be zero or nonzero.\n", + fprintf(stderr, + "scan_icount %" PRIu32 " and scan_table_count %" PRIu32 + " must both be zero or nonzero.\n", opts->scan_icount, opts->scan_table_count); return (EINVAL); } @@ -799,17 +781,15 @@ config_sanity(WTPERF *wtperf) if (wtperf->workload != NULL) for (i = 0, workp = wtperf->workload; i < wtperf->workload_cnt; ++i, ++workp) { - if (opts->readonly && (workp->insert != 0 || workp->modify != 0 || - workp->truncate != 0 || workp->update != 0)) { + if (opts->readonly && + (workp->insert != 0 || workp->modify != 0 || workp->truncate != 0 || + workp->update != 0)) { fprintf(stderr, - "Invalid workload: insert, modify, truncate or update specified with " - "readonly\n"); + "Invalid workload: insert, modify, truncate or update specified with readonly\n"); return (EINVAL); } if (workp->insert != 0 && workp->table_index != INT32_MAX) { - fprintf(stderr, - "Invalid workload: Cannot insert into " - "specific table only\n"); + fprintf(stderr, "Invalid workload: Cannot insert into specific table only\n"); return (EINVAL); } if (workp->table_index != INT32_MAX && @@ -905,8 +885,7 @@ config_opt_log(CONFIG_OPTS *opts, const char *path) config_consolidate(opts); fprintf(fp, - "# Warning: This config includes " - "unwritten, implicit configuration defaults.\n" + "# Warning: This config includes unwritten, implicit configuration defaults.\n" "# Changes to those values may cause differences in behavior.\n"); TAILQ_FOREACH (config_line, &opts->config_head, q) fprintf(fp, "%s\n", config_line->string); @@ -1027,12 +1006,10 @@ config_opt_usage(void) const char *defaultval, *typestr; pretty_print( - "The following are options settable using -o or -O, showing the " - "type and default value.\n", + "The following are options settable using -o or -O, showing the type and default value.\n", NULL); pretty_print( - "String values must be enclosed in \" quotes, boolean values must " - "be either true or false.\n", + "String values must be enclosed in \" quotes, boolean values must be either true or false.\n", NULL); for (i = 0; i < WT_ELEMENTS(config_opts_desc); i++) { diff --git a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c index d2be4f5454e..e5f8d97bc9a 100644 --- a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c +++ b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c @@ -29,21 +29,21 @@ #include "wtperf.h" static int -check_timing(WTPERF *wtperf, const char *name, struct timespec start, struct timespec *stop) +check_timing(WTPERF *wtperf, const char *name, uint64_t start, uint64_t *stop) { CONFIG_OPTS *opts; uint64_t last_interval; opts = wtperf->opts; - __wt_epoch(NULL, stop); + *stop = __wt_clock(NULL); - last_interval = (uint64_t)(WT_TIMEDIFF_SEC(*stop, start)); + last_interval = WT_CLOCKDIFF_SEC(*stop, start); if (last_interval > opts->idle_table_cycle) { - lprintf(wtperf, ETIMEDOUT, 0, "Cycling idle table failed because %s took %" PRIu64 - " seconds which is longer than configured acceptable" - " maximum of %" PRIu32 ".", + lprintf(wtperf, ETIMEDOUT, 0, + "Cycling idle table failed because %s took %" PRIu64 + " seconds which is longer than configured acceptable maximum of %" PRIu32 ".", name, last_interval, opts->idle_table_cycle); wtperf->error = true; return (ETIMEDOUT); @@ -57,11 +57,11 @@ check_timing(WTPERF *wtperf, const char *name, struct timespec start, struct tim static WT_THREAD_RET cycle_idle_tables(void *arg) { - struct timespec start, stop; CONFIG_OPTS *opts; WTPERF *wtperf; WT_CURSOR *cursor; WT_SESSION *session; + uint64_t start, stop; int cycle_count, ret; char uri[512]; @@ -81,7 +81,7 @@ cycle_idle_tables(void *arg) __wt_sleep(1, 0); /* Setup a start timer. */ - __wt_epoch(NULL, &start); + start = __wt_clock(NULL); /* Create a table. */ if ((ret = session->create(session, uri, opts->table_config)) != 0) { diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 6328b408fc8..6eefbbf373f 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -499,7 +499,6 @@ pre_load_data(WTPERF *wtperf) static WT_THREAD_RET worker(void *arg) { - struct timespec start, stop; CONFIG_OPTS *opts; TRACK *trk; WORKLOAD *workload; @@ -512,7 +511,7 @@ worker(void *arg) WT_SESSION *session; size_t i, iter, modify_offset, modify_size, total_modify_size, value_len; int64_t delta, ops, ops_per_txn; - uint64_t log_id, next_val, usecs; + uint64_t log_id, next_val, start, stop, usecs; uint32_t rand_val, total_table_count; uint8_t *op, *op_end; int measure_latency, nmodify, ret, truncated; @@ -654,7 +653,7 @@ worker(void *arg) */ measure_latency = opts->sample_interval != 0 && trk != NULL && trk->ops != 0 && (trk->ops % opts->sample_rate == 0); - __wt_epoch(NULL, &start); /* [-Werror=maybe-uninitialized] */ + start = __wt_clock(NULL); cursor->set_key(cursor, key_buf); switch (*op) { @@ -868,15 +867,11 @@ op_err: if (ret == WT_ROLLBACK && ops_per_txn == 0) { lprintf(wtperf, ret, 1, "log-table: ROLLBACK"); if ((ret = session->rollback_transaction(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, - "Failed" - " rollback_transaction"); + lprintf(wtperf, ret, 0, "Failed rollback_transaction"); goto err; } if ((ret = session->begin_transaction(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, - "Worker begin " - "transaction failed"); + lprintf(wtperf, ret, 0, "Worker begin transaction failed"); goto err; } } else @@ -895,9 +890,9 @@ op_err: /* Gather statistics */ if (!wtperf->in_warmup) { if (measure_latency) { - __wt_epoch(NULL, &stop); + stop = __wt_clock(NULL); ++trk->latency_ops; - usecs = WT_TIMEDIFF_US(stop, start); + usecs = WT_CLOCKDIFF_US(stop, start); track_operation(trk, usecs); } /* Increment operation count */ @@ -1065,7 +1060,6 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) static WT_THREAD_RET populate_thread(void *arg) { - struct timespec start, stop; CONFIG_OPTS *opts; TRACK *trk; WTPERF *wtperf; @@ -1074,7 +1068,7 @@ populate_thread(void *arg) WT_CURSOR **cursors, *cursor; WT_SESSION *session; size_t i; - uint64_t op, usecs; + uint64_t op, start, stop, usecs; uint32_t opcount, total_table_count; int intxn, measure_latency, ret, stress_checkpoint_due; char *value_buf, *key_buf; @@ -1087,6 +1081,7 @@ populate_thread(void *arg) session = NULL; cursors = NULL; ret = stress_checkpoint_due = 0; + start = 0; trk = &thread->insert; total_table_count = opts->table_count + opts->scan_table_count; @@ -1131,7 +1126,7 @@ populate_thread(void *arg) measure_latency = opts->sample_interval != 0 && trk->ops != 0 && (trk->ops % opts->sample_rate == 0); if (measure_latency) - __wt_epoch(NULL, &start); + start = __wt_clock(NULL); cursor->set_key(cursor, key_buf); if (opts->random_value) randomize_value(thread, value_buf, 0); @@ -1153,9 +1148,9 @@ populate_thread(void *arg) * multiple tables, it is the time for insertion into all of them. */ if (measure_latency) { - __wt_epoch(NULL, &stop); + stop = __wt_clock(NULL); ++trk->latency_ops; - usecs = WT_TIMEDIFF_US(stop, start); + usecs = WT_CLOCKDIFF_US(stop, start); track_operation(trk, usecs); } ++thread->insert.ops; /* Same as trk->ops */ @@ -1201,7 +1196,6 @@ err: static WT_THREAD_RET populate_async(void *arg) { - struct timespec start, stop; CONFIG_OPTS *opts; TRACK *trk; WTPERF *wtperf; @@ -1209,7 +1203,7 @@ populate_async(void *arg) WT_ASYNC_OP *asyncop; WT_CONNECTION *conn; WT_SESSION *session; - uint64_t op, usecs; + uint64_t op, start, stop, usecs; int measure_latency, ret; char *value_buf, *key_buf; @@ -1219,6 +1213,7 @@ populate_async(void *arg) conn = wtperf->conn; session = NULL; ret = 0; + start = 0; trk = &thread->insert; key_buf = thread->key_buf; @@ -1236,7 +1231,7 @@ populate_async(void *arg) measure_latency = opts->sample_interval != 0 && trk->ops != 0 && (trk->ops % opts->sample_rate == 0); if (measure_latency) - __wt_epoch(NULL, &start); + start = __wt_clock(NULL); /* Populate the databases. */ for (;;) { @@ -1277,9 +1272,9 @@ populate_async(void *arg) goto err; } if (measure_latency) { - __wt_epoch(NULL, &stop); + stop = __wt_clock(NULL); ++trk->latency_ops; - usecs = WT_TIMEDIFF_US(stop, start); + usecs = WT_CLOCKDIFF_US(stop, start); track_operation(trk, usecs); } if ((ret = session->close(session, NULL)) != 0) { @@ -1409,10 +1404,9 @@ monitor(void *arg) cur_updates = (updates - last_updates) / opts->sample_interval; (void)fprintf(fp, - "%s,%" PRIu32 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 - ",%c,%c" + "%s,%" PRIu32 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%c,%c,%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 - ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 "\n", + ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 "\n", buf, wtperf->totalsec, cur_inserts, cur_modifies, cur_reads, cur_updates, wtperf->ckpt ? 'Y' : 'N', wtperf->scan ? 'Y' : 'N', insert_avg, insert_min, insert_max, modify_avg, modify_min, modify_max, read_avg, read_min, read_max, update_avg, update_min, @@ -1437,8 +1431,9 @@ monitor(void *arg) ",\"modify\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", cur_modifies, modify_avg, modify_min, modify_max); - (void)fprintf(jfp, ",\"read\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 - ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", + (void)fprintf(jfp, + ",\"read\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 + ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", cur_reads, read_avg, read_min, read_max); (void)fprintf(jfp, ",\"update\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 @@ -1447,8 +1442,9 @@ monitor(void *arg) fprintf(jfp, "}}\n"); } - if (latency_max != 0 && (insert_max > latency_max || modify_max > latency_max || - read_max > latency_max || update_max > latency_max)) { + if (latency_max != 0 && + (insert_max > latency_max || modify_max > latency_max || read_max > latency_max || + update_max > latency_max)) { if (opts->max_latency_fatal) { level = 1; msg_err = WT_PANIC; @@ -1463,10 +1459,11 @@ monitor(void *arg) " modify max %" PRIu32 " read max %" PRIu32 " update max %" PRIu32, str, latency_max, insert_max, modify_max, read_max, update_max); } - if (min_thr != 0 && ((cur_inserts != 0 && cur_inserts < min_thr) || - (cur_modifies != 0 && cur_modifies < min_thr) || - (cur_reads != 0 && cur_reads < min_thr) || - (cur_updates != 0 && cur_updates < min_thr))) { + if (min_thr != 0 && + ((cur_inserts != 0 && cur_inserts < min_thr) || + (cur_modifies != 0 && cur_modifies < min_thr) || + (cur_reads != 0 && cur_reads < min_thr) || + (cur_updates != 0 && cur_updates < min_thr))) { if (opts->min_throughput_fatal) { level = 1; msg_err = WT_PANIC; @@ -1510,7 +1507,6 @@ checkpoint_worker(void *arg) WTPERF_THREAD *thread; WT_CONNECTION *conn; WT_SESSION *session; - struct timespec e, s; uint32_t i; int ret; @@ -1536,8 +1532,6 @@ checkpoint_worker(void *arg) if (wtperf->stop) break; - __wt_epoch(NULL, &s); - wtperf->ckpt = true; if ((ret = session->checkpoint(session, NULL)) != 0) { lprintf(wtperf, ret, 0, "Checkpoint failed."); @@ -1545,8 +1539,6 @@ checkpoint_worker(void *arg) } wtperf->ckpt = false; ++thread->ckpt.ops; - - __wt_epoch(NULL, &e); } if (session != NULL && ((ret = session->close(session, NULL)) != 0)) { @@ -1573,7 +1565,6 @@ scan_worker(void *arg) WT_CURSOR *cursor, **cursors; WT_SESSION *session; char *key_buf; - struct timespec e, s; uint32_t i, ntables, pct, table_start; uint64_t cur_id, end_id, incr, items, start_id, tot_items; int ret; @@ -1636,8 +1627,6 @@ scan_worker(void *arg) if (wtperf->stop) break; - __wt_epoch(NULL, &s); - wtperf->scan = true; items = 0; while (items < tot_items && !wtperf->stop) { @@ -1645,10 +1634,7 @@ scan_worker(void *arg) generate_key(opts, key_buf, cur_id); cursor->set_key(cursor, key_buf); if ((ret = cursor->search(cursor)) != 0) { - lprintf(wtperf, ret, 0, - "Failed scan search " - "key %s, items %d", - key_buf, (int)items); + lprintf(wtperf, ret, 0, "Failed scan search key %s, items %d", key_buf, (int)items); goto err; } @@ -1665,7 +1651,6 @@ scan_worker(void *arg) } wtperf->scan = false; ++thread->scan.ops; - __wt_epoch(NULL, &e); } if (session != NULL && ((ret = session->close(session, NULL)) != 0)) { @@ -1685,13 +1670,12 @@ err: static int execute_populate(WTPERF *wtperf) { - struct timespec start, stop; CONFIG_OPTS *opts; WT_ASYNC_OP *asyncop; WTPERF_THREAD *popth; WT_THREAD_CALLBACK (*pfunc)(void *); size_t i; - uint64_t last_ops, msecs, print_ops_sec, max_key; + uint64_t last_ops, max_key, msecs, print_ops_sec, start, stop; uint32_t interval, tables; wt_thread_t idle_table_cycle_thread; double print_secs; @@ -1716,7 +1700,7 @@ execute_populate(WTPERF *wtperf) pfunc = populate_thread; start_threads(wtperf, NULL, wtperf->popthreads, opts->populate_threads, pfunc); - __wt_epoch(NULL, &start); + start = __wt_clock(NULL); for (elapsed = 0, interval = 0, last_ops = 0; wtperf->insert_key < max_key && !wtperf->error;) { /* * Sleep for 100th of a second, report_interval is in second granularity, each 100th @@ -1731,13 +1715,14 @@ execute_populate(WTPERF *wtperf) interval = 0; wtperf->totalsec += opts->report_interval; wtperf->insert_ops = sum_pop_ops(wtperf); - lprintf(wtperf, 0, 1, "%" PRIu64 " populate inserts (%" PRIu64 " of %" PRIu32 - ") in %" PRIu32 " secs (%" PRIu32 " total secs)", + lprintf(wtperf, 0, 1, + "%" PRIu64 " populate inserts (%" PRIu64 " of %" PRIu32 ") in %" PRIu32 " secs (%" PRIu32 + " total secs)", wtperf->insert_ops - last_ops, wtperf->insert_ops, opts->icount, opts->report_interval, wtperf->totalsec); last_ops = wtperf->insert_ops; } - __wt_epoch(NULL, &stop); + stop = __wt_clock(NULL); /* * Move popthreads aside to narrow possible race with the monitor thread. The latency tracking @@ -1756,7 +1741,7 @@ execute_populate(WTPERF *wtperf) } lprintf(wtperf, 0, 1, "Finished load of %" PRIu32 " items", opts->icount); - msecs = WT_TIMEDIFF_MS(stop, start); + msecs = WT_CLOCKDIFF_MS(stop, start); /* * This is needed as the divisions will fail if the insert takes no time which will only be the @@ -1781,7 +1766,7 @@ execute_populate(WTPERF *wtperf) if (opts->compact) { assert(opts->async_threads > 0); lprintf(wtperf, 0, 1, "Compact after populate"); - __wt_epoch(NULL, &start); + start = __wt_clock(NULL); tables = opts->table_count; for (i = 0; i < opts->table_count; i++) { /* @@ -1803,9 +1788,9 @@ execute_populate(WTPERF *wtperf) lprintf(wtperf, ret, 0, "Populate async flush failed."); return (ret); } - __wt_epoch(NULL, &stop); + stop = __wt_clock(NULL); lprintf(wtperf, 0, 1, "Compact completed in %" PRIu64 " seconds", - (uint64_t)(WT_TIMEDIFF_SEC(stop, start))); + (uint64_t)(WT_CLOCKDIFF_SEC(stop, start))); assert(tables == 0); } @@ -2523,9 +2508,7 @@ extern char *__wt_optarg; static void usage(void) { - printf( - "wtperf [-C config] " - "[-h home] [-O file] [-o option] [-T config]\n"); + printf("wtperf [-C config] [-h home] [-O file] [-o option] [-T config]\n"); printf("\t-C <string> additional connection configuration\n"); printf("\t (added to option conn_config)\n"); printf("\t-h <string> Wired Tiger home must exist, default WT_TEST\n"); @@ -2903,12 +2886,11 @@ recreate_dir(const char *name) static int drop_all_tables(WTPERF *wtperf) { - struct timespec start, stop; CONFIG_OPTS *opts; WT_SESSION *session; size_t i; uint32_t total_table_count; - uint64_t msecs; + uint64_t msecs, start, stop; int ret, t_ret; opts = wtperf->opts; @@ -2919,15 +2901,15 @@ drop_all_tables(WTPERF *wtperf) lprintf(wtperf, ret, 0, "Error opening a session on %s", wtperf->home); return (ret); } - __wt_epoch(NULL, &start); + start = __wt_clock(NULL); for (i = 0; i < total_table_count; i++) { if ((ret = session->drop(session, wtperf->uris[i], NULL)) != 0) { lprintf(wtperf, ret, 0, "Error dropping table %s", wtperf->uris[i]); goto err; } } - __wt_epoch(NULL, &stop); - msecs = WT_TIMEDIFF_MS(stop, start); + stop = __wt_clock(NULL); + msecs = WT_CLOCKDIFF_MS(stop, start); lprintf(wtperf, 0, 1, "Executed %" PRIu32 " drop operations average time %" PRIu64 "ms", total_table_count, msecs / total_table_count); diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i index 2706558c887..a9066f58174 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i @@ -84,71 +84,61 @@ DEF_OPT_AS_UINT32(async_threads, 0, "number of async worker threads") DEF_OPT_AS_UINT32( checkpoint_interval, 120, "checkpoint every interval seconds during the workload phase.") DEF_OPT_AS_UINT32(checkpoint_stress_rate, 0, - "checkpoint every rate operations during the populate phase in the " - "populate thread(s), 0 to disable") + "checkpoint every rate operations during the populate phase in the populate thread(s), 0 to " + "disable") DEF_OPT_AS_UINT32(checkpoint_threads, 0, "number of checkpoint threads") DEF_OPT_AS_CONFIG_STRING(conn_config, "create,statistics=(fast),statistics_log=(json,wait=1)", "connection configuration string") DEF_OPT_AS_BOOL(close_conn, 1, - "properly close connection at end of test. " - "Setting to false does not sync data to disk and can result in lost " - "data after test exits.") + "properly close connection at end of test. Setting to false does not sync data to disk and can " + "result in lost data after test exits.") DEF_OPT_AS_BOOL(compact, 0, "post-populate compact for LSM merging activity") DEF_OPT_AS_STRING(compression, "none", - "compression extension. Allowed configuration values are: " - "'none', 'lz4', 'snappy', 'zlib', 'zstd'") + "compression extension. Allowed configuration values are: 'none', 'lz4', 'snappy', 'zlib', " + "'zstd'") DEF_OPT_AS_BOOL(create, 1, "do population phase; false to use existing database") DEF_OPT_AS_UINT32(database_count, 1, - "number of WiredTiger databases to use. Each database will execute the" - " workload using a separate home directory and complete set of worker" - " threads") + "number of WiredTiger databases to use. Each database will execute the workload using a separate " + "home directory and complete set of worker threads") DEF_OPT_AS_BOOL(drop_tables, 0, - "Whether to drop all tables at the end of the run, and report time taken" - " to do the drop.") + "Whether to drop all tables at the end of the run, and report time taken to do the drop.") DEF_OPT_AS_BOOL(in_memory, 0, "Whether to create the database in-memory.") DEF_OPT_AS_UINT32(icount, 5000, - "number of records to initially populate. If multiple tables are " - "configured the count is spread evenly across all tables.") + "number of records to initially populate. If multiple tables are configured the count is spread " + "evenly across all tables.") DEF_OPT_AS_UINT32(idle_table_cycle, 0, - "Enable regular create and drop of idle tables, value is the maximum " - "number of seconds a create or drop is allowed before flagging an error. " - "Default 0 which means disabled.") + "Enable regular create and drop of idle tables, value is the maximum number of seconds a create " + "or drop is allowed before flagging an error. Default 0 which means disabled.") DEF_OPT_AS_BOOL(index, 0, "Whether to create an index on the value field.") DEF_OPT_AS_BOOL(insert_rmw, 0, "execute a read prior to each insert in workload phase") DEF_OPT_AS_UINT32(key_sz, 20, "key size") DEF_OPT_AS_BOOL(log_partial, 0, "perform partial logging on first table only.") DEF_OPT_AS_BOOL(log_like_table, 0, "Append all modification operations to another shared table.") DEF_OPT_AS_UINT32(min_throughput, 0, - "notify if any throughput measured is less than this amount. " - "Aborts or prints warning based on min_throughput_fatal setting. " - "Requires sample_interval to be configured") + "notify if any throughput measured is less than this amount. Aborts or prints warning based on " + "min_throughput_fatal setting. Requires sample_interval to be configured") DEF_OPT_AS_BOOL( min_throughput_fatal, 0, "print warning (false) or abort (true) of min_throughput failure.") DEF_OPT_AS_UINT32(max_latency, 0, - "notify if any latency measured exceeds this number of milliseconds." - "Aborts or prints warning based on min_throughput_fatal setting. " - "Requires sample_interval to be configured") + "notify if any latency measured exceeds this number of milliseconds.Aborts or prints warning " + "based on min_throughput_fatal setting. Requires sample_interval to be configured") DEF_OPT_AS_BOOL( max_latency_fatal, 0, "print warning (false) or abort (true) of max_latency failure.") DEF_OPT_AS_UINT32(pareto, 0, - "use pareto distribution for random numbers. Zero " - "to disable, otherwise a percentage indicating how aggressive the " - "distribution should be.") + "use pareto distribution for random numbers. Zero to disable, otherwise a percentage indicating " + "how aggressive the distribution should be.") DEF_OPT_AS_UINT32(populate_ops_per_txn, 0, - "number of operations to group into each transaction in the populate " - "phase, zero for auto-commit") + "number of operations to group into each transaction in the populate phase, zero for auto-commit") DEF_OPT_AS_UINT32(populate_threads, 1, "number of populate threads, 1 for bulk load") DEF_OPT_AS_BOOL( pre_load_data, 0, "Scan all data prior to starting the workload phase to warm the cache") DEF_OPT_AS_UINT32(random_range, 0, - "if non zero choose a value from within this range as the key for " - "insert operations") + "if non zero choose a value from within this range as the key for insert operations") DEF_OPT_AS_BOOL(random_value, 0, "generate random content for the value") DEF_OPT_AS_BOOL(range_partition, 0, "partition data by range (vs hash)") DEF_OPT_AS_BOOL(readonly, 0, - "reopen the connection between populate and workload phases in readonly " - "mode. Requires reopen_connection turned on (default). Requires that " - "read be the only workload specified") + "reopen the connection between populate and workload phases in readonly mode. Requires " + "reopen_connection turned on (default). Requires that read be the only workload specified") DEF_OPT_AS_BOOL( reopen_connection, 1, "close and reopen the connection between populate and workload phases") DEF_OPT_AS_UINT32( @@ -157,49 +147,43 @@ DEF_OPT_AS_UINT32(run_ops, 0, "total insert, modify, read and update workload op DEF_OPT_AS_UINT32(run_time, 0, "total workload seconds") DEF_OPT_AS_UINT32(sample_interval, 0, "performance logging every interval seconds, 0 to disable") DEF_OPT_AS_UINT32(sample_rate, 50, - "how often the latency of operations is measured. One for every operation," - "two for every second operation, three for every third operation etc.") + "how often the latency of operations is measured. One for every operation, two for every second " + "operation, three for every third operation etc.") DEF_OPT_AS_UINT32(scan_icount, 0, "number of records in scan tables to populate") -DEF_OPT_AS_UINT32(scan_interval, 0, - "scan tables every interval seconds during the workload phase," - " 0 to disable") +DEF_OPT_AS_UINT32( + scan_interval, 0, "scan tables every interval seconds during the workload phase, 0 to disable") DEF_OPT_AS_UINT32( scan_pct, 10, "percentage of entire data set scanned, if scan_interval is enabled") DEF_OPT_AS_UINT32(scan_table_count, 0, - "number of separate tables to be used for scanning. Zero indicates " - "that tables are shared with other operations") + "number of separate tables to be used for scanning. Zero indicates that tables are shared with " + "other operations") DEF_OPT_AS_CONFIG_STRING(sess_config, "", "session configuration string") DEF_OPT_AS_UINT32(session_count_idle, 0, "number of idle sessions to create. Default 0.") /* The following table configuration is based on the configuration MongoDB uses for collections. */ DEF_OPT_AS_CONFIG_STRING(table_config, - "key_format=S,value_format=S,type=file,exclusive=true," - "leaf_value_max=64MB,memory_page_max=10m,split_pct=90,checksum=on", + "key_format=S,value_format=S,type=file,exclusive=true,leaf_value_max=64MB,memory_page_max=10m," + "split_pct=90,checksum=on", "table configuration string") DEF_OPT_AS_UINT32(table_count, 1, - "number of tables to run operations over. Keys are divided evenly " - "over the tables. Cursors are held open on all tables. Default 1, maximum " - "99999.") + "number of tables to run operations over. Keys are divided evenly over the tables. Cursors are " + "held open on all tables. Default 1, maximum 99999.") DEF_OPT_AS_UINT32( table_count_idle, 0, "number of tables to create, that won't be populated. Default 0.") DEF_OPT_AS_STRING(threads, "", - "workload configuration: each 'count' " - "entry is the total number of threads, and the 'insert', 'modify', 'read' and " - "'update' entries are the ratios of insert, modify, read and update operations " - "done by each worker thread; If a throttle value is provided each thread " - "will do a maximum of that number of operations per second; multiple " - "workload configurations may be specified per threads configuration; " - "for example, a more complex threads configuration might be " - "'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' " - "which would create 2 threads doing nothing but reads and 8 threads " - "each doing 50% inserts and 25% reads and updates. Allowed configuration " - "values are 'count', 'throttle', 'inserts', 'reads', 'read_range', 'modify', " - "'modify_delta', 'modify_distribute', 'modify_force_update', 'updates', 'update_delta', " - "'truncate', 'truncate_pct' and 'truncate_count'. " - "There are also behavior modifiers, supported modifiers are " - "'ops_per_txn'") + "workload configuration: each 'count' entry is the total number of threads, and the 'insert', " + "'modify', 'read' and 'update' entries are the ratios of insert, modify, read and update " + "operations done by each worker thread; If a throttle value is provided each thread will do a " + "maximum of that number of operations per second; multiple workload configurations may be " + "specified per threads configuration; for example, a more complex threads configuration might be " + "'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' which would create 2 threads " + "doing nothing but reads and 8 threads each doing 50% inserts and 25% reads and updates. " + "Allowed configuration values are 'count', 'throttle', 'inserts', 'reads', 'read_range', " + "'modify', 'modify_delta', 'modify_distribute', 'modify_force_update', 'updates', " + "'update_delta', 'truncate', 'truncate_pct' and 'truncate_count'. There are also behavior " + "modifiers, supported modifiers are 'ops_per_txn'") DEF_OPT_AS_CONFIG_STRING(transaction_config, "", - "WT_SESSION.begin_transaction configuration string, applied during the " - "populate phase when populate_ops_per_txn is nonzero") + "WT_SESSION.begin_transaction configuration string, applied during the populate phase when " + "populate_ops_per_txn is nonzero") DEF_OPT_AS_STRING(table_name, "test", "table name") DEF_OPT_AS_BOOL( truncate_single_ops, 0, "Implement truncate via cursor remove instead of session API") diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 2286b64857c..c2139a07202 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -452,18 +452,6 @@ connection_runtime_config = [ for space to be available in cache before giving up. Default will wait forever''', min=0), - Config('cache_overflow', '', r''' - cache overflow configuration options''', - type='category', subconfig=[ - Config('file_max', '0', r''' - The maximum number of bytes that WiredTiger is allowed to use for - its cache overflow mechanism. If the cache overflow file exceeds - this size, a panic will be triggered. The default value means that - the cache overflow file is unbounded and may use as much space as - the filesystem will accommodate. The minimum non-zero setting is - 100MB.''', # !!! TODO: WT-5585 To be removed when we switch to history_store config - min='0') - ]), Config('history_store', '', r''' history store configuration options''', type='category', subconfig=[ @@ -722,9 +710,9 @@ connection_runtime_config = [ intended for use with internal stress testing of WiredTiger.''', type='list', undoc=True, choices=[ - 'aggressive_sweep', 'checkpoint_slow', 'history_store_checkpoint_delay', - 'history_store_sweep_race', 'split_1', 'split_2', 'split_3', 'split_4', 'split_5', - 'split_6', 'split_7', 'split_8']), + 'aggressive_sweep', 'backup_rename', 'checkpoint_slow', 'history_store_checkpoint_delay', + 'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', 'split_2', + 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8']), Config('verbose', '', r''' enable messages for various events. Options are given as a list, such as <code>"verbose=[evictserver,read]"</code>''', @@ -1016,6 +1004,18 @@ wiredtiger_open_common =\ modified file is always flushed to storage when closing file handles to acquire exclusive access to the table''', type='boolean'), + Config('hash', '', r''' + manage resources around hash bucket arrays. All values must be a power of two. + Note that setting large values can significantly increase memory usage inside + WiredTiger''', + type='category', subconfig=[ + Config('buckets', 512, r''' + configure the number of hash buckets for most system hash arrays''', + min='64', max='65536'), + Config('dhandle_buckets', 512, r''' + configure the number of hash buckets for hash arrays relating to data handles''', + min='64', max='65536'), + ]), Config('hazard_max', '1000', r''' maximum number of simultaneous hazard pointers per session handle''', @@ -1450,11 +1450,11 @@ methods = { Config('name', '', r''' name of the transaction for tracing and debugging'''), Config('operation_timeout_ms', '0', r''' - when non-zero, a requested limit on the number of elapsed real time milliseconds taken - to complete database operations in this transaction. Time is measured from the start - of each WiredTiger API call. There is no guarantee any operation will not take longer - than this amount of time. If WiredTiger notices the limit has been exceeded, an operation - may return a WT_ROLLBACK error. Default is to have no limit''', + when non-zero, a requested limit on the time taken to complete operations in this + transaction. Time is measured in real time milliseconds from the start of each WiredTiger + API call. There is no guarantee any operation will not take longer than this amount of time. + If WiredTiger notices the limit has been exceeded, an operation may return a WT_ROLLBACK + error. Default is to have no limit''', min=1), Config('priority', 0, r''' priority of the transaction for resolving conflicts. @@ -1501,6 +1501,13 @@ methods = { current transaction. The value must also not be older than the current stable timestamp. See @ref transaction_timestamps'''), + Config('operation_timeout_ms', '0', r''' + when non-zero, a requested limit on the time taken to complete operations in this + transaction. Time is measured in real time milliseconds from the start of each WiredTiger + API call. There is no guarantee any operation will not take longer than this amount of time. + If WiredTiger notices the limit has been exceeded, an operation may return a WT_ROLLBACK + error. Default is to have no limit''', + min=1), Config('sync', '', r''' override whether to sync log records when the transaction commits, inherited from ::wiredtiger_open \c transaction_sync. @@ -1542,7 +1549,15 @@ methods = { for a transaction. See @ref transaction_timestamps'''), ]), -'WT_SESSION.rollback_transaction' : Method([]), +'WT_SESSION.rollback_transaction' : Method([ + Config('operation_timeout_ms', '0', r''' + when non-zero, a requested limit on the time taken to complete operations in this + transaction. Time is measured in real time milliseconds from the start of each WiredTiger + API call. There is no guarantee any operation will not take longer than this amount of time. + If WiredTiger notices the limit has been exceeded, an operation may return a WT_ROLLBACK + error. Default is to have no limit''', + min=1), +]), 'WT_SESSION.checkpoint' : Method([ Config('drop', '', r''' diff --git a/src/third_party/wiredtiger/dist/s_clang-format b/src/third_party/wiredtiger/dist/s_clang-format index 1673946ee08..be1611fa1e7 100755 --- a/src/third_party/wiredtiger/dist/s_clang-format +++ b/src/third_party/wiredtiger/dist/s_clang-format @@ -7,11 +7,11 @@ trap 'rm -rf $t' 0 1 2 3 13 15 download_clang_format() { if [ `uname` = "Linux" ]; then - curl https://s3.amazonaws.com/boxes.10gen.com/build/clang-format-3.8-rhel55.tar.gz -o dist/clang-format.tar.gz - tar --strip=2 -C dist/ -xf dist/clang-format.tar.gz build/bin/clang-format && rm dist/clang-format.tar.gz + curl https://s3.amazonaws.com/boxes.10gen.com/build/build/clang-format-llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.gz -o dist/clang-format.tar.gz + tar --strip=1 -C dist/ -xf dist/clang-format.tar.gz clang-format-llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04/clang-format && rm dist/clang-format.tar.gz elif [ `uname` = "Darwin" ]; then - curl https://s3.amazonaws.com/boxes.10gen.com/build/clang%2Bllvm-3.8.0-x86_64-apple-darwin.tar.xz -o dist/clang-format.tar.gz - tar --strip=2 -C dist/ -xf dist/clang-format.tar.gz clang+llvm-3.8.0-x86_64-apple-darwin/bin/clang-format && rm dist/clang-format.tar.gz + curl https://s3.amazonaws.com/boxes.10gen.com/build/build/clang-format-llvm-10.0.0-x86_64-apple-darwin.tar.gz -o dist/clang-format.tar.gz + tar --strip=1 -C dist/ -xf dist/clang-format.tar.gz clang-format-llvm-10.0.0-x86_64-apple-darwin/clang-format && rm dist/clang-format.tar.gz else echo "$0: unsupported environment $(uname)" exit 1 @@ -24,9 +24,17 @@ cd `git rev-parse --show-toplevel` || exit 1 # Override existing Clang Format versions in the PATH. export PATH="${PWD}/dist":$PATH +# Download the clang-format binary if it's not in place. +[ ! -x "$(command -v clang-format)" ] && download_clang_format + # Ensure that we have the correct version of clang-format. -clang_format_version="3.8.0" -clang-format --version | grep "version $clang_format_version" >/dev/null 2>&1 +desired_version="10.0.0" + +# On macOS Catalina, users need to manually approve binaries. +# If we're not allowed to run clang-format, let's exit (should be obvious from the dialog). +current_version=`clang-format --version` || exit 1 + +echo $current_version | grep "version $desired_version" >/dev/null 2>&1 if test $? -ne 0; then download_clang_format fi @@ -50,7 +58,6 @@ esac for f in $search; do cat "$f" | \ clang-format --fallback-style=none | \ - python dist/s_goto.py | \ python dist/s_comment.py > "$t" || exit 1 cmp --silent "$f" "$t" if test $? -ne 0; then diff --git a/src/third_party/wiredtiger/dist/s_copyright.list b/src/third_party/wiredtiger/dist/s_copyright.list index 03475f5c602..b79fae5fc6e 100644 --- a/src/third_party/wiredtiger/dist/s_copyright.list +++ b/src/third_party/wiredtiger/dist/s_copyright.list @@ -14,7 +14,6 @@ skip dist/log_data.py skip dist/prototypes.py skip dist/s_comment.py skip dist/s_function_loop.py -skip dist/s_goto.py skip dist/stat.py skip dist/stat_data.py skip dist/style.py diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index d23e37ed9fa..095ada474d3 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -97,7 +97,7 @@ spellchk() (cd ../src/docs && cat *.dox | - aspell --encoding=iso-8859-1 --lang=en --personal=./spell.ok list) | + aspell --encoding=iso-8859-1 --lang=en_US --personal=./spell.ok list) | sort -u > $t test -s $t && { echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" diff --git a/src/third_party/wiredtiger/dist/s_goto.py b/src/third_party/wiredtiger/dist/s_goto.py deleted file mode 100644 index a7f35488603..00000000000 --- a/src/third_party/wiredtiger/dist/s_goto.py +++ /dev/null @@ -1,18 +0,0 @@ -# Trim all trailing whitespace in front of goto labels. -# This is a workaround for a Clang Format limitation where goto labels are -# automatically indented according to nesting. -import re, sys - -# 1. Zero or more whitespace characters. -# 2. One or more lowercase ASCII characters. -# 3. Colon character. -p = re.compile('^\s*[a-z_]+:$') -for line in sys.stdin: - m = p.search(line) - if m is not None: - sline = line.lstrip() - # The "default" tag in a switch statement looks identical so we need - # to filter these out here. - if not sline.startswith('default'): - line = sline - sys.stdout.write(line) diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 689aa30c192..52c2e5ee652 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -233,7 +233,7 @@ LeafGreen LevelDB Levyx LmRrSVv -LmT +LmsT LoadLoad LockFile Lookaside @@ -387,6 +387,7 @@ TCMalloc TESTUTIL TIMESTAMP TIMESTAMPS +TMP TODO TORTIOUS TSO diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 8befa46c629..84d2bb1fec2 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -157,6 +157,8 @@ connection_stats = [ ########################################## # System statistics ########################################## + ConnStat('buckets', 'hash bucket array size general', 'no_clear,no_scale,size'), + ConnStat('buckets_dh', 'hash bucket array size for data handles', 'no_clear,no_scale,size'), ConnStat('cond_auto_wait', 'auto adjusting condition wait calls'), ConnStat('cond_auto_wait_reset', 'auto adjusting condition resets'), ConnStat('cond_auto_wait_skipped', 'auto adjusting condition wait raced to update timeout and skipped updating'), @@ -622,6 +624,12 @@ connection_stats = [ TxnStat('txn_checkpoint', 'transaction checkpoints'), TxnStat('txn_checkpoint_fsync_post', 'transaction fsync calls for checkpoint after allocating the transaction ID'), TxnStat('txn_checkpoint_fsync_post_duration', 'transaction fsync duration for checkpoint after allocating the transaction ID (usecs)', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_handle_applied', 'transaction checkpoint most recent handles applied'), + TxnStat('txn_checkpoint_handle_skipped', 'transaction checkpoint most recent handles skipped'), + TxnStat('txn_checkpoint_handle_walked', 'transaction checkpoint most recent handles walked'), + TxnStat('txn_checkpoint_handle_duration', 'transaction checkpoint most recent duration for gathering all handles (usecs)', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_handle_duration_apply', 'transaction checkpoint most recent duration for gathering applied handles (usecs)', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_handle_duration_skip', 'transaction checkpoint most recent duration for gathering skipped handles (usecs)', 'no_clear,no_scale'), TxnStat('txn_hs_ckpt_duration', 'transaction checkpoint history store file duration (usecs)'), TxnStat('txn_checkpoint_generation', 'transaction checkpoint generation', 'no_clear,no_scale'), TxnStat('txn_checkpoint_prep_max', 'transaction checkpoint prepare max time (msecs)', 'no_clear,no_scale'), diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index 7360bf92862..163c9cfddda 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -571,22 +571,19 @@ session_ops_create(WT_SESSION *session) * integer, unsigned 16-bit integer). */ error_check(session->create(session, "table:mytable", - "key_format=r,value_format=SiH," - "columns=(id,department,salary,year-started)")); + "key_format=r,value_format=SiH,columns=(id,department,salary,year-started)")); /*! [Create a table with columns] */ error_check(session->drop(session, "table:mytable", NULL)); /*! [Create a table and configure the page size] */ error_check(session->create(session, "table:mytable", - "key_format=S,value_format=S," - "internal_page_max=16KB,leaf_page_max=1MB,leaf_value_max=64KB")); + "key_format=S,value_format=S,internal_page_max=16KB,leaf_page_max=1MB,leaf_value_max=64KB")); /*! [Create a table and configure the page size] */ error_check(session->drop(session, "table:mytable", NULL)); /*! [Create a table and configure a large leaf value max] */ error_check(session->create(session, "table:mytable", - "key_format=S,value_format=S," - "leaf_page_max=16KB,leaf_value_max=256KB")); + "key_format=S,value_format=S,leaf_page_max=16KB,leaf_value_max=256KB")); /*! [Create a table and configure a large leaf value max] */ error_check(session->drop(session, "table:mytable", NULL)); @@ -687,7 +684,7 @@ session_ops(WT_SESSION *session) /*! [Compact a table] */ error_check(session->compact(session, "table:mytable", NULL)); -/*! [Compact a table] */ + /*! [Compact a table] */ #ifdef MIGHT_NOT_RUN /*! [Import a file] */ @@ -1145,9 +1142,9 @@ main(int argc, char *argv[]) /*! [Open a connection] */ connection_ops(conn); -/* - * The connection has been closed. - */ + /* + * The connection has been closed. + */ #ifdef MIGHT_NOT_RUN /* @@ -1155,51 +1152,39 @@ main(int argc, char *argv[]) * open to fail. The documentation requires the code snippets, use #ifdef's to avoid running it. */ /*! [Configure lz4 extension] */ - error_check(wiredtiger_open(home, NULL, - "create," - "extensions=[/usr/local/lib/libwiredtiger_lz4.so]", - &conn)); + error_check(wiredtiger_open( + home, NULL, "create,extensions=[/usr/local/lib/libwiredtiger_lz4.so]", &conn)); /*! [Configure lz4 extension] */ error_check(conn->close(conn, NULL)); /*! [Configure snappy extension] */ - error_check(wiredtiger_open(home, NULL, - "create," - "extensions=[/usr/local/lib/libwiredtiger_snappy.so]", - &conn)); + error_check(wiredtiger_open( + home, NULL, "create,extensions=[/usr/local/lib/libwiredtiger_snappy.so]", &conn)); /*! [Configure snappy extension] */ error_check(conn->close(conn, NULL)); /*! [Configure zlib extension] */ - error_check(wiredtiger_open(home, NULL, - "create," - "extensions=[/usr/local/lib/libwiredtiger_zlib.so]", - &conn)); + error_check(wiredtiger_open( + home, NULL, "create,extensions=[/usr/local/lib/libwiredtiger_zlib.so]", &conn)); /*! [Configure zlib extension] */ error_check(conn->close(conn, NULL)); /*! [Configure zlib extension with compression level] */ error_check(wiredtiger_open(home, NULL, - "create," - "extensions=[/usr/local/lib/" - "libwiredtiger_zlib.so=[config=[compression_level=3]]]", + "create,extensions=[/usr/local/lib/libwiredtiger_zlib.so=[config=[compression_level=3]]]", &conn)); /*! [Configure zlib extension with compression level] */ error_check(conn->close(conn, NULL)); /*! [Configure zstd extension] */ - error_check(wiredtiger_open(home, NULL, - "create," - "extensions=[/usr/local/lib/libwiredtiger_zstd.so]", - &conn)); + error_check(wiredtiger_open( + home, NULL, "create,extensions=[/usr/local/lib/libwiredtiger_zstd.so]", &conn)); /*! [Configure zstd extension] */ error_check(conn->close(conn, NULL)); /*! [Configure zstd extension with compression level] */ error_check(wiredtiger_open(home, NULL, - "create," - "extensions=[/usr/local/lib/" - "libwiredtiger_zstd.so=[config=[compression_level=9]]]", + "create,extensions=[/usr/local/lib/libwiredtiger_zstd.so=[config=[compression_level=9]]]", &conn)); /*! [Configure zstd extension with compression level] */ error_check(conn->close(conn, NULL)); @@ -1257,9 +1242,7 @@ main(int argc, char *argv[]) */ /*! [Statistics logging with a table] */ error_check(wiredtiger_open(home, NULL, - "create, statistics_log=(" - "sources=(\"table:table1\",\"table:table2\"), wait=5)", - &conn)); + "create, statistics_log=(sources=(\"table:table1\",\"table:table2\"), wait=5)", &conn)); /*! [Statistics logging with a table] */ error_check(conn->close(conn, NULL)); diff --git a/src/third_party/wiredtiger/examples/c/ex_async.c b/src/third_party/wiredtiger/examples/c/ex_async.c index 54df30e3ef2..b4d01d6d57f 100644 --- a/src/third_party/wiredtiger/examples/c/ex_async.c +++ b/src/third_party/wiredtiger/examples/c/ex_async.c @@ -110,10 +110,8 @@ main(int argc, char *argv[]) home = example_setup(argc, argv); /*! [async example connection] */ - error_check(wiredtiger_open(home, NULL, - "create,cache_size=100MB," - "async=(enabled=true,ops_max=20,threads=2)", - &conn)); + error_check(wiredtiger_open( + home, NULL, "create,cache_size=100MB,async=(enabled=true,ops_max=20,threads=2)", &conn)); /*! [async example connection] */ /*! [async example table create] */ diff --git a/src/third_party/wiredtiger/examples/c/ex_backup_block.c b/src/third_party/wiredtiger/examples/c/ex_backup_block.c index 24ec718af53..be21c76051d 100644 --- a/src/third_party/wiredtiger/examples/c/ex_backup_block.c +++ b/src/third_party/wiredtiger/examples/c/ex_backup_block.c @@ -387,9 +387,16 @@ take_incr_backup(WT_SESSION *session, int i) first = false; } - error_sys_check(lseek(rfd, (wt_off_t)offset, SEEK_SET)); + /* + * Don't use the system checker for lseek. The system check macro uses an int which + * is often 4 bytes and checks for any negative value. The offset returned from + * lseek is 8 bytes and we can have a false positive error check. + */ + if (lseek(rfd, (wt_off_t)offset, SEEK_SET) == -1) + testutil_die(errno, "lseek: read"); error_sys_check(rdsize = (size_t)read(rfd, tmp, (size_t)size)); - error_sys_check(lseek(wfd, (wt_off_t)offset, SEEK_SET)); + if (lseek(wfd, (wt_off_t)offset, SEEK_SET) == -1) + testutil_die(errno, "lseek: write"); /* Use the read size since we may have read less than the granularity. */ error_sys_check(write(wfd, tmp, rdsize)); } else { diff --git a/src/third_party/wiredtiger/examples/c/ex_call_center.c b/src/third_party/wiredtiger/examples/c/ex_call_center.c index c3475535ab2..b73ba5d7282 100644 --- a/src/third_party/wiredtiger/examples/c/ex_call_center.c +++ b/src/third_party/wiredtiger/examples/c/ex_call_center.c @@ -95,10 +95,7 @@ main(int argc, char *argv[]) * in two groups: "main" and "address", created below. */ error_check(session->create(session, "table:customers", - "key_format=r," - "value_format=SSS," - "columns=(id,name,address,phone)," - "colgroups=(main,address)")); + "key_format=r,value_format=SSS,columns=(id,name,address,phone),colgroups=(main,address)")); /* Create the main column group with value columns except address. */ error_check(session->create(session, "colgroup:customers:main", "columns=(name,phone)")); @@ -122,9 +119,7 @@ main(int argc, char *argv[]) * together, so no column groups are declared. */ error_check(session->create(session, "table:calls", - "key_format=r," - "value_format=qrrSS," - "columns=(id,call_date,cust_id,emp_id,call_type,notes)")); + "key_format=r,value_format=qrrSS,columns=(id,call_date,cust_id,emp_id,call_type,notes)")); /* * Create an index on the calls table with a composite key of cust_id and call_date. diff --git a/src/third_party/wiredtiger/examples/c/ex_cursor.c b/src/third_party/wiredtiger/examples/c/ex_cursor.c index beb4c9cc007..61861a05264 100644 --- a/src/third_party/wiredtiger/examples/c/ex_cursor.c +++ b/src/third_party/wiredtiger/examples/c/ex_cursor.c @@ -171,8 +171,7 @@ main(int argc, char *argv[]) error_check(conn->open_session(conn, NULL, NULL, &session)); error_check(session->create(session, "table:world", - "key_format=r,value_format=5sii," - "columns=(id,country,population,area)")); + "key_format=r,value_format=5sii,columns=(id,country,population,area)")); /*! [open cursor #1] */ error_check(session->open_cursor(session, "table:world", NULL, NULL, &cursor)); diff --git a/src/third_party/wiredtiger/examples/c/ex_encrypt.c b/src/third_party/wiredtiger/examples/c/ex_encrypt.c index 9512ad89600..66fb2504aba 100644 --- a/src/third_party/wiredtiger/examples/c/ex_encrypt.c +++ b/src/third_party/wiredtiger/examples/c/ex_encrypt.c @@ -431,25 +431,21 @@ main(int argc, char *argv[]) * Create and open some encrypted and not encrypted tables. Also use column store and * compression for some tables. */ - error_check( - session->create(session, "table:crypto1", "encryption=(name=rotn,keyid=" USER1_KEYID ")," - "columns=(key0,value0)," - "key_format=S,value_format=S")); - error_check(session->create(session, - "index:crypto1:byvalue", "encryption=(name=rotn,keyid=" USER1_KEYID ")," - "columns=(value0,key0)")); - error_check( - session->create(session, "table:crypto2", "encryption=(name=rotn,keyid=" USER2_KEYID ")," - "key_format=S,value_format=S")); + error_check(session->create(session, "table:crypto1", + "encryption=(name=rotn,keyid=" USER1_KEYID + "),columns=(key0,value0),key_format=S,value_format=S")); + error_check(session->create(session, "index:crypto1:byvalue", + "encryption=(name=rotn,keyid=" USER1_KEYID "),columns=(value0,key0)")); + error_check(session->create(session, "table:crypto2", + "encryption=(name=rotn,keyid=" USER2_KEYID "),key_format=S,value_format=S")); error_check(session->create(session, "table:nocrypto", "key_format=S,value_format=S")); /* * Send in an unknown keyid. WiredTiger will try to add in the new keyid, but the customize * function above will return an error since it is unrecognized. */ - ret = session->create(session, "table:cryptobad", "encryption=(name=rotn,keyid=" USERBAD_KEYID - ")," - "key_format=S,value_format=S"); + ret = session->create(session, "table:cryptobad", + "encryption=(name=rotn,keyid=" USERBAD_KEYID "),key_format=S,value_format=S"); if (ret == 0) { fprintf(stderr, "Did not detect bad/unknown keyid error\n"); exit(EXIT_FAILURE); diff --git a/src/third_party/wiredtiger/examples/c/ex_extractor.c b/src/third_party/wiredtiger/examples/c/ex_extractor.c index b4c74c35cfd..4530b97a395 100644 --- a/src/third_party/wiredtiger/examples/c/ex_extractor.c +++ b/src/third_party/wiredtiger/examples/c/ex_extractor.c @@ -183,8 +183,7 @@ setup_table(WT_SESSION *session) /* Create the primary table. It has a key of the unique ID. */ error_check(session->create(session, "table:presidents", - "key_format=I,value_format=SSHH," - "columns=(ID,last_name,first_name,term_begin,term_end)")); + "key_format=I,value_format=SSHH,columns=(ID,last_name,first_name,term_begin,term_end)")); /* * Create the index that is generated with an extractor. The index will generate an entry in the diff --git a/src/third_party/wiredtiger/examples/c/ex_file_system.c b/src/third_party/wiredtiger/examples/c/ex_file_system.c index 993e5b5f30c..1f213a07ff6 100644 --- a/src/third_party/wiredtiger/examples/c/ex_file_system.c +++ b/src/third_party/wiredtiger/examples/c/ex_file_system.c @@ -852,10 +852,8 @@ main(void) * function as the "config" value. */ open_config = - "create,log=(enabled=true),extensions=(local={" - "entry=demo_file_system_create,early_load=true," - "config={config_string=\"demo-file-system\",config_value=37}" - "})"; + "create,log=(enabled=true),extensions=(local={entry=demo_file_system_create,early_load=true," + "config={config_string=\"demo-file-system\",config_value=37}})"; /* Open a connection to the database, creating it if necessary. */ if ((ret = wiredtiger_open(home, NULL, open_config, &conn)) != 0) { fprintf(stderr, "Error connecting to %s: %s\n", home == NULL ? "." : home, diff --git a/src/third_party/wiredtiger/examples/c/ex_log.c b/src/third_party/wiredtiger/examples/c/ex_log.c index f02ffb15678..14c96fd839d 100644 --- a/src/third_party/wiredtiger/examples/c/ex_log.c +++ b/src/third_party/wiredtiger/examples/c/ex_log.c @@ -64,10 +64,8 @@ compare_tables(WT_SESSION *session, WT_SESSION *sess_copy) error_check(curs_copy->get_key(curs_copy, &key_copy)); error_check(curs_copy->get_value(curs_copy, &value_copy)); if (strcmp(key, key_copy) != 0 || strcmp(value, value_copy) != 0) { - fprintf(stderr, - "Mismatched: key %s, key_copy %s " - "value %s value_copy %s\n", - key, key_copy, value, value_copy); + fprintf(stderr, "Mismatched: key %s, key_copy %s value %s value_copy %s\n", key, + key_copy, value, value_copy); exit(1); } } diff --git a/src/third_party/wiredtiger/examples/c/ex_schema.c b/src/third_party/wiredtiger/examples/c/ex_schema.c index d9249efecc6..4d9e1aa0edc 100644 --- a/src/third_party/wiredtiger/examples/c/ex_schema.c +++ b/src/third_party/wiredtiger/examples/c/ex_schema.c @@ -72,10 +72,8 @@ main(int argc, char *argv[]) * string, uint16_t, uint64_t). See ::wiredtiger_struct_pack for details of the format strings. */ error_check(session->create(session, "table:poptable", - "key_format=r," - "value_format=5sHQ," - "columns=(id,country,year,population)," - "colgroups=(main,population)")); + "key_format=r,value_format=5sHQ,columns=(id,country,year,population),colgroups=(main," + "population)")); /* * Create two column groups: a primary column group with the country code, year and population diff --git a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c index 28f1d41bcfc..111c9081c73 100644 --- a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c @@ -181,9 +181,8 @@ lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, uint8_t *src, siz lz4_prefix_swap(&prefix); #endif if (prefix.compressed_len + sizeof(LZ4_PREFIX) > src_len) { - (void)wt_api->err_printf(wt_api, session, - "WT_COMPRESSOR.decompress: stored size exceeds source " - "size"); + (void)wt_api->err_printf( + wt_api, session, "WT_COMPRESSOR.decompress: stored size exceeds source size"); return (WT_ERROR); } diff --git a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c index 5e37a01147d..bc8a5576472 100644 --- a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c @@ -168,9 +168,8 @@ snappy_decompression(WT_COMPRESSOR *compressor, WT_SESSION *session, uint8_t *sr snaplen = snappy_bswap64(snaplen); #endif if (snaplen + SNAPPY_PREFIX > src_len) { - (void)wt_api->err_printf(wt_api, session, - "WT_COMPRESSOR.decompress: stored size exceeds source " - "size"); + (void)wt_api->err_printf( + wt_api, session, "WT_COMPRESSOR.decompress: stored size exceeds source size"); return (WT_ERROR); } diff --git a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c index 80db373a597..42eed08304c 100644 --- a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c @@ -257,10 +257,8 @@ zlib_init_config(WT_CONNECTION *connection, WT_CONFIG_ARG *config, int *zlib_lev */ zlib_level = (int)v.val; if (zlib_level < 0 || zlib_level > 9) { - (void)wt_api->err_printf(wt_api, NULL, - "zlib_init_config: " - "unsupported compression level %d", - zlib_level); + (void)wt_api->err_printf( + wt_api, NULL, "zlib_init_config: unsupported compression level %d", zlib_level); return (EINVAL); } *zlib_levelp = zlib_level; diff --git a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c index f6a0e09a809..f4e48ad61fd 100644 --- a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c @@ -157,9 +157,8 @@ zstd_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, uint8_t *src, si zstd_len = zstd_bswap64(zstd_len); #endif if (zstd_len + ZSTD_PREFIX > src_len) { - (void)wt_api->err_printf(wt_api, session, - "WT_COMPRESSOR.decompress: stored size exceeds source " - "size"); + (void)wt_api->err_printf( + wt_api, session, "WT_COMPRESSOR.decompress: stored size exceeds source size"); return (WT_ERROR); } diff --git a/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c b/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c index 02ca4fab2d1..15799625e1b 100644 --- a/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c +++ b/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c @@ -763,9 +763,8 @@ wiredtiger_extension_init(WT_CONNECTION *conn, WT_CONFIG_ARG *config) continue; (void)wtext->err_printf(wtext, NULL, - "WT_CONFIG_PARSER.next: unexpected configuration " - "information: %.*s=%.*s: %s", - (int)k.len, k.str, (int)v.len, v.str, wtext->strerror(wtext, NULL, ret)); + "WT_CONFIG_PARSER.next: unexpected configuration information: %.*s=%.*s: %s", (int)k.len, + k.str, (int)v.len, v.str, wtext->strerror(wtext, NULL, ret)); goto err; } if (ret != WT_NOTFOUND) { diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index a2af96455f4..a848a6155d8 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "579966149b9094922f22d9bd7e8f3b8b9db1ccf7" + "commit": "6a7db4f96fe828fdb1b9a31d83460c0573a2c3b1" } diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c index 17054e0b852..06f8291b08b 100644 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ b/src/third_party/wiredtiger/src/async/async_api.c @@ -70,7 +70,7 @@ __async_get_format( TAILQ_INSERT_HEAD(&async->formatqh, af, q); __wt_spin_unlock(session, &async->ops_lock); - WT_ERR(wt_session->close(wt_session, NULL)); + WT_ERR(__wt_session_close_internal(session)); setup: op->format = af; @@ -292,7 +292,6 @@ __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) WT_ASYNC *async; WT_CONNECTION_IMPL *conn, tmp_conn; WT_DECL_RET; - WT_SESSION *wt_session; uint32_t i, session_flags; bool run; @@ -383,8 +382,7 @@ __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) WT_ASSERT(session, async->worker_sessions[i] != NULL); F_CLR(async->worker_sessions[i], WT_SESSION_SERVER_ASYNC); WT_TRET(__wt_thread_join(session, &async->worker_tids[i])); - wt_session = &async->worker_sessions[i]->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(async->worker_sessions[i])); async->worker_sessions[i] = NULL; } conn->async_workers = tmp_conn.async_workers; @@ -405,7 +403,6 @@ __wt_async_destroy(WT_SESSION_IMPL *session) WT_ASYNC_OP *op; WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION *wt_session; uint32_t i; conn = S2C(session); @@ -422,8 +419,7 @@ __wt_async_destroy(WT_SESSION_IMPL *session) /* Close the server threads' sessions. */ for (i = 0; i < conn->async_workers; i++) if (async->worker_sessions[i] != NULL) { - wt_session = &async->worker_sessions[i]->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(async->worker_sessions[i])); async->worker_sessions[i] = NULL; } /* Free any op key/value buffers. */ diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 6a10df06fff..7199a6b4c24 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -197,8 +197,8 @@ __wt_block_checkpoint_start(WT_SESSION_IMPL *session, WT_BLOCK *block) case WT_CKPT_PANIC_ON_FAILURE: case WT_CKPT_SALVAGE: ret = __wt_panic(session, EINVAL, - "%s: an unexpected checkpoint start: the checkpoint " - "has already started or was configured for salvage", + "%s: an unexpected checkpoint start: the checkpoint has already started or was " + "configured for salvage", block->name); __wt_block_set_readonly(session); break; @@ -506,8 +506,8 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) case WT_CKPT_NONE: case WT_CKPT_PANIC_ON_FAILURE: ret = __wt_panic(session, EINVAL, - "%s: an unexpected checkpoint attempt: the checkpoint " - "was never started or has already completed", + "%s: an unexpected checkpoint attempt: the checkpoint was never started or has already " + "completed", block->name); __wt_block_set_readonly(session); break; @@ -751,9 +751,8 @@ live_update: if ((a = ckpt->bpriv) == NULL) a = &block->live; if (a->discard.entries != 0) - WT_ERR_MSG(session, WT_ERROR, - "first checkpoint incorrectly has blocks on the discard " - "list"); + WT_ERR_MSG( + session, WT_ERROR, "first checkpoint incorrectly has blocks on the discard list"); #endif err: @@ -913,8 +912,8 @@ __wt_block_checkpoint_resolve(WT_SESSION_IMPL *session, WT_BLOCK *block, bool fa case WT_CKPT_NONE: case WT_CKPT_SALVAGE: ret = __wt_panic(session, EINVAL, - "%s: an unexpected checkpoint resolution: the checkpoint " - "was never started or completed, or configured for salvage", + "%s: an unexpected checkpoint resolution: the checkpoint was never started or completed, " + "or configured for salvage", block->name); __wt_block_set_readonly(session); break; diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 0189da70acb..19ee0b664f6 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -108,17 +108,15 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) block->compact_pct_tenths = 1; } - __wt_verbose(session, WT_VERB_COMPACT, "%s: %" PRIuMAX "MB (%" PRIuMAX - ") available space in the first " - "80%% of the file", + __wt_verbose(session, WT_VERB_COMPACT, + "%s: %" PRIuMAX "MB (%" PRIuMAX ") available space in the first 80%% of the file", block->name, (uintmax_t)avail_eighty / WT_MEGABYTE, (uintmax_t)avail_eighty); - __wt_verbose(session, WT_VERB_COMPACT, "%s: %" PRIuMAX "MB (%" PRIuMAX - ") available space in the first " - "90%% of the file", + __wt_verbose(session, WT_VERB_COMPACT, + "%s: %" PRIuMAX "MB (%" PRIuMAX ") available space in the first 90%% of the file", block->name, (uintmax_t)avail_ninety / WT_MEGABYTE, (uintmax_t)avail_ninety); - __wt_verbose(session, WT_VERB_COMPACT, "%s: require 10%% or %" PRIuMAX "MB (%" PRIuMAX - ") in the first " - "90%% of the file to perform compaction, compaction %s", + __wt_verbose(session, WT_VERB_COMPACT, + "%s: require 10%% or %" PRIuMAX "MB (%" PRIuMAX + ") in the first 90%% of the file to perform compaction, compaction %s", block->name, (uintmax_t)(block->size / 10) / WT_MEGABYTE, (uintmax_t)block->size / 10, *skipp ? "skipped" : "proceeding"); @@ -206,8 +204,9 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) session, WT_VERB_COMPACT, "pages written: %" PRIu64, block->compact_pages_written); } - __wt_verbose(session, WT_VERB_COMPACT, "file size %" PRIuMAX "MB (%" PRIuMAX ") with %" PRIuMAX - "%% space available %" PRIuMAX "MB (%" PRIuMAX ")", + __wt_verbose(session, WT_VERB_COMPACT, + "file size %" PRIuMAX "MB (%" PRIuMAX ") with %" PRIuMAX "%% space available %" PRIuMAX + "MB (%" PRIuMAX ")", (uintmax_t)size / WT_MEGABYTE, (uintmax_t)size, ((uintmax_t)el->bytes * 100) / (uintmax_t)size, (uintmax_t)el->bytes / WT_MEGABYTE, (uintmax_t)el->bytes); diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index b11cdf9efff..06c44680f1a 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -13,12 +13,13 @@ * Handle extension list errors that would normally panic the system but * which should fail gracefully when verifying. */ -#define WT_BLOCK_RET(session, block, v, ...) \ - do { \ - int __ret = (v); \ - __wt_err(session, __ret, __VA_ARGS__); \ - return ((block)->verify ? __ret : __wt_panic(session, WT_PANIC, \ - "block manager extension list failure")); \ +#define WT_BLOCK_RET(session, block, v, ...) \ + do { \ + int __ret = (v); \ + __wt_err(session, __ret, __VA_ARGS__); \ + return ((block)->verify ? \ + __ret : \ + __wt_panic(session, WT_PANIC, "block manager extension list failure")); \ } while (0) static int __block_append(WT_SESSION_IMPL *, WT_BLOCK *, WT_EXTLIST *, wt_off_t, wt_off_t); @@ -494,9 +495,9 @@ __wt_block_alloc(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t *offp, wt_o WT_STAT_DATA_INCR(session, block_alloc); if (size % block->allocsize != 0) - WT_RET_MSG(session, EINVAL, "cannot allocate a block size %" PRIdMAX - " that is not " - "a multiple of the allocation size %" PRIu32, + WT_RET_MSG(session, EINVAL, + "cannot allocate a block size %" PRIdMAX + " that is not a multiple of the allocation size %" PRIu32, (intmax_t)size, block->allocsize); /* diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index e6b59847fcf..63deadcfac8 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -93,7 +93,7 @@ __block_destroy(WT_SESSION_IMPL *session, WT_BLOCK *block) uint64_t bucket; conn = S2C(session); - bucket = block->name_hash % WT_HASH_ARRAY_SIZE; + bucket = block->name_hash & (conn->hash_size - 1); WT_CONN_BLOCK_REMOVE(conn, block, bucket); __wt_free(session, block->name); @@ -147,7 +147,7 @@ __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[ conn = S2C(session); hash = __wt_hash_city64(filename, strlen(filename)); - bucket = hash % WT_HASH_ARRAY_SIZE; + bucket = hash & (conn->hash_size - 1); __wt_spin_lock(session, &conn->block_lock); TAILQ_FOREACH (block, &conn->blockhash[bucket], hashq) { if (strcmp(filename, block->name) == 0) { @@ -383,9 +383,8 @@ __desc_read(WT_SESSION_IMPL *session, uint32_t allocsize, WT_BLOCK *block) if (desc->majorv > WT_BLOCK_MAJOR_VERSION || (desc->majorv == WT_BLOCK_MAJOR_VERSION && desc->minorv > WT_BLOCK_MINOR_VERSION)) WT_ERR_MSG(session, WT_ERROR, - "unsupported WiredTiger file version: this build only " - "supports major/minor versions up to %d/%d, and the file " - "is version %" PRIu16 "/%" PRIu16, + "unsupported WiredTiger file version: this build only supports major/minor versions up " + "to %d/%d, and the file is version %" PRIu16 "/%" PRIu16, WT_BLOCK_MAJOR_VERSION, WT_BLOCK_MINOR_VERSION, desc->majorv, desc->minorv); __wt_verbose(session, WT_VERB_BLOCK, "%s: magic %" PRIu32 ", major/minor: %" PRIu32 "/%" PRIu32, diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 0226ede60e1..8680f1f90f8 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -238,9 +238,8 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_ * test. */ if (size < block->allocsize) - WT_RET_MSG(session, EINVAL, "%s: impossibly small block size of %" PRIu32 - "B, less than " - "allocation size of %" PRIu32, + WT_RET_MSG(session, EINVAL, + "%s: impossibly small block size of %" PRIu32 "B, less than allocation size of %" PRIu32, block->name, size, block->allocsize); WT_RET(__wt_buf_init(session, buf, bufsize)); @@ -266,20 +265,14 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_ } if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) - __wt_errx(session, "%s: read checksum error for %" PRIu32 - "B block at " - "offset %" PRIuMAX - ": calculated block checksum " - " doesn't match expected checksum", + __wt_errx(session, + "%s: read checksum error for %" PRIu32 "B block at offset %" PRIuMAX + ": calculated block checksum doesn't match expected checksum", block->name, size, (uintmax_t)offset); } else if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) - __wt_errx(session, "%s: read checksum error for %" PRIu32 - "B block at " - "offset %" PRIuMAX - ": block header checksum " - "of %#" PRIx32 - " doesn't match expected checksum " - "of %#" PRIx32, + __wt_errx(session, + "%s: read checksum error for %" PRIu32 "B block at offset %" PRIuMAX + ": block header checksum of %#" PRIx32 " doesn't match expected checksum of %#" PRIx32, block->name, size, (uintmax_t)offset, swap.checksum, checksum); if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index bb00d280419..57bba04e5da 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -361,9 +361,8 @@ __verify_filefrag_add(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *typ /* Check each chunk against the total file size. */ if (offset + size > block->size) - WT_RET_MSG(session, WT_ERROR, "fragment %" PRIuMAX "-%" PRIuMAX - " references " - "non-existent file blocks", + WT_RET_MSG(session, WT_ERROR, + "fragment %" PRIuMAX "-%" PRIuMAX " references non-existent file blocks", (uintmax_t)offset, (uintmax_t)(offset + size)); frag = (uint64_t)WT_wt_off_TO_FRAG(block, offset); @@ -373,10 +372,8 @@ __verify_filefrag_add(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *typ if (nodup) for (f = frag, i = 0; i < frags; ++f, ++i) if (__bit_test(block->fragfile, f)) - WT_RET_MSG(session, WT_ERROR, "file fragment at %" PRIuMAX - " referenced " - "multiple times", - (uintmax_t)offset); + WT_RET_MSG(session, WT_ERROR, + "file fragment at %" PRIuMAX " referenced multiple times", (uintmax_t)offset); /* Add fragments to the file's fragment list. */ __bit_nset(block->fragfile, frag, frag + (frags - 1)); @@ -458,9 +455,8 @@ __verify_ckptfrag_add(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset * outside of the checkpoint's stored size. */ if (offset + size > block->verify_size) - WT_RET_MSG(session, WT_ERROR, "fragment %" PRIuMAX "-%" PRIuMAX - " references " - "file blocks outside the checkpoint", + WT_RET_MSG(session, WT_ERROR, + "fragment %" PRIuMAX "-%" PRIuMAX " references file blocks outside the checkpoint", (uintmax_t)offset, (uintmax_t)(offset + size)); frag = (uint64_t)WT_wt_off_TO_FRAG(block, offset); @@ -469,11 +465,10 @@ __verify_ckptfrag_add(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset /* It is illegal to reference a particular chunk more than once. */ for (f = frag, i = 0; i < frags; ++f, ++i) if (!__bit_test(block->fragckpt, f)) - WT_RET_MSG(session, WT_ERROR, "fragment at %" PRIuMAX - " referenced multiple " - "times in a single checkpoint or found in the " - "checkpoint but not listed in the checkpoint's " - "allocation list", + WT_RET_MSG(session, WT_ERROR, + "fragment at %" PRIuMAX + " referenced multiple times in a single checkpoint or found in the checkpoint but " + "not listed in the checkpoint's allocation list", (uintmax_t)offset); /* Remove fragments from the checkpoint's allocation list. */ diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index a8a0091d854..c5890efd038 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -118,7 +118,7 @@ __wt_block_extend(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_FH *fh, wt_off_t */ if (block->extend_size > block->size && (offset > block->extend_size || - offset + block->extend_len + (wt_off_t)align_size < block->extend_size)) + offset + block->extend_len + (wt_off_t)align_size < block->extend_size)) return (0); /* diff --git a/src/third_party/wiredtiger/src/bloom/bloom.c b/src/third_party/wiredtiger/src/bloom/bloom.c index 99aacc24bea..ff48eb2db54 100644 --- a/src/third_party/wiredtiger/src/bloom/bloom.c +++ b/src/third_party/wiredtiger/src/bloom/bloom.c @@ -63,8 +63,7 @@ __bloom_setup(WT_BLOOM *bloom, uint64_t n, uint64_t m, uint32_t factor, uint32_t { if (k < 2) WT_RET_MSG(bloom->session, EINVAL, - "bloom filter hash values to be set/tested must be " - "greater than 2"); + "bloom filter hash values to be set/tested must be greater than 2"); bloom->k = k; bloom->factor = factor; @@ -350,10 +349,8 @@ __wt_bloom_intersection(WT_BLOOM *bloom, WT_BLOOM *other) if (bloom->k != other->k || bloom->factor != other->factor || bloom->m != other->m || bloom->n != other->n) WT_RET_MSG(bloom->session, EINVAL, - "bloom filter intersection configuration mismatch: (" - "%" PRIu32 "/%" PRIu32 ", %" PRIu32 "/%" PRIu32 - ", " - "%" PRIu64 "/%" PRIu64 ", %" PRIu64 "/%" PRIu64 ")", + "bloom filter intersection configuration mismatch: (%" PRIu32 "/%" PRIu32 ", %" PRIu32 + "/%" PRIu32 ", %" PRIu64 "/%" PRIu64 ", %" PRIu64 "/%" PRIu64 ")", bloom->k, other->k, bloom->factor, other->factor, bloom->m, other->m, bloom->n, other->n); nbytes = __bitstr_size(bloom->m); diff --git a/src/third_party/wiredtiger/src/btree/bt_compact.c b/src/third_party/wiredtiger/src/btree/bt_compact.c index 1f657fdd931..c205710684b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_compact.c +++ b/src/third_party/wiredtiger/src/btree/bt_compact.c @@ -104,10 +104,8 @@ __compact_progress(WT_SESSION_IMPL *session) time_diff = WT_TIMEDIFF_SEC(cur_time, session->compact->begin); if (time_diff / WT_PROGRESS_MSG_PERIOD > session->compact->prog_msg_count) { __wt_verbose(session, WT_VERB_COMPACT_PROGRESS, - "Compact running" - " for %" PRIu64 " seconds; reviewed %" PRIu64 " pages, skipped %" PRIu64 - " pages," - " wrote %" PRIu64 " pages", + "Compact running for %" PRIu64 " seconds; reviewed %" PRIu64 " pages, skipped %" PRIu64 + " pages, wrote %" PRIu64 " pages", time_diff, bm->block->compact_pages_reviewed, bm->block->compact_pages_skipped, bm->block->compact_pages_written); session->compact->prog_msg_count++; diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index e6047060faf..00ae7165454 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -116,7 +116,7 @@ new_page: __wt_upd_value_clear(cbt->upd_value); if (cbt->ins != NULL) restart_read: - WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd, NULL)); + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd, NULL)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); cbt->iface.value.data = &cbt->v; @@ -444,9 +444,8 @@ __cursor_key_order_check_col(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo return (0); } - WT_RET_PANIC(session, EINVAL, "WT_CURSOR.%s out-of-order returns: returned key %" PRIu64 - " then " - "key %" PRIu64, + WT_RET_PANIC(session, EINVAL, + "WT_CURSOR.%s out-of-order returns: returned key %" PRIu64 " then key %" PRIu64, next ? "next" : "prev", cbt->lastrecno, cbt->recno); } @@ -478,10 +477,10 @@ __cursor_key_order_check_row(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo WT_ERR(__wt_scr_alloc(session, 512, &b)); WT_ERR_PANIC(session, EINVAL, - "WT_CURSOR.%s out-of-order returns: returned key %.1024s then " - "key %.1024s", - next ? "next" : "prev", __wt_buf_set_printable_format(session, cbt->lastkey->data, - cbt->lastkey->size, btree->key_format, a), + "WT_CURSOR.%s out-of-order returns: returned key %.1024s then key %.1024s", + next ? "next" : "prev", + __wt_buf_set_printable_format( + session, cbt->lastkey->data, cbt->lastkey->size, btree->key_format, a), __wt_buf_set_printable_format(session, key->data, key->size, btree->key_format, b)); err: @@ -719,8 +718,9 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) * from reconciliation getting rid of the obsolete content. Hence mark the page dirty to * force it through reconciliation. */ - if (page != NULL && (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD || - (newpage && cbt->page_deleted_count > 0))) { + if (page != NULL && + (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD || + (newpage && cbt->page_deleted_count > 0))) { WT_ERR(__wt_page_dirty_and_evict_soon(session, cbt->ref)); WT_STAT_CONN_INCR(session, cache_eviction_force_delete); } diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index f3fcbd24144..c71fa5d25e8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -256,7 +256,7 @@ new_page: __wt_upd_value_clear(cbt->upd_value); if (cbt->ins != NULL) restart_read: - WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd, NULL)); + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd, NULL)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); cbt->iface.value.data = &cbt->v; @@ -675,8 +675,9 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) * from reconciliation getting rid of the obsolete content. Hence mark the page dirty to * force it through reconciliation. */ - if (page != NULL && (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD || - (newpage && cbt->page_deleted_count > 0))) { + if (page != NULL && + (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD || + (newpage && cbt->page_deleted_count > 0))) { WT_ERR(__wt_page_dirty_and_evict_soon(session, cbt->ref)); WT_STAT_CONN_INCR(session, cache_eviction_force_delete); } diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 66f525ff862..e5e2efada27 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -120,9 +120,9 @@ __cursor_size_chk(WT_SESSION_IMPL *session, WT_ITEM *kv) if (btree->type == BTREE_COL_FIX) { /* Fixed-size column-stores take a single byte. */ if (kv->size != 1) - WT_RET_MSG(session, EINVAL, "item size of %" WT_SIZET_FMT - " does not match " - "fixed-length file requirement of 1 byte", + WT_RET_MSG(session, EINVAL, + "item size of %" WT_SIZET_FMT + " does not match fixed-length file requirement of 1 byte", kv->size); return (0); } @@ -133,9 +133,9 @@ __cursor_size_chk(WT_SESSION_IMPL *session, WT_ITEM *kv) /* Check what we are willing to store in the tree. */ if (kv->size > WT_BTREE_MAX_OBJECT_SIZE) - WT_RET_MSG(session, EINVAL, "item size of %" WT_SIZET_FMT - " exceeds the maximum " - "supported WiredTiger size of %" PRIu32, + WT_RET_MSG(session, EINVAL, + "item size of %" WT_SIZET_FMT + " exceeds the maximum supported WiredTiger size of %" PRIu32, kv->size, WT_BTREE_MAX_OBJECT_SIZE); /* Check what the block manager can actually write. */ @@ -1433,9 +1433,8 @@ __wt_btcur_modify(WT_CURSOR_BTREE *cbt, WT_MODIFY *entries, int nentries) * operations. */ if (session->txn->isolation != WT_ISO_SNAPSHOT) - WT_ERR_MSG(session, ENOTSUP, - "not supported in read-committed or read-uncommitted " - "transactions"); + WT_ERR_MSG( + session, ENOTSUP, "not supported in read-committed or read-uncommitted transactions"); if (F_ISSET(session->txn, WT_TXN_AUTOCOMMIT)) WT_ERR_MSG(session, ENOTSUP, "not supported in implicit transactions"); diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 33f4bc3ab3b..631c6b4644d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -20,8 +20,6 @@ struct __wt_dbg { WT_ITEM *hs_key; /* History store lookups */ WT_ITEM *hs_value; - uint32_t session_flags; - bool hs_is_local; /* * When using the standard event handlers, the debugging output has to do its own message @@ -287,8 +285,8 @@ __debug_wrapup(WT_DBG *ds) session = ds->session; msg = ds->msg; - if (ds->hs_is_local) - WT_TRET(__wt_hs_cursor_close(session, ds->session_flags)); + if (session->hs_cursor != NULL) + WT_TRET(__wt_hs_cursor_close(session)); __wt_scr_free(session, &ds->key); __wt_scr_free(session, &ds->hs_key); @@ -414,12 +412,14 @@ err: static int __debug_hs_cursor(WT_DBG *ds, WT_CURSOR *hs_cursor) { + WT_CURSOR_BTREE *cbt; WT_SESSION_IMPL *session; WT_TIME_WINDOW tw; uint64_t hs_counter, hs_upd_type; uint32_t hs_btree_id; char time_string[WT_TIME_STRING_SIZE]; + cbt = (WT_CURSOR_BTREE *)hs_cursor; session = ds->session; WT_TIME_WINDOW_INIT(&tw); @@ -433,7 +433,7 @@ __debug_hs_cursor(WT_DBG *ds, WT_CURSOR *hs_cursor) WT_RET(ds->f(ds, "\t" "hs-modify: %s\n", - __wt_time_window_to_string(&tw, time_string))); + __wt_time_window_to_string(&cbt->upd_value->tw, time_string))); WT_RET(ds->f(ds, "\tV ")); WT_RET(__debug_modify(ds, ds->hs_value->data)); WT_RET(ds->f(ds, "\n")); @@ -442,7 +442,7 @@ __debug_hs_cursor(WT_DBG *ds, WT_CURSOR *hs_cursor) WT_RET(ds->f(ds, "\t" "hs-update: %s\n", - __wt_time_window_to_string(&tw, time_string))); + __wt_time_window_to_string(&cbt->upd_value->tw, time_string))); WT_RET(__debug_item_value(ds, "V", ds->hs_value->data, ds->hs_value->size)); break; default: @@ -974,15 +974,13 @@ __wt_debug_cursor_tree_hs(void *cursor_arg, const char *ofile) WT_CURSOR_BTREE *cbt; WT_DECL_RET; WT_SESSION_IMPL *session; - uint32_t session_flags; session = CUR2S(cursor_arg); - session_flags = 0; /* [-Werror=maybe-uninitialized] */ - WT_RET(__wt_hs_cursor_open(session, &session_flags)); + WT_RET(__wt_hs_cursor_open(session)); cbt = (WT_CURSOR_BTREE *)session->hs_cursor; WT_WITH_BTREE(session, CUR2BT(cbt), ret = __wt_debug_tree_all(session, NULL, NULL, ofile)); - WT_TRET(__wt_hs_cursor_close(session, session_flags)); + WT_TRET(__wt_hs_cursor_close(session)); return (ret); } @@ -1028,9 +1026,7 @@ __debug_page(WT_DBG *ds, WT_REF *ref, uint32_t flags) * doesn't work, we may be running in-memory. */ if (!WT_IS_HS(S2BT(session))) { - if (session->hs_cursor == NULL && __wt_hs_cursor_open(session, &ds->session_flags) == 0) - ds->hs_is_local = true; - if (session->hs_cursor != NULL) { + if (session->hs_cursor != NULL || __wt_hs_cursor_open(session) == 0) { WT_RET(__wt_scr_alloc(session, 0, &ds->hs_key)); WT_RET(__wt_scr_alloc(session, 0, &ds->hs_value)); } diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index 5c0e658d80d..f1f397d926f 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -244,7 +244,7 @@ __wt_delete_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool visible_all) */ if (skip && ref->page_del != NULL && (visible_all || - __wt_txn_visible_all(session, ref->page_del->txnid, ref->page_del->timestamp))) { + __wt_txn_visible_all(session, ref->page_del->txnid, ref->page_del->timestamp))) { __wt_free(session, ref->page_del->update_list); __wt_free(session, ref->page_del); } diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index b6f8a31034f..d9b30f9483a 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -177,8 +177,10 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) * eviction, it must either clear the evict-disabled-open flag or restore the eviction * configuration when finished so that handle close behaves correctly. */ - if (btree->original || F_ISSET(btree, WT_BTREE_IN_MEMORY | WT_BTREE_REBALANCE | - WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) { + if (btree->original || + F_ISSET(btree, + WT_BTREE_IN_MEMORY | WT_BTREE_REBALANCE | WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | + WT_BTREE_VERIFY)) { WT_ERR(__wt_evict_file_exclusive_on(session)); btree->evict_disabled_open = true; } @@ -222,7 +224,8 @@ __wt_btree_close(WT_SESSION_IMPL *session) * Verify the history store state. If the history store is open and this btree has history store * entries, it can't be a metadata file, nor can it be the history store file. */ - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_HS_OPEN) || !btree->hs_entries || + WT_ASSERT(session, + !F_ISSET(S2C(session), WT_CONN_HS_OPEN) || !btree->hs_entries || (!WT_IS_METADATA(btree->dhandle) && !WT_IS_HS(btree))); /* @@ -366,8 +369,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) if (fixed) { if (bitcnt == 0 || bitcnt > 8) WT_RET_MSG(session, EINVAL, - "fixed-width field sizes must be greater " - "than 0 and less than or equal to 8"); + "fixed-width field sizes must be greater than 0 and less than or equal to 8"); btree->bitcnt = (uint8_t)bitcnt; btree->type = BTREE_COL_FIX; } @@ -386,8 +388,8 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) if (cval.val) { if (!F_ISSET(conn, WT_CONN_IN_MEMORY)) WT_RET_MSG(session, EINVAL, - "ignore_in_memory_cache_size setting is only valid " - "with databases configured to run in-memory"); + "ignore_in_memory_cache_size setting is only valid with databases configured to run " + "in-memory"); F_SET(btree, WT_BTREE_IGNORE_CACHE); } else F_CLR(btree, WT_BTREE_IGNORE_CACHE); @@ -603,13 +605,11 @@ __wt_btree_tree_open(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_ if (ret != 0 && WT_IS_METADATA(session->dhandle)) { __wt_err(session, ret, "WiredTiger has failed to open its metadata"); __wt_err(session, ret, - "This may be due to the database" - " files being encrypted, being from an older" - " version or due to corruption on disk"); + "This may be due to the database files being encrypted, being from an older version or " + "due to corruption on disk"); __wt_err(session, ret, - "You should confirm that you have" - " opened the database with the correct options including" - " all encryption and compression options"); + "You should confirm that you have opened the database with the correct options including " + "all encryption and compression options"); } WT_ERR(ret); @@ -832,8 +832,7 @@ __btree_page_sizes(WT_SESSION_IMPL *session) if (btree->maxintlpage < btree->allocsize || btree->maxintlpage % btree->allocsize != 0 || btree->maxleafpage < btree->allocsize || btree->maxleafpage % btree->allocsize != 0) WT_RET_MSG(session, EINVAL, - "page sizes must be a multiple of the page allocation " - "size (%" PRIu32 "B)", + "page sizes must be a multiple of the page allocation size (%" PRIu32 "B)", btree->allocsize); /* @@ -847,8 +846,8 @@ __btree_page_sizes(WT_SESSION_IMPL *session) btree->maxmempage_image = 4 * max; else if (btree->maxmempage_image < max) WT_RET_MSG(session, EINVAL, - "in-memory page image size must be larger than the maximum " - "page size (%" PRIu32 "B < %" PRIu32 "B)", + "in-memory page image size must be larger than the maximum page size (%" PRIu32 + "B < %" PRIu32 "B)", btree->maxmempage_image, max); /* @@ -884,9 +883,7 @@ __btree_page_sizes(WT_SESSION_IMPL *session) WT_RET(__wt_config_gets(session, cfg, "split_pct", &cval)); if (cval.val < WT_BTREE_MIN_SPLIT_PCT) { btree->split_pct = WT_BTREE_MIN_SPLIT_PCT; - WT_RET(__wt_msg(session, - "Re-setting split_pct for %s to the minimum allowed of " - "%d%%", + WT_RET(__wt_msg(session, "Re-setting split_pct for %s to the minimum allowed of %d%%", session->dhandle->name, WT_BTREE_MIN_SPLIT_PCT)); } else btree->split_pct = (int)cval.val; diff --git a/src/third_party/wiredtiger/src/btree/bt_huffman.c b/src/third_party/wiredtiger/src/btree/bt_huffman.c index 0e5597da756..8496339d666 100644 --- a/src/third_party/wiredtiger/src/btree/bt_huffman.c +++ b/src/third_party/wiredtiger/src/btree/bt_huffman.c @@ -219,8 +219,7 @@ __wt_btree_huffman_open(WT_SESSION_IMPL *session) case BTREE_COL_VAR: if (key_conf.len != 0) WT_RET_MSG(session, EINVAL, - "the keys of variable-length column-store files " - "may not be Huffman encoded"); + "the keys of variable-length column-store files may not be Huffman encoded"); break; case BTREE_ROW: break; @@ -330,27 +329,22 @@ __wt_huffman_read(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *ip, struct __wt_huff */ if (lineno > entries + 1) WT_ERR_MSG(session, EINVAL, - "Huffman table file %.*s is corrupted, " - "more than %" PRIu32 " entries", - (int)ip->len, ip->str, entries + 1); + "Huffman table file %.*s is corrupted, more than %" PRIu32 " entries", (int)ip->len, + ip->str, entries + 1); if (n != 2) WT_ERR_MSG(session, EINVAL, - "line %u of Huffman table file %.*s is corrupted: " - "expected two unsigned integral values", + "line %u of Huffman table file %.*s is corrupted: expected two unsigned integral " + "values", lineno, (int)ip->len, ip->str); if (symbol < 0 || symbol > entries) WT_ERR_MSG(session, EINVAL, - "line %u of Huffman file %.*s is corrupted; " - "symbol %" PRId64 - " not in range, maximum " - "value is %u", + "line %u of Huffman file %.*s is corrupted; symbol %" PRId64 + " not in range, maximum value is %u", lineno, (int)ip->len, ip->str, symbol, entries); if (frequency < 0 || frequency > UINT32_MAX) WT_ERR_MSG(session, EINVAL, - "line %u of Huffman file %.*s is corrupted; " - "frequency %" PRId64 - " not in range, maximum " - "value is %" PRIu32, + "line %u of Huffman file %.*s is corrupted; frequency %" PRId64 + " not in range, maximum value is %" PRIu32, lineno, (int)ip->len, ip->str, frequency, (uint32_t)UINT32_MAX); tp->symbol = (uint32_t)symbol; diff --git a/src/third_party/wiredtiger/src/btree/bt_import.c b/src/third_party/wiredtiger/src/btree/bt_import.c index 91fa8c471cf..ba3335d9ae7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_import.c +++ b/src/third_party/wiredtiger/src/btree/bt_import.c @@ -58,8 +58,8 @@ __wt_import(WT_SESSION_IMPL *session, const char *uri) WT_ERR(__wt_btree_config_encryptor(session, filecfg, &kencryptor)); if ((kencryptor == NULL && v.val != 0) || (kencryptor != NULL && v.val == 0)) WT_ERR_MSG(session, EINVAL, - "%s: loaded object's encryption configuration doesn't " - "match the database's encryption configuration", + "%s: loaded object's encryption configuration doesn't match the database's encryption " + "configuration", filename); /* * The metadata was quoted to avoid configuration string characters acting as separators. diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index 7fac3deabd5..6e76441d527 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -52,9 +52,7 @@ __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t if (F_ISSET(dsk, WT_PAGE_ENCRYPTED)) { if (btree->kencryptor == NULL || (encryptor = btree->kencryptor->encryptor) == NULL || encryptor->decrypt == NULL) { - fail_msg = - "encrypted block in file for which no encryption " - "configured"; + fail_msg = "encrypted block in file for which no encryption configured"; goto corrupt; } @@ -73,9 +71,7 @@ __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t if (F_ISSET(dsk, WT_PAGE_COMPRESSED)) { if (btree->compressor == NULL || btree->compressor->decompress == NULL) { - fail_msg = - "compressed block in file for which no compression " - "configured"; + fail_msg = "compressed block in file for which no compression configured"; goto corrupt; } @@ -181,7 +177,8 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *add time_start = time_stop = 0; /* Checkpoint calls are different than standard calls. */ - WT_ASSERT(session, (!checkpoint && addr != NULL && addr_sizep != NULL) || + WT_ASSERT(session, + (!checkpoint && addr != NULL && addr_sizep != NULL) || (checkpoint && addr == NULL && addr_sizep == NULL)); /* In-memory databases shouldn't write pages. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index 7a325ffe3eb..5a9cc9cea41 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -531,32 +531,36 @@ __inmem_row_leaf_entries(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, ui static int __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) { + enum { PREPARE_INSTANTIATE, PREPARE_INITIALIZED, PREPARE_IGNORE } prepare; WT_BTREE *btree; WT_CELL_UNPACK_KV unpack; WT_DECL_ITEM(value); WT_DECL_RET; WT_ROW *rip; - WT_UPDATE *tombstone, *upd, **upd_array; + WT_UPDATE *tombstone, *upd; size_t size, total_size; - uint32_t i; - bool instantiate_prepared, prepare; btree = S2BT(session); tombstone = upd = NULL; - prepare = false; + size = total_size = 0; - instantiate_prepared = F_ISSET(session, WT_SESSION_INSTANTIATE_PREPARE); + /* + * Optionally instantiate prepared updates. In-memory databases restore non-obsolete updates on + * the page as part of the __split_multi_inmem function. + */ + prepare = F_ISSET(session, WT_SESSION_INSTANTIATE_PREPARE) && + !F_ISSET(S2C(session), WT_CONN_IN_MEMORY) ? + PREPARE_INSTANTIATE : + PREPARE_IGNORE; /* Walk the page, building indices. */ rip = page->pg_row; WT_CELL_FOREACH_KV (session, page->dsk, unpack) { - if (instantiate_prepared && !prepare && unpack.tw.prepare) - prepare = true; switch (unpack.type) { case WT_CELL_KEY_OVFL: __wt_row_leaf_key_set_cell(page, rip, unpack.cell); ++rip; - break; + continue; case WT_CELL_KEY: /* * Simple keys without compression (not Huffman encoded or prefix compressed), can be @@ -567,7 +571,7 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) else __wt_row_leaf_key_set_cell(page, rip, unpack.cell); ++rip; - break; + continue; case WT_CELL_VALUE: /* * Simple values without compression can be directly referenced on the page to avoid @@ -576,80 +580,87 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) * The visibility information is not referenced on the page so we need to ensure that * the value is globally visible at the point in time where we read the page into cache. */ - if (!btree->huffman_value && (WT_TIME_WINDOW_IS_EMPTY(&unpack.tw) || - (!WT_TIME_WINDOW_HAS_STOP(&unpack.tw) && - __wt_txn_tw_start_visible_all(session, &unpack.tw)))) + if (!btree->huffman_value && + (WT_TIME_WINDOW_IS_EMPTY(&unpack.tw) || + (!WT_TIME_WINDOW_HAS_STOP(&unpack.tw) && + __wt_txn_tw_start_visible_all(session, &unpack.tw)))) __wt_row_leaf_value_set(page, rip - 1, &unpack); break; case WT_CELL_VALUE_OVFL: break; default: - return (__wt_illegal_value(session, unpack.type)); + WT_ERR(__wt_illegal_value(session, unpack.type)); } - } - WT_CELL_FOREACH_END; - /* - * Instantiate prepared updates on leaf pages when the page is loaded. For in-memory databases, - * all non obsolete updates will retain on the page as part of __split_multi_inmem function. - */ - if (prepare && !F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) { - WT_RET(__wt_page_modify_init(session, page)); - if (!F_ISSET(btree, WT_BTREE_READONLY)) - __wt_page_modify_set(session, page); - - /* Allocate the per-page update array if one doesn't already exist. */ - if (page->entries != 0 && page->modify->mod_row_update == NULL) - WT_PAGE_ALLOC_AND_SWAP( - session, page, page->modify->mod_row_update, upd_array, page->entries); - - /* For each entry in the page */ - size = total_size = 0; - upd_array = page->modify->mod_row_update; - WT_ROW_FOREACH (page, rip, i) { - /* Unpack the on-page value cell. */ - __wt_row_leaf_value_cell(session, page, rip, NULL, &unpack); - if (unpack.tw.prepare) { - /* Take the value from the original page cell. */ - if (value == NULL) - WT_ERR(__wt_scr_alloc(session, 0, &value)); - WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value)); - - WT_ERR(__wt_upd_alloc(session, value, WT_UPDATE_STANDARD, &upd, &size)); - total_size += size; - upd->durable_ts = unpack.tw.durable_start_ts; - upd->start_ts = unpack.tw.start_ts; - upd->txnid = unpack.tw.start_txn; - - /* - * Instantiating both update and tombstone if the prepared update is a tombstone. - * This is required to ensure that written prepared delete operation must be removed - * from the data store, when the prepared transaction gets rollback. - */ - if (WT_TIME_WINDOW_HAS_STOP(&unpack.tw)) { - WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, &size)); - total_size += size; - tombstone->durable_ts = WT_TS_NONE; - tombstone->start_ts = unpack.tw.stop_ts; - tombstone->txnid = unpack.tw.stop_txn; - tombstone->prepare_state = WT_PREPARE_INPROGRESS; - F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS); - F_SET(upd, WT_UPDATE_RESTORED_FROM_DS); - tombstone->next = upd; - } else { - upd->durable_ts = WT_TS_NONE; - upd->prepare_state = WT_PREPARE_INPROGRESS; - F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS); - tombstone = upd; - } - - upd_array[WT_ROW_SLOT(page, rip)] = tombstone; - tombstone = upd = NULL; + if (!unpack.tw.prepare || prepare == PREPARE_IGNORE) + continue; + + /* First prepared transaction setup. */ + if (prepare == PREPARE_INSTANTIATE) { + WT_ERR(__wt_page_modify_init(session, page)); + if (!F_ISSET(btree, WT_BTREE_READONLY)) + __wt_page_modify_set(session, page); + + /* Allocate the per-page update array. */ + WT_ERR(__wt_calloc_def(session, page->entries, &page->modify->mod_row_update)); + total_size += page->entries * sizeof(*page->modify->mod_row_update); + + WT_ERR(__wt_scr_alloc(session, 0, &value)); + + prepare = PREPARE_INITIALIZED; + } + + /* Take the value from the page cell. */ + WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value)); + + WT_ERR(__wt_upd_alloc(session, value, WT_UPDATE_STANDARD, &upd, &size)); + total_size += size; + upd->durable_ts = unpack.tw.durable_start_ts; + upd->start_ts = unpack.tw.start_ts; + upd->txnid = unpack.tw.start_txn; + + /* + * Instantiate both update and tombstone if the prepared update is a tombstone. This is + * required to ensure that written prepared delete operation must be removed from the data + * store, when the prepared transaction gets rollback. + */ + if (WT_TIME_WINDOW_HAS_STOP(&unpack.tw)) { + WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, &size)); + total_size += size; + tombstone->durable_ts = WT_TS_NONE; + tombstone->start_ts = unpack.tw.stop_ts; + tombstone->txnid = unpack.tw.stop_txn; + tombstone->prepare_state = WT_PREPARE_INPROGRESS; + F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS); + F_SET(upd, WT_UPDATE_RESTORED_FROM_DS); + + /* + * Mark the update also as in-progress if the update and tombstone are from same + * transaction by comparing both the transaction and timestamps as the transaction + * information gets lost after restart. + */ + if (unpack.tw.start_ts == unpack.tw.stop_ts && + unpack.tw.durable_start_ts == unpack.tw.durable_stop_ts && + unpack.tw.start_txn == unpack.tw.stop_txn) { + upd->durable_ts = WT_TS_NONE; + upd->prepare_state = WT_PREPARE_INPROGRESS; + F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS); } + + tombstone->next = upd; + } else { + upd->durable_ts = WT_TS_NONE; + upd->prepare_state = WT_PREPARE_INPROGRESS; + F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS); + tombstone = upd; } - __wt_cache_page_inmem_incr(session, page, total_size); + page->modify->mod_row_update[WT_ROW_SLOT(page, rip - 1)] = tombstone; + tombstone = upd = NULL; } + WT_CELL_FOREACH_END; + + __wt_cache_page_inmem_incr(session, page, total_size); err: __wt_free(session, tombstone); diff --git a/src/third_party/wiredtiger/src/btree/bt_random.c b/src/third_party/wiredtiger/src/btree/bt_random.c index cb79127188d..4e4eaedaa85 100644 --- a/src/third_party/wiredtiger/src/btree/bt_random.c +++ b/src/third_party/wiredtiger/src/btree/bt_random.c @@ -429,12 +429,12 @@ restart: return (WT_NOTFOUND); } - /* - * Swap the current page for the child page. If the page splits while we're retrieving it, - * restart the search at the root. - * - * On other error, simply return, the swap call ensures we're holding nothing on failure. - */ + /* + * Swap the current page for the child page. If the page splits while we're retrieving it, + * restart the search at the root. + * + * On other error, simply return, the swap call ensures we're holding nothing on failure. + */ descend: if ((ret = __wt_page_swap(session, current, descent, flags)) == 0) { current = descent; diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 286bb4939db..9684094c27e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -206,7 +206,7 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags , const char *func, int line #endif - ) +) { WT_BTREE *btree; WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/btree/bt_rebalance.c b/src/third_party/wiredtiger/src/btree/bt_rebalance.c index 9ab576812a1..44b379ba804 100644 --- a/src/third_party/wiredtiger/src/btree/bt_rebalance.c +++ b/src/third_party/wiredtiger/src/btree/bt_rebalance.c @@ -130,9 +130,9 @@ __rebalance_internal(WT_SESSION_IMPL *session, WT_REBALANCE_STUFF *rs) * a 4B quantity and it's technically possible there could be more pages than that in a tree. */ if (rs->leaf_next > UINT32_MAX) - WT_RET_MSG(session, ENOTSUP, "too many leaf pages to rebalance, %" WT_SIZET_FMT - " pages " - "exceeds the maximum of %" PRIu32, + WT_RET_MSG(session, ENOTSUP, + "too many leaf pages to rebalance, %" WT_SIZET_FMT + " pages exceeds the maximum of %" PRIu32, rs->leaf_next, UINT32_MAX); leaf_next = (uint32_t)rs->leaf_next; diff --git a/src/third_party/wiredtiger/src/btree/bt_ret.c b/src/third_party/wiredtiger/src/btree/bt_ret.c index 1a7b42e39c5..7ef66f2d572 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ret.c +++ b/src/third_party/wiredtiger/src/btree/bt_ret.c @@ -256,8 +256,10 @@ __wt_value_return(WT_CURSOR_BTREE *cbt, WT_UPDATE_VALUE *upd_value) * We're passed a "standard" update that's visible to us. Our caller should have already * checked for deleted items (we're too far down the call stack to return not-found) and any * modify updates should be have been reconstructed into a full standard update. + * + * We are here to return a value to the caller. Make sure we don't skip the buf. */ - WT_ASSERT(session, upd_value->type == WT_UPDATE_STANDARD); + WT_ASSERT(session, upd_value->type == WT_UPDATE_STANDARD && !upd_value->skip_buf); cursor->value.data = upd_value->buf.data; cursor->value.size = upd_value->buf.size; } diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index a652d68727f..0dd24eee0f8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -509,8 +509,7 @@ __slvg_read(WT_SESSION_IMPL *session, WT_STUFF *ss) ss->page_type = dsk->type; if (ss->page_type != dsk->type) WT_ERR_MSG(session, WT_ERROR, - "file contains multiple file formats (both " - "%s and %s), and cannot be salvaged", + "file contains multiple file formats (both %s and %s), and cannot be salvaged", __wt_page_type_string(ss->page_type), __wt_page_type_string(dsk->type)); WT_ERR(__slvg_trk_leaf(session, dsk, addr, addr_size, ss)); @@ -1240,9 +1239,8 @@ __slvg_col_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref) cookie->skip = skip = trk->col_start - recno; cookie->take = take = (trk->col_stop - trk->col_start) + 1; - __wt_verbose(session, WT_VERB_SALVAGE, "%s merge discarding first %" PRIu64 - " records, " - "then taking %" PRIu64 " records", + __wt_verbose(session, WT_VERB_SALVAGE, + "%s merge discarding first %" PRIu64 " records, then taking %" PRIu64 " records", __wt_addr_string(session, trk->trk_addr, trk->trk_addr_size, trk->ss->tmp1), skip, take); /* Set the referenced flag on overflow pages we're using. */ @@ -2253,8 +2251,7 @@ __slvg_ovfl_ref(WT_SESSION_IMPL *session, WT_TRACK *trk, bool multi_panic) if (!multi_panic) return (__wt_set_return(session, EBUSY)); WT_RET_PANIC(session, EINVAL, - "overflow record unexpectedly referenced multiple times " - "during leaf page merge"); + "overflow record unexpectedly referenced multiple times during leaf page merge"); } F_SET(trk, WT_TRACK_OVFL_REFD); diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 56f91eb883f..291f1a1be1c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -1652,7 +1652,8 @@ __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi, WT_R session, multi->disk_image != NULL || (multi->supd_entries == 0 && !multi->supd_restore)); /* Verify any disk image we have. */ - WT_ASSERT(session, multi->disk_image == NULL || + WT_ASSERT(session, + multi->disk_image == NULL || __wt_verify_dsk_image( session, "[page instantiate]", multi->disk_image, 0, &multi->addr, true) == 0); diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index b897ce6bbbd..461acf4ec90 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -259,7 +259,7 @@ __sync_ref_obsolete_check(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF_LIST *rl "obsolete, stop time point %s", (void *)ref, obsolete ? "" : "not ", __wt_time_point_to_string( - newest_stop_ts, newest_stop_durable_ts, newest_stop_txn, tp_string)); + newest_stop_ts, newest_stop_durable_ts, newest_stop_txn, tp_string)); return (0); } WT_REF_UNLOCK(ref, previous_state); @@ -334,7 +334,7 @@ __sync_ref_obsolete_check(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF_LIST *rl "obsolete, stop time point %s", (void *)ref, tag, obsolete ? "" : "not ", __wt_time_point_to_string( - newest_stop_ts, newest_stop_durable_ts, newest_stop_txn, tp_string)); + newest_stop_ts, newest_stop_durable_ts, newest_stop_txn, tp_string)); err: if (hazard) diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 8a22ea1032f..626f916cdd8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -172,7 +172,6 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) #if 0 /* FIXME-WT-6263: Temporarily disable history store verification. */ - uint32_t session_flags; bool skip_hs; #endif @@ -184,7 +183,6 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) #if 0 /* FIXME-WT-6263: Temporarily disable history store verification. */ - session_flags = 0; /* -Wuninitialized */ /* * Skip the history store explicit call if we're performing a metadata verification. The @@ -282,9 +280,9 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) */ if (ret == 0 && (ckpt + 1)->name == NULL && !skip_hs) { /* Open a history store cursor. */ - WT_ERR(__wt_hs_cursor_open(session, &session_flags); + WT_ERR(__wt_hs_cursor_open(session); WT_TRET(__wt_history_store_verify_one(session)); - WT_TRET(__wt_hs_cursor_close(session, session_flags); + WT_TRET(__wt_hs_cursor_close(session); /* * We cannot error out here. If we got an error verifying the history store, we need * to follow through with reacquiring the exclusive call below. We'll error out @@ -474,8 +472,9 @@ __verify_tree( case WT_PAGE_COL_INT: case WT_PAGE_COL_VAR: if (ref->ref_recno != vs->records_so_far + 1) - WT_RET_MSG(session, WT_ERROR, "page at %s has a starting record of %" PRIu64 - " when the expected starting record is %" PRIu64, + WT_RET_MSG(session, WT_ERROR, + "page at %s has a starting record of %" PRIu64 + " when the expected starting record is %" PRIu64, __verify_addr_string(session, ref, vs->tmp1), ref->ref_recno, vs->records_so_far + 1); break; } @@ -524,10 +523,10 @@ __verify_tree( case WT_PAGE_ROW_INT: if (addr_unpack->raw != WT_CELL_ADDR_INT) celltype_err: - WT_RET_MSG(session, WT_ERROR, - "page at %s, of type %s, is referenced in its parent by a cell of type %s", - __verify_addr_string(session, ref, vs->tmp1), __wt_page_type_string(page->type), - __wt_cell_type_string(addr_unpack->raw)); + WT_RET_MSG(session, WT_ERROR, + "page at %s, of type %s, is referenced in its parent by a cell of type %s", + __verify_addr_string(session, ref, vs->tmp1), __wt_page_type_string(page->type), + __wt_cell_type_string(addr_unpack->raw)); break; } @@ -625,9 +624,10 @@ __verify_row_int_key_order( /* Compare the key against the largest key we've seen so far. */ WT_RET(__wt_compare(session, btree->collator, &item, vs->max_key, &cmp)); if (cmp <= 0) - WT_RET_MSG(session, WT_ERROR, "the internal key in entry %" PRIu32 - " on the page at %s sorts before the last key appearing on " - "page %s, earlier in the tree: %s, %s", + WT_RET_MSG(session, WT_ERROR, + "the internal key in entry %" PRIu32 + " on the page at %s sorts before the last key appearing on page %s, earlier in the tree: " + "%s, %s", entry, __verify_addr_string(session, ref, vs->tmp1), (char *)vs->max_addr->data, __wt_buf_set_printable(session, item.data, item.size, vs->tmp2), __wt_buf_set_printable(session, vs->max_key->data, vs->max_key->size, vs->tmp3)); @@ -877,8 +877,9 @@ __verify_page_content_int( ++cell_num; if (!__wt_cell_type_check(unpack.type, dsk->type)) - WT_RET_MSG(session, WT_ERROR, "illegal cell and page type combination: cell %" PRIu32 - " on page at %s is a %s cell on a %s page", + WT_RET_MSG(session, WT_ERROR, + "illegal cell and page type combination: cell %" PRIu32 + " on page at %s is a %s cell on a %s page", cell_num - 1, __verify_addr_string(session, ref, vs->tmp1), __wt_cell_type_string(unpack.type), __wt_page_type_string(dsk->type)); @@ -952,8 +953,9 @@ __verify_page_content_leaf( ++cell_num; if (!__wt_cell_type_check(unpack.type, dsk->type)) - WT_RET_MSG(session, WT_ERROR, "illegal cell and page type combination: cell %" PRIu32 - " on page at %s is a %s cell on a %s page", + WT_RET_MSG(session, WT_ERROR, + "illegal cell and page type combination: cell %" PRIu32 + " on page at %s is a %s cell on a %s page", cell_num - 1, __verify_addr_string(session, ref, vs->tmp1), __wt_cell_type_string(unpack.type), __wt_page_type_string(dsk->type)); diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index 5ca3f562d9e..d8d6b4e899d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -561,9 +561,10 @@ __verify_dsk_row_leaf( */ prefix = unpack->prefix; if (last_pfx->size == 0 && prefix != 0) - WT_ERR_VRFY(session, "the %" PRIu32 - " key on page at %s is the first non-overflow key on the page and " - "has a non-zero prefix compression value", + WT_ERR_VRFY(session, + "the %" PRIu32 + " key on page at %s is the first non-overflow key on the page and has a non-zero " + "prefix compression value", cell_num, tag); /* Confirm the prefix compression count is possible. */ @@ -795,9 +796,10 @@ __verify_dsk_col_var( } else if (cell_type == WT_CELL_VALUE && last.data != NULL && last.size == unpack->size && memcmp(last.data, unpack->data, last.size) == 0) match_err: - WT_RET_VRFY(session, "data entries %" PRIu32 " and %" PRIu32 - " on page at %s are identical and should have been run-length encoded", - cell_num - 1, cell_num, tag); + WT_RET_VRFY(session, + "data entries %" PRIu32 " and %" PRIu32 + " on page at %s are identical and should have been run-length encoded", + cell_num - 1, cell_num, tag); WT_TIME_WINDOW_COPY(&last.tw, &unpack->tw); switch (cell_type) { @@ -954,8 +956,9 @@ __err_cell_type(WT_SESSION_IMPL *session, uint32_t entry_num, const char *tag, u uint8_t dsk_type) { if (!__wt_cell_type_check(cell_type, dsk_type)) - WT_RET_VRFY(session, "illegal cell and page type combination: cell %" PRIu32 - " on page at %s is a %s cell on a %s page", + WT_RET_VRFY(session, + "illegal cell and page type combination: cell %" PRIu32 + " on page at %s is a %s cell on a %s page", entry_num, tag, __wt_cell_type_string(cell_type), __wt_page_type_string(dsk_type)); return (0); } diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index 4d40d9d2ce6..49e57bd51d8 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -48,11 +48,11 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U * A "full update list" is distinguished from "an update" by checking whether it has a "next" * update. */ - WT_ASSERT( - session, ((modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && - value == NULL && upd_arg == NULL) || + WT_ASSERT(session, + ((modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && value == NULL && + upd_arg == NULL) || (!(modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && - ((value == NULL && upd_arg != NULL) || (value != NULL && upd_arg == NULL)))); + ((value == NULL && upd_arg != NULL) || (value != NULL && upd_arg == NULL)))); /* If we don't yet have a modify structure, we'll need one. */ WT_RET(__wt_page_modify_init(session, page)); @@ -74,8 +74,8 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_ASSERT(session, recno != WT_RECNO_OOB || cbt->compare != 0); if (cbt->compare != 0 && (recno == WT_RECNO_OOB || - recno > (btree->type == BTREE_COL_VAR ? __col_var_last_recno(cbt->ref) : - __col_fix_last_recno(cbt->ref)))) { + recno > (btree->type == BTREE_COL_VAR ? __col_var_last_recno(cbt->ref) : + __col_fix_last_recno(cbt->ref)))) { append = true; cbt->ins = NULL; cbt->ins_head = NULL; @@ -179,7 +179,8 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U * Check for insert split and checkpoint races in column-store: it's easy (as opposed to in * row-store) and a difficult bug to otherwise diagnose. */ - WT_ASSERT(session, mod->mod_col_split_recno == WT_RECNO_OOB || + WT_ASSERT(session, + mod->mod_col_split_recno == WT_RECNO_OOB || (recno != WT_RECNO_OOB && mod->mod_col_split_recno > recno)); if (upd_arg == NULL) { diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index e71f96daa9a..caef5a88c52 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -32,7 +32,7 @@ __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) __wt_cache_page_inmem_incr(session, page, sizeof(*modify)); else err: - __wt_free(session, modify); + __wt_free(session, modify); return (ret); } @@ -73,11 +73,11 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, * A "full update list" is distinguished from "an update" by checking whether it has a "next" * update. The modify type should only be set if no update list provided. */ - WT_ASSERT( - session, ((modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && - value == NULL && upd_arg == NULL) || + WT_ASSERT(session, + ((modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && value == NULL && + upd_arg == NULL) || (!(modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && - ((value == NULL && upd_arg != NULL) || (value != NULL && upd_arg == NULL)))); + ((value == NULL && upd_arg != NULL) || (value != NULL && upd_arg == NULL)))); WT_ASSERT(session, upd_arg == NULL || modify_type == WT_UPDATE_INVALID); /* If we don't yet have a modify structure, we'll need one. */ diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index 75bf4b9aa44..c19323c7e30 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -53,9 +53,9 @@ __search_insert_append(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT * serialized insert function. */ for (i = WT_SKIP_MAXDEPTH - 1; i >= 0; i--) { - cbt->ins_stack[i] = (i == 0) ? &ins->next[0] : (ins_head->tail[i] != NULL) ? - &ins_head->tail[i]->next[i] : - &ins_head->head[i]; + cbt->ins_stack[i] = (i == 0) ? + &ins->next[0] : + (ins_head->tail[i] != NULL) ? &ins_head->tail[i]->next[i] : &ins_head->head[i]; cbt->next_stack[i] = NULL; } cbt->compare = -cmp; diff --git a/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c b/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c index afbab3182de..91d59615fdd 100644 --- a/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c +++ b/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c @@ -39,13 +39,13 @@ #endif #define CRC32CX(crc, value) \ - __asm__("crc32cx %w[c], %w[c], %x[v]" : [c] "+r"(*&crc) : [v] "r"(+value)) + __asm__("crc32cx %w[c], %w[c], %x[v]" : [ c ] "+r"(*&crc) : [ v ] "r"(+value)) #define CRC32CW(crc, value) \ - __asm__("crc32cw %w[c], %w[c], %w[v]" : [c] "+r"(*&crc) : [v] "r"(+value)) + __asm__("crc32cw %w[c], %w[c], %w[v]" : [ c ] "+r"(*&crc) : [ v ] "r"(+value)) #define CRC32CH(crc, value) \ - __asm__("crc32ch %w[c], %w[c], %w[v]" : [c] "+r"(*&crc) : [v] "r"(+value)) + __asm__("crc32ch %w[c], %w[c], %w[v]" : [ c ] "+r"(*&crc) : [ v ] "r"(+value)) #define CRC32CB(crc, value) \ - __asm__("crc32cb %w[c], %w[c], %w[v]" : [c] "+r"(*&crc) : [v] "r"(+value)) + __asm__("crc32cb %w[c], %w[c], %w[v]" : [ c ] "+r"(*&crc) : [ v ] "r"(+value)) /* * __wt_checksum_hw -- diff --git a/src/third_party/wiredtiger/src/checksum/power8/crc32_constants.h b/src/third_party/wiredtiger/src/checksum/power8/crc32_constants.h index 886858ef2b5..bcdf3a8b8a3 100644 --- a/src/third_party/wiredtiger/src/checksum/power8/crc32_constants.h +++ b/src/third_party/wiredtiger/src/checksum/power8/crc32_constants.h @@ -5,38 +5,262 @@ #ifndef __ASSEMBLY__ #ifdef CRC_TABLE static const unsigned int crc_table[] = { - 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, - 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, - 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, - 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, - 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, - 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, - 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, - 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, - 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, - 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, - 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, - 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, - 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, - 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, - 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, - 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, - 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, - 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, - 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, - 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, - 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, - 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, - 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, - 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, - 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, - 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, - 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, - 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, - 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, - 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, - 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, - 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351, + 0x00000000, + 0xf26b8303, + 0xe13b70f7, + 0x1350f3f4, + 0xc79a971f, + 0x35f1141c, + 0x26a1e7e8, + 0xd4ca64eb, + 0x8ad958cf, + 0x78b2dbcc, + 0x6be22838, + 0x9989ab3b, + 0x4d43cfd0, + 0xbf284cd3, + 0xac78bf27, + 0x5e133c24, + 0x105ec76f, + 0xe235446c, + 0xf165b798, + 0x030e349b, + 0xd7c45070, + 0x25afd373, + 0x36ff2087, + 0xc494a384, + 0x9a879fa0, + 0x68ec1ca3, + 0x7bbcef57, + 0x89d76c54, + 0x5d1d08bf, + 0xaf768bbc, + 0xbc267848, + 0x4e4dfb4b, + 0x20bd8ede, + 0xd2d60ddd, + 0xc186fe29, + 0x33ed7d2a, + 0xe72719c1, + 0x154c9ac2, + 0x061c6936, + 0xf477ea35, + 0xaa64d611, + 0x580f5512, + 0x4b5fa6e6, + 0xb93425e5, + 0x6dfe410e, + 0x9f95c20d, + 0x8cc531f9, + 0x7eaeb2fa, + 0x30e349b1, + 0xc288cab2, + 0xd1d83946, + 0x23b3ba45, + 0xf779deae, + 0x05125dad, + 0x1642ae59, + 0xe4292d5a, + 0xba3a117e, + 0x4851927d, + 0x5b016189, + 0xa96ae28a, + 0x7da08661, + 0x8fcb0562, + 0x9c9bf696, + 0x6ef07595, + 0x417b1dbc, + 0xb3109ebf, + 0xa0406d4b, + 0x522bee48, + 0x86e18aa3, + 0x748a09a0, + 0x67dafa54, + 0x95b17957, + 0xcba24573, + 0x39c9c670, + 0x2a993584, + 0xd8f2b687, + 0x0c38d26c, + 0xfe53516f, + 0xed03a29b, + 0x1f682198, + 0x5125dad3, + 0xa34e59d0, + 0xb01eaa24, + 0x42752927, + 0x96bf4dcc, + 0x64d4cecf, + 0x77843d3b, + 0x85efbe38, + 0xdbfc821c, + 0x2997011f, + 0x3ac7f2eb, + 0xc8ac71e8, + 0x1c661503, + 0xee0d9600, + 0xfd5d65f4, + 0x0f36e6f7, + 0x61c69362, + 0x93ad1061, + 0x80fde395, + 0x72966096, + 0xa65c047d, + 0x5437877e, + 0x4767748a, + 0xb50cf789, + 0xeb1fcbad, + 0x197448ae, + 0x0a24bb5a, + 0xf84f3859, + 0x2c855cb2, + 0xdeeedfb1, + 0xcdbe2c45, + 0x3fd5af46, + 0x7198540d, + 0x83f3d70e, + 0x90a324fa, + 0x62c8a7f9, + 0xb602c312, + 0x44694011, + 0x5739b3e5, + 0xa55230e6, + 0xfb410cc2, + 0x092a8fc1, + 0x1a7a7c35, + 0xe811ff36, + 0x3cdb9bdd, + 0xceb018de, + 0xdde0eb2a, + 0x2f8b6829, + 0x82f63b78, + 0x709db87b, + 0x63cd4b8f, + 0x91a6c88c, + 0x456cac67, + 0xb7072f64, + 0xa457dc90, + 0x563c5f93, + 0x082f63b7, + 0xfa44e0b4, + 0xe9141340, + 0x1b7f9043, + 0xcfb5f4a8, + 0x3dde77ab, + 0x2e8e845f, + 0xdce5075c, + 0x92a8fc17, + 0x60c37f14, + 0x73938ce0, + 0x81f80fe3, + 0x55326b08, + 0xa759e80b, + 0xb4091bff, + 0x466298fc, + 0x1871a4d8, + 0xea1a27db, + 0xf94ad42f, + 0x0b21572c, + 0xdfeb33c7, + 0x2d80b0c4, + 0x3ed04330, + 0xccbbc033, + 0xa24bb5a6, + 0x502036a5, + 0x4370c551, + 0xb11b4652, + 0x65d122b9, + 0x97baa1ba, + 0x84ea524e, + 0x7681d14d, + 0x2892ed69, + 0xdaf96e6a, + 0xc9a99d9e, + 0x3bc21e9d, + 0xef087a76, + 0x1d63f975, + 0x0e330a81, + 0xfc588982, + 0xb21572c9, + 0x407ef1ca, + 0x532e023e, + 0xa145813d, + 0x758fe5d6, + 0x87e466d5, + 0x94b49521, + 0x66df1622, + 0x38cc2a06, + 0xcaa7a905, + 0xd9f75af1, + 0x2b9cd9f2, + 0xff56bd19, + 0x0d3d3e1a, + 0x1e6dcdee, + 0xec064eed, + 0xc38d26c4, + 0x31e6a5c7, + 0x22b65633, + 0xd0ddd530, + 0x0417b1db, + 0xf67c32d8, + 0xe52cc12c, + 0x1747422f, + 0x49547e0b, + 0xbb3ffd08, + 0xa86f0efc, + 0x5a048dff, + 0x8ecee914, + 0x7ca56a17, + 0x6ff599e3, + 0x9d9e1ae0, + 0xd3d3e1ab, + 0x21b862a8, + 0x32e8915c, + 0xc083125f, + 0x144976b4, + 0xe622f5b7, + 0xf5720643, + 0x07198540, + 0x590ab964, + 0xab613a67, + 0xb831c993, + 0x4a5a4a90, + 0x9e902e7b, + 0x6cfbad78, + 0x7fab5e8c, + 0x8dc0dd8f, + 0xe330a81a, + 0x115b2b19, + 0x020bd8ed, + 0xf0605bee, + 0x24aa3f05, + 0xd6c1bc06, + 0xc5914ff2, + 0x37faccf1, + 0x69e9f0d5, + 0x9b8273d6, + 0x88d28022, + 0x7ab90321, + 0xae7367ca, + 0x5c18e4c9, + 0x4f48173d, + 0xbd23943e, + 0xf36e6f75, + 0x0105ec76, + 0x12551f82, + 0xe03e9c81, + 0x34f4f86a, + 0xc69f7b69, + 0xd5cf889d, + 0x27a40b9e, + 0x79b737ba, + 0x8bdcb4b9, + 0x988c474d, + 0x6ae7c44e, + 0xbe2da0a5, + 0x4c4623a6, + 0x5f16d052, + 0xad7d5351, }; #endif @@ -44,827 +268,827 @@ static const unsigned int crc_table[] = { #define MAX_SIZE 32768 .constants : - /* Reduce 262144 kbits to 1024 bits */ - /* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */ - .octa 0x00000000b6ca9e20000000009c37c408 + /* Reduce 262144 kbits to 1024 bits */ + /* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */ + .octa 0x00000000b6ca9e20000000009c37c408 - /* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */ - .octa 0x00000000350249a800000001b51df26c + /* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */ + .octa 0x00000000350249a800000001b51df26c - /* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */ - .octa 0x00000001862dac54000000000724b9d0 + /* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */ + .octa 0x00000001862dac54000000000724b9d0 - /* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */ - .octa 0x00000001d87fb48c00000001c00532fe + /* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */ + .octa 0x00000001d87fb48c00000001c00532fe - /* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */ - .octa 0x00000001f39b699e00000000f05a9362 + /* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */ + .octa 0x00000001f39b699e00000000f05a9362 - /* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */ - .octa 0x0000000101da11b400000001e1007970 + /* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */ + .octa 0x0000000101da11b400000001e1007970 - /* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */ - .octa 0x00000001cab571e000000000a57366ee + /* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */ + .octa 0x00000001cab571e000000000a57366ee - /* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */ - .octa 0x00000000c7020cfe0000000192011284 + /* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */ + .octa 0x00000000c7020cfe0000000192011284 - /* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */ - .octa 0x00000000cdaed1ae0000000162716d9a + /* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */ + .octa 0x00000000cdaed1ae0000000162716d9a - /* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */ - .octa 0x00000001e804effc00000000cd97ecde + /* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */ + .octa 0x00000001e804effc00000000cd97ecde - /* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */ - .octa 0x0000000077c3ea3a0000000058812bc0 + /* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */ + .octa 0x0000000077c3ea3a0000000058812bc0 - /* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */ - .octa 0x0000000068df31b40000000088b8c12e + /* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */ + .octa 0x0000000068df31b40000000088b8c12e - /* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */ - .octa 0x00000000b059b6c200000001230b234c + /* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */ + .octa 0x00000000b059b6c200000001230b234c - /* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */ - .octa 0x0000000145fb8ed800000001120b416e + /* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */ + .octa 0x0000000145fb8ed800000001120b416e - /* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */ - .octa 0x00000000cbc0916800000001974aecb0 + /* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */ + .octa 0x00000000cbc0916800000001974aecb0 - /* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */ - .octa 0x000000005ceeedc2000000008ee3f226 + /* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */ + .octa 0x000000005ceeedc2000000008ee3f226 - /* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */ - .octa 0x0000000047d74e8600000001089aba9a + /* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */ + .octa 0x0000000047d74e8600000001089aba9a - /* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */ - .octa 0x00000001407e9e220000000065113872 + /* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */ + .octa 0x00000001407e9e220000000065113872 - /* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */ - .octa 0x00000001da967bda000000005c07ec10 + /* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */ + .octa 0x00000001da967bda000000005c07ec10 - /* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */ - .octa 0x000000006c8983680000000187590924 + /* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */ + .octa 0x000000006c8983680000000187590924 - /* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */ - .octa 0x00000000f2d14c9800000000e35da7c6 + /* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */ + .octa 0x00000000f2d14c9800000000e35da7c6 - /* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */ - .octa 0x00000001993c6ad4000000000415855a + /* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */ + .octa 0x00000001993c6ad4000000000415855a - /* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */ - .octa 0x000000014683d1ac0000000073617758 + /* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */ + .octa 0x000000014683d1ac0000000073617758 - /* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */ - .octa 0x00000001a7c93e6c0000000176021d28 + /* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */ + .octa 0x00000001a7c93e6c0000000176021d28 - /* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */ - .octa 0x000000010211e90a00000001c358fd0a + /* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */ + .octa 0x000000010211e90a00000001c358fd0a - /* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */ - .octa 0x000000001119403e00000001ff7a2c18 + /* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */ + .octa 0x000000001119403e00000001ff7a2c18 - /* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */ - .octa 0x000000001c3261aa00000000f2d9f7e4 + /* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */ + .octa 0x000000001c3261aa00000000f2d9f7e4 - /* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */ - .octa 0x000000014e37a634000000016cf1f9c8 + /* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */ + .octa 0x000000014e37a634000000016cf1f9c8 - /* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */ - .octa 0x0000000073786c0c000000010af9279a + /* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */ + .octa 0x0000000073786c0c000000010af9279a - /* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */ - .octa 0x000000011dc037f80000000004f101e8 + /* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */ + .octa 0x000000011dc037f80000000004f101e8 - /* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */ - .octa 0x0000000031433dfc0000000070bcf184 + /* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */ + .octa 0x0000000031433dfc0000000070bcf184 - /* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */ - .octa 0x000000009cde8348000000000a8de642 + /* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */ + .octa 0x000000009cde8348000000000a8de642 - /* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */ - .octa 0x0000000038d3c2a60000000062ea130c + /* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */ + .octa 0x0000000038d3c2a60000000062ea130c - /* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */ - .octa 0x000000011b25f26000000001eb31cbb2 + /* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */ + .octa 0x000000011b25f26000000001eb31cbb2 - /* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */ - .octa 0x000000001629e6f00000000170783448 + /* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */ + .octa 0x000000001629e6f00000000170783448 - /* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */ - .octa 0x0000000160838b4c00000001a684b4c6 + /* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */ + .octa 0x0000000160838b4c00000001a684b4c6 - /* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */ - .octa 0x000000007a44011c00000000253ca5b4 + /* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */ + .octa 0x000000007a44011c00000000253ca5b4 - /* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */ - .octa 0x00000000226f417a0000000057b4b1e2 + /* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */ + .octa 0x00000000226f417a0000000057b4b1e2 - /* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */ - .octa 0x0000000045eb2eb400000000b6bd084c + /* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */ + .octa 0x0000000045eb2eb400000000b6bd084c - /* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */ - .octa 0x000000014459d70c0000000123c2d592 + /* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */ + .octa 0x000000014459d70c0000000123c2d592 - /* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */ - .octa 0x00000001d406ed8200000000159dafce + /* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */ + .octa 0x00000001d406ed8200000000159dafce - /* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */ - .octa 0x0000000160c8e1a80000000127e1a64e + /* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */ + .octa 0x0000000160c8e1a80000000127e1a64e - /* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */ - .octa 0x0000000027ba80980000000056860754 + /* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */ + .octa 0x0000000027ba80980000000056860754 - /* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */ - .octa 0x000000006d92d01800000001e661aae8 + /* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */ + .octa 0x000000006d92d01800000001e661aae8 - /* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */ - .octa 0x000000012ed7e3f200000000f82c6166 + /* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */ + .octa 0x000000012ed7e3f200000000f82c6166 - /* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */ - .octa 0x000000002dc8778800000000c4f9c7ae + /* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */ + .octa 0x000000002dc8778800000000c4f9c7ae - /* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */ - .octa 0x0000000018240bb80000000074203d20 + /* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */ + .octa 0x0000000018240bb80000000074203d20 - /* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */ - .octa 0x000000001ad381580000000198173052 + /* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */ + .octa 0x000000001ad381580000000198173052 - /* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */ - .octa 0x00000001396b78f200000001ce8aba54 + /* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */ + .octa 0x00000001396b78f200000001ce8aba54 - /* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */ - .octa 0x000000011a68133400000001850d5d94 + /* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */ + .octa 0x000000011a68133400000001850d5d94 - /* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */ - .octa 0x000000012104732e00000001d609239c + /* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */ + .octa 0x000000012104732e00000001d609239c - /* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */ - .octa 0x00000000a140d90c000000001595f048 + /* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */ + .octa 0x00000000a140d90c000000001595f048 - /* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */ - .octa 0x00000001b7215eda0000000042ccee08 + /* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */ + .octa 0x00000001b7215eda0000000042ccee08 - /* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */ - .octa 0x00000001aaf1df3c000000010a389d74 + /* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */ + .octa 0x00000001aaf1df3c000000010a389d74 - /* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */ - .octa 0x0000000029d15b8a000000012a840da6 + /* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */ + .octa 0x0000000029d15b8a000000012a840da6 - /* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */ - .octa 0x00000000f1a96922000000001d181c0c + /* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */ + .octa 0x00000000f1a96922000000001d181c0c - /* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */ - .octa 0x00000001ac80d03c0000000068b7d1f6 + /* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */ + .octa 0x00000001ac80d03c0000000068b7d1f6 - /* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */ - .octa 0x000000000f11d56a000000005b0f14fc + /* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */ + .octa 0x000000000f11d56a000000005b0f14fc - /* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */ - .octa 0x00000001f1c022a20000000179e9e730 + /* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */ + .octa 0x00000001f1c022a20000000179e9e730 - /* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */ - .octa 0x0000000173d00ae200000001ce1368d6 + /* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */ + .octa 0x0000000173d00ae200000001ce1368d6 - /* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */ - .octa 0x00000001d4ffe4ac0000000112c3a84c + /* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */ + .octa 0x00000001d4ffe4ac0000000112c3a84c - /* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */ - .octa 0x000000016edc5ae400000000de940fee + /* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */ + .octa 0x000000016edc5ae400000000de940fee - /* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */ - .octa 0x00000001f1a0214000000000fe896b7e + /* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */ + .octa 0x00000001f1a0214000000000fe896b7e - /* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */ - .octa 0x00000000ca0b28a000000001f797431c + /* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */ + .octa 0x00000000ca0b28a000000001f797431c - /* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */ - .octa 0x00000001928e30a20000000053e989ba + /* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */ + .octa 0x00000001928e30a20000000053e989ba - /* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */ - .octa 0x0000000097b1b002000000003920cd16 + /* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */ + .octa 0x0000000097b1b002000000003920cd16 - /* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */ - .octa 0x00000000b15bf90600000001e6f579b8 + /* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */ + .octa 0x00000000b15bf90600000001e6f579b8 - /* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */ - .octa 0x00000000411c5d52000000007493cb0a + /* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */ + .octa 0x00000000411c5d52000000007493cb0a - /* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */ - .octa 0x00000001c36f330000000001bdd376d8 + /* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */ + .octa 0x00000001c36f330000000001bdd376d8 - /* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */ - .octa 0x00000001119227e0000000016badfee6 + /* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */ + .octa 0x00000001119227e0000000016badfee6 - /* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */ - .octa 0x00000000114d47020000000071de5c58 + /* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */ + .octa 0x00000000114d47020000000071de5c58 - /* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */ - .octa 0x00000000458b5b9800000000453f317c + /* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */ + .octa 0x00000000458b5b9800000000453f317c - /* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */ - .octa 0x000000012e31fb8e0000000121675cce + /* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */ + .octa 0x000000012e31fb8e0000000121675cce - /* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */ - .octa 0x000000005cf619d800000001f409ee92 + /* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */ + .octa 0x000000005cf619d800000001f409ee92 - /* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */ - .octa 0x0000000063f4d8b200000000f36b9c88 + /* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */ + .octa 0x0000000063f4d8b200000000f36b9c88 - /* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */ - .octa 0x000000004138dc8a0000000036b398f4 + /* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */ + .octa 0x000000004138dc8a0000000036b398f4 - /* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */ - .octa 0x00000001d29ee8e000000001748f9adc + /* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */ + .octa 0x00000001d29ee8e000000001748f9adc - /* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */ - .octa 0x000000006a08ace800000001be94ec00 + /* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */ + .octa 0x000000006a08ace800000001be94ec00 - /* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */ - .octa 0x0000000127d4201000000000b74370d6 + /* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */ + .octa 0x0000000127d4201000000000b74370d6 - /* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */ - .octa 0x0000000019d76b6200000001174d0b98 + /* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */ + .octa 0x0000000019d76b6200000001174d0b98 - /* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */ - .octa 0x00000001b1471f6e00000000befc06a4 + /* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */ + .octa 0x00000001b1471f6e00000000befc06a4 - /* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */ - .octa 0x00000001f64c19cc00000001ae125288 + /* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */ + .octa 0x00000001f64c19cc00000001ae125288 - /* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */ - .octa 0x00000000003c0ea00000000095c19b34 + /* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */ + .octa 0x00000000003c0ea00000000095c19b34 - /* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */ - .octa 0x000000014d73abf600000001a78496f2 + /* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */ + .octa 0x000000014d73abf600000001a78496f2 - /* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */ - .octa 0x00000001620eb84400000001ac5390a0 + /* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */ + .octa 0x00000001620eb84400000001ac5390a0 - /* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */ - .octa 0x0000000147655048000000002a80ed6e + /* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */ + .octa 0x0000000147655048000000002a80ed6e - /* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */ - .octa 0x0000000067b5077e00000001fa9b0128 + /* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */ + .octa 0x0000000067b5077e00000001fa9b0128 - /* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */ - .octa 0x0000000010ffe20600000001ea94929e + /* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */ + .octa 0x0000000010ffe20600000001ea94929e - /* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */ - .octa 0x000000000fee8f1e0000000125f4305c + /* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */ + .octa 0x000000000fee8f1e0000000125f4305c - /* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */ - .octa 0x00000001da26fbae00000001471e2002 + /* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */ + .octa 0x00000001da26fbae00000001471e2002 - /* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */ - .octa 0x00000001b3a8bd880000000132d2253a + /* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */ + .octa 0x00000001b3a8bd880000000132d2253a - /* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */ - .octa 0x00000000e8f3898e00000000f26b3592 + /* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */ + .octa 0x00000000e8f3898e00000000f26b3592 - /* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */ - .octa 0x00000000b0d0d28c00000000bc8b67b0 + /* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */ + .octa 0x00000000b0d0d28c00000000bc8b67b0 - /* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */ - .octa 0x0000000030f2a798000000013a826ef2 + /* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */ + .octa 0x0000000030f2a798000000013a826ef2 - /* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */ - .octa 0x000000000fba10020000000081482c84 + /* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */ + .octa 0x000000000fba10020000000081482c84 - /* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */ - .octa 0x00000000bdb9bd7200000000e77307c2 + /* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */ + .octa 0x00000000bdb9bd7200000000e77307c2 - /* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */ - .octa 0x0000000075d3bf5a00000000d4a07ec8 + /* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */ + .octa 0x0000000075d3bf5a00000000d4a07ec8 - /* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */ - .octa 0x00000000ef1f98a00000000017102100 + /* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */ + .octa 0x00000000ef1f98a00000000017102100 - /* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */ - .octa 0x00000000689c760200000000db406486 + /* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */ + .octa 0x00000000689c760200000000db406486 - /* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */ - .octa 0x000000016d5fa5fe0000000192db7f88 + /* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */ + .octa 0x000000016d5fa5fe0000000192db7f88 - /* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */ - .octa 0x00000001d0d2b9ca000000018bf67b1e + /* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */ + .octa 0x00000001d0d2b9ca000000018bf67b1e - /* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */ - .octa 0x0000000041e7b470000000007c09163e + /* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */ + .octa 0x0000000041e7b470000000007c09163e - /* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */ - .octa 0x00000001cbb6495e000000000adac060 + /* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */ + .octa 0x00000001cbb6495e000000000adac060 - /* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */ - .octa 0x000000010052a0b000000000bd8316ae + /* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */ + .octa 0x000000010052a0b000000000bd8316ae - /* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */ - .octa 0x00000001d8effb5c000000019f09ab54 + /* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */ + .octa 0x00000001d8effb5c000000019f09ab54 - /* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */ - .octa 0x00000001d969853c0000000125155542 + /* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */ + .octa 0x00000001d969853c0000000125155542 - /* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */ - .octa 0x00000000523ccce2000000018fdb5882 + /* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */ + .octa 0x00000000523ccce2000000018fdb5882 - /* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */ - .octa 0x000000001e2436bc00000000e794b3f4 + /* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */ + .octa 0x000000001e2436bc00000000e794b3f4 - /* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */ - .octa 0x00000000ddd1c3a2000000016f9bb022 + /* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */ + .octa 0x00000000ddd1c3a2000000016f9bb022 - /* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */ - .octa 0x0000000019fcfe3800000000290c9978 + /* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */ + .octa 0x0000000019fcfe3800000000290c9978 - /* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */ - .octa 0x00000001ce95db640000000083c0f350 + /* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */ + .octa 0x00000001ce95db640000000083c0f350 - /* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */ - .octa 0x00000000af5828060000000173ea6628 + /* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */ + .octa 0x00000000af5828060000000173ea6628 - /* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */ - .octa 0x00000001006388f600000001c8b4e00a + /* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */ + .octa 0x00000001006388f600000001c8b4e00a - /* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */ - .octa 0x0000000179eca00a00000000de95d6aa + /* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */ + .octa 0x0000000179eca00a00000000de95d6aa - /* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */ - .octa 0x0000000122410a6a000000010b7f7248 + /* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */ + .octa 0x0000000122410a6a000000010b7f7248 - /* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */ - .octa 0x000000004288e87c00000001326e3a06 + /* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */ + .octa 0x000000004288e87c00000001326e3a06 - /* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */ - .octa 0x000000016c5490da00000000bb62c2e6 + /* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */ + .octa 0x000000016c5490da00000000bb62c2e6 - /* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */ - .octa 0x00000000d1c71f6e0000000156a4b2c2 + /* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */ + .octa 0x00000000d1c71f6e0000000156a4b2c2 - /* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */ - .octa 0x00000001b4ce08a6000000011dfe763a + /* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */ + .octa 0x00000001b4ce08a6000000011dfe763a - /* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */ - .octa 0x00000001466ba60c000000007bcca8e2 + /* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */ + .octa 0x00000001466ba60c000000007bcca8e2 - /* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */ - .octa 0x00000001f6c488a40000000186118faa + /* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */ + .octa 0x00000001f6c488a40000000186118faa - /* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */ - .octa 0x000000013bfb06820000000111a65a88 + /* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */ + .octa 0x000000013bfb06820000000111a65a88 - /* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */ - .octa 0x00000000690e9e54000000003565e1c4 + /* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */ + .octa 0x00000000690e9e54000000003565e1c4 - /* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */ - .octa 0x00000000281346b6000000012ed02a82 + /* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */ + .octa 0x00000000281346b6000000012ed02a82 - /* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */ - .octa 0x000000015646402400000000c486ecfc + /* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */ + .octa 0x000000015646402400000000c486ecfc - /* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */ - .octa 0x000000016063a8dc0000000001b951b2 + /* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */ + .octa 0x000000016063a8dc0000000001b951b2 - /* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */ - .octa 0x0000000116a663620000000048143916 + /* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */ + .octa 0x0000000116a663620000000048143916 - /* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */ - .octa 0x000000017e8aa4d200000001dc2ae124 + /* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */ + .octa 0x000000017e8aa4d200000001dc2ae124 - /* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */ - .octa 0x00000001728eb10c00000001416c58d6 + /* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */ + .octa 0x00000001728eb10c00000001416c58d6 - /* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */ - .octa 0x00000001b08fd7fa00000000a479744a + /* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */ + .octa 0x00000001b08fd7fa00000000a479744a - /* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */ - .octa 0x00000001092a16e80000000096ca3a26 + /* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */ + .octa 0x00000001092a16e80000000096ca3a26 - /* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */ - .octa 0x00000000a505637c00000000ff223d4e + /* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */ + .octa 0x00000000a505637c00000000ff223d4e - /* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */ - .octa 0x00000000d94869b2000000010e84da42 + /* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */ + .octa 0x00000000d94869b2000000010e84da42 - /* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */ - .octa 0x00000001c8b203ae00000001b61ba3d0 + /* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */ + .octa 0x00000001c8b203ae00000001b61ba3d0 - /* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */ - .octa 0x000000005704aea000000000680f2de8 + /* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */ + .octa 0x000000005704aea000000000680f2de8 - /* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */ - .octa 0x000000012e295fa2000000008772a9a8 + /* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */ + .octa 0x000000012e295fa2000000008772a9a8 - /* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */ - .octa 0x000000011d0908bc0000000155f295bc + /* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */ + .octa 0x000000011d0908bc0000000155f295bc - /* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */ - .octa 0x0000000193ed97ea00000000595f9282 + /* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */ + .octa 0x0000000193ed97ea00000000595f9282 - /* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */ - .octa 0x000000013a0f1c520000000164b1c25a + /* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */ + .octa 0x000000013a0f1c520000000164b1c25a - /* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */ - .octa 0x000000010c2c40c000000000fbd67c50 + /* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */ + .octa 0x000000010c2c40c000000000fbd67c50 - /* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */ - .octa 0x00000000ff6fac3e0000000096076268 + /* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */ + .octa 0x00000000ff6fac3e0000000096076268 - /* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */ - .octa 0x000000017b3609c000000001d288e4cc + /* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */ + .octa 0x000000017b3609c000000001d288e4cc - /* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */ - .octa 0x0000000088c8c92200000001eaac1bdc + /* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */ + .octa 0x0000000088c8c92200000001eaac1bdc - /* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */ - .octa 0x00000001751baae600000001f1ea39e2 + /* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */ + .octa 0x00000001751baae600000001f1ea39e2 - /* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */ - .octa 0x000000010795297200000001eb6506fc + /* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */ + .octa 0x000000010795297200000001eb6506fc - /* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */ - .octa 0x0000000162b00abe000000010f806ffe + /* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */ + .octa 0x0000000162b00abe000000010f806ffe - /* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */ - .octa 0x000000000d7b404c000000010408481e + /* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */ + .octa 0x000000000d7b404c000000010408481e - /* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */ - .octa 0x00000000763b13d40000000188260534 + /* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */ + .octa 0x00000000763b13d40000000188260534 - /* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */ - .octa 0x00000000f6dc22d80000000058fc73e0 + /* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */ + .octa 0x00000000f6dc22d80000000058fc73e0 - /* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */ - .octa 0x000000007daae06000000000391c59b8 + /* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */ + .octa 0x000000007daae06000000000391c59b8 - /* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */ - .octa 0x000000013359ab7c000000018b638400 + /* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */ + .octa 0x000000013359ab7c000000018b638400 - /* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */ - .octa 0x000000008add438a000000011738f5c4 + /* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */ + .octa 0x000000008add438a000000011738f5c4 - /* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */ - .octa 0x00000001edbefdea000000008cf7c6da + /* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */ + .octa 0x00000001edbefdea000000008cf7c6da - /* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */ - .octa 0x000000004104e0f800000001ef97fb16 + /* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */ + .octa 0x000000004104e0f800000001ef97fb16 - /* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */ - .octa 0x00000000b48a82220000000102130e20 + /* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */ + .octa 0x00000000b48a82220000000102130e20 - /* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */ - .octa 0x00000001bcb4684400000000db968898 + /* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */ + .octa 0x00000001bcb4684400000000db968898 - /* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */ - .octa 0x000000013293ce0a00000000b5047b5e + /* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */ + .octa 0x000000013293ce0a00000000b5047b5e - /* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */ - .octa 0x00000001710d0844000000010b90fdb2 + /* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */ + .octa 0x00000001710d0844000000010b90fdb2 - /* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */ - .octa 0x0000000117907f6e000000004834a32e + /* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */ + .octa 0x0000000117907f6e000000004834a32e - /* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */ - .octa 0x0000000087ddf93e0000000059c8f2b0 + /* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */ + .octa 0x0000000087ddf93e0000000059c8f2b0 - /* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */ - .octa 0x000000005970e9b00000000122cec508 + /* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */ + .octa 0x000000005970e9b00000000122cec508 - /* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */ - .octa 0x0000000185b2b7d0000000000a330cda + /* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */ + .octa 0x0000000185b2b7d0000000000a330cda - /* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */ - .octa 0x00000001dcee0efc000000014a47148c + /* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */ + .octa 0x00000001dcee0efc000000014a47148c - /* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */ - .octa 0x0000000030da27220000000042c61cb8 + /* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */ + .octa 0x0000000030da27220000000042c61cb8 - /* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */ - .octa 0x000000012f925a180000000012fe6960 + /* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */ + .octa 0x000000012f925a180000000012fe6960 - /* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */ - .octa 0x00000000dd2e357c00000000dbda2c20 + /* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */ + .octa 0x00000000dd2e357c00000000dbda2c20 - /* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */ - .octa 0x00000000071c80de000000011122410c + /* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */ + .octa 0x00000000071c80de000000011122410c - /* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */ - .octa 0x000000011513140a00000000977b2070 + /* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */ + .octa 0x000000011513140a00000000977b2070 - /* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */ - .octa 0x00000001df876e8e000000014050438e + /* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */ + .octa 0x00000001df876e8e000000014050438e - /* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */ - .octa 0x000000015f81d6ce0000000147c840e8 + /* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */ + .octa 0x000000015f81d6ce0000000147c840e8 - /* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */ - .octa 0x000000019dd94dbe00000001cc7c88ce + /* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */ + .octa 0x000000019dd94dbe00000001cc7c88ce - /* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */ - .octa 0x00000001373d206e00000001476b35a4 + /* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */ + .octa 0x00000001373d206e00000001476b35a4 - /* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */ - .octa 0x00000000668ccade000000013d52d508 + /* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */ + .octa 0x00000000668ccade000000013d52d508 - /* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */ - .octa 0x00000001b192d268000000008e4be32e + /* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */ + .octa 0x00000001b192d268000000008e4be32e - /* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */ - .octa 0x00000000e30f3a7800000000024120fe + /* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */ + .octa 0x00000000e30f3a7800000000024120fe - /* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */ - .octa 0x000000010ef1f7bc00000000ddecddb4 + /* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */ + .octa 0x000000010ef1f7bc00000000ddecddb4 - /* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */ - .octa 0x00000001f5ac738000000000d4d403bc + /* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */ + .octa 0x00000001f5ac738000000000d4d403bc - /* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */ - .octa 0x000000011822ea7000000001734b89aa + /* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */ + .octa 0x000000011822ea7000000001734b89aa - /* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */ - .octa 0x00000000c3a33848000000010e7a58d6 + /* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */ + .octa 0x00000000c3a33848000000010e7a58d6 - /* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */ - .octa 0x00000001bd151c2400000001f9f04e9c + /* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */ + .octa 0x00000001bd151c2400000001f9f04e9c - /* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */ - .octa 0x0000000056002d7600000000b692225e + /* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */ + .octa 0x0000000056002d7600000000b692225e - /* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */ - .octa 0x000000014657c4f4000000019b8d3f3e + /* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */ + .octa 0x000000014657c4f4000000019b8d3f3e - /* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */ - .octa 0x0000000113742d7c00000001a874f11e + /* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */ + .octa 0x0000000113742d7c00000001a874f11e - /* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */ - .octa 0x000000019c5920ba000000010d5a4254 + /* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */ + .octa 0x000000019c5920ba000000010d5a4254 - /* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */ - .octa 0x000000005216d2d600000000bbb2f5d6 + /* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */ + .octa 0x000000005216d2d600000000bbb2f5d6 - /* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */ - .octa 0x0000000136f5ad8a0000000179cc0e36 + /* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */ + .octa 0x0000000136f5ad8a0000000179cc0e36 - /* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */ - .octa 0x000000018b07beb600000001dca1da4a + /* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */ + .octa 0x000000018b07beb600000001dca1da4a - /* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */ - .octa 0x00000000db1e93b000000000feb1a192 + /* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */ + .octa 0x00000000db1e93b000000000feb1a192 - /* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */ - .octa 0x000000000b96fa3a00000000d1eeedd6 + /* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */ + .octa 0x000000000b96fa3a00000000d1eeedd6 - /* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */ - .octa 0x00000001d9968af0000000008fad9bb4 + /* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */ + .octa 0x00000001d9968af0000000008fad9bb4 - /* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */ - .octa 0x000000000e4a77a200000001884938e4 + /* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */ + .octa 0x000000000e4a77a200000001884938e4 - /* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */ - .octa 0x00000000508c2ac800000001bc2e9bc0 + /* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */ + .octa 0x00000000508c2ac800000001bc2e9bc0 - /* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */ - .octa 0x0000000021572a8000000001f9658a68 + /* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */ + .octa 0x0000000021572a8000000001f9658a68 - /* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */ - .octa 0x00000001b859daf2000000001b9224fc + /* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */ + .octa 0x00000001b859daf2000000001b9224fc - /* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */ - .octa 0x000000016f7884740000000055b2fb84 + /* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */ + .octa 0x000000016f7884740000000055b2fb84 - /* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */ - .octa 0x00000001b438810e000000018b090348 + /* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */ + .octa 0x00000001b438810e000000018b090348 - /* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */ - .octa 0x0000000095ddc6f2000000011ccbd5ea + /* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */ + .octa 0x0000000095ddc6f2000000011ccbd5ea - /* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */ - .octa 0x00000001d977c20c0000000007ae47f8 + /* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */ + .octa 0x00000001d977c20c0000000007ae47f8 - /* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */ - .octa 0x00000000ebedb99a0000000172acbec0 + /* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */ + .octa 0x00000000ebedb99a0000000172acbec0 - /* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */ - .octa 0x00000001df9e9e9200000001c6e3ff20 + /* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */ + .octa 0x00000001df9e9e9200000001c6e3ff20 - /* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */ - .octa 0x00000001a4a3f95200000000e1b38744 + /* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */ + .octa 0x00000001a4a3f95200000000e1b38744 - /* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */ - .octa 0x00000000e2f5122000000000791585b2 + /* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */ + .octa 0x00000000e2f5122000000000791585b2 - /* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */ - .octa 0x000000004aa01f3e00000000ac53b894 + /* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */ + .octa 0x000000004aa01f3e00000000ac53b894 - /* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */ - .octa 0x00000000b3e90a5800000001ed5f2cf4 + /* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */ + .octa 0x00000000b3e90a5800000001ed5f2cf4 - /* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */ - .octa 0x000000000c9ca2aa00000001df48b2e0 + /* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */ + .octa 0x000000000c9ca2aa00000001df48b2e0 - /* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */ - .octa 0x000000015168231600000000049c1c62 + /* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */ + .octa 0x000000015168231600000000049c1c62 - /* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */ - .octa 0x0000000036fce78c000000017c460c12 + /* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */ + .octa 0x0000000036fce78c000000017c460c12 - /* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */ - .octa 0x000000009037dc10000000015be4da7e + /* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */ + .octa 0x000000009037dc10000000015be4da7e - /* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */ - .octa 0x00000000d3298582000000010f38f668 + /* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */ + .octa 0x00000000d3298582000000010f38f668 - /* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */ - .octa 0x00000001b42e8ad60000000039f40a00 + /* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */ + .octa 0x00000001b42e8ad60000000039f40a00 - /* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */ - .octa 0x00000000142a983800000000bd4c10c4 + /* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */ + .octa 0x00000000142a983800000000bd4c10c4 - /* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */ - .octa 0x0000000109c7f1900000000042db1d98 + /* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */ + .octa 0x0000000109c7f1900000000042db1d98 - /* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */ - .octa 0x0000000056ff931000000001c905bae6 + /* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */ + .octa 0x0000000056ff931000000001c905bae6 - /* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */ - .octa 0x00000001594513aa00000000069d40ea + /* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */ + .octa 0x00000001594513aa00000000069d40ea - /* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */ - .octa 0x00000001e3b5b1e8000000008e4fbad0 + /* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */ + .octa 0x00000001e3b5b1e8000000008e4fbad0 - /* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */ - .octa 0x000000011dd5fc080000000047bedd46 + /* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */ + .octa 0x000000011dd5fc080000000047bedd46 - /* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */ - .octa 0x00000001675f0cc20000000026396bf8 + /* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */ + .octa 0x00000001675f0cc20000000026396bf8 - /* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */ - .octa 0x00000000d1c8dd4400000000379beb92 + /* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */ + .octa 0x00000000d1c8dd4400000000379beb92 - /* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */ - .octa 0x0000000115ebd3d8000000000abae54a + /* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */ + .octa 0x0000000115ebd3d8000000000abae54a - /* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */ - .octa 0x00000001ecbd0dac0000000007e6a128 + /* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */ + .octa 0x00000001ecbd0dac0000000007e6a128 - /* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */ - .octa 0x00000000cdf67af2000000000ade29d2 + /* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */ + .octa 0x00000000cdf67af2000000000ade29d2 - /* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */ - .octa 0x000000004c01ff4c00000000f974c45c + /* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */ + .octa 0x000000004c01ff4c00000000f974c45c - /* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */ - .octa 0x00000000f2d8657e00000000e77ac60a + /* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */ + .octa 0x00000000f2d8657e00000000e77ac60a - /* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */ - .octa 0x000000006bae74c40000000145895816 + /* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */ + .octa 0x000000006bae74c40000000145895816 - /* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */ - .octa 0x0000000152af8aa00000000038e362be + /* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */ + .octa 0x0000000152af8aa00000000038e362be - /* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */ - .octa 0x0000000004663802000000007f991a64 + /* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */ + .octa 0x0000000004663802000000007f991a64 - /* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */ - .octa 0x00000001ab2f5afc00000000fa366d3a + /* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */ + .octa 0x00000001ab2f5afc00000000fa366d3a - /* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */ - .octa 0x0000000074a4ebd400000001a2bb34f0 + /* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */ + .octa 0x0000000074a4ebd400000001a2bb34f0 - /* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */ - .octa 0x00000001d7ab3a4c0000000028a9981e + /* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */ + .octa 0x00000001d7ab3a4c0000000028a9981e - /* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */ - .octa 0x00000001a8da60c600000001dbc672be + /* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */ + .octa 0x00000001a8da60c600000001dbc672be - /* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */ - .octa 0x000000013cf6382000000000b04d77f6 + /* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */ + .octa 0x000000013cf6382000000000b04d77f6 - /* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */ - .octa 0x00000000bec12e1e0000000124400d96 + /* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */ + .octa 0x00000000bec12e1e0000000124400d96 - /* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */ - .octa 0x00000001c6368010000000014ca4b414 + /* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */ + .octa 0x00000001c6368010000000014ca4b414 - /* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */ - .octa 0x00000001e6e78758000000012fe2c938 + /* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */ + .octa 0x00000001e6e78758000000012fe2c938 - /* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */ - .octa 0x000000008d7f2b3c00000001faed01e6 + /* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */ + .octa 0x000000008d7f2b3c00000001faed01e6 - /* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */ - .octa 0x000000016b4a156e000000007e80ecfe + /* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */ + .octa 0x000000016b4a156e000000007e80ecfe - /* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */ - .octa 0x00000001c63cfeb60000000098daee94 + /* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */ + .octa 0x00000001c63cfeb60000000098daee94 - /* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */ - .octa 0x000000015f902670000000010a04edea + /* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */ + .octa 0x000000015f902670000000010a04edea - /* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */ - .octa 0x00000001cd5de11e00000001c00b4524 + /* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */ + .octa 0x00000001cd5de11e00000001c00b4524 - /* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */ - .octa 0x000000001acaec540000000170296550 + /* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */ + .octa 0x000000001acaec540000000170296550 - /* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */ - .octa 0x000000002bd0ca780000000181afaa48 + /* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */ + .octa 0x000000002bd0ca780000000181afaa48 - /* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */ - .octa 0x0000000032d63d5c0000000185a31ffa + /* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */ + .octa 0x0000000032d63d5c0000000185a31ffa - /* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */ - .octa 0x000000001c6d4e4c000000002469f608 + /* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */ + .octa 0x000000001c6d4e4c000000002469f608 - /* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */ - .octa 0x0000000106a60b92000000006980102a + /* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */ + .octa 0x0000000106a60b92000000006980102a - /* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */ - .octa 0x00000000d3855e120000000111ea9ca8 + /* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */ + .octa 0x00000000d3855e120000000111ea9ca8 - /* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */ - .octa 0x00000000e312563600000001bd1d29ce + /* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */ + .octa 0x00000000e312563600000001bd1d29ce - /* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */ - .octa 0x000000009e8f7ea400000001b34b9580 + /* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */ + .octa 0x000000009e8f7ea400000001b34b9580 - /* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */ - .octa 0x00000001c82e562c000000003076054e + /* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */ + .octa 0x00000001c82e562c000000003076054e - /* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */ - .octa 0x00000000ca9f09ce000000012a608ea4 + /* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */ + .octa 0x00000000ca9f09ce000000012a608ea4 - /* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */ - .octa 0x00000000c63764e600000000784d05fe + /* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */ + .octa 0x00000000c63764e600000000784d05fe - /* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */ - .octa 0x0000000168d2e49e000000016ef0d82a + /* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */ + .octa 0x0000000168d2e49e000000016ef0d82a - /* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */ - .octa 0x00000000e986c1480000000075bda454 + /* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */ + .octa 0x00000000e986c1480000000075bda454 - /* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */ - .octa 0x00000000cfb65894000000003dc0a1c4 + /* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */ + .octa 0x00000000cfb65894000000003dc0a1c4 - /* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */ - .octa 0x0000000111cadee400000000e9a5d8be + /* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */ + .octa 0x0000000111cadee400000000e9a5d8be - /* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */ - .octa 0x0000000171fb63ce00000001609bc4b4 + /* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */ + .octa 0x0000000171fb63ce00000001609bc4b4 - .short_constants : + .short_constants : - /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of - zeros */ - /* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, x^2048 mod p(x)` */ - .octa 0x7fec2963e5bf80485cf015c388e56f72 + /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of + zeros */ + /* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, x^2048 mod p(x)` */ + .octa 0x7fec2963e5bf80485cf015c388e56f72 - /* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, x^1920 mod p(x)` */ - .octa 0x38e888d4844752a9963a18920246e2e6 + /* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, x^1920 mod p(x)` */ + .octa 0x38e888d4844752a9963a18920246e2e6 - /* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, x^1792 mod p(x)` */ - .octa 0x42316c00730206ad419a441956993a31 + /* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, x^1792 mod p(x)` */ + .octa 0x42316c00730206ad419a441956993a31 - /* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, x^1664 mod p(x)` */ - .octa 0x543d5c543e65ddf9924752ba2b830011 + /* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, x^1664 mod p(x)` */ + .octa 0x543d5c543e65ddf9924752ba2b830011 - /* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, x^1536 mod p(x)` */ - .octa 0x78e87aaf56767c9255bd7f9518e4a304 + /* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, x^1536 mod p(x)` */ + .octa 0x78e87aaf56767c9255bd7f9518e4a304 - /* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, x^1408 mod p(x)` */ - .octa 0x8f68fcec1903da7f6d76739fe0553f1e + /* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, x^1408 mod p(x)` */ + .octa 0x8f68fcec1903da7f6d76739fe0553f1e - /* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, x^1280 mod p(x)` */ - .octa 0x3f4840246791d588c133722b1fe0b5c3 + /* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, x^1280 mod p(x)` */ + .octa 0x3f4840246791d588c133722b1fe0b5c3 - /* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, x^1152 mod p(x)` */ - .octa 0x34c96751b04de25a64b67ee0e55ef1f3 + /* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, x^1152 mod p(x)` */ + .octa 0x34c96751b04de25a64b67ee0e55ef1f3 - /* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, x^1024 mod p(x)` */ - .octa 0x156c8e180b4a395b069db049b8fdb1e7 + /* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, x^1024 mod p(x)` */ + .octa 0x156c8e180b4a395b069db049b8fdb1e7 - /* x^800 mod p(x)`, x^832 mod p(x)`, x^864 mod p(x)`, x^896 mod p(x)` */ - .octa 0xe0b99ccbe661f7bea11bfaf3c9e90b9e + /* x^800 mod p(x)`, x^832 mod p(x)`, x^864 mod p(x)`, x^896 mod p(x)` */ + .octa 0xe0b99ccbe661f7bea11bfaf3c9e90b9e - /* x^672 mod p(x)`, x^704 mod p(x)`, x^736 mod p(x)`, x^768 mod p(x)` */ - .octa 0x041d37768cd75659817cdc5119b29a35 + /* x^672 mod p(x)`, x^704 mod p(x)`, x^736 mod p(x)`, x^768 mod p(x)` */ + .octa 0x041d37768cd75659817cdc5119b29a35 - /* x^544 mod p(x)`, x^576 mod p(x)`, x^608 mod p(x)`, x^640 mod p(x)` */ - .octa 0x3a0777818cfaa9651ce9d94b36c41f1c + /* x^544 mod p(x)`, x^576 mod p(x)`, x^608 mod p(x)`, x^640 mod p(x)` */ + .octa 0x3a0777818cfaa9651ce9d94b36c41f1c - /* x^416 mod p(x)`, x^448 mod p(x)`, x^480 mod p(x)`, x^512 mod p(x)` */ - .octa 0x0e148e8252377a554f256efcb82be955 + /* x^416 mod p(x)`, x^448 mod p(x)`, x^480 mod p(x)`, x^512 mod p(x)` */ + .octa 0x0e148e8252377a554f256efcb82be955 - /* x^288 mod p(x)`, x^320 mod p(x)`, x^352 mod p(x)`, x^384 mod p(x)` */ - .octa 0x9c25531d19e65ddeec1631edb2dea967 + /* x^288 mod p(x)`, x^320 mod p(x)`, x^352 mod p(x)`, x^384 mod p(x)` */ + .octa 0x9c25531d19e65ddeec1631edb2dea967 - /* x^160 mod p(x)`, x^192 mod p(x)`, x^224 mod p(x)`, x^256 mod p(x)` */ - .octa 0x790606ff9957c0a65d27e147510ac59a + /* x^160 mod p(x)`, x^192 mod p(x)`, x^224 mod p(x)`, x^256 mod p(x)` */ + .octa 0x790606ff9957c0a65d27e147510ac59a - /* x^32 mod p(x)`, x^64 mod p(x)`, x^96 mod p(x)`, x^128 mod p(x)` */ - .octa 0x82f63b786ea2d55ca66805eb18b8ea18 + /* x^32 mod p(x)`, x^64 mod p(x)`, x^96 mod p(x)`, x^128 mod p(x)` */ + .octa 0x82f63b786ea2d55ca66805eb18b8ea18 - .barrett_constants : - /* 33 bit reflected Barrett constant m - (4^32)/n */ - .octa 0x000000000000000000000000dea713f1 /* x^64 div p(x)` */ - /* 33 bit reflected Barrett constant n */ - .octa 0x00000000000000000000000105ec76f1 + .barrett_constants : + /* 33 bit reflected Barrett constant m - (4^32)/n */ + .octa 0x000000000000000000000000dea713f1 /* x^64 div p(x)` */ + /* 33 bit reflected Barrett constant n */ + .octa 0x00000000000000000000000105ec76f1 #endif diff --git a/src/third_party/wiredtiger/src/config/config_api.c b/src/third_party/wiredtiger/src/config/config_api.c index 6c8bdd54f30..f0c1d799089 100644 --- a/src/third_party/wiredtiger/src/config/config_api.c +++ b/src/third_party/wiredtiger/src/config/config_api.c @@ -103,8 +103,7 @@ wiredtiger_config_validate( */ if (session != NULL && event_handler != NULL) WT_RET_MSG(session, EINVAL, - "wiredtiger_config_validate event handler ignored when " - "a session also specified"); + "wiredtiger_config_validate event handler ignored when a session also specified"); /* * If we're not given a session, but we do have an event handler, build a fake @@ -231,9 +230,8 @@ __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char * WT_RET_MSG(session, EINVAL, "no configuration type specified"); if (strcmp(type, "boolean") != 0 && strcmp(type, "int") != 0 && strcmp(type, "list") != 0 && strcmp(type, "string") != 0) - WT_RET_MSG(session, EINVAL, - "type must be one of \"boolean\", \"int\", \"list\" or " - "\"string\""); + WT_RET_MSG( + session, EINVAL, "type must be one of \"boolean\", \"int\", \"list\" or \"string\""); /* * Translate the method name to our configuration names, then find a match. diff --git a/src/third_party/wiredtiger/src/config/config_check.c b/src/third_party/wiredtiger/src/config/config_check.c index 61582a0c88f..6925eeb41fc 100644 --- a/src/third_party/wiredtiger/src/config/config_check.c +++ b/src/third_party/wiredtiger/src/config/config_check.c @@ -134,15 +134,13 @@ config_check(WT_SESSION_IMPL *session, const WT_CONFIG_CHECK *checks, u_int chec if (WT_STRING_MATCH("min", ck.str, ck.len)) { if (v.val < cv.val) WT_RET_MSG(session, EINVAL, - "Value too small for key '%.*s' " - "the minimum is %.*s", - (int)k.len, k.str, (int)cv.len, cv.str); + "Value too small for key '%.*s' the minimum is %.*s", (int)k.len, k.str, + (int)cv.len, cv.str); } else if (WT_STRING_MATCH("max", ck.str, ck.len)) { if (v.val > cv.val) WT_RET_MSG(session, EINVAL, - "Value too large for key '%.*s' " - "the maximum is %.*s", - (int)k.len, k.str, (int)cv.len, cv.str); + "Value too large for key '%.*s' the maximum is %.*s", (int)k.len, k.str, + (int)cv.len, cv.str); } else if (WT_STRING_MATCH("choices", ck.str, ck.len)) { if (v.len == 0) WT_RET_MSG(session, EINVAL, "Key '%.*s' requires a value", (int)k.len, k.str); @@ -165,13 +163,10 @@ config_check(WT_SESSION_IMPL *session, const WT_CONFIG_CHECK *checks, u_int chec return (ret); if (!found) WT_RET_MSG(session, EINVAL, - "Value '%.*s' not a " - "permitted choice for key '%.*s'", - (int)v.len, v.str, (int)k.len, k.str); + "Value '%.*s' not a permitted choice for key '%.*s'", (int)v.len, v.str, + (int)k.len, k.str); } else - WT_RET_MSG(session, EINVAL, - "unexpected configuration description " - "keyword %.*s", + WT_RET_MSG(session, EINVAL, "unexpected configuration description keyword %.*s", (int)ck.len, ck.str); } } diff --git a/src/third_party/wiredtiger/src/config/config_collapse.c b/src/third_party/wiredtiger/src/config/config_collapse.c index 29bf8702781..4e30c1a1832 100644 --- a/src/third_party/wiredtiger/src/config/config_collapse.c +++ b/src/third_party/wiredtiger/src/config/config_collapse.c @@ -143,9 +143,7 @@ __config_merge_scan( */ for (len = 0; len < k.len; ++len) if (k.str[len] == SEPC) - WT_ERR_MSG(session, EINVAL, - "key %.*s contains a '%c' separator " - "character", + WT_ERR_MSG(session, EINVAL, "key %.*s contains a '%c' separator character", (int)k.len, (char *)k.str, SEPC); /* Build the key/value strings. */ diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 62ec99ac07a..6c7931360fb 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -42,9 +42,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_async_subconfigs[] = { {"enabled", "boolean", NULL, NULL, NULL, 0}, {"ops_max", "int", NULL, "min=1,max=4096", NULL, 0}, {"threads", "int", NULL, "min=1,max=20", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; -static const WT_CONFIG_CHECK confchk_wiredtiger_open_cache_overflow_subconfigs[] = { - {"file_max", "int", NULL, "min=0", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; - static const WT_CONFIG_CHECK confchk_wiredtiger_open_checkpoint_subconfigs[] = { {"log_size", "int", NULL, "min=0,max=2GB", NULL, 0}, {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -104,7 +101,6 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_statistics_log_su static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"async", "category", NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3}, {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, - {"cache_overflow", "category", NULL, NULL, confchk_wiredtiger_open_cache_overflow_subconfigs, 1}, {"cache_overhead", "int", NULL, "min=0,max=30", NULL, 0}, {"cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0}, {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, @@ -136,8 +132,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"statistics_log", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, @@ -215,6 +212,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_checkpoint[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_commit_transaction[] = { {"commit_timestamp", "string", NULL, NULL, NULL, 0}, {"durable_timestamp", "string", NULL, NULL, NULL, 0}, + {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"sync", "string", NULL, "choices=[\"background\",\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -345,6 +343,9 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_reconfigure[] = { NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_WT_SESSION_rollback_transaction[] = { + {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_salvage[] = { {"force", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -509,6 +510,10 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_encryption_subconfigs[] = { {"keyid", "string", NULL, NULL, NULL, 0}, {"name", "string", NULL, NULL, NULL, 0}, {"secretkey", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_wiredtiger_open_hash_subconfigs[] = { + {"buckets", "int", NULL, "min=64,max=65536", NULL, 0}, + {"dhandle_buckets", "int", NULL, "min=64,max=65536", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_wiredtiger_open_log_subconfigs[] = { {"archive", "boolean", NULL, NULL, NULL, 0}, {"compressor", "string", NULL, NULL, NULL, 0}, {"enabled", "boolean", NULL, NULL, NULL, 0}, @@ -535,7 +540,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, - {"cache_overflow", "category", NULL, NULL, confchk_wiredtiger_open_cache_overflow_subconfigs, 1}, {"cache_overhead", "int", NULL, "min=0,max=30", NULL, 0}, {"cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0}, {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, @@ -558,6 +562,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"file_close_sync", "boolean", NULL, NULL, NULL, 0}, {"file_extend", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {"file_manager", "category", NULL, NULL, confchk_wiredtiger_open_file_manager_subconfigs, 3}, + {"hash", "category", NULL, NULL, confchk_wiredtiger_open_hash_subconfigs, 2}, {"hazard_max", "int", NULL, "min=15", NULL, 0}, {"history_store", "category", NULL, NULL, confchk_wiredtiger_open_history_store_subconfigs, 1}, {"in_memory", "boolean", NULL, NULL, NULL, 0}, @@ -580,8 +585,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, @@ -611,7 +617,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, - {"cache_overflow", "category", NULL, NULL, confchk_wiredtiger_open_cache_overflow_subconfigs, 1}, {"cache_overhead", "int", NULL, "min=0,max=30", NULL, 0}, {"cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0}, {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, @@ -634,6 +639,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"file_close_sync", "boolean", NULL, NULL, NULL, 0}, {"file_extend", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {"file_manager", "category", NULL, NULL, confchk_wiredtiger_open_file_manager_subconfigs, 3}, + {"hash", "category", NULL, NULL, confchk_wiredtiger_open_hash_subconfigs, 2}, {"hazard_max", "int", NULL, "min=15", NULL, 0}, {"history_store", "category", NULL, NULL, confchk_wiredtiger_open_history_store_subconfigs, 1}, {"in_memory", "boolean", NULL, NULL, NULL, 0}, @@ -656,8 +662,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, @@ -687,7 +694,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, - {"cache_overflow", "category", NULL, NULL, confchk_wiredtiger_open_cache_overflow_subconfigs, 1}, {"cache_overhead", "int", NULL, "min=0,max=30", NULL, 0}, {"cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0}, {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, @@ -708,6 +714,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"extensions", "list", NULL, NULL, NULL, 0}, {"file_close_sync", "boolean", NULL, NULL, NULL, 0}, {"file_extend", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {"file_manager", "category", NULL, NULL, confchk_wiredtiger_open_file_manager_subconfigs, 3}, + {"hash", "category", NULL, NULL, confchk_wiredtiger_open_hash_subconfigs, 2}, {"hazard_max", "int", NULL, "min=15", NULL, 0}, {"history_store", "category", NULL, NULL, confchk_wiredtiger_open_history_store_subconfigs, 1}, {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, @@ -729,8 +736,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, @@ -758,7 +766,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, - {"cache_overflow", "category", NULL, NULL, confchk_wiredtiger_open_cache_overflow_subconfigs, 1}, {"cache_overhead", "int", NULL, "min=0,max=30", NULL, 0}, {"cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0}, {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, @@ -779,6 +786,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"extensions", "list", NULL, NULL, NULL, 0}, {"file_close_sync", "boolean", NULL, NULL, NULL, 0}, {"file_extend", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {"file_manager", "category", NULL, NULL, confchk_wiredtiger_open_file_manager_subconfigs, 3}, + {"hash", "category", NULL, NULL, confchk_wiredtiger_open_hash_subconfigs, 2}, {"hazard_max", "int", NULL, "min=15", NULL, 0}, {"history_store", "category", NULL, NULL, confchk_wiredtiger_open_history_store_subconfigs, 1}, {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, @@ -800,8 +808,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, @@ -844,8 +853,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_CONNECTION.query_timestamp", "get=all_durable", confchk_WT_CONNECTION_query_timestamp, 1}, {"WT_CONNECTION.reconfigure", "async=(enabled=false,ops_max=1024,threads=2),cache_max_wait_ms=0" - ",cache_overflow=(file_max=0),cache_overhead=8,cache_size=100MB," - "checkpoint=(log_size=0,wait=0),compatibility=(release=)," + ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," + "wait=0),compatibility=(release=)," "debug_mode=(checkpoint_retention=0,cursor_copy=false," "eviction=false,log_retention=0,realloc_exact=false," "rollback_error=0,slow_checkpoint=false,table_logging=false)," @@ -863,7 +872,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,verbose=", - confchk_WT_CONNECTION_reconfigure, 29}, + confchk_WT_CONNECTION_reconfigure, 28}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", "commit_timestamp=,durable_timestamp=,force=false," @@ -886,8 +895,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.checkpoint", "drop=,force=false,name=,target=,use_timestamp=true", confchk_WT_SESSION_checkpoint, 5}, {"WT_SESSION.close", "", NULL, 0}, - {"WT_SESSION.commit_transaction", "commit_timestamp=,durable_timestamp=,sync=", - confchk_WT_SESSION_commit_transaction, 3}, + {"WT_SESSION.commit_transaction", + "commit_timestamp=,durable_timestamp=,operation_timeout_ms=0," + "sync=", + confchk_WT_SESSION_commit_transaction, 4}, {"WT_SESSION.compact", "timeout=1200", confchk_WT_SESSION_compact, 1}, {"WT_SESSION.create", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," @@ -933,17 +944,20 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.prepare_transaction", "prepare_timestamp=", confchk_WT_SESSION_prepare_transaction, 1}, {"WT_SESSION.query_timestamp", "get=read", confchk_WT_SESSION_query_timestamp, 1}, - {"WT_SESSION.rebalance", "", NULL, 0}, {"WT_SESSION.reconfigure", - "cache_cursors=true,ignore_cache_size=false," - "isolation=read-committed", - confchk_WT_SESSION_reconfigure, 3}, + {"WT_SESSION.rebalance", "", NULL, 0}, + {"WT_SESSION.reconfigure", + "cache_cursors=true,ignore_cache_size=false," + "isolation=read-committed", + confchk_WT_SESSION_reconfigure, 3}, {"WT_SESSION.rename", "", NULL, 0}, {"WT_SESSION.reset", "", NULL, 0}, - {"WT_SESSION.rollback_transaction", "", NULL, 0}, + {"WT_SESSION.rollback_transaction", "operation_timeout_ms=0", + confchk_WT_SESSION_rollback_transaction, 1}, {"WT_SESSION.salvage", "force=false", confchk_WT_SESSION_salvage, 1}, - {"WT_SESSION.strerror", "", NULL, 0}, {"WT_SESSION.timestamp_transaction", - "commit_timestamp=,durable_timestamp=,prepare_timestamp=," - "read_timestamp=", - confchk_WT_SESSION_timestamp_transaction, 4}, + {"WT_SESSION.strerror", "", NULL, 0}, + {"WT_SESSION.timestamp_transaction", + "commit_timestamp=,durable_timestamp=,prepare_timestamp=," + "read_timestamp=", + confchk_WT_SESSION_timestamp_transaction, 4}, {"WT_SESSION.transaction_sync", "timeout_ms=1200000", confchk_WT_SESSION_transaction_sync, 1}, {"WT_SESSION.truncate", "", NULL, 0}, {"WT_SESSION.upgrade", "", NULL, 0}, {"WT_SESSION.verify", @@ -1017,89 +1031,87 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"wiredtiger_open", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," - "cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8" - ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," - "checkpoint_sync=true,compatibility=(release=,require_max=," - "require_min=),config_base=true,create=false," - "debug_mode=(checkpoint_retention=0,cursor_copy=false," - "eviction=false,log_retention=0,realloc_exact=false," - "rollback_error=0,slow_checkpoint=false,table_logging=false)," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "compatibility=(release=,require_max=,require_min=)," + "config_base=true,create=false,debug_mode=(checkpoint_retention=0" + ",cursor_copy=false,eviction=false,log_retention=0," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000," - "history_store=(file_max=0),in_memory=false,io_capacity=(total=0)" - ",log=(archive=true,compressor=,enabled=false,file_max=100MB," - "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," - "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," - "mmap=true,mmap_all=false,multiprocess=false," - "operation_timeout_ms=0,operation_tracking=(enabled=false," - "path=\".\"),readonly=false,salvage=false,session_max=100," - "session_scratch_max=2MB,session_table_cache=true," - "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," - "statistics=none,statistics_log=(json=false,on_close=false," - "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),use_environment=true,use_environment_priv=false," + "close_scan_interval=10),hash=(buckets=512,dhandle_buckets=512)," + "hazard_max=1000,history_store=(file_max=0),in_memory=false," + "io_capacity=(total=0),log=(archive=true,compressor=," + "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," + "prealloc=true,recover=on,zero_fill=false)," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "mmap_all=false,multiprocess=false,operation_timeout_ms=0," + "operation_tracking=(enabled=false,path=\".\"),readonly=false," + "salvage=false,session_max=100,session_scratch_max=2MB," + "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," + "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" + ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" + ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" + ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,verify_metadata=false,write_through=", confchk_wiredtiger_open, 56}, {"wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," - "cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8" - ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," - "checkpoint_sync=true,compatibility=(release=,require_max=," - "require_min=),config_base=true,create=false," - "debug_mode=(checkpoint_retention=0,cursor_copy=false," - "eviction=false,log_retention=0,realloc_exact=false," - "rollback_error=0,slow_checkpoint=false,table_logging=false)," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "compatibility=(release=,require_max=,require_min=)," + "config_base=true,create=false,debug_mode=(checkpoint_retention=0" + ",cursor_copy=false,eviction=false,log_retention=0," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000," - "history_store=(file_max=0),in_memory=false,io_capacity=(total=0)" - ",log=(archive=true,compressor=,enabled=false,file_max=100MB," - "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," - "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," - "mmap=true,mmap_all=false,multiprocess=false," - "operation_timeout_ms=0,operation_tracking=(enabled=false," - "path=\".\"),readonly=false,salvage=false,session_max=100," - "session_scratch_max=2MB,session_table_cache=true," - "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," - "statistics=none,statistics_log=(json=false,on_close=false," - "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),use_environment=true,use_environment_priv=false," + "close_scan_interval=10),hash=(buckets=512,dhandle_buckets=512)," + "hazard_max=1000,history_store=(file_max=0),in_memory=false," + "io_capacity=(total=0),log=(archive=true,compressor=," + "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," + "prealloc=true,recover=on,zero_fill=false)," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "mmap_all=false,multiprocess=false,operation_timeout_ms=0," + "operation_tracking=(enabled=false,path=\".\"),readonly=false," + "salvage=false,session_max=100,session_scratch_max=2MB," + "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," + "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" + ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" + ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" + ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,verify_metadata=false,version=(major=0,minor=0)," "write_through=", confchk_wiredtiger_open_all, 57}, {"wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," - "cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8" - ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," - "checkpoint_sync=true,compatibility=(release=,require_max=," - "require_min=),debug_mode=(checkpoint_retention=0," - "cursor_copy=false,eviction=false,log_retention=0," - "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "compatibility=(release=,require_max=,require_min=)," + "debug_mode=(checkpoint_retention=0,cursor_copy=false," + "eviction=false,log_retention=0,realloc_exact=false," + "rollback_error=0,slow_checkpoint=false,table_logging=false)," + "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000," - "history_store=(file_max=0),io_capacity=(total=0)," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "close_scan_interval=10),hash=(buckets=512,dhandle_buckets=512)," + "hazard_max=1000,history_store=(file_max=0),io_capacity=(total=0)" + ",log=(archive=true,compressor=,enabled=false,file_max=100MB," "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," "mmap=true,mmap_all=false,multiprocess=false," @@ -1116,22 +1128,22 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," - "cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8" - ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," - "checkpoint_sync=true,compatibility=(release=,require_max=," - "require_min=),debug_mode=(checkpoint_retention=0," - "cursor_copy=false,eviction=false,log_retention=0," - "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "compatibility=(release=,require_max=,require_min=)," + "debug_mode=(checkpoint_retention=0,cursor_copy=false," + "eviction=false,log_retention=0,realloc_exact=false," + "rollback_error=0,slow_checkpoint=false,table_logging=false)," + "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000," - "history_store=(file_max=0),io_capacity=(total=0)," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "close_scan_interval=10),hash=(buckets=512,dhandle_buckets=512)," + "hazard_max=1000,history_store=(file_max=0),io_capacity=(total=0)" + ",log=(archive=true,compressor=,enabled=false,file_max=100MB," "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," "mmap=true,mmap_all=false,multiprocess=false," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 59f0a3bc27d..7caef79173d 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -397,9 +397,7 @@ __wt_encryptor_config(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_CONFIG_ WT_ERR(__encryptor_confchk(session, cval, &nenc)); if (nenc == NULL) { if (keyid->len != 0) - WT_ERR_MSG(session, EINVAL, - "encryption.keyid " - "requires encryption.name to be set"); + WT_ERR_MSG(session, EINVAL, "encryption.keyid requires encryption.name to be set"); goto out; } @@ -408,11 +406,9 @@ __wt_encryptor_config(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_CONFIG_ * needs to be configured on the database as well. */ if (conn->kencryptor == NULL && kencryptorp != &conn->kencryptor) - WT_ERR_MSG(session, EINVAL, - "table encryption " - "requires connection encryption to be set"); + WT_ERR_MSG(session, EINVAL, "table encryption requires connection encryption to be set"); hash = __wt_hash_city64(keyid->str, keyid->len); - bucket = hash % WT_HASH_ARRAY_SIZE; + bucket = hash & (conn->hash_size - 1); TAILQ_FOREACH (kenc, &nenc->keyedhashqh[bucket], q) if (WT_STRING_MATCH(kenc->keyid, keyid->str, keyid->len)) goto out; @@ -459,7 +455,7 @@ __conn_add_encryptor( WT_DECL_RET; WT_NAMED_ENCRYPTOR *nenc; WT_SESSION_IMPL *session; - int i; + uint64_t i; nenc = NULL; @@ -485,7 +481,8 @@ __conn_add_encryptor( WT_ERR(__wt_strdup(session, name, &nenc->name)); nenc->encryptor = encryptor; TAILQ_INIT(&nenc->keyedqh); - for (i = 0; i < WT_HASH_ARRAY_SIZE; i++) + WT_ERR(__wt_calloc_def(session, conn->hash_size, &nenc->keyedhashqh)); + for (i = 0; i < conn->hash_size; i++) TAILQ_INIT(&nenc->keyedhashqh[i]); TAILQ_INSERT_TAIL(&conn->encryptqh, nenc, q); @@ -493,6 +490,7 @@ __conn_add_encryptor( err: if (nenc != NULL) { + __wt_free(session, nenc->keyedhashqh); __wt_free(session, nenc->name); __wt_free(session, nenc); } @@ -532,6 +530,7 @@ __wt_conn_remove_encryptor(WT_SESSION_IMPL *session) if (nenc->encryptor->terminate != NULL) WT_TRET(nenc->encryptor->terminate(nenc->encryptor, (WT_SESSION *)session)); + __wt_free(session, nenc->keyedhashqh); __wt_free(session, nenc->name); __wt_free(session, nenc); } @@ -1044,7 +1043,7 @@ err: */ if (s->event_handler->handle_close != NULL) WT_TRET(s->event_handler->handle_close(s->event_handler, wt_session, NULL)); - WT_TRET(wt_session->close(wt_session, config)); + WT_TRET(__wt_session_close_internal(s)); } /* Wait for in-flight operations to complete. */ @@ -1081,7 +1080,7 @@ err: WT_TRET(__wt_checkpoint_server_destroy(session)); /* Perform a final checkpoint and shut down the global transaction state. */ - WT_TRET(__wt_txn_global_shutdown(session, config, cfg)); + WT_TRET(__wt_txn_global_shutdown(session, cfg)); if (ret != 0) { __wt_err(session, ret, "failure during close, disabling further writes"); @@ -1432,8 +1431,9 @@ __conn_config_file( WT_ERR(__conn_config_check_version(session, cbuf->data)); /* Check the configuration information. */ - WT_ERR(__wt_config_check(session, is_user ? WT_CONFIG_REF(session, wiredtiger_open_usercfg) : - WT_CONFIG_REF(session, wiredtiger_open_basecfg), + WT_ERR(__wt_config_check(session, + is_user ? WT_CONFIG_REF(session, wiredtiger_open_usercfg) : + WT_CONFIG_REF(session, wiredtiger_open_basecfg), cbuf->data, 0)); /* Append it to the stack. */ @@ -1489,8 +1489,8 @@ __conn_env_var(WT_SESSION_IMPL *session, const char *cfg[], const char *name, co WT_ERR(__wt_config_gets(session, cfg, "use_environment_priv", &cval)); if (cval.val == 0) WT_ERR_MSG(session, WT_ERROR, - "privileged process has %s environment variable set, " - "without having \"use_environment_priv\" configured", + "privileged process has %s environment variable set, without having " + "\"use_environment_priv\" configured", name); return (0); @@ -1533,6 +1533,46 @@ err: } /* + * __conn_hash_config -- + * Configure and allocate hash buckets in the connection. + */ +static int +__conn_hash_config(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + uint64_t i; + + conn = S2C(session); + WT_RET(__wt_config_gets(session, cfg, "hash.buckets", &cval)); + if (!__wt_ispo2((uint32_t)cval.val)) + WT_RET_MSG(session, EINVAL, "Hash bucket size %" PRIu64 " invalid. Must be power of 2", + (uint64_t)cval.val); + conn->hash_size = (uint64_t)cval.val; + WT_RET(__wt_config_gets(session, cfg, "hash.dhandle_buckets", &cval)); + if (!__wt_ispo2((uint32_t)cval.val)) + WT_RET_MSG(session, EINVAL, + "Data handle hash bucket size %" PRIu64 " invalid. Must be power of 2", + (uint64_t)cval.val); + conn->dh_hash_size = (uint64_t)cval.val; + /* Don't set the values in the statistics here. They're set after the connection is set up. */ + + /* Hash bucket arrays. */ + WT_RET(__wt_calloc_def(session, conn->hash_size, &conn->blockhash)); + WT_RET(__wt_calloc_def(session, conn->hash_size, &conn->fhhash)); + for (i = 0; i < conn->hash_size; ++i) { + TAILQ_INIT(&conn->blockhash[i]); + TAILQ_INIT(&conn->fhhash[i]); + } + WT_RET(__wt_calloc_def(session, conn->dh_hash_size, &conn->dh_bucket_count)); + WT_RET(__wt_calloc_def(session, conn->dh_hash_size, &conn->dhhash)); + for (i = 0; i < conn->dh_hash_size; ++i) + TAILQ_INIT(&conn->dhhash[i]); + + return (0); +} + +/* * __conn_home -- * Set the database home directory. */ @@ -1599,8 +1639,7 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) } if (match) WT_ERR_MSG(session, EBUSY, - "WiredTiger database is already being managed by another " - "thread in this process"); + "WiredTiger database is already being managed by another thread in this process"); /* * !!! @@ -1672,9 +1711,8 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) * locking past the end-of-file. */ if (__wt_file_lock(session, conn->lock_fh, true) != 0) - WT_ERR_MSG(session, EBUSY, - "WiredTiger database is already being managed by " - "another process"); + WT_ERR_MSG( + session, EBUSY, "WiredTiger database is already being managed by another process"); /* * If the size of the lock file is non-zero, we created it (or won a locking race with the thread @@ -1716,9 +1754,8 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) * Immediately release the lock, it's just a test. */ if (__wt_file_lock(session, fh, true) != 0) { - WT_ERR_MSG(session, EBUSY, - "WiredTiger database is already being managed by " - "another process"); + WT_ERR_MSG( + session, EBUSY, "WiredTiger database is already being managed by another process"); } WT_ERR(__wt_file_lock(session, fh, false)); } @@ -1736,8 +1773,8 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) if (conn->is_new) { if (F_ISSET(conn, WT_CONN_READONLY)) WT_ERR_MSG(session, EINVAL, - "The database directory is empty or needs recovery, cannot continue with a" - " read only connection"); + "The database directory is empty or needs recovery, cannot continue with a read only " + "connection"); WT_ERR(__wt_snprintf_len_set( buf, sizeof(buf), &len, "%s\n%s\n", WT_WIREDTIGER, WIREDTIGER_VERSION_STRING)); WT_ERR(__wt_write(session, fh, (wt_off_t)0, len, buf)); @@ -1751,8 +1788,7 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_config_gets(session, cfg, "exclusive", &cval)); if (cval.val != 0) WT_ERR_MSG(session, EEXIST, - "WiredTiger database already exists and exclusive " - "option configured"); + "WiredTiger database already exists and exclusive option configured"); } err: @@ -1935,14 +1971,15 @@ __wt_verbose_dump_sessions(WT_SESSION_IMPL *session, bool show_cursors) session, " Current dhandle: %s", s->dhandle == NULL ? "NONE" : s->dhandle->name)); WT_ERR( __wt_msg(session, " Backup in progress: %s", s->bkp_cursor == NULL ? "no" : "yes")); - WT_ERR(__wt_msg(session, " Compact state: %s", s->compact_state == WT_COMPACT_NONE ? + WT_ERR(__wt_msg(session, " Compact state: %s", + s->compact_state == WT_COMPACT_NONE ? "none" : (s->compact_state == WT_COMPACT_RUNNING ? "running" : "success"))); WT_ERR(__wt_msg(session, " Flags: 0x%" PRIx32, s->flags)); - WT_ERR( - __wt_msg(session, " Isolation level: %s", s->isolation == WT_ISO_READ_COMMITTED ? - "read-committed" : - (s->isolation == WT_ISO_READ_UNCOMMITTED ? "read-uncommitted" : "snapshot"))); + WT_ERR(__wt_msg(session, " Isolation level: %s", + s->isolation == WT_ISO_READ_COMMITTED ? + "read-committed" : + (s->isolation == WT_ISO_READ_UNCOMMITTED ? "read-uncommitted" : "snapshot"))); WT_ERR(__wt_msg(session, " Transaction:")); WT_ERR(__wt_verbose_dump_txn_one(session, s, 0, NULL)); } else { @@ -1998,9 +2035,11 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) */ static const WT_NAME_FLAG stress_types[] = { {"aggressive_sweep", WT_TIMING_STRESS_AGGRESSIVE_SWEEP}, + {"backup_rename", WT_TIMING_STRESS_BACKUP_RENAME}, {"checkpoint_slow", WT_TIMING_STRESS_CHECKPOINT_SLOW}, {"history_store_checkpoint_delay", WT_TIMING_STRESS_HS_CHECKPOINT_DELAY}, {"history_store_sweep_race", WT_TIMING_STRESS_HS_SWEEP}, + {"prepare_checkpoint_delay", WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY}, {"split_1", WT_TIMING_STRESS_SPLIT_1}, {"split_2", WT_TIMING_STRESS_SPLIT_2}, {"split_3", WT_TIMING_STRESS_SPLIT_3}, {"split_4", WT_TIMING_STRESS_SPLIT_4}, {"split_5", WT_TIMING_STRESS_SPLIT_5}, {"split_6", WT_TIMING_STRESS_SPLIT_6}, @@ -2129,7 +2168,7 @@ __conn_write_base_config(WT_SESSION_IMPL *session, const char *cfg[]) ret = __wt_sync_and_rename(session, &fs, WT_BASECONFIG_SET, WT_BASECONFIG); if (0) { - /* Close open file handle, remove any temporary file. */ + /* Close open file handle, remove any temporary file. */ err: WT_TRET(__wt_fclose(session, &fs)); WT_TRET(__wt_remove_if_exists(session, WT_BASECONFIG_SET, false)); @@ -2162,8 +2201,8 @@ __conn_set_file_system(WT_CONNECTION *wt_conn, WT_FILE_SYSTEM *file_system, cons */ if (conn->file_system != NULL) WT_ERR_MSG(session, EPERM, - "filesystem already configured; custom filesystems should " - "enable \"early_load\" configuration"); + "filesystem already configured; custom filesystems should enable \"early_load\" " + "configuration"); conn->file_system = file_system; @@ -2426,6 +2465,12 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c WT_ERR(__conn_home(session, home, cfg)); /* + * Configure and allocate hash buckets. This must be done before the call to load extensions. + * Some extensions like encryption or file systems may allocate hash arrays. + */ + WT_ERR(__conn_hash_config(session, cfg)); + + /* * Load early extensions before doing further initialization (one early extension is to * configure a file system). */ @@ -2767,13 +2812,19 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c if (verify_meta) { WT_ERR(__wt_open_internal_session(conn, "verify hs", false, 0, &verify_session)); ret = __wt_history_store_verify(verify_session); - wt_session = &verify_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(verify_session)); WT_ERR(ret); } #endif /* + * The hash array sizes needed to be set up very early. Set them in the statistics here. Setting + * them in the early configuration function makes them get zeroed out. + */ + WT_STAT_CONN_SET(session, buckets, conn->hash_size); + WT_STAT_CONN_SET(session, buckets_dh, conn->dh_hash_size); + + /* * The default session should not open data handles after this point: since it can be shared * between threads, relying on session->dhandle is not safe. */ diff --git a/src/third_party/wiredtiger/src/conn/conn_cache.c b/src/third_party/wiredtiger/src/conn/conn_cache.c index 3b212d94352..59cc7426599 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache.c @@ -35,9 +35,7 @@ __cache_config_abs_to_pct( */ if (shared) WT_RET_MSG(session, EINVAL, - "Shared cache configuration requires a percentage " - "value for %s", - param_name); + "Shared cache configuration requires a percentage value for %s", param_name); /* An absolute value can't exceed the cache size. */ if (input > conn->cache_size) WT_RET_MSG(session, EINVAL, "%s should not exceed cache size", param_name); @@ -148,9 +146,8 @@ __cache_config_local(WT_SESSION_IMPL *session, bool shared, const char *cfg[]) evict_threads_min = (uint32_t)cval.val; if (evict_threads_min > evict_threads_max) - WT_RET_MSG(session, EINVAL, - "eviction=(threads_min) cannot be greater than " - "eviction=(threads_max)"); + WT_RET_MSG( + session, EINVAL, "eviction=(threads_min) cannot be greater than eviction=(threads_max)"); conn->evict_threads_max = evict_threads_max; conn->evict_threads_min = evict_threads_min; @@ -352,7 +349,6 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION *wt_session; int i; conn = S2C(session); @@ -363,14 +359,11 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) /* The cache should be empty at this point. Complain if not. */ if (cache->pages_inmem != cache->pages_evicted) - __wt_errx(session, "cache server: exiting with %" PRIu64 - " pages in " - "memory and %" PRIu64 " pages evicted", + __wt_errx(session, + "cache server: exiting with %" PRIu64 " pages in memory and %" PRIu64 " pages evicted", cache->pages_inmem, cache->pages_evicted); if (cache->bytes_image_intl + cache->bytes_image_leaf != 0) - __wt_errx(session, "cache server: exiting with %" PRIu64 - " image bytes in " - "memory", + __wt_errx(session, "cache server: exiting with %" PRIu64 " image bytes in memory", cache->bytes_image_intl + cache->bytes_image_leaf); if (cache->bytes_inmem != 0) __wt_errx( @@ -386,9 +379,8 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) __wt_spin_destroy(session, &cache->evict_pass_lock); __wt_spin_destroy(session, &cache->evict_queue_lock); __wt_spin_destroy(session, &cache->evict_walk_lock); - wt_session = &cache->walk_session->iface; - if (wt_session != NULL) - WT_TRET(wt_session->close(wt_session, NULL)); + if (cache->walk_session != NULL) + WT_TRET(__wt_session_close_internal(cache->walk_session)); for (i = 0; i < WT_EVICT_QUEUE_MAX; ++i) { __wt_spin_destroy(session, &cache->evict_queues[i].evict_lock); diff --git a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c index b01617de75b..1b9742a5b62 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c @@ -67,16 +67,13 @@ __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) * pool. */ if (__wt_config_gets(session, &cfg[1], "shared_cache.size", &cval) != WT_NOTFOUND) - WT_RET_MSG(session, EINVAL, - "Shared cache configuration requires a " - "pool name"); + WT_RET_MSG(session, EINVAL, "Shared cache configuration requires a pool name"); return (0); } if (__wt_config_gets(session, &cfg[1], "cache_size", &cval_cache_size) != WT_NOTFOUND) WT_RET_MSG(session, EINVAL, - "Only one of cache_size and shared_cache can be " - "in the configuration"); + "Only one of cache_size and shared_cache can be in the configuration"); /* * NOTE: The allocations made when configuring and opening a cache pool don't really belong @@ -185,8 +182,8 @@ __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) used_cache -= conn->cache->cp_reserved; if (used_cache + reserve > size) WT_ERR_MSG(session, EINVAL, - "Shared cache unable to accommodate this configuration. " - "Shared cache size: %" PRIu64 ", requested min: %" PRIu64, + "Shared cache unable to accommodate this configuration. Shared cache size: %" PRIu64 + ", requested min: %" PRIu64, size, used_cache + reserve); /* The configuration is verified - it's safe to update the pool. */ @@ -284,7 +281,6 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) WT_CACHE_POOL *cp; WT_CONNECTION_IMPL *conn, *entry; WT_DECL_RET; - WT_SESSION *wt_session; bool cp_locked, found; conn = S2C(session); @@ -328,8 +324,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) __wt_cond_signal(session, cp->cache_pool_cond); WT_TRET(__wt_thread_join(session, &cache->cp_tid)); - wt_session = &cache->cp_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(cache->cp_session)); /* * Grab the lock again now to stop other threads joining the pool while we are figuring out diff --git a/src/third_party/wiredtiger/src/conn/conn_capacity.c b/src/third_party/wiredtiger/src/conn/conn_capacity.c index 8b5f7299fd4..2213f696e4f 100644 --- a/src/third_party/wiredtiger/src/conn/conn_capacity.c +++ b/src/third_party/wiredtiger/src/conn/conn_capacity.c @@ -193,7 +193,6 @@ __wt_capacity_server_destroy(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION *wt_session; conn = S2C(session); @@ -206,10 +205,8 @@ __wt_capacity_server_destroy(WT_SESSION_IMPL *session) __wt_cond_destroy(session, &conn->capacity_cond); /* Close the server thread's session. */ - if (conn->capacity_session != NULL) { - wt_session = &conn->capacity_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); - } + if (conn->capacity_session != NULL) + WT_TRET(__wt_session_close_internal(conn->capacity_session)); /* * Ensure capacity settings are cleared - so that reconfigure doesn't get confused. diff --git a/src/third_party/wiredtiger/src/conn/conn_ckpt.c b/src/third_party/wiredtiger/src/conn/conn_ckpt.c index 5c8ad02b01e..b71ac6c28f4 100644 --- a/src/third_party/wiredtiger/src/conn/conn_ckpt.c +++ b/src/third_party/wiredtiger/src/conn/conn_ckpt.c @@ -46,8 +46,7 @@ __ckpt_server_config(WT_SESSION_IMPL *session, const char **cfg, bool *startp) WT_RET(__wt_config_gets(session, cfg, "in_memory", &cval)); if (cval.val != 0) WT_RET_MSG(session, EINVAL, - "checkpoint configuration incompatible with " - "in-memory configuration"); + "checkpoint configuration incompatible with in-memory configuration"); __wt_log_written_reset(session); @@ -199,7 +198,6 @@ __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION *wt_session; conn = S2C(session); @@ -212,10 +210,8 @@ __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) __wt_cond_destroy(session, &conn->ckpt_cond); /* Close the server thread's session. */ - if (conn->ckpt_session != NULL) { - wt_session = &conn->ckpt_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); - } + if (conn->ckpt_session != NULL) + WT_TRET(__wt_session_close_internal(conn->ckpt_session)); /* * Ensure checkpoint settings are cleared - so that reconfigure doesn't get confused. diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index beb222d08bb..3f506bec283 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -198,7 +198,7 @@ __wt_conn_dhandle_alloc(WT_SESSION_IMPL *session, const char *uri, const char *c * Prepend the handle to the connection list, assuming we're likely to need new files again * soon, until they are cached by all sessions. */ - bucket = dhandle->name_hash % WT_HASH_ARRAY_SIZE; + bucket = dhandle->name_hash & (S2C(session)->dh_hash_size - 1); WT_CONN_DHANDLE_INSERT(S2C(session), dhandle, bucket); session->dhandle = dhandle; @@ -225,7 +225,7 @@ __wt_conn_dhandle_find(WT_SESSION_IMPL *session, const char *uri, const char *ch /* We must be holding the handle list lock at a higher level. */ WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); - bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; + bucket = __wt_hash_city64(uri, strlen(uri)) & (conn->dh_hash_size - 1); if (checkpoint == NULL) { TAILQ_FOREACH (dhandle, &conn->dhhash[bucket], hashq) { if (F_ISSET(dhandle, WT_DHANDLE_DEAD)) @@ -512,9 +512,14 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) { + WT_CONNECTION_IMPL *conn; WT_DECL_RET; + uint64_t time_diff, time_start, time_stop; bool skip; + conn = S2C(session); + time_start = time_stop = 0; + /* Always apply the name function, if supplied. */ skip = false; if (name_func != NULL) @@ -531,7 +536,21 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, if ((ret = __wt_session_get_dhandle(session, dhandle->name, dhandle->checkpoint, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); + if (WT_SESSION_IS_CHECKPOINT(session)) + time_start = __wt_clock(session); WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); + /* We need to gather this information before releasing the dhandle. */ + if (WT_SESSION_IS_CHECKPOINT(session)) { + time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); + if (F_ISSET(S2BT(session), WT_BTREE_SKIP_CKPT)) { + ++conn->ckpt_skip; + conn->ckpt_skip_time += time_diff; + } else { + ++conn->ckpt_apply; + conn->ckpt_apply_time += time_diff; + } + } WT_TRET(__wt_session_release_dhandle(session)); return (ret); } @@ -549,15 +568,16 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, WT_DATA_HANDLE *dhandle; WT_DECL_RET; uint64_t bucket; + uint64_t time_diff, time_start, time_stop; conn = S2C(session); - + time_start = time_stop = 0; /* * If we're given a URI, then we walk only the hash list for that name. If we don't have a URI * we walk the entire dhandle list. */ if (uri != NULL) { - bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; + bucket = __wt_hash_city64(uri, strlen(uri)) & (conn->dh_hash_size - 1); for (dhandle = NULL;;) { WT_WITH_HANDLE_LIST_READ_LOCK( @@ -571,11 +591,16 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, WT_ERR(__conn_btree_apply_internal(session, dhandle, file_func, name_func, cfg)); } } else { + if (WT_SESSION_IS_CHECKPOINT(session)) { + conn->ckpt_apply = conn->ckpt_skip = 0; + conn->ckpt_apply_time = conn->ckpt_skip_time = 0; + time_start = __wt_clock(session); + } for (dhandle = NULL;;) { WT_WITH_HANDLE_LIST_READ_LOCK( session, WT_DHANDLE_NEXT(session, dhandle, &conn->dhqh, q)); if (dhandle == NULL) - return (0); + goto done; if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || F_ISSET(dhandle, WT_DHANDLE_DEAD) || dhandle->type != WT_DHANDLE_TYPE_BTREE || dhandle->checkpoint != NULL || @@ -583,6 +608,18 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, continue; WT_ERR(__conn_btree_apply_internal(session, dhandle, file_func, name_func, cfg)); } +done: + if (WT_SESSION_IS_CHECKPOINT(session)) { + time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); + WT_STAT_CONN_SET(session, txn_checkpoint_handle_applied, conn->ckpt_apply); + WT_STAT_CONN_SET(session, txn_checkpoint_handle_skipped, conn->ckpt_skip); + WT_STAT_CONN_SET(session, txn_checkpoint_handle_walked, conn->dhandle_count); + WT_STAT_CONN_SET(session, txn_checkpoint_handle_duration, time_diff); + WT_STAT_CONN_SET(session, txn_checkpoint_handle_duration_apply, conn->ckpt_apply_time); + WT_STAT_CONN_SET(session, txn_checkpoint_handle_duration_skip, conn->ckpt_skip_time); + } + return (0); } err: @@ -656,7 +693,7 @@ __wt_conn_dhandle_close_all(WT_SESSION_IMPL *session, const char *uri, bool remo */ WT_ERR(__conn_dhandle_close_one(session, uri, NULL, removed, mark_dead)); - bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; + bucket = __wt_hash_city64(uri, strlen(uri)) & (conn->dh_hash_size - 1); TAILQ_FOREACH (dhandle, &conn->dhhash[bucket], hashq) { if (strcmp(dhandle->name, uri) != 0 || dhandle->checkpoint == NULL || F_ISSET(dhandle, WT_DHANDLE_DEAD)) @@ -684,7 +721,7 @@ __conn_dhandle_remove(WT_SESSION_IMPL *session, bool final) conn = S2C(session); dhandle = session->dhandle; - bucket = dhandle->name_hash % WT_HASH_ARRAY_SIZE; + bucket = dhandle->name_hash & (conn->dh_hash_size - 1); WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); WT_ASSERT(session, dhandle != conn->cache->walk_tree); diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index db2b085e8c5..f0d861d16b4 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -16,15 +16,9 @@ int __wt_connection_init(WT_CONNECTION_IMPL *conn) { WT_SESSION_IMPL *session; - u_int i; session = conn->default_session; - for (i = 0; i < WT_HASH_ARRAY_SIZE; i++) { - TAILQ_INIT(&conn->dhhash[i]); /* Data handle hash lists */ - TAILQ_INIT(&conn->fhhash[i]); /* File handle hash lists */ - } - TAILQ_INIT(&conn->dhqh); /* Data handle list */ TAILQ_INIT(&conn->dlhqh); /* Library list */ TAILQ_INIT(&conn->dsrcqh); /* Data source list */ @@ -79,9 +73,7 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) * opaque, but for now this is simpler. */ WT_RET(__wt_spin_init(session, &conn->block_lock, "block manager")); - for (i = 0; i < WT_HASH_ARRAY_SIZE; i++) - TAILQ_INIT(&conn->blockhash[i]); /* Block handle hash lists */ - TAILQ_INIT(&conn->blockqh); /* Block manager list */ + TAILQ_INIT(&conn->blockqh); /* Block manager list */ conn->ckpt_prep_min = UINT64_MAX; conn->ckpt_time_min = UINT64_MAX; @@ -133,6 +125,12 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &conn->lsm_manager.manager_lock); __wt_cond_destroy(session, &conn->lsm_manager.work_cond); + /* Free allocated hash buckets. */ + __wt_free(session, conn->blockhash); + __wt_free(session, conn->dh_bucket_count); + __wt_free(session, conn->dhhash); + __wt_free(session, conn->fhhash); + /* Free allocated memory. */ __wt_free(session, conn->cfg); __wt_free(session, conn->debug_ckpt); diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 95e7b263925..c22be09e3db 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -74,7 +74,7 @@ __logmgr_force_archive(WT_SESSION_IMPL *session, uint32_t lognum) WT_RET(WT_SESSION_CHECK_PANIC(tmp_session)); WT_RET(__wt_log_truncate_files(tmp_session, NULL, true)); } - WT_RET(tmp_session->iface.close(&tmp_session->iface, NULL)); + WT_RET(__wt_session_close_internal(tmp_session)); return (0); } @@ -230,19 +230,18 @@ __wt_logmgr_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfig) * * See above: should never happen. */ - if (reconfig && ((enabled && !FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) || - (!enabled && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)))) - WT_RET_MSG(session, EINVAL, - "log manager reconfigure: enabled mismatch with existing " - "setting"); + if (reconfig && + ((enabled && !FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) || + (!enabled && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)))) + WT_RET_MSG( + session, EINVAL, "log manager reconfigure: enabled mismatch with existing setting"); /* Logging is incompatible with in-memory */ if (enabled) { WT_RET(__wt_config_gets(session, cfg, "in_memory", &cval)); if (cval.val != 0) - WT_RET_MSG(session, EINVAL, - "In-memory configuration incompatible with " - "log=(enabled=true)"); + WT_RET_MSG( + session, EINVAL, "In-memory configuration incompatible with log=(enabled=true)"); } if (enabled) @@ -321,9 +320,8 @@ __wt_logmgr_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfig) WT_RET(__wt_config_gets(session, cfg, "log.zero_fill", &cval)); if (cval.val != 0) { if (F_ISSET(conn, WT_CONN_READONLY)) - WT_RET_MSG(session, EINVAL, - "Read-only configuration incompatible with " - "zero-filling log files"); + WT_RET_MSG( + session, EINVAL, "Read-only configuration incompatible with zero-filling log files"); FLD_SET(conn->log_flags, WT_CONN_LOG_ZERO_FILL); } @@ -440,7 +438,7 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) if (0) err: - __wt_err(session, ret, "log archive server error"); + __wt_err(session, ret, "log archive server error"); WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); return (ret); } @@ -504,7 +502,7 @@ __log_prealloc_once(WT_SESSION_IMPL *session) if (0) err: - __wt_err(session, ret, "log pre-alloc server error"); + __wt_err(session, ret, "log pre-alloc server error"); WT_TRET(__wt_fs_directory_list_free(session, &recfiles, reccount)); return (ret); } @@ -605,8 +603,8 @@ __log_file_server(void *arg) */ if (conn->hot_backup_start == 0 && conn->log_cursors == 0) { WT_WITH_HOTBACKUP_READ_LOCK(session, - WT_ERR_ERROR_OK(__wt_ftruncate(session, close_fh, close_end_lsn.l.offset), - ENOTSUP, false), + WT_ERR_ERROR_OK( + __wt_ftruncate(session, close_fh, close_end_lsn.l.offset), ENOTSUP, false), NULL); } WT_SET_LSN(&close_end_lsn, close_end_lsn.l.file + 1, 0); @@ -947,8 +945,7 @@ __log_server(void *arg) WT_ERR(ret); } else __wt_verbose(session, WT_VERB_LOG, "%s", - "log_archive: Blocked due to open " - "log cursor holding archive lock"); + "log_archive: Blocked due to open log cursor holding archive lock"); } time_start = __wt_clock(session); } @@ -1104,7 +1101,6 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION *wt_session; conn = S2C(session); @@ -1129,8 +1125,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) conn->log_file_tid_set = false; } if (conn->log_file_session != NULL) { - wt_session = &conn->log_file_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(conn->log_file_session)); conn->log_file_session = NULL; } if (conn->log_wrlsn_tid_set) { @@ -1139,8 +1134,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) conn->log_wrlsn_tid_set = false; } if (conn->log_wrlsn_session != NULL) { - wt_session = &conn->log_wrlsn_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(conn->log_wrlsn_session)); conn->log_wrlsn_session = NULL; } @@ -1149,8 +1143,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) /* Close the server thread's session. */ if (conn->log_session != NULL) { - wt_session = &conn->log_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(conn->log_session)); conn->log_session = NULL; } diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 68f370fd829..2840c6dc2e8 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -157,7 +157,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) * error messages from the remaining operations while destroying the connection handle. */ if (session != &conn->dummy_session) { - WT_TRET(session->iface.close(&session->iface, NULL)); + WT_TRET(__wt_session_close_internal(session)); session = conn->default_session = &conn->dummy_session; } diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c index 1f53ceecfd7..18c974e61a3 100644 --- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c +++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c @@ -77,9 +77,7 @@ __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfi */ WT_RET(__wt_txn_activity_check(session, &txn_active)); if (txn_active) - WT_RET_MSG(session, ENOTSUP, - "system must be quiescent" - " for upgrade or downgrade"); + WT_RET_MSG(session, ENOTSUP, "system must be quiescent for upgrade or downgrade"); } F_SET(conn, WT_CONN_COMPATIBILITY); } @@ -132,10 +130,11 @@ __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfi */ if (reconfig && conn->req_max_major != WT_CONN_COMPAT_NONE && (conn->req_max_major < rel_major || - (conn->req_max_major == rel_major && conn->req_max_minor < rel_minor))) - WT_RET_MSG(session, ENOTSUP, WT_COMPAT_MSG_PREFIX - "required max of %" PRIu16 ".%" PRIu16 - "cannot be smaller than requested compatibility release %" PRIu16 ".%" PRIu16, + (conn->req_max_major == rel_major && conn->req_max_minor < rel_minor))) + WT_RET_MSG(session, ENOTSUP, + WT_COMPAT_MSG_PREFIX "required max of %" PRIu16 ".%" PRIu16 + "cannot be smaller than requested compatibility release %" PRIu16 + ".%" PRIu16, conn->req_max_major, conn->req_max_minor, rel_major, rel_minor); /* @@ -144,10 +143,11 @@ __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfi */ if (reconfig && conn->req_min_major != WT_CONN_COMPAT_NONE && (conn->req_min_major > rel_major || - (conn->req_min_major == rel_major && conn->req_min_minor > rel_minor))) - WT_RET_MSG(session, ENOTSUP, WT_COMPAT_MSG_PREFIX - "required min of %" PRIu16 ".%" PRIu16 - "cannot be larger than requested compatibility release %" PRIu16 ".%" PRIu16, + (conn->req_min_major == rel_major && conn->req_min_minor > rel_minor))) + WT_RET_MSG(session, ENOTSUP, + WT_COMPAT_MSG_PREFIX "required min of %" PRIu16 ".%" PRIu16 + "cannot be larger than requested compatibility release %" PRIu16 + ".%" PRIu16, conn->req_min_major, conn->req_min_minor, rel_major, rel_minor); conn->compat_major = rel_major; @@ -236,9 +236,8 @@ __wt_conn_optrack_setup(WT_SESSION_IMPL *session, const char *cfg[], bool reconf } if (F_ISSET(conn, WT_CONN_READONLY)) /* Operation tracking isn't supported in read-only mode */ - WT_RET_MSG(session, EINVAL, - "Operation tracking is incompatible with read only " - "configuration"); + WT_RET_MSG( + session, EINVAL, "Operation tracking is incompatible with read only configuration"); if (F_ISSET(conn, WT_CONN_OPTRACK)) /* Already enabled, nothing else to do */ return (0); @@ -338,9 +337,8 @@ __wt_conn_statistics_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET_NOTFOUND_OK(ret); if (set > 1) - WT_RET_MSG(session, EINVAL, - "Only one of all, fast, none configuration values should " - "be specified"); + WT_RET_MSG( + session, EINVAL, "Only one of all, fast, none configuration values should be specified"); /* * Now that we've parsed general statistics categories, process sub-categories. @@ -365,8 +363,7 @@ __wt_conn_statistics_config(WT_SESSION_IMPL *session, const char *cfg[]) if (!LF_ISSET(WT_STAT_TYPE_ALL | WT_STAT_TYPE_CACHE_WALK | WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK)) WT_RET_MSG(session, EINVAL, - "the value \"clear\" can only be specified if " - "statistics are enabled"); + "the value \"clear\" can only be specified if statistics are enabled"); LF_SET(WT_STAT_CLEAR); } WT_RET_NOTFOUND_OK(ret); diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index b3a1b3e979a..cf54572f70a 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -164,8 +164,7 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) */ if (!WT_PREFIX_MATCH(k.str, "file:") && !WT_PREFIX_MATCH(k.str, "lsm:")) WT_ERR_MSG(session, EINVAL, - "statistics_log sources configuration only " - "supports objects of type \"file\" or " + "statistics_log sources configuration only supports objects of type \"file\" or " "\"lsm\""); WT_ERR(__wt_strndup(session, k.str, k.len, &sources[cnt])); } @@ -683,7 +682,6 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION *wt_session; conn = S2C(session); @@ -705,8 +703,7 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) /* Close the server thread's session. */ if (conn->stat_session != NULL) { - wt_session = &conn->stat_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(conn->stat_session)); conn->stat_session = NULL; } diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index 934acd228e8..0ab2b97e21e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -80,7 +80,7 @@ __sweep_expire_one(WT_SESSION_IMPL *session) /* Only sweep clean trees where all updates are visible. */ if (btree != NULL && (btree->modified || - !__wt_txn_visible_all(session, btree->rec_max_txn, btree->rec_max_timestamp))) + !__wt_txn_visible_all(session, btree->rec_max_txn, btree->rec_max_timestamp))) goto err; /* @@ -403,7 +403,6 @@ __wt_sweep_destroy(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION *wt_session; conn = S2C(session); @@ -416,8 +415,7 @@ __wt_sweep_destroy(WT_SESSION_IMPL *session) __wt_cond_destroy(session, &conn->sweep_cond); if (conn->sweep_session != NULL) { - wt_session = &conn->sweep_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(conn->sweep_session)); conn->sweep_session = NULL; } diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c index f30bd1c4d3b..c71676b2082 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c @@ -70,6 +70,15 @@ __curbackup_incr_blkmod(WT_SESSION_IMPL *session, WT_BTREE *btree, WT_CURSOR_BAC cb->nbits = (uint64_t)b.val; WT_ERR(__wt_config_subgets(session, &v, "offset", &b)); cb->offset = (uint64_t)b.val; + /* + * The rename configuration string component was added later. So don't error if we don't + * find it in the string. If we don't have it, we're not doing a rename. + */ + WT_ERR_NOTFOUND_OK(__wt_config_subgets(session, &v, "rename", &b), true); + if (ret == 0 && b.val) + F_SET(cb, WT_CURBACKUP_RENAME); + else + F_CLR(cb, WT_CURBACKUP_RENAME); /* * We found a match. Load the block information into the cursor. @@ -112,7 +121,7 @@ __curbackup_incr_next(WT_CURSOR *cursor) F_CLR(cursor, WT_CURSTD_RAW); if (!F_ISSET(cb, WT_CURBACKUP_INCR_INIT) && - (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL))) { + (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL | WT_CURBACKUP_RENAME))) { /* * We don't have this object's incremental information or it's a forced file copy. If this * is a log file, use the full pathname that may include the log path. @@ -155,19 +164,21 @@ __curbackup_incr_next(WT_CURSOR *cursor) */ WT_ERR(__curbackup_incr_blkmod(session, btree, cb)); /* - * There are three cases where we do not have block modification information for + * There are several cases where we do not have block modification information for * the file. They are described and handled as follows: * - * 1. Newly created file without checkpoint information. Return the whole file - * information. - * 2. File created and checkpointed before incremental backups were configured. + * 1. Renamed file. Always return the whole file information. + * 2. Newly created file without checkpoint information. Return the whole + * file information. + * 3. File created and checkpointed before incremental backups were configured. * Return no file information as it was copied in the initial full backup. - * 3. File that has not been modified since the previous incremental backup. + * 4. File that has not been modified since the previous incremental backup. * Return no file information as there is no new information. */ - if (cb->bitstring.mem == NULL) { + if (cb->bitstring.mem == NULL || F_ISSET(cb, WT_CURBACKUP_RENAME)) { F_SET(cb, WT_CURBACKUP_INCR_INIT); - if (F_ISSET(cb, WT_CURBACKUP_CKPT_FAKE) && F_ISSET(cb, WT_CURBACKUP_HAS_CB_INFO)) { + if (F_ISSET(cb, WT_CURBACKUP_RENAME) || + (F_ISSET(cb, WT_CURBACKUP_CKPT_FAKE) && F_ISSET(cb, WT_CURBACKUP_HAS_CB_INFO))) { WT_ERR(__wt_fs_size(session, cb->incr_file, &size)); __wt_cursor_set_key(cursor, 0, size, WT_BACKUP_FILE); goto done; diff --git a/src/third_party/wiredtiger/src/cursor/cur_bulk.c b/src/third_party/wiredtiger/src/cursor/cur_bulk.c index 8302330edd1..f10ff01d1b0 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_bulk.c +++ b/src/third_party/wiredtiger/src/cursor/cur_bulk.c @@ -21,9 +21,9 @@ __bulk_col_keycmp_err(WT_CURSOR_BULK *cbulk) cursor = &cbulk->cbt.iface; session = CUR2S(cbulk); - WT_RET_MSG(session, EINVAL, "bulk-load presented with out-of-order keys: %" PRIu64 - " is less " - "than previously inserted key %" PRIu64, + WT_RET_MSG(session, EINVAL, + "bulk-load presented with out-of-order keys: %" PRIu64 + " is less than previously inserted key %" PRIu64, cursor->recno, cbulk->recno); } @@ -203,8 +203,8 @@ __bulk_row_keycmp_err(WT_CURSOR_BULK *cbulk) WT_ERR(__wt_scr_alloc(session, 512, &b)); WT_ERR_MSG(session, EINVAL, - "bulk-load presented with out-of-order keys: %s compares smaller " - "than previously inserted key %s", + "bulk-load presented with out-of-order keys: %s compares smaller than previously inserted " + "key %s", __wt_buf_set_printable(session, cursor->key.data, cursor->key.size, a), __wt_buf_set_printable(session, cbulk->last.data, cbulk->last.size, b)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index 21ed7e4fa75..a2c0d1873e5 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -135,7 +135,7 @@ str2recno(WT_SESSION_IMPL *session, const char *p, uint64_t *recnop) WT_RET_MSG(session, ERANGE, "%s: invalid record number", p); if (endptr[0] != '\0') format: - WT_RET_MSG(session, EINVAL, "%s: invalid record number", p); + WT_RET_MSG(session, EINVAL, "%s: invalid record number", p); *recnop = recno; return (0); @@ -420,8 +420,9 @@ __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) cdump->child = child; /* Copy the dump flags from the child cursor. */ - F_SET(cursor, F_MASK(child, WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_JSON | WT_CURSTD_DUMP_PRETTY | - WT_CURSTD_DUMP_PRINT)); + F_SET(cursor, + F_MASK(child, + WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_JSON | WT_CURSTD_DUMP_PRETTY | WT_CURSTD_DUMP_PRINT)); if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) { WT_ERR(__wt_calloc_one(session, &json)); cursor->json_private = child->json_private = json; diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 19e69d6dc3e..b1b39c604fe 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -93,8 +93,8 @@ __curfile_next(WT_CURSOR *cursor) WT_ERR(__wt_btcur_next(cbt, false)); /* Next maintains a position, key and value. */ - WT_ASSERT(session, F_ISSET(cbt, WT_CBT_ACTIVE) && - F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + WT_ASSERT(session, + F_ISSET(cbt, WT_CBT_ACTIVE) && F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: @@ -120,8 +120,8 @@ __wt_curfile_next_random(WT_CURSOR *cursor) WT_ERR(__wt_btcur_next_random(cbt)); /* Next-random maintains a position, key and value. */ - WT_ASSERT(session, F_ISSET(cbt, WT_CBT_ACTIVE) && - F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + WT_ASSERT(session, + F_ISSET(cbt, WT_CBT_ACTIVE) && F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: @@ -146,8 +146,8 @@ __curfile_prev(WT_CURSOR *cursor) WT_ERR(__wt_btcur_prev(cbt, false)); /* Prev maintains a position, key and value. */ - WT_ASSERT(session, F_ISSET(cbt, WT_CBT_ACTIVE) && - F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + WT_ASSERT(session, + F_ISSET(cbt, WT_CBT_ACTIVE) && F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: @@ -172,7 +172,8 @@ __curfile_reset(WT_CURSOR *cursor) ret = __wt_btcur_reset(cbt); /* Reset maintains no position, key or value. */ - WT_ASSERT(session, !F_ISSET(cbt, WT_CBT_ACTIVE) && F_MASK(cursor, WT_CURSTD_KEY_SET) == 0 && + WT_ASSERT(session, + !F_ISSET(cbt, WT_CBT_ACTIVE) && F_MASK(cursor, WT_CURSTD_KEY_SET) == 0 && F_MASK(cursor, WT_CURSTD_VALUE_SET) == 0); err: @@ -202,8 +203,8 @@ __curfile_search(WT_CURSOR *cursor) __wt_stat_usecs_hist_incr_opread(session, WT_CLOCKDIFF_US(time_stop, time_start)); /* Search maintains a position, key and value. */ - WT_ASSERT(session, F_ISSET(cbt, WT_CBT_ACTIVE) && - F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + WT_ASSERT(session, + F_ISSET(cbt, WT_CBT_ACTIVE) && F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: @@ -233,8 +234,8 @@ __curfile_search_near(WT_CURSOR *cursor, int *exact) __wt_stat_usecs_hist_incr_opread(session, WT_CLOCKDIFF_US(time_stop, time_start)); /* Search-near maintains a position, key and value. */ - WT_ASSERT(session, F_ISSET(cbt, WT_CBT_ACTIVE) && - F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + WT_ASSERT(session, + F_ISSET(cbt, WT_CBT_ACTIVE) && F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: @@ -270,10 +271,11 @@ __curfile_insert(WT_CURSOR *cursor) * Insert maintains no position, key or value (except for column-store appends, where we are * returning a key). */ - WT_ASSERT(session, !F_ISSET(cbt, WT_CBT_ACTIVE) && + WT_ASSERT(session, + !F_ISSET(cbt, WT_CBT_ACTIVE) && ((F_ISSET(cursor, WT_CURSTD_APPEND) && F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_EXT) || - (!F_ISSET(cursor, WT_CURSTD_APPEND) && F_MASK(cursor, WT_CURSTD_KEY_SET) == 0))); + (!F_ISSET(cursor, WT_CURSTD_APPEND) && F_MASK(cursor, WT_CURSTD_KEY_SET) == 0))); WT_ASSERT(session, F_MASK(cursor, WT_CURSTD_VALUE_SET) == 0); err: @@ -368,8 +370,8 @@ __curfile_update(WT_CURSOR *cursor) __wt_stat_usecs_hist_incr_opwrite(session, WT_CLOCKDIFF_US(time_stop, time_start)); /* Update maintains a position, key and value. */ - WT_ASSERT(session, F_ISSET(cbt, WT_CBT_ACTIVE) && - F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + WT_ASSERT(session, + F_ISSET(cbt, WT_CBT_ACTIVE) && F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: @@ -685,9 +687,8 @@ __curfile_create(WT_SESSION_IMPL *session, WT_CURSOR *owner, const char *cfg[], WT_ERR(__wt_config_gets_def(session, cfg, "next_random", 0, &cval)); if (cval.val != 0) { if (WT_CURSOR_RECNO(cursor)) - WT_ERR_MSG(session, ENOTSUP, - "next_random configuration not supported for " - "column-store objects"); + WT_ERR_MSG( + session, ENOTSUP, "next_random configuration not supported for column-store objects"); __wt_cursor_set_notsup(cursor); cursor->next = __wt_curfile_next_random; diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index 9cc1ba83a4f..b286f6f9e43 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -509,8 +509,7 @@ __wt_curindex_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, */ if (WT_CURSOR_RECNO(cursor)) WT_ERR_MSG(session, WT_ERROR, - "Column store indexes based on a record number primary " - "key are not supported"); + "Column store indexes based on a record number primary key are not supported"); /* Handle projections. */ if (columns != NULL) { diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 53923282fc0..dfb65b0f2bc 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -589,7 +589,7 @@ __curjoin_entry_member( if (entry->subjoin == NULL && iter != NULL && (iter->end_pos + iter->end_skip >= entry->ends_next || - (iter->end_skip > 0 && F_ISSET(entry, WT_CURJOIN_ENTRY_DISJUNCTION)))) + (iter->end_skip > 0 && F_ISSET(entry, WT_CURJOIN_ENTRY_DISJUNCTION)))) return (0); /* no checks to make */ entry->stats.membership_check++; @@ -884,9 +884,7 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, bool iterab mainbuf = NULL; if (cjoin->entries_next == 0) - WT_RET_MSG(session, EINVAL, - "join cursor has not yet been joined with any other " - "cursors"); + WT_RET_MSG(session, EINVAL, "join cursor has not yet been joined with any other cursors"); /* Get a consistent view of our subordinate cursors if appropriate. */ __wt_txn_cursor_op(session); @@ -935,8 +933,7 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, bool iterab if (!iterable && F_ISSET(je, WT_CURJOIN_ENTRY_BLOOM)) { if (session->txn->isolation == WT_ISO_READ_UNCOMMITTED) WT_ERR_MSG(session, EINVAL, - "join cursors with Bloom filters cannot be " - "used with read-uncommitted isolation"); + "join cursors with Bloom filters cannot be used with read-uncommitted isolation"); if (je->bloom == NULL) { /* * Look for compatible filters to be shared, pick compatible numbers for bit counts @@ -1353,19 +1350,15 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_INDEX *idx } else { /* Merge the join into an existing entry for this index */ if (count != 0 && entry->count != 0 && entry->count != count) - WT_RET_MSG(session, EINVAL, "count=%" PRIu64 - " does not match " - "previous count=%" PRIu64 " for this index", - count, entry->count); - if (LF_MASK(WT_CURJOIN_ENTRY_BLOOM) != F_MASK(entry, WT_CURJOIN_ENTRY_BLOOM)) WT_RET_MSG(session, EINVAL, - "join has incompatible strategy " - "values for the same index"); + "count=%" PRIu64 " does not match previous count=%" PRIu64 " for this index", count, + entry->count); + if (LF_MASK(WT_CURJOIN_ENTRY_BLOOM) != F_MASK(entry, WT_CURJOIN_ENTRY_BLOOM)) + WT_RET_MSG(session, EINVAL, "join has incompatible strategy values for the same index"); if (LF_MASK(WT_CURJOIN_ENTRY_FALSE_POSITIVES) != F_MASK(entry, WT_CURJOIN_ENTRY_FALSE_POSITIVES)) WT_RET_MSG(session, EINVAL, - "join has incompatible bloom_false_positives " - "values for the same index"); + "join has incompatible bloom_false_positives values for the same index"); /* * Check against other comparisons (we call them endpoints) @@ -1391,20 +1384,19 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_INDEX *idx ((range & WT_CURJOIN_END_GT) != 0 || range_eq)) || (F_ISSET(end, WT_CURJOIN_END_LT) && ((range & WT_CURJOIN_END_LT) != 0 || range_eq)) || (endrange == WT_CURJOIN_END_EQ && - (range & (WT_CURJOIN_END_LT | WT_CURJOIN_END_GT)) != 0)) + (range & (WT_CURJOIN_END_LT | WT_CURJOIN_END_GT)) != 0)) WT_RET_MSG(session, EINVAL, "join has overlapping ranges"); if (range == WT_CURJOIN_END_EQ && endrange == WT_CURJOIN_END_EQ && !F_ISSET(entry, WT_CURJOIN_ENTRY_DISJUNCTION)) - WT_RET_MSG(session, EINVAL, - "compare=eq can only be combined " - "using operation=or"); + WT_RET_MSG(session, EINVAL, "compare=eq can only be combined using operation=or"); /* * Sort "gt"/"ge" to the front, followed by any number of "eq", and finally "lt"/"le". */ - if (!hasins && ((range & WT_CURJOIN_END_GT) != 0 || - (range == WT_CURJOIN_END_EQ && endrange != WT_CURJOIN_END_EQ && - !F_ISSET(end, WT_CURJOIN_END_GT)))) { + if (!hasins && + ((range & WT_CURJOIN_END_GT) != 0 || + (range == WT_CURJOIN_END_EQ && endrange != WT_CURJOIN_END_EQ && + !F_ISSET(end, WT_CURJOIN_END_GT)))) { ins = i; hasins = true; } diff --git a/src/third_party/wiredtiger/src/cursor/cur_json.c b/src/third_party/wiredtiger/src/cursor/cur_json.c index 7e9ac93eea6..89e5ecba4d3 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_json.c +++ b/src/third_party/wiredtiger/src/cursor/cur_json.c @@ -869,10 +869,8 @@ __wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, const char WT_RET_MSG(session, EINVAL, "invalid Unicode within JSON string"); src += 4; if (hi != 0) - WT_RET_MSG(session, EINVAL, - "Unicode \"%6.6s\" byte out of " - "range in JSON", - src - 6); + WT_RET_MSG( + session, EINVAL, "Unicode \"%6.6s\" byte out of range in JSON", src - 6); *dst++ = (char)lo; break; case 'f': diff --git a/src/third_party/wiredtiger/src/cursor/cur_metadata.c b/src/third_party/wiredtiger/src/cursor/cur_metadata.c index 5967bd2bf74..543f6221333 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_metadata.c +++ b/src/third_party/wiredtiger/src/cursor/cur_metadata.c @@ -46,9 +46,7 @@ __schema_source_config( WT_ERR(__wt_buf_fmt(session, buf, "%.*s", (int)cval.len, cval.str)); srch->set_key(srch, buf->data); if ((ret = srch->search(srch)) != 0) - WT_ERR_MSG(session, ret, - "metadata information for source configuration" - " \"%s\" not found", + WT_ERR_MSG(session, ret, "metadata information for source configuration \"%s\" not found", (const char *)buf->data); WT_ERR(srch->get_value(srch, &v)); WT_ERR(__wt_strdup(session, v, result)); @@ -100,8 +98,7 @@ __schema_create_collapse(WT_SESSION_IMPL *session, WT_CURSOR_METADATA *mdc, cons c->set_key(c, buf->data); if ((ret = c->search(c)) != 0) WT_ERR_MSG(session, ret, - "metadata information for source configuration" - " \"%s\" not found", + "metadata information for source configuration \"%s\" not found", (const char *)buf->data); WT_ERR(c->get_value(c, &v)); WT_ERR(__wt_strdup(session, v, --cfg)); @@ -167,9 +164,10 @@ err: * Check if a key matches the metadata. The public value is "metadata:", but also check for the * internal version of the URI. */ -#define WT_KEY_IS_METADATA(key) \ - ((key)->size > 0 && (WT_STRING_MATCH(WT_METADATA_URI, (key)->data, (key)->size - 1) || \ - WT_STRING_MATCH(WT_METAFILE_URI, (key)->data, (key)->size - 1))) +#define WT_KEY_IS_METADATA(key) \ + ((key)->size > 0 && \ + (WT_STRING_MATCH(WT_METADATA_URI, (key)->data, (key)->size - 1) || \ + WT_STRING_MATCH(WT_METAFILE_URI, (key)->data, (key)->size - 1))) /* * __curmetadata_metadata_search -- diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index f3e2b2930dc..a02f5e5f28d 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -635,15 +635,15 @@ __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, c if ((ret = __wt_config_subgets(session, &cval, "all", &sval)) == 0 && sval.val != 0) { if (!FLD_ISSET(conn->stat_flags, WT_STAT_TYPE_ALL)) goto config_err; - F_SET(cst, WT_STAT_TYPE_ALL | WT_STAT_TYPE_CACHE_WALK | WT_STAT_TYPE_FAST | + F_SET(cst, + WT_STAT_TYPE_ALL | WT_STAT_TYPE_CACHE_WALK | WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK); } WT_ERR_NOTFOUND_OK(ret, false); if ((ret = __wt_config_subgets(session, &cval, "fast", &sval)) == 0 && sval.val != 0) { if (F_ISSET(cst, WT_STAT_TYPE_ALL)) WT_ERR_MSG(session, EINVAL, - "Only one of all, fast, none " - "configuration values should be specified"); + "Only one of all, fast, none configuration values should be specified"); F_SET(cst, WT_STAT_TYPE_FAST); } WT_ERR_NOTFOUND_OK(ret, false); @@ -670,16 +670,13 @@ __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, c if ((ret = __wt_config_subgets(session, &cval, "size", &sval)) == 0 && sval.val != 0) { if (F_ISSET(cst, WT_STAT_TYPE_FAST | WT_STAT_TYPE_ALL)) WT_ERR_MSG(session, EINVAL, - "Only one of all, fast, none " - "configuration values should be specified"); + "Only one of all, fast, none configuration values should be specified"); F_SET(cst, WT_STAT_TYPE_SIZE); } WT_ERR_NOTFOUND_OK(ret, false); if ((ret = __wt_config_subgets(session, &cval, "clear", &sval)) == 0 && sval.val != 0) { if (F_ISSET(cst, WT_STAT_TYPE_SIZE)) - WT_ERR_MSG(session, EINVAL, - "clear is incompatible with size " - "statistics"); + WT_ERR_MSG(session, EINVAL, "clear is incompatible with size statistics"); F_SET(cst, WT_STAT_CLEAR); } WT_ERR_NOTFOUND_OK(ret, false); @@ -734,8 +731,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, c if (0) { config_err: WT_ERR_MSG(session, EINVAL, - "cursor's statistics configuration doesn't match the " - "database statistics configuration"); + "cursor's statistics configuration doesn't match the database statistics configuration"); } if (0) { diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index fb68d4681a6..364a6dffbdb 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -143,9 +143,8 @@ __wt_cursor_modify_value_format_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, in if (cursor->value_format != NULL && strlen(cursor->value_format) != 0) { session = CUR2S(cursor); - WT_RET_MSG(session, ENOTSUP, - "WT_CURSOR.modify only supported for 'S' and 'u' value " - "formats"); + WT_RET_MSG( + session, ENOTSUP, "WT_CURSOR.modify only supported for 'S' and 'u' value formats"); } return (__wt_cursor_notsup(cursor)); } @@ -669,7 +668,7 @@ __wt_cursor_cache(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle) /* Move the cursor from the open list to the caching hash table. */ if (cursor->uri_hash == 0) cursor->uri_hash = __wt_hash_city64(cursor->uri, strlen(cursor->uri)); - bucket = cursor->uri_hash % WT_HASH_ARRAY_SIZE; + bucket = cursor->uri_hash & (S2C(session)->hash_size - 1); TAILQ_REMOVE(&session->cursors, cursor, q); TAILQ_INSERT_HEAD(&session->cursor_cache[bucket], cursor, q); @@ -702,7 +701,7 @@ __wt_cursor_reopen(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle) WT_STAT_CONN_DECR_ATOMIC(session, cursor_cached_count); WT_STAT_DATA_INCR(session, cursor_open_count); - bucket = cursor->uri_hash % WT_HASH_ARRAY_SIZE; + bucket = cursor->uri_hash & (S2C(session)->hash_size - 1); TAILQ_REMOVE(&session->cursor_cache[bucket], cursor, q); TAILQ_INSERT_HEAD(&session->cursors, cursor, q); F_CLR(cursor, WT_CURSTD_CACHED); @@ -732,20 +731,28 @@ __wt_cursor_cache_release(WT_SESSION_IMPL *session, WT_CURSOR *cursor, bool *rel WT_RET(__wt_session_cursor_cache_sweep(session)); } - WT_ERR(cursor->cache(cursor)); + /* + * Caching the cursor releases its data handle. So we have to update statistics first. If + * caching fails, we'll decrement the statistics after reopening the cursor (and getting the + * data handle back). + */ WT_STAT_CONN_INCR(session, cursor_cache); WT_STAT_DATA_INCR(session, cursor_cache); + WT_ERR(cursor->cache(cursor)); WT_ASSERT(session, F_ISSET(cursor, WT_CURSTD_CACHED)); *released = true; if (0) { - /* - * If caching fails, we must restore the state of the cursor back to open so that the close - * works from a known state. The reopen may also fail, but that doesn't matter at this point. - */ + /* + * If caching fails, we must restore the state of the cursor back to open so that the close + * works from a known state. The reopen may also fail, but that doesn't matter at this + * point. + */ err: WT_TRET(cursor->reopen(cursor, false)); WT_ASSERT(session, !F_ISSET(cursor, WT_CURSTD_CACHED)); + WT_STAT_CONN_DECR(session, cursor_cache); + WT_STAT_DATA_DECR(session, cursor_cache); } return (ret); @@ -825,7 +832,7 @@ __wt_cursor_cache_get(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *to_d * Walk through all cursors, if there is a cached cursor that matches uri and configuration, use * it. */ - bucket = hash_value % WT_HASH_ARRAY_SIZE; + bucket = hash_value & (S2C(session)->hash_size - 1); TAILQ_FOREACH (cursor, &session->cursor_cache[bucket], q) { if (cursor->uri_hash == hash_value && strcmp(cursor->uri, uri) == 0) { if ((ret = cursor->reopen(cursor, false)) != 0) { @@ -953,9 +960,8 @@ __cursor_modify(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) * for consistency. */ if (session->txn->isolation != WT_ISO_SNAPSHOT) - WT_ERR_MSG(session, ENOTSUP, - "not supported in read-committed or read-uncommitted " - "transactions"); + WT_ERR_MSG( + session, ENOTSUP, "not supported in read-committed or read-uncommitted transactions"); if (F_ISSET(session->txn, WT_TXN_AUTOCOMMIT)) WT_ERR_MSG(session, ENOTSUP, "not supported in implicit transactions"); diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index 6f6274523b2..a05136d4f0c 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -175,9 +175,9 @@ QT_AUTOBRIEF = YES # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen # treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. +# comments) as a brief description. This used to be the default behavior. # The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. +# description. Set this tag to YES if you prefer the old behavior instead. MULTILINE_CPP_IS_BRIEF = NO diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox index 40ad6ffb043..813377be455 100644 --- a/src/third_party/wiredtiger/src/docs/command-line.dox +++ b/src/third_party/wiredtiger/src/docs/command-line.dox @@ -142,7 +142,7 @@ The \c downgrade command downgrades the database to the specified compatibility The following are command-specific options for the \c downgrade command: @par <code>-V version</code> -The \c -V option is required, and specfies the version to which the database is downgraded. +The \c -V option is required, and specifies the version to which the database is downgraded. <hr> @section util_drop wt drop diff --git a/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox b/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox index f92238db424..8d1253dd903 100644 --- a/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox +++ b/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox @@ -318,7 +318,7 @@ click on the white bar following the current, red-highlighted, bar.  Next you see a legend that shows all functions that were called during this -execution interval and their corresponding colours. +execution interval and their corresponding colors.  diff --git a/src/third_party/wiredtiger/src/docs/devdoc-xray.dox b/src/third_party/wiredtiger/src/docs/devdoc-xray.dox index e46a9d590f6..145c3ac0a19 100644 --- a/src/third_party/wiredtiger/src/docs/devdoc-xray.dox +++ b/src/third_party/wiredtiger/src/docs/devdoc-xray.dox @@ -41,7 +41,7 @@ In general the usage is:

 wtperf_xray.sh <wtperf-config-file> [-h output-directory] [wtperf other arguments] @endcode -The \c wtperf_xray.sh produces a few outputs to help analyse performance: +The \c wtperf_xray.sh produces a few outputs to help analyze performance: - \c wtperf_account.txt: The top 10 functions where the workload is spending the most time with a count, min, max and some percentiles for each one. @@ -50,15 +50,15 @@ most time with a count, min, max and some percentiles for each one. the most time. This calculation is done separately per thread. - \c wtperf_graph.svg: A function call graph showing what functions call each -other. The edges are labelled and coloured proportionally to represent the ratio +other. The edges are labeled and colored proportionally to represent the ratio of time spent in each function call. -- \c wtperf_flame.svg: A graph visualising stack traces and the time spent +- \c wtperf_flame.svg: A graph visualizing stack traces and the time spent within each stack frame. If \c FLAME_GRAPH_PATH is not specified, this graph won't be generated. The \c wtperf_xray.sh script uses a few optional environment variables to modify -its behaviour: +its behavior: - \c XRAY_BINARY: The binary to use to inspect the XRay log. The script defaults to using \c llvm-xray however, if you compiled with a particular \c clang @@ -118,7 +118,7 @@ dependency to LLVM. $ ../configure --enable-llvm @endcode -Take care NOT to customise \c CC or \c CXX. Customising either of these +Take care NOT to customize \c CC or \c CXX. Customizing either of these variables will cause C++ programs such as \c workgen or \c xray_to_optrack to be skipped since we can't reliably link object files emitted by C and C++ compilers unless they are the system's default \c cc and \c c++. diff --git a/src/third_party/wiredtiger/src/docs/eviction.dox b/src/third_party/wiredtiger/src/docs/eviction.dox index 1d04b3ba76d..37d400f31cc 100644 --- a/src/third_party/wiredtiger/src/docs/eviction.dox +++ b/src/third_party/wiredtiger/src/docs/eviction.dox @@ -1,7 +1,7 @@ /*! @m_page{{c,java},eviction, Eviction} All the operations performed in WiredTiger are on the data read into -preconfigured amount of memory. WiredTiger uses memory as a cache for +pre-configured amount of memory. WiredTiger uses memory as a cache for all the data on the disk and the data in memory forms the current working set. @@ -17,7 +17,7 @@ The WiredTiger eviction runs in the background with one eviction server thread and several eviction worker threads. 1) The eviction server thread walks the btrees and adds the least -recently used pages to the eviction queues until the queus are full. +recently used pages to the eviction queues until the queues are full. 2) The eviction workers continuously remove pages from the eviction queue and try to evict them. @@ -42,7 +42,7 @@ remains solely on the disk unchanged. For history store eviction, the page is modified but the changes are all committed. Therefore, the page should be clean and can be removed from memory after the dirty changes are written to the new disk image, -which is the data fomat WiredTiger stores on disk. The process that +which is the data format WiredTiger stores on disk. The process that builds the disk image is called reconciliation. In reconciliation, WiredTiger writes the newest committed value of each key to the disk image. All the older values of the key are moved to the history store diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index ed61b2321b2..d229012b626 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -5,11 +5,13 @@ ActiveState Adler's Atomicity BLOBs +BLRrVv CFLAGS CPPFLAGS CPUs CRC CSV +CXX Cheng Christoph Collet's @@ -72,6 +74,7 @@ MVCC's Makefiles Mewhort MongoDB +MongoDB's Multithreaded NOTFOUND NSEC @@ -278,6 +281,7 @@ hb hotbackup href hrow +hs hsearch html htmlinclude @@ -298,6 +302,7 @@ je jemalloc jitter jni +jprx jrx json jxf @@ -313,6 +318,7 @@ lastname latencies le len +lgnd li libdir libhe @@ -371,9 +377,11 @@ mutex mutexes mutexing mvcc +mx mygcc mytable namespace +nav ndary ndbm newsite @@ -419,6 +427,7 @@ png posix pre prepends +prev primary's printf printlog @@ -503,7 +512,6 @@ svg sys syscalls sz -t2 tRuE tablename tcl @@ -551,12 +559,14 @@ valuefmt valuev vec versa +viewable vm vpmsum warmup wget whitespace wiredtiger +workgen workQ writelock writelocks diff --git a/src/third_party/wiredtiger/src/docs/tune-capacity.dox b/src/third_party/wiredtiger/src/docs/tune-capacity.dox index 3aad4997576..b8a8dadb54a 100644 --- a/src/third_party/wiredtiger/src/docs/tune-capacity.dox +++ b/src/third_party/wiredtiger/src/docs/tune-capacity.dox @@ -12,7 +12,7 @@ An example of setting a capacity limit to 40MB per second: @snippet ex_all.c Configure capacity -When a total capacity is set the volume of system reads and writes totalled +When a total capacity is set the volume of system reads and writes totaled will not exceed the given I/O capacity. If a read or write is scheduled and would overflow the capacity, the issuing thread will sleep to guarantee the capacity ceiling. The policy used is diff --git a/src/third_party/wiredtiger/src/docs/tune-durability.dox b/src/third_party/wiredtiger/src/docs/tune-durability.dox index 0b4644f3ec5..a16c2938d9c 100644 --- a/src/third_party/wiredtiger/src/docs/tune-durability.dox +++ b/src/third_party/wiredtiger/src/docs/tune-durability.dox @@ -100,7 +100,7 @@ logging subsystem to write any outstanding in-memory buffers to the file system before returning. If \c sync=background is configured then this flush operation will force -the signalling of a background synchronization operation. +the signaling of a background synchronization operation. The caller may then check the status of that background synchronization with the WT_SESSION::transaction_sync method. */ diff --git a/src/third_party/wiredtiger/src/docs/wtperf.dox b/src/third_party/wiredtiger/src/docs/wtperf.dox index b2d7fd16a0e..d11a606dbdc 100644 --- a/src/third_party/wiredtiger/src/docs/wtperf.dox +++ b/src/third_party/wiredtiger/src/docs/wtperf.dox @@ -133,29 +133,29 @@ number of async worker threads @par checkpoint_interval (unsigned int, default=120) checkpoint every interval seconds during the workload phase. @par checkpoint_stress_rate (unsigned int, default=0) -checkpoint every rate operations during the populate phase in the populate thread(s), 0 to disable +checkpoint every rate operations during the populate phase in the populate thread(s), 0 to disable @par checkpoint_threads (unsigned int, default=0) number of checkpoint threads @par conn_config (string, default="create,statistics=(fast),statistics_log=(json,wait=1)") connection configuration string @par close_conn (boolean, default=true) -properly close connection at end of test. Setting to false does not sync data to disk and can result in lost data after test exits. +properly close connection at end of test. Setting to false does not sync data to disk and can result in lost data after test exits. @par compact (boolean, default=false) post-populate compact for LSM merging activity @par compression (string, default="none") -compression extension. Allowed configuration values are: 'none', 'lz4', 'snappy', 'zlib', 'zstd' +compression extension. Allowed configuration values are: 'none', 'lz4', 'snappy', 'zlib', 'zstd' @par create (boolean, default=true) do population phase; false to use existing database @par database_count (unsigned int, default=1) -number of WiredTiger databases to use. Each database will execute the workload using a separate home directory and complete set of worker threads +number of WiredTiger databases to use. Each database will execute the workload using a separate home directory and complete set of worker threads @par drop_tables (boolean, default=false) -Whether to drop all tables at the end of the run, and report time taken to do the drop. +Whether to drop all tables at the end of the run, and report time taken to do the drop. @par in_memory (boolean, default=false) Whether to create the database in-memory. @par icount (unsigned int, default=5000) -number of records to initially populate. If multiple tables are configured the count is spread evenly across all tables. +number of records to initially populate. If multiple tables are configured the count is spread evenly across all tables. @par idle_table_cycle (unsigned int, default=0) -Enable regular create and drop of idle tables, value is the maximum number of seconds a create or drop is allowed before flagging an error. Default 0 which means disabled. +Enable regular create and drop of idle tables, value is the maximum number of seconds a create or drop is allowed before flagging an error. Default 0 which means disabled. @par index (boolean, default=false) Whether to create an index on the value field. @par insert_rmw (boolean, default=false) @@ -167,29 +167,29 @@ perform partial logging on first table only. @par log_like_table (boolean, default=false) Append all modification operations to another shared table. @par min_throughput (unsigned int, default=0) -notify if any throughput measured is less than this amount. Aborts or prints warning based on min_throughput_fatal setting. Requires sample_interval to be configured +notify if any throughput measured is less than this amount. Aborts or prints warning based on min_throughput_fatal setting. Requires sample_interval to be configured @par min_throughput_fatal (boolean, default=false) print warning (false) or abort (true) of min_throughput failure. @par max_latency (unsigned int, default=0) -notify if any latency measured exceeds this number of milliseconds. Aborts or prints warning based on min_throughput_fatal setting. Requires sample_interval to be configured +notify if any latency measured exceeds this number of milliseconds.Aborts or prints warning based on min_throughput_fatal setting. Requires sample_interval to be configured @par max_latency_fatal (boolean, default=false) print warning (false) or abort (true) of max_latency failure. @par pareto (unsigned int, default=0) -use pareto distribution for random numbers. Zero to disable, otherwise a percentage indicating how aggressive the distribution should be. +use pareto distribution for random numbers. Zero to disable, otherwise a percentage indicating how aggressive the distribution should be. @par populate_ops_per_txn (unsigned int, default=0) -number of operations to group into each transaction in the populate phase, zero for auto-commit +number of operations to group into each transaction in the populate phase, zero for auto-commit @par populate_threads (unsigned int, default=1) number of populate threads, 1 for bulk load @par pre_load_data (boolean, default=false) Scan all data prior to starting the workload phase to warm the cache @par random_range (unsigned int, default=0) -if non zero choose a value from within this range as the key for insert operations +if non zero choose a value from within this range as the key for insert operations @par random_value (boolean, default=false) generate random content for the value @par range_partition (boolean, default=false) partition data by range (vs hash) @par readonly (boolean, default=false) -reopen the connection between populate and workload phases in readonly mode. Requires reopen_connection turned on (default). Requires that read be the only workload specified +reopen the connection between populate and workload phases in readonly mode. Requires reopen_connection turned on (default). Requires that read be the only workload specified @par reopen_connection (boolean, default=true) close and reopen the connection between populate and workload phases @par report_interval (unsigned int, default=2) @@ -201,29 +201,29 @@ total workload seconds @par sample_interval (unsigned int, default=0) performance logging every interval seconds, 0 to disable @par sample_rate (unsigned int, default=50) -how often the latency of operations is measured. One for every operation, two for every second operation, three for every third operation etc. +how often the latency of operations is measured. One for every operation, two for every second operation, three for every third operation etc. @par scan_icount (unsigned int, default=0) number of records in scan tables to populate @par scan_interval (unsigned int, default=0) -scan tables every interval seconds during the workload phase, 0 to disable +scan tables every interval seconds during the workload phase, 0 to disable @par scan_pct (unsigned int, default=10) percentage of entire data set scanned, if scan_interval is enabled @par scan_table_count (unsigned int, default=0) -number of separate tables to be used for scanning. Zero indicates that tables are shared with other operations +number of separate tables to be used for scanning. Zero indicates that tables are shared with other operations @par sess_config (string, default="") session configuration string @par session_count_idle (unsigned int, default=0) number of idle sessions to create. Default 0. -@par table_config (string, default="key_format=S,value_format=S,type=file,exclusive=true, leaf_value_max=64MB,memory_page_max=10m,split_pct=90,checksum=on") +@par table_config (string, default="key_format=S,value_format=S,type=file,exclusive=true,leaf_value_max=64MB,memory_page_max=10m, split_pct=90,checksum=on") table configuration string @par table_count (unsigned int, default=1) -number of tables to run operations over. Keys are divided evenly over the tables. Cursors are held open on all tables. Default 1, maximum 99999. +number of tables to run operations over. Keys are divided evenly over the tables. Cursors are held open on all tables. Default 1, maximum 99999. @par table_count_idle (unsigned int, default=0) number of tables to create, that won't be populated. Default 0. @par threads (string, default="") -workload configuration: each 'count' entry is the total number of threads, and the 'insert', 'modify', 'read' and 'update' entries are the ratios of insert, modify, read and update operations done by each worker thread; If a throttle value is provided each thread will do a maximum of that number of operations per second; multiple workload configurations may be specified per threads configuration; for example, a more complex threads configuration might be 'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' which would create 2 threads doing nothing but reads and 8 threads each doing 50% inserts and 25% reads and updates. Allowed configuration values are 'count', 'throttle', 'inserts', 'reads', 'read_range', 'modify', 'modify_delta', 'modify_distribute', 'modify_force_update', 'updates', 'update_delta', 'truncate', 'truncate_pct' and 'truncate_count'. There are also behavior modifiers, supported modifiers are 'ops_per_txn' +workload configuration: each 'count' entry is the total number of threads, and the 'insert', 'modify', 'read' and 'update' entries are the ratios of insert, modify, read and update operations done by each worker thread; If a throttle value is provided each thread will do a maximum of that number of operations per second; multiple workload configurations may be specified per threads configuration; for example, a more complex threads configuration might be 'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' which would create 2 threads doing nothing but reads and 8 threads each doing 50% inserts and 25% reads and updates. Allowed configuration values are 'count', 'throttle', 'inserts', 'reads', 'read_range', 'modify', 'modify_delta', 'modify_distribute', 'modify_force_update', 'updates', 'update_delta', 'truncate', 'truncate_pct' and 'truncate_count'. There are also behavior modifiers, supported modifiers are 'ops_per_txn' @par transaction_config (string, default="") -WT_SESSION.begin_transaction configuration string, applied during the populate phase when populate_ops_per_txn is nonzero +WT_SESSION.begin_transaction configuration string, applied during the populate phase when populate_ops_per_txn is nonzero @par table_name (string, default="test") table name @par truncate_single_ops (boolean, default=false) diff --git a/src/third_party/wiredtiger/src/evict/evict_file.c b/src/third_party/wiredtiger/src/evict/evict_file.c index edeb226471d..3f44c152caf 100644 --- a/src/third_party/wiredtiger/src/evict/evict_file.c +++ b/src/third_party/wiredtiger/src/evict/evict_file.c @@ -91,8 +91,9 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) * If the page has a page deleted structure, we are discarding the page that is cleaned * by a checkpoint. */ - WT_ASSERT(session, F_ISSET(dhandle, WT_DHANDLE_DEAD) || - F_ISSET(S2C(session), WT_CONN_CLOSING) || __wt_page_can_evict(session, ref, NULL) || + WT_ASSERT(session, + F_ISSET(dhandle, WT_DHANDLE_DEAD) || F_ISSET(S2C(session), WT_CONN_CLOSING) || + __wt_page_can_evict(session, ref, NULL) || (ref->page_del != NULL && page->modify->page_state == WT_PAGE_CLEAN)); __wt_ref_out(session, ref); break; diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 6cd75bd55e1..749050edf6e 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -272,7 +272,6 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; - uint32_t session_flags; bool did_work, was_intr; conn = S2C(session); @@ -283,10 +282,9 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) * busy and then opens a different file (in this case, the HS file), it can deadlock with a * thread waiting for the first file to drain from the eviction queue. See WT-5946 for details. */ - if (session->hs_cursor == NULL && !F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_READONLY)) { - session_flags = 0; /* [-Werror=maybe-uninitialized] */ - WT_RET(__wt_hs_cursor_open(session, &session_flags)); - WT_RET(__wt_hs_cursor_close(session, session_flags)); + if (session->hs_cursor == NULL && !F_ISSET(conn, WT_CONN_IN_MEMORY)) { + WT_RET(__wt_hs_cursor_open(session)); + WT_RET(__wt_hs_cursor_close(session)); } if (conn->evict_server_running && __wt_spin_trylock(session, &cache->evict_pass_lock) == 0) { @@ -1325,7 +1323,7 @@ __evict_walk_choose_dhandle(WT_SESSION_IMPL *session, WT_DATA_HANDLE **dhandle_p * If we don't have many dhandles, most hash buckets will be empty. Just pick a random dhandle * from the list in that case. */ - if (conn->dhandle_count < WT_HASH_ARRAY_SIZE / 4) { + if (conn->dhandle_count < conn->dh_hash_size / 4) { rnd_dh = __wt_random(&session->rnd) % conn->dhandle_count; dhandle = TAILQ_FIRST(&conn->dhqh); for (; rnd_dh > 0; rnd_dh--) @@ -1338,7 +1336,7 @@ __evict_walk_choose_dhandle(WT_SESSION_IMPL *session, WT_DATA_HANDLE **dhandle_p * Keep picking up a random bucket until we find one that is not empty. */ do { - rnd_bucket = __wt_random(&session->rnd) % WT_HASH_ARRAY_SIZE; + rnd_bucket = __wt_random(&session->rnd) & (conn->dh_hash_size - 1); } while ((dh_bucket_count = conn->dh_bucket_count[rnd_bucket]) == 0); /* We can't pick up an empty bucket with a non zero bucket count. */ @@ -1538,7 +1536,7 @@ retry: */ if (slot < max_entries && (retries < 2 || - (retries < WT_RETRY_MAX && (slot == queue->evict_entries || slot > start_slot)))) { + (retries < WT_RETRY_MAX && (slot == queue->evict_entries || slot > start_slot)))) { start_slot = slot; ++retries; goto retry; @@ -1924,6 +1922,17 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_ent if (__wt_page_is_empty(page) || F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) goto fast; + /* + * Do not evict a clean metadata page that contains historical data needed to satisfy a + * reader. Since there is no history store for metadata, we won't be able to serve an older + * reader if we evict this page. + */ + if (WT_IS_METADATA(session->dhandle) && F_ISSET(cache, WT_CACHE_EVICT_CLEAN_HARD) && + F_ISSET(ref, WT_REF_FLAG_LEAF) && !modified && page->modify != NULL && + !__wt_txn_visible_all( + session, page->modify->rec_max_txn, page->modify->rec_max_timestamp)) + continue; + /* Skip pages we don't want. */ want_page = (F_ISSET(cache, WT_CACHE_EVICT_CLEAN) && !modified) || (F_ISSET(cache, WT_CACHE_EVICT_DIRTY) && modified) || @@ -2091,7 +2100,7 @@ __evict_get_ref(WT_SESSION_IMPL *session, bool is_server, WT_BTREE **btreep, WT_ !__evict_queue_full(cache->evict_current_queue) && !__evict_queue_full(cache->evict_fill_queue) && (cache->evict_empty_score > WT_EVICT_SCORE_CUTOFF || - __evict_queue_empty(cache->evict_fill_queue, false))) + __evict_queue_empty(cache->evict_fill_queue, false))) return (WT_NOTFOUND); __wt_spin_lock(session, &cache->evict_queue_lock); @@ -2323,6 +2332,16 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d } /* + * Check if we've exceeded our operation timeout, this would also get called from the + * previous txn is blocking call, however it won't pickup transactions that have been + * committed or rolled back as their mod count is 0, and that txn needs to be the oldest. + * + * Additionally we don't return rollback which could confuse the caller. + */ + if (__wt_op_timer_fired(session)) + break; + + /* * Check if we have become busy. * * If we're busy (because of the transaction check we just did or because our caller is @@ -2590,8 +2609,9 @@ __verbose_dump_cache_apply(WT_SESSION_IMPL *session, uint64_t *total_bytesp, F_ISSET(dhandle, WT_DHANDLE_DISCARD)) continue; - WT_WITH_DHANDLE(session, dhandle, ret = __verbose_dump_cache_single(session, total_bytesp, - total_dirty_bytesp, total_updates_bytesp)); + WT_WITH_DHANDLE(session, dhandle, + ret = __verbose_dump_cache_single( + session, total_bytesp, total_dirty_bytesp, total_updates_bytesp)); if (ret != 0) WT_RET(ret); } @@ -2628,8 +2648,9 @@ __wt_verbose_dump_cache(WT_SESSION_IMPL *session) needed = __wt_eviction_updates_needed(session, &pct); WT_RET(__wt_msg(session, "cache updates check: %s (%2.3f%%)", needed ? "yes" : "no", pct)); - WT_WITH_HANDLE_LIST_READ_LOCK(session, ret = __verbose_dump_cache_apply(session, &total_bytes, - &total_dirty_bytes, &total_updates_bytes)); + WT_WITH_HANDLE_LIST_READ_LOCK(session, + ret = __verbose_dump_cache_apply( + session, &total_bytes, &total_dirty_bytes, &total_updates_bytes)); WT_RET(ret); /* @@ -2637,10 +2658,9 @@ __wt_verbose_dump_cache(WT_SESSION_IMPL *session) */ total_bytes = __wt_cache_bytes_plus_overhead(conn->cache, total_bytes); - WT_RET(__wt_msg(session, - "cache dump: " - "total found: %" PRIu64 "MB vs tracked inuse %" PRIu64 "MB", - total_bytes / WT_MEGABYTE, cache->bytes_inmem / WT_MEGABYTE)); + WT_RET( + __wt_msg(session, "cache dump: total found: %" PRIu64 "MB vs tracked inuse %" PRIu64 "MB", + total_bytes / WT_MEGABYTE, cache->bytes_inmem / WT_MEGABYTE)); WT_RET(__wt_msg(session, "total dirty bytes: %" PRIu64 "MB vs tracked dirty %" PRIu64 "MB", total_dirty_bytes / WT_MEGABYTE, (cache->bytes_dirty_intl + cache->bytes_dirty_leaf) / WT_MEGABYTE)); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index ff21df8b519..7b432308b5c 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -92,10 +92,10 @@ int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32_t flags) { WT_CONNECTION_IMPL *conn; + WT_CURSOR *hs_cursor_saved; WT_DECL_RET; WT_PAGE *page; uint64_t time_start, time_stop; - uint32_t session_flags; bool clean_page, closing, force_evict_hs, inmem_split, local_gen, tree_dead; conn = S2C(session); @@ -108,6 +108,15 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32 __wt_verbose( session, WT_VERB_EVICT, "page %p (%s)", (void *)page, __wt_page_type_string(page->type)); + /* + * If we have a history store cursor, save it. This ensures that if eviction needs to access the + * history store, it will get its own cursor, avoiding potential problems if it were to + * reposition or reset a history store cursor that we're in the middle of using for something + * else. + */ + hs_cursor_saved = session->hs_cursor; + session->hs_cursor = NULL; + tree_dead = F_ISSET(session->dhandle, WT_DHANDLE_DEAD); if (tree_dead) LF_SET(WT_EVICT_CALL_NO_SPLIT); @@ -133,9 +142,8 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32 if (!WT_IS_METADATA(S2BT(session)->dhandle) && !F_ISSET(conn, WT_CONN_IN_MEMORY) && session->hs_cursor == NULL && !F_ISSET(session, WT_SESSION_NO_RECONCILE) && session != conn->default_session) { - session_flags = 0; /* [-Werror=maybe-uninitialized] */ - WT_RET(__wt_hs_cursor_open(session, &session_flags)); - WT_RET(__wt_hs_cursor_close(session, session_flags)); + WT_RET(__wt_hs_cursor_open(session)); + WT_RET(__wt_hs_cursor_close(session)); } /* @@ -274,6 +282,12 @@ done: if (local_gen) __wt_session_gen_leave(session, WT_GEN_EVICT); + /* If the caller was using a history store cursor they should have closed it by now. */ + WT_ASSERT(session, session->hs_cursor == NULL); + + /* Restore caller's history store cursor. */ + session->hs_cursor = hs_cursor_saved; + return (ret); } diff --git a/src/third_party/wiredtiger/src/history/hs.c b/src/third_party/wiredtiger/src/history/hs.c index 9ad0f9aab28..f47305d341c 100644 --- a/src/third_party/wiredtiger/src/history/hs.c +++ b/src/third_party/wiredtiger/src/history/hs.c @@ -18,12 +18,6 @@ typedef struct { uint64_t txnid; } WT_HS_TIME_POINT; -/* - * When an operation is accessing the history store table, it should ignore the cache size (since - * the cache is already full). - */ -#define WT_HS_SESSION_FLAGS WT_SESSION_IGNORE_CACHE_SIZE - static int __hs_delete_key_from_pos( WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, uint32_t btree_id, const WT_ITEM *key); static int __hs_fixup_out_of_order_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, @@ -48,10 +42,7 @@ __hs_start_internal_session(WT_SESSION_IMPL *session, WT_SESSION_IMPL **int_sess static int __hs_release_internal_session(WT_SESSION_IMPL *int_session) { - WT_SESSION *wt_session; - - wt_session = &int_session->iface; - return (wt_session->close(wt_session, NULL)); + return (__wt_session_close_internal(int_session)); } /* @@ -62,17 +53,15 @@ int __wt_hs_get_btree(WT_SESSION_IMPL *session, WT_BTREE **hs_btreep) { WT_DECL_RET; - uint32_t session_flags; *hs_btreep = NULL; - session_flags = 0; /* [-Werror=maybe-uninitialized] */ - WT_RET(__wt_hs_cursor_open(session, &session_flags)); + WT_RET(__wt_hs_cursor_open(session)); *hs_btreep = CUR2BT(session->hs_cursor); WT_ASSERT(session, *hs_btreep != NULL); - WT_TRET(__wt_hs_cursor_close(session, session_flags)); + WT_TRET(__wt_hs_cursor_close(session)); return (ret); } @@ -98,14 +87,6 @@ __wt_hs_config(WT_SESSION_IMPL *session, const char **cfg) WT_ERR_MSG(session, EINVAL, "max history store size %" PRId64 " below minimum %d", cval.val, WT_HS_FILE_MIN); - /* TODO: WT-5585 Remove after we switch to using history_store config in MongoDB. */ - if (cval.val == 0) { - WT_ERR(__wt_config_gets(session, cfg, "cache_overflow.file_max", &cval)); - if (cval.val != 0 && cval.val < WT_HS_FILE_MIN) - WT_ERR_MSG(session, EINVAL, "max history store size %" PRId64 " below minimum %d", - cval.val, WT_HS_FILE_MIN); - } - /* in-memory or readonly configurations do not have a history store. */ if (F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_READONLY)) return (0); @@ -211,14 +192,14 @@ __wt_hs_destroy(WT_SESSION_IMPL *session) * Open a new history store table cursor. */ int -__wt_hs_cursor_open(WT_SESSION_IMPL *session, uint32_t *session_flags) +__wt_hs_cursor_open(WT_SESSION_IMPL *session) { WT_CURSOR *cursor; WT_DECL_RET; const char *open_cursor_cfg[] = {WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL}; /* Not allowed to open a cursor if you already have one */ - WT_ASSERT(session, session->hs_cursor == NULL && !F_ISSET(session, WT_SESSION_HS_CURSOR)); + WT_ASSERT(session, session->hs_cursor == NULL); WT_WITHOUT_DHANDLE( session, ret = __wt_open_cursor(session, WT_HS_URI, NULL, open_cursor_cfg, &cursor)); @@ -227,16 +208,7 @@ __wt_hs_cursor_open(WT_SESSION_IMPL *session, uint32_t *session_flags) /* History store cursors should always ignore tombstones. */ F_SET(cursor, WT_CURSTD_IGNORE_TOMBSTONE); - /* - * We don't want to get tapped for eviction after we start using the history store cursor. Save - * a copy of the current flag values. We'll restore them when the cursor is closed. - */ - *session_flags = F_MASK(session, WT_HS_SESSION_FLAGS); - F_SET(session, WT_HS_SESSION_FLAGS); - session->hs_cursor = cursor; - F_SET(session, WT_SESSION_HS_CURSOR); - return (0); } @@ -245,21 +217,13 @@ __wt_hs_cursor_open(WT_SESSION_IMPL *session, uint32_t *session_flags) * Discard a history store cursor. */ int -__wt_hs_cursor_close(WT_SESSION_IMPL *session, uint32_t session_flags) +__wt_hs_cursor_close(WT_SESSION_IMPL *session) { /* Should only be called when session has an open history store cursor */ - WT_ASSERT(session, session->hs_cursor != NULL && F_ISSET(session, WT_SESSION_HS_CURSOR)); - - /* - * Restore previous values of history store session flags. - */ - F_CLR(session, WT_HS_SESSION_FLAGS); - F_SET(session, session_flags); + WT_ASSERT(session, session->hs_cursor != NULL); WT_RET(session->hs_cursor->close(session->hs_cursor)); session->hs_cursor = NULL; - F_CLR(session, WT_SESSION_HS_CURSOR); - return (0); } @@ -291,8 +255,9 @@ __hs_row_search(WT_CURSOR_BTREE *hs_cbt, WT_ITEM *srch_key, bool insert) * than the page's boundary slots, if that's not the case, the record might belong on an * entirely different page. */ - if (leaf_found && (hs_cbt->compare != 0 && - (hs_cbt->slot == 0 || hs_cbt->slot == hs_cbt->ref->page->entries - 1))) + if (leaf_found && + (hs_cbt->compare != 0 && + (hs_cbt->slot == 0 || hs_cbt->slot == hs_cbt->ref->page->entries - 1))) leaf_found = false; if (!leaf_found) hs_cursor->reset(hs_cursor); @@ -360,17 +325,14 @@ __hs_insert_updates_verbose(WT_SESSION_IMPL *session, WT_BTREE *btree) */ if (WT_VERBOSE_ISSET(session, WT_VERB_HS) || (ckpt_gen_current > ckpt_gen_last && - __wt_atomic_casv64(&cache->hs_verb_gen_write, ckpt_gen_last, ckpt_gen_current))) { + __wt_atomic_casv64(&cache->hs_verb_gen_write, ckpt_gen_last, ckpt_gen_current))) { WT_IGNORE_RET_BOOL(__wt_eviction_clean_needed(session, &pct_full)); WT_IGNORE_RET_BOOL(__wt_eviction_dirty_needed(session, &pct_dirty)); __wt_verbose(session, WT_VERB_HS | WT_VERB_HS_ACTIVITY, "Page reconciliation triggered history store write: file ID %" PRIu32 - ". " - "Current history store file size: %" PRId64 - ", " - "cache dirty: %2.3f%% , " - "cache use: %2.3f%%", + ". Current history store file size: %" PRId64 + ", cache dirty: %2.3f%% , cache use: %2.3f%%", btree_id, WT_STAT_READ(conn->stats, cache_hs_ondisk), pct_dirty, pct_full); } @@ -599,8 +561,9 @@ __hs_insert_record(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, WT_DECL_RET; cbt = (WT_CURSOR_BTREE *)cursor; - WT_WITH_BTREE(session, CUR2BT(cbt), ret = __hs_insert_record_with_btree(session, cursor, btree, - key, upd, type, hs_value, stop_time_point, clear_hs)); + WT_WITH_BTREE(session, CUR2BT(cbt), + ret = __hs_insert_record_with_btree( + session, cursor, btree, key, upd, type, hs_value, stop_time_point, clear_hs)); return (ret); } @@ -660,7 +623,7 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) WT_MODIFY_VECTOR modifies; WT_SAVE_UPD *list; WT_UPDATE *first_globally_visible_upd, *first_non_ts_upd; - WT_UPDATE *non_aborted_upd, *oldest_upd, *prev_upd, *upd; + WT_UPDATE *non_aborted_upd, *oldest_upd, *prev_upd, *tombstone, *upd; WT_HS_TIME_POINT stop_time_point; wt_off_t hs_size; wt_timestamp_t min_insert_ts; @@ -669,7 +632,7 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) uint8_t *p; int nentries; char ts_string[3][WT_TS_INT_STRING_SIZE]; - bool clear_hs, enable_reverse_modify, squashed, ts_updates_in_hs; + bool clear_hs, enable_reverse_modify, hs_inserted, squashed, ts_updates_in_hs; btree = S2BT(session); cursor = session->hs_cursor; @@ -884,7 +847,7 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) WT_ERR(__hs_next_upd_full_value(session, &modifies, NULL, full_value, &upd)); - squashed = false; + hs_inserted = squashed = false; /* * Flush the updates on stack. Stopping once we run out or we reach the onpage upd start @@ -892,11 +855,12 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) */ for (; modifies.size > 0 && !(upd->txnid == list->onpage_upd->txnid && - upd->start_ts == list->onpage_upd->start_ts); + upd->start_ts == list->onpage_upd->start_ts); tmp = full_value, full_value = prev_full_value, prev_full_value = tmp, upd = prev_upd) { WT_ASSERT(session, upd->type == WT_UPDATE_STANDARD || upd->type == WT_UPDATE_MODIFY); + tombstone = NULL; __wt_modify_vector_peek(&modifies, &prev_upd); /* @@ -905,7 +869,8 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) * removal by checkpoint garbage collection until the data store update is committed. */ if (prev_upd->prepare_state == WT_PREPARE_INPROGRESS) { - WT_ASSERT(session, list->onpage_upd->txnid == prev_upd->txnid && + WT_ASSERT(session, + list->onpage_upd->txnid == prev_upd->txnid && list->onpage_upd->start_ts == prev_upd->start_ts); stop_time_point.durable_ts = stop_time_point.ts = WT_TS_MAX; stop_time_point.txnid = WT_TXN_MAX; @@ -919,6 +884,9 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) stop_time_point.durable_ts = prev_upd->durable_ts; stop_time_point.ts = prev_upd->start_ts; stop_time_point.txnid = prev_upd->txnid; + + if (prev_upd->type == WT_UPDATE_TOMBSTONE) + tombstone = prev_upd; } WT_ERR( @@ -931,8 +899,13 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) } /* Skip updates that are already in the history store or are obsolete. */ - if (F_ISSET(upd, WT_UPDATE_HS | WT_UPDATE_OBSOLETE)) + if (F_ISSET(upd, WT_UPDATE_HS | WT_UPDATE_OBSOLETE)) { + if (hs_inserted) + WT_ERR_PANIC(session, WT_PANIC, + "Inserting updates older than obsolete updates or updates that are already " + "in the history store to the history store may corrupt the data."); continue; + } /* * If the time points are out of order (which can happen if the application performs @@ -985,6 +958,9 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) clear_hs = false; /* Flag the update as now in the history store. */ F_SET(upd, WT_UPDATE_HS); + if (tombstone != NULL) + F_SET(tombstone, WT_UPDATE_HS); + hs_inserted = true; ++insert_cnt; if (squashed) { WT_STAT_CONN_INCR(session, cache_hs_write_squash); @@ -1010,8 +986,9 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) * removed from the history store. U@1 will be removed from the history store once U@0 is * moved to the history store. */ - if (clear_hs && (first_non_ts_upd->txnid != list->onpage_upd->txnid || - first_non_ts_upd->start_ts != list->onpage_upd->start_ts)) { + if (clear_hs && + (first_non_ts_upd->txnid != list->onpage_upd->txnid || + first_non_ts_upd->start_ts != list->onpage_upd->start_ts)) { /* We can only delete history store entries that have timestamps. */ WT_ERR(__wt_hs_delete_key_from_ts(session, btree->id, key, 1)); WT_STAT_CONN_INCR(session, cache_hs_key_truncate_mix_ts); @@ -1187,7 +1164,7 @@ __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma wt_timestamp_t durable_timestamp, durable_timestamp_tmp, hs_start_ts, hs_start_ts_tmp; wt_timestamp_t hs_stop_durable_ts, hs_stop_durable_ts_tmp, read_timestamp; uint64_t hs_counter, hs_counter_tmp, upd_type_full; - uint32_t hs_btree_id, session_flags; + uint32_t hs_btree_id; uint8_t *p, recno_key_buf[WT_INTPACK64_MAXSIZE], upd_type; int cmp; bool modify; @@ -1200,7 +1177,6 @@ __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma txn = session->txn; txn_shared = WT_SESSION_TXN_SHARED(session); hs_btree_id = S2BT(session)->id; - session_flags = 0; /* [-Werror=maybe-uninitialized] */ WT_NOT_READ(modify, false); WT_STAT_CONN_INCR(session, cursor_search_hs); @@ -1222,7 +1198,7 @@ __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma WT_ERR(__wt_scr_alloc(session, 0, &hs_value)); /* Open a history store table cursor. */ - WT_ERR(__wt_hs_cursor_open(session, &session_flags)); + WT_ERR(__wt_hs_cursor_open(session)); hs_cursor = session->hs_cursor; hs_cbt = (WT_CURSOR_BTREE *)hs_cursor; @@ -1402,7 +1378,7 @@ err: __wt_scr_free(session, &hs_value); WT_ASSERT(session, hs_key.mem == NULL && hs_key.memsize == 0); - WT_TRET(__wt_hs_cursor_close(session, session_flags)); + WT_TRET(__wt_hs_cursor_close(session)); __wt_free_update_list(session, &mod_upd); while (modifies.size > 0) { @@ -1880,7 +1856,7 @@ __wt_history_store_verify(WT_SESSION_IMPL *session) WT_ITEM hs_key; wt_timestamp_t hs_start_ts; uint64_t hs_counter; - uint32_t btree_id, session_flags; + uint32_t btree_id; char *uri_data; bool stop; @@ -1890,11 +1866,10 @@ __wt_history_store_verify(WT_SESSION_IMPL *session) cursor = data_cursor = NULL; WT_CLEAR(hs_key); btree_id = WT_BTREE_ID_INVALID; - session_flags = 0; /* [-Wconditional-uninitialized] */ uri_data = NULL; WT_ERR(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__wt_hs_cursor_open(session, &session_flags)); + WT_ERR(__wt_hs_cursor_open(session)); cursor = session->hs_cursor; WT_ERR_NOTFOUND_OK(__wt_hs_cursor_next(session, cursor), true); stop = ret == WT_NOTFOUND ? true : false; @@ -1926,7 +1901,7 @@ __wt_history_store_verify(WT_SESSION_IMPL *session) WT_ERR_NOTFOUND_OK(ret, false); } err: - WT_TRET(__wt_hs_cursor_close(session, session_flags)); + WT_TRET(__wt_hs_cursor_close(session)); __wt_scr_free(session, &buf); WT_ASSERT(session, hs_key.mem == NULL && hs_key.memsize == 0); diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index f74fc0a5cd6..0acb3c4097f 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -202,7 +202,7 @@ #define CURSOR_API_CALL(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - if (!F_ISSET(s, WT_SESSION_HS_CURSOR)) \ + if ((s)->hs_cursor == NULL) \ SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ API_CALL_NOCONF(s, WT_CURSOR, n, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ if (F_ISSET(cur, WT_CURSTD_CACHED)) \ diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 325ba9f05c1..baa8fbbdb50 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -1298,10 +1298,12 @@ struct __wt_insert_head { /* WT_FIX_FOREACH walks fixed-length bit-fields on a disk page. */ #define WT_FIX_FOREACH(btree, dsk, v, i) \ - for ((i) = 0, (v) = (i) < (dsk)->u.entries ? \ + for ((i) = 0, \ + (v) = (i) < (dsk)->u.entries ? \ __bit_getv(WT_PAGE_HEADER_BYTE(btree, dsk), 0, (btree)->bitcnt) : \ 0; \ - (i) < (dsk)->u.entries; ++(i), (v) = (i) < (dsk)->u.entries ? \ + (i) < (dsk)->u.entries; ++(i), \ + (v) = (i) < (dsk)->u.entries ? \ __bit_getv(WT_PAGE_HEADER_BYTE(btree, dsk), i, (btree)->bitcnt) : \ 0) diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 84b4f7d2aa0..9a18ae24d7b 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -141,8 +141,9 @@ __wt_btree_bytes_evictable(WT_SESSION_IMPL *session) bytes_inmem = btree->bytes_inmem; bytes_root = root_page == NULL ? 0 : root_page->memory_footprint; - return (bytes_inmem <= bytes_root ? 0 : __wt_cache_bytes_plus_overhead( - cache, bytes_inmem - bytes_root)); + return (bytes_inmem <= bytes_root ? + 0 : + __wt_cache_bytes_plus_overhead(cache, bytes_inmem - bytes_root)); } /* @@ -1509,6 +1510,11 @@ __wt_page_can_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool *inmem_splitp) __wt_gen_active(session, WT_GEN_SPLIT, page->pg_intl_split_gen)) return (false); + /* If the metadata page is clean but has modifications that appear too new to evict, skip it. */ + if (WT_IS_METADATA(S2BT(session)->dhandle) && !modified && + !__wt_txn_visible_all(session, mod->rec_max_txn, mod->rec_max_timestamp)) + return (false); + return (true); } @@ -1718,7 +1724,7 @@ __wt_page_swap_func(WT_SESSION_IMPL *session, WT_REF *held, WT_REF *want, uint32 , const char *func, int line #endif - ) +) { WT_DECL_RET; bool acquired; @@ -1742,7 +1748,7 @@ __wt_page_swap_func(WT_SESSION_IMPL *session, WT_REF *held, WT_REF *want, uint32 , func, line #endif - ); + ); /* * Expected failures: page not found or restart. Our callers list the errors they're expecting diff --git a/src/third_party/wiredtiger/src/include/cache.i b/src/third_party/wiredtiger/src/include/cache.i index 204da30753c..4061d5b52f6 100644 --- a/src/third_party/wiredtiger/src/include/cache.i +++ b/src/third_party/wiredtiger/src/include/cache.i @@ -413,9 +413,11 @@ __wt_eviction_needed(WT_SESSION_IMPL *session, bool busy, bool readonly, double * application thread. */ if (pct_fullp != NULL) - *pct_fullp = WT_MAX(0.0, 100.0 - WT_MIN(WT_MIN(cache->eviction_trigger - pct_full, - cache->eviction_dirty_trigger - pct_dirty), - cache->eviction_updates_trigger - pct_updates)); + *pct_fullp = WT_MAX(0.0, + 100.0 - + WT_MIN( + WT_MIN(cache->eviction_trigger - pct_full, cache->eviction_dirty_trigger - pct_dirty), + cache->eviction_updates_trigger - pct_updates)); /* * Only check the dirty trigger when the session is not busy. @@ -458,6 +460,10 @@ __wt_cache_eviction_check(WT_SESSION_IMPL *session, bool busy, bool readonly, bo if (didworkp != NULL) *didworkp = false; + /* Eviction causes reconciliation. So don't evict if we can't reconcile */ + if (F_ISSET(session, WT_SESSION_NO_RECONCILE)) + return (0); + /* * If the current transaction is keeping the oldest ID pinned, it is in the middle of an * operation. This may prevent the oldest ID from moving forward, leading to deadlock, so only @@ -474,8 +480,9 @@ __wt_cache_eviction_check(WT_SESSION_IMPL *session, bool busy, bool readonly, bo * holding the handle list, schema or table locks (which can block checkpoints and eviction), * don't block the thread for eviction. */ - if (F_ISSET(session, WT_SESSION_IGNORE_CACHE_SIZE | WT_SESSION_LOCKED_HANDLE_LIST | - WT_SESSION_LOCKED_SCHEMA | WT_SESSION_LOCKED_TABLE)) + if (F_ISSET(session, + WT_SESSION_IGNORE_CACHE_SIZE | WT_SESSION_LOCKED_HANDLE_LIST | WT_SESSION_LOCKED_SCHEMA | + WT_SESSION_LOCKED_TABLE)) return (0); /* In memory configurations don't block when the cache is full. */ diff --git a/src/third_party/wiredtiger/src/include/cell.h b/src/third_party/wiredtiger/src/include/cell.h index 3c63fe4f49d..e17a0370425 100644 --- a/src/third_party/wiredtiger/src/include/cell.h +++ b/src/third_party/wiredtiger/src/include/cell.h @@ -49,10 +49,10 @@ * Bits 1 and 2 are reserved for "short" key and value cells (that is, a cell * carrying data less than 64B, where we can store the data length in the cell * descriptor byte): - * 0x00 Not a short key/data cell - * 0x01 Short key cell - * 0x10 Short key cell, with a following prefix-compression byte - * 0x11 Short value cell + * 0b00 Not a short key/data cell + * 0b01 Short key cell + * 0b10 Short key cell, with a following prefix-compression byte + * 0b11 Short value cell * In the "short" variants, the other 6 bits of the descriptor byte are the * data length. * diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index 070e80e8718..f60cc4e874f 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -17,8 +17,9 @@ __cell_check_value_validity(WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw, bool e WT_DECL_RET; if ((ret = __wt_time_value_validate(session, tw, NULL, false)) != 0) - return (expected_error ? WT_ERROR : __wt_panic(session, ret, - "value timestamp window failed validation")); + return (expected_error ? + WT_ERROR : + __wt_panic(session, ret, "value timestamp window failed validation")); #else WT_UNUSED(session); WT_UNUSED(tw); @@ -100,8 +101,9 @@ __wt_check_addr_validity(WT_SESSION_IMPL *session, WT_TIME_AGGREGATE *ta, bool e WT_DECL_RET; if ((ret = __wt_time_aggregate_validate(session, ta, NULL, false)) != 0) - return (expected_error ? WT_ERROR : __wt_panic(session, ret, - "address timestamp window failed validation")); + return (expected_error ? + WT_ERROR : + __wt_panic(session, ret, "address timestamp window failed validation")); #else WT_UNUSED(session); WT_UNUSED(ta); @@ -885,7 +887,7 @@ copy_cell_restart: * Set overflow flag. */ F_SET(unpack, WT_CELL_UNPACK_OVERFLOW); - /* FALLTHROUGH */ + /* FALLTHROUGH */ case WT_CELL_ADDR_DEL: case WT_CELL_ADDR_INT: @@ -905,7 +907,7 @@ copy_cell_restart: */ if (unpack->raw == WT_CELL_KEY || unpack->raw == WT_CELL_KEY_PFX || (unpack->raw == WT_CELL_VALUE && unpack->v == 0 && - (cell->__chunk[0] & WT_CELL_SECOND_DESC) == 0)) + (cell->__chunk[0] & WT_CELL_SECOND_DESC) == 0)) v += WT_CELL_SIZE_ADJUST; unpack->data = p; @@ -1113,8 +1115,9 @@ __cell_data_ref(WT_SESSION_IMPL *session, WT_PAGE *page, int page_type, return (__wt_illegal_value(session, unpack->type)); } - return (huffman == NULL || store->size == 0 ? 0 : __wt_huffman_decode(session, huffman, - store->data, store->size, store)); + return (huffman == NULL || store->size == 0 ? + 0 : + __wt_huffman_decode(session, huffman, store->data, store->size, store)); } /* diff --git a/src/third_party/wiredtiger/src/include/column.i b/src/third_party/wiredtiger/src/include/column.i index fbe2c80f8a3..b76b1a728ff 100644 --- a/src/third_party/wiredtiger/src/include/column.i +++ b/src/third_party/wiredtiger/src/include/column.i @@ -176,9 +176,9 @@ __col_insert_search( /* Fast path appends. */ if (recno >= WT_INSERT_RECNO(ret_ins)) { for (i = 0; i < WT_SKIP_MAXDEPTH; i++) { - ins_stack[i] = (i == 0) ? &ret_ins->next[0] : (ins_head->tail[i] != NULL) ? - &ins_head->tail[i]->next[i] : - &ins_head->head[i]; + ins_stack[i] = (i == 0) ? + &ret_ins->next[0] : + (ins_head->tail[i] != NULL) ? &ins_head->tail[i]->next[i] : &ins_head->head[i]; next_stack[i] = NULL; } return (ret_ins); diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 9dce40a60eb..48b7784d23f 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -85,8 +85,7 @@ struct __wt_named_encryptor { const char *name; /* Name of encryptor */ WT_ENCRYPTOR *encryptor; /* User supplied callbacks */ /* Locked: list of encryptors by key */ - TAILQ_HEAD(__wt_keyedhash, __wt_keyed_encryptor) - keyedhashqh[WT_HASH_ARRAY_SIZE]; + TAILQ_HEAD(__wt_keyedhash, __wt_keyed_encryptor) * keyedhashqh; TAILQ_HEAD(__wt_keyed_qh, __wt_keyed_encryptor) keyedqh; /* Linked list of encryptors */ TAILQ_ENTRY(__wt_named_encryptor) q; @@ -186,6 +185,8 @@ struct __wt_connection_impl { const char *home; /* Database home */ const char *error_prefix; /* Database error prefix */ + uint64_t dh_hash_size; /* Data handle hash bucket array size */ + uint64_t hash_size; /* General hash bucket array size */ int is_new; /* Connection created database */ uint16_t compat_major; /* Compatibility major version */ @@ -220,24 +221,24 @@ struct __wt_connection_impl { * of the table URI. */ /* Locked: data handle hash array */ - TAILQ_HEAD(__wt_dhhash, __wt_data_handle) dhhash[WT_HASH_ARRAY_SIZE]; + TAILQ_HEAD(__wt_dhhash, __wt_data_handle) * dhhash; /* Locked: data handle list */ TAILQ_HEAD(__wt_dhandle_qh, __wt_data_handle) dhqh; /* Locked: LSM handle list. */ TAILQ_HEAD(__wt_lsm_qh, __wt_lsm_tree) lsmqh; /* Locked: file list */ - TAILQ_HEAD(__wt_fhhash, __wt_fh) fhhash[WT_HASH_ARRAY_SIZE]; + TAILQ_HEAD(__wt_fhhash, __wt_fh) * fhhash; TAILQ_HEAD(__wt_fh_qh, __wt_fh) fhqh; /* Locked: library list */ TAILQ_HEAD(__wt_dlh_qh, __wt_dlh) dlhqh; WT_SPINLOCK block_lock; /* Locked: block manager list */ - TAILQ_HEAD(__wt_blockhash, __wt_block) blockhash[WT_HASH_ARRAY_SIZE]; + TAILQ_HEAD(__wt_blockhash, __wt_block) * blockhash; TAILQ_HEAD(__wt_block_qh, __wt_block) blockqh; /* Locked: handles in each bucket */ - u_int dh_bucket_count[WT_HASH_ARRAY_SIZE]; - u_int dhandle_count; /* Locked: handles in the queue */ + uint64_t *dh_bucket_count; + uint64_t dhandle_count; /* Locked: handles in the queue */ u_int open_btree_count; /* Locked: open writable btree count */ uint32_t next_file_id; /* Locked: file ID counter */ uint32_t open_file_count; /* Atomic: open file handle count */ @@ -278,8 +279,12 @@ struct __wt_connection_impl { wt_off_t ckpt_logsize; /* Checkpoint log size period */ bool ckpt_signalled; /* Checkpoint signalled */ - uint64_t ckpt_usecs; /* Checkpoint timer */ - uint64_t ckpt_prep_max; /* Checkpoint prepare time min/max */ + uint64_t ckpt_apply; /* Checkpoint handles applied */ + uint64_t ckpt_apply_time; /* Checkpoint applied handles gather time */ + uint64_t ckpt_skip; /* Checkpoint handles skipped */ + uint64_t ckpt_skip_time; /* Checkpoint skipped handles gather time */ + uint64_t ckpt_usecs; /* Checkpoint timer */ + uint64_t ckpt_prep_max; /* Checkpoint prepare time min/max */ uint64_t ckpt_prep_min; uint64_t ckpt_prep_recent; /* Checkpoint prepare time recent/total */ uint64_t ckpt_prep_total; @@ -503,18 +508,20 @@ struct __wt_connection_impl { * Variable with flags for which subsystems the diagnostic stress timing delays have been requested. */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_TIMING_STRESS_AGGRESSIVE_SWEEP 0x001u -#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x002u -#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x004u -#define WT_TIMING_STRESS_HS_SWEEP 0x008u -#define WT_TIMING_STRESS_SPLIT_1 0x010u -#define WT_TIMING_STRESS_SPLIT_2 0x020u -#define WT_TIMING_STRESS_SPLIT_3 0x040u -#define WT_TIMING_STRESS_SPLIT_4 0x080u -#define WT_TIMING_STRESS_SPLIT_5 0x100u -#define WT_TIMING_STRESS_SPLIT_6 0x200u -#define WT_TIMING_STRESS_SPLIT_7 0x400u -#define WT_TIMING_STRESS_SPLIT_8 0x800u +#define WT_TIMING_STRESS_AGGRESSIVE_SWEEP 0x0001u +#define WT_TIMING_STRESS_BACKUP_RENAME 0x0002u +#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x0004u +#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x0008u +#define WT_TIMING_STRESS_HS_SWEEP 0x0010u +#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x0020u +#define WT_TIMING_STRESS_SPLIT_1 0x0040u +#define WT_TIMING_STRESS_SPLIT_2 0x0080u +#define WT_TIMING_STRESS_SPLIT_3 0x0100u +#define WT_TIMING_STRESS_SPLIT_4 0x0200u +#define WT_TIMING_STRESS_SPLIT_5 0x0400u +#define WT_TIMING_STRESS_SPLIT_6 0x0800u +#define WT_TIMING_STRESS_SPLIT_7 0x1000u +#define WT_TIMING_STRESS_SPLIT_8 0x2000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t timing_stress_flags; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index be3672f1ac5..c344710120d 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -12,27 +12,27 @@ /* * Initialize a static WT_CURSOR structure. */ -#define WT_CURSOR_STATIC_INIT(n, get_key, get_value, set_key, set_value, compare, equals, next, \ - prev, reset, search, search_near, insert, modify, update, remove, \ - reserve, reconfigure, cache, reopen, close) \ - static const WT_CURSOR n = { \ - NULL, /* session */ \ - NULL, /* uri */ \ - NULL, /* key_format */ \ - NULL, /* value_format */ \ - get_key, get_value, set_key, set_value, compare, equals, next, prev, reset, search, \ - search_near, insert, modify, update, remove, reserve, close, reconfigure, cache, reopen, \ - 0, /* uri_hash */ \ - {NULL, NULL}, /* TAILQ_ENTRY q */ \ - 0, /* recno key */ \ - {0}, /* recno raw buffer */ \ - NULL, /* json_private */ \ - NULL, /* lang_private */ \ - {NULL, 0, NULL, 0, 0}, /* WT_ITEM key */ \ - {NULL, 0, NULL, 0, 0}, /* WT_ITEM value */ \ - 0, /* int saved_err */ \ - NULL, /* internal_uri */ \ - 0 /* uint32_t flags */ \ +#define WT_CURSOR_STATIC_INIT(n, get_key, get_value, set_key, set_value, compare, equals, next, \ + prev, reset, search, search_near, insert, modify, update, remove, reserve, reconfigure, cache, \ + reopen, close) \ + static const WT_CURSOR n = { \ + NULL, /* session */ \ + NULL, /* uri */ \ + NULL, /* key_format */ \ + NULL, /* value_format */ \ + get_key, get_value, set_key, set_value, compare, equals, next, prev, reset, search, \ + search_near, insert, modify, update, remove, reserve, close, reconfigure, cache, reopen, \ + 0, /* uri_hash */ \ + {NULL, NULL}, /* TAILQ_ENTRY q */ \ + 0, /* recno key */ \ + {0}, /* recno raw buffer */ \ + NULL, /* json_private */ \ + NULL, /* lang_private */ \ + {NULL, 0, NULL, 0, 0}, /* WT_ITEM key */ \ + {NULL, 0, NULL, 0, 0}, /* WT_ITEM value */ \ + 0, /* int saved_err */ \ + NULL, /* internal_uri */ \ + 0 /* uint32_t flags */ \ } struct __wt_cursor_backup { @@ -61,15 +61,16 @@ struct __wt_cursor_backup { uint64_t granularity; /* Length, transfer size */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CURBACKUP_CKPT_FAKE 0x01u /* Object has fake checkpoint */ -#define WT_CURBACKUP_DUP 0x02u /* Duplicated backup cursor */ -#define WT_CURBACKUP_FORCE_FULL 0x04u /* Force full file copy for this cursor */ -#define WT_CURBACKUP_FORCE_STOP 0x08u /* Force stop incremental backup */ -#define WT_CURBACKUP_HAS_CB_INFO 0x10u /* Object has checkpoint backup info */ -#define WT_CURBACKUP_INCR 0x20u /* Incremental backup cursor */ -#define WT_CURBACKUP_INCR_INIT 0x40u /* Cursor traversal initialized */ -#define WT_CURBACKUP_LOCKER 0x80u /* Hot-backup started */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_CURBACKUP_CKPT_FAKE 0x001u /* Object has fake checkpoint */ +#define WT_CURBACKUP_DUP 0x002u /* Duplicated backup cursor */ +#define WT_CURBACKUP_FORCE_FULL 0x004u /* Force full file copy for this cursor */ +#define WT_CURBACKUP_FORCE_STOP 0x008u /* Force stop incremental backup */ +#define WT_CURBACKUP_HAS_CB_INFO 0x010u /* Object has checkpoint backup info */ +#define WT_CURBACKUP_INCR 0x020u /* Incremental backup cursor */ +#define WT_CURBACKUP_INCR_INIT 0x040u /* Cursor traversal initialized */ +#define WT_CURBACKUP_LOCKER 0x080u /* Hot-backup started */ +#define WT_CURBACKUP_RENAME 0x100u /* Object had a rename */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; @@ -221,7 +222,7 @@ struct __wt_cursor_btree { #define WT_CBT_READ_ONCE 0x100u /* Page in with WT_READ_WONT_NEED */ #define WT_CBT_SEARCH_SMALLEST 0x200u /* Row-store: small-key insert list */ #define WT_CBT_VAR_ONPAGE_MATCH 0x400u /* Var-store: on-page recno match */ -/* AUTOMATIC FLAG VALUE GENERATION STOP */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ #define WT_CBT_POSITION_MASK /* Flags associated with position */ \ (WT_CBT_ITERATE_APPEND | WT_CBT_ITERATE_NEXT | WT_CBT_ITERATE_PREV | \ diff --git a/src/third_party/wiredtiger/src/include/cursor.i b/src/third_party/wiredtiger/src/include/cursor.i index 7bba5af25a0..ad2ffcc6538 100644 --- a/src/third_party/wiredtiger/src/include/cursor.i +++ b/src/third_party/wiredtiger/src/include/cursor.i @@ -483,10 +483,10 @@ __cursor_row_slot_key_return( memcpy((uint8_t *)cbt->row_key->data + cbt->row_key->size, kpack->data, kpack->size); cbt->row_key->size += kpack->size; } else { - /* - * Call __wt_row_leaf_key_work instead of __wt_row_leaf_key: we already did __wt_row_leaf_key's - * fast-path checks inline. - */ + /* + * Call __wt_row_leaf_key_work instead of __wt_row_leaf_key: we already did + * __wt_row_leaf_key's fast-path checks inline. + */ slow: WT_RET(__wt_row_leaf_key_work(session, page, rip, cbt->row_key, false)); } diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index 39cce1e5b34..7ef6f387176 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -86,19 +86,20 @@ #define WT_RET_BUSY_OK(a) WT_RET_ERROR_OK(a, EBUSY) #define WT_RET_NOTFOUND_OK(a) WT_RET_ERROR_OK(a, WT_NOTFOUND) /* Set "ret" if not already set. */ -#define WT_TRET(a) \ - do { \ - int __ret; \ - if ((__ret = (a)) != 0 && (__ret == WT_PANIC || ret == 0 || ret == WT_DUPLICATE_KEY || \ - ret == WT_NOTFOUND || ret == WT_RESTART)) \ - ret = __ret; \ +#define WT_TRET(a) \ + do { \ + int __ret; \ + if ((__ret = (a)) != 0 && \ + (__ret == WT_PANIC || ret == 0 || ret == WT_DUPLICATE_KEY || ret == WT_NOTFOUND || \ + ret == WT_RESTART)) \ + ret = __ret; \ } while (0) #define WT_TRET_ERROR_OK(a, e) \ do { \ int __ret; \ if ((__ret = (a)) != 0 && __ret != (e) && \ (__ret == WT_PANIC || ret == 0 || ret == WT_DUPLICATE_KEY || ret == WT_NOTFOUND || \ - ret == WT_RESTART)) \ + ret == WT_RESTART)) \ ret = __ret; \ } while (0) #define WT_TRET_BUSY_OK(a) WT_TRET_ERROR_OK(a, EBUSY) diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 7a37bf638e8..4939b3d3952 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -358,6 +358,8 @@ extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_close_connection_close(WT_SESSION_IMPL *session) @@ -753,11 +755,11 @@ extern int __wt_hs_config(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_create(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_hs_cursor_close(WT_SESSION_IMPL *session, uint32_t session_flags) +extern int __wt_hs_cursor_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_cursor_next(WT_SESSION_IMPL *session, WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_hs_cursor_open(WT_SESSION_IMPL *session, uint32_t *session_flags) +extern int __wt_hs_cursor_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_cursor_position(WT_SESSION_IMPL *session, WT_CURSOR *cursor, uint32_t btree_id, const WT_ITEM *key, wt_timestamp_t timestamp, WT_ITEM *user_srch_key) @@ -842,7 +844,7 @@ extern int __wt_log_reset(WT_SESSION_IMPL *session, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, int (*func)(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN *next_lsnp, - void *cookie, int firstrecord), + void *cookie, int firstrecord), void *cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_set_version(WT_SESSION_IMPL *session, uint16_t version, uint32_t first_rec, bool downgrade, bool live_chg, uint32_t *lognump) @@ -1032,6 +1034,8 @@ extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_blk_mods_load(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt, + bool rename) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, @@ -1325,6 +1329,8 @@ extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_breakpoint(WT_SESSION *wt_session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_close_internal(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_compact(WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) @@ -1477,7 +1483,7 @@ extern int __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_global_shutdown(WT_SESSION_IMPL *session, const char *config, const char **cfg) +extern int __wt_txn_global_shutdown(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_init(WT_SESSION_IMPL *session, WT_SESSION_IMPL *session_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index bd99c32ca6e..26dbdc51d57 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -170,9 +170,10 @@ union __wt_lsn { /* Slot is in use */ #define WT_LOG_SLOT_ACTIVE(state) (WT_LOG_SLOT_JOINED(state) != WT_LOG_SLOT_JOIN_MASK) /* Slot is in use, but closed to new joins */ -#define WT_LOG_SLOT_CLOSED(state) \ - (WT_LOG_SLOT_ACTIVE(state) && (FLD_LOG_SLOT_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ - !FLD_LOG_SLOT_ISSET((uint64_t)(state), WT_LOG_SLOT_RESERVED))) +#define WT_LOG_SLOT_CLOSED(state) \ + (WT_LOG_SLOT_ACTIVE(state) && \ + (FLD_LOG_SLOT_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ + !FLD_LOG_SLOT_ISSET((uint64_t)(state), WT_LOG_SLOT_RESERVED))) /* Slot is in use, all data copied into buffer */ #define WT_LOG_SLOT_INPROGRESS(state) (WT_LOG_SLOT_RELEASED(state) != WT_LOG_SLOT_JOINED(state)) #define WT_LOG_SLOT_DONE(state) (WT_LOG_SLOT_CLOSED(state) && !WT_LOG_SLOT_INPROGRESS(state)) diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index dac2cc59573..a0241a22ce7 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -98,8 +98,9 @@ struct __wt_block_mods { uint64_t offset; /* Zero bit offset for bitstring */ uint64_t granularity; /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_BLOCK_MODS_VALID 0x1u /* Entry is valid */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_BLOCK_MODS_RENAME 0x1u /* Entry is from a rename */ +#define WT_BLOCK_MODS_VALID 0x2u /* Entry is valid */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 053ed2d6a82..77c4088a363 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -96,12 +96,6 @@ #define WT_ENCRYPT_LEN_SIZE sizeof(uint32_t) /* - * Default hash table size; we don't need a prime number of buckets because we always use a good - * hash function. - */ -#define WT_HASH_ARRAY_SIZE 512 - -/* * __wt_calloc_def, __wt_calloc_one -- * Most calloc calls don't need separate count or sizeof arguments. */ diff --git a/src/third_party/wiredtiger/src/include/os.h b/src/third_party/wiredtiger/src/include/os.h index 668e8daf781..965baab05e8 100644 --- a/src/third_party/wiredtiger/src/include/os.h +++ b/src/third_party/wiredtiger/src/include/os.h @@ -71,12 +71,12 @@ #define WT_CLOCKDIFF_MS(end, begin) (WT_CLOCKDIFF_NS(end, begin) / WT_MILLION) #define WT_CLOCKDIFF_SEC(end, begin) (WT_CLOCKDIFF_NS(end, begin) / WT_BILLION) -#define WT_TIMECMP(t1, t2) \ - ((t1).tv_sec < (t2).tv_sec ? -1 : (t1).tv_sec == (t2).tv_sec ? \ - (t1).tv_nsec < (t2).tv_nsec ? -1 : (t1).tv_nsec == (t2).tv_nsec ? \ - 0 : \ - 1 : \ - 1) +#define WT_TIMECMP(t1, t2) \ + ((t1).tv_sec < (t2).tv_sec ? \ + -1 : \ + (t1).tv_sec == (t2).tv_sec ? \ + (t1).tv_nsec < (t2).tv_nsec ? -1 : (t1).tv_nsec == (t2).tv_nsec ? 0 : 1 : \ + 1) /* * Macros to ensure a file handle is inserted or removed from both the main and the hashed queue, diff --git a/src/third_party/wiredtiger/src/include/os_fhandle.i b/src/third_party/wiredtiger/src/include/os_fhandle.i index a391327d9a1..ae0f750ac00 100644 --- a/src/third_party/wiredtiger/src/include/os_fhandle.i +++ b/src/third_party/wiredtiger/src/include/os_fhandle.i @@ -36,8 +36,9 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh, bool block) if (block) ret = (handle->fh_sync == NULL ? 0 : handle->fh_sync(handle, (WT_SESSION *)session)); else - ret = (handle->fh_sync_nowait == NULL ? 0 : handle->fh_sync_nowait( - handle, (WT_SESSION *)session)); + ret = + (handle->fh_sync_nowait == NULL ? 0 : + handle->fh_sync_nowait(handle, (WT_SESSION *)session)); WT_STAT_CONN_DECR_ATOMIC(session, thread_fsync_active); return (ret); } @@ -160,7 +161,8 @@ __wt_write(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, size_t len, con WT_DECL_RET; uint64_t time_start, time_stop; - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || + WT_ASSERT(session, + !F_ISSET(S2C(session), WT_CONN_READONLY) || WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, strlen(WT_SINGLETHREAD))); __wt_verbose(session, WT_VERB_HANDLEOPS, "%s: handle-write: %" WT_SIZET_FMT " at %" PRIuMAX, diff --git a/src/third_party/wiredtiger/src/include/packing.i b/src/third_party/wiredtiger/src/include/packing.i index b18555ca0b4..e09bf051acb 100644 --- a/src/third_party/wiredtiger/src/include/packing.i +++ b/src/third_party/wiredtiger/src/include/packing.i @@ -158,8 +158,7 @@ next: case 's': if (pv->size < 1) WT_RET_MSG(pack->session, EINVAL, - "Fixed length strings must be at least 1 byte " - "in format '%.*s'", + "Fixed length strings must be at least 1 byte in format '%.*s'", (int)(pack->end - pack->orig), pack->orig); return (0); case 'x': @@ -167,8 +166,7 @@ next: case 't': if (pv->size < 1 || pv->size > 8) WT_RET_MSG(pack->session, EINVAL, - "Bitfield sizes must be between 1 and 8 bits " - "in format '%.*s'", + "Bitfield sizes must be between 1 and 8 bits in format '%.*s'", (int)(pack->end - pack->orig), pack->orig); return (0); case 'u': diff --git a/src/third_party/wiredtiger/src/include/reconcile.i b/src/third_party/wiredtiger/src/include/reconcile.i index ba74a54f429..07b7abf80cc 100644 --- a/src/third_party/wiredtiger/src/include/reconcile.i +++ b/src/third_party/wiredtiger/src/include/reconcile.i @@ -217,10 +217,12 @@ __wt_rec_need_split(WT_RECONCILE *r, size_t len) * updates) associated with the page. If there are barely any items on the page, then it's not * worth splitting. We also want to temper this effect to avoid in-memory updates from * dominating the calculation and causing excessive splitting. Therefore, we'll limit the impact - * to a tenth of the cache usage occupied by those updates. + * to a tenth of the cache usage occupied by those updates. For small updates the overhead of + * the update structure could skew the calculation, so we subtract the overhead before + * considering the cache usage by the updates. */ if (r->page->type == WT_PAGE_ROW_LEAF && page_items > WT_REC_SPLIT_MIN_ITEMS_USE_MEM) - len += r->supd_memsize / 10; + len += (r->supd_memsize - (r->supd_next * WT_UPDATE_SIZE)) / 10; /* Check for the disk image crossing a boundary. */ return (WT_CHECK_CROSSING_BND(r, len)); diff --git a/src/third_party/wiredtiger/src/include/schema.h b/src/third_party/wiredtiger/src/include/schema.h index 83827896f43..5974dbfbd15 100644 --- a/src/third_party/wiredtiger/src/include/schema.h +++ b/src/third_party/wiredtiger/src/include/schema.h @@ -185,18 +185,22 @@ struct __wt_table { */ #define WT_WITH_SCHEMA_LOCK(session, op) \ do { \ - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || \ - !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST | WT_SESSION_NO_SCHEMA_LOCK | \ - WT_SESSION_LOCKED_TABLE)); \ + WT_ASSERT(session, \ + F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || \ + !F_ISSET(session, \ + WT_SESSION_LOCKED_HANDLE_LIST | WT_SESSION_NO_SCHEMA_LOCK | \ + WT_SESSION_LOCKED_TABLE)); \ WT_WITH_LOCK_WAIT(session, &S2C(session)->schema_lock, WT_SESSION_LOCKED_SCHEMA, op); \ } while (0) -#define WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, op) \ - do { \ - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || \ - !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST | WT_SESSION_NO_SCHEMA_LOCK | \ - WT_SESSION_LOCKED_TABLE)); \ - WT_WITH_LOCK_NOWAIT( \ - session, ret, &S2C(session)->schema_lock, WT_SESSION_LOCKED_SCHEMA, op); \ +#define WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, op) \ + do { \ + WT_ASSERT(session, \ + F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || \ + !F_ISSET(session, \ + WT_SESSION_LOCKED_HANDLE_LIST | WT_SESSION_NO_SCHEMA_LOCK | \ + WT_SESSION_LOCKED_TABLE)); \ + WT_WITH_LOCK_NOWAIT( \ + session, ret, &S2C(session)->schema_lock, WT_SESSION_LOCKED_SCHEMA, op); \ } while (0) /* @@ -240,7 +244,8 @@ struct __wt_table { } while (0) #define WT_WITH_TABLE_WRITE_LOCK_NOWAIT(session, ret, op) \ do { \ - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE) || \ + WT_ASSERT(session, \ + F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE) || \ !F_ISSET(session, WT_SESSION_LOCKED_TABLE_READ | WT_SESSION_LOCKED_HANDLE_LIST)); \ if (F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE)) { \ op; \ diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 5da9919e15a..355cc5f1343 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -169,33 +169,32 @@ struct __wt_session_impl { #define WT_SESSION_BACKUP_DUP 0x00000002u #define WT_SESSION_CACHE_CURSORS 0x00000004u #define WT_SESSION_CAN_WAIT 0x00000008u -#define WT_SESSION_HS_CURSOR 0x00000010u -#define WT_SESSION_IGNORE_CACHE_SIZE 0x00000020u -#define WT_SESSION_INSTANTIATE_PREPARE 0x00000040u -#define WT_SESSION_INTERNAL 0x00000080u -#define WT_SESSION_LOCKED_CHECKPOINT 0x00000100u -#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x00000200u -#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x00000400u -#define WT_SESSION_LOCKED_HOTBACKUP_READ 0x00000800u -#define WT_SESSION_LOCKED_HOTBACKUP_WRITE 0x00001000u -#define WT_SESSION_LOCKED_METADATA 0x00002000u -#define WT_SESSION_LOCKED_PASS 0x00004000u -#define WT_SESSION_LOCKED_SCHEMA 0x00008000u -#define WT_SESSION_LOCKED_SLOT 0x00010000u -#define WT_SESSION_LOCKED_TABLE_READ 0x00020000u -#define WT_SESSION_LOCKED_TABLE_WRITE 0x00040000u -#define WT_SESSION_LOCKED_TURTLE 0x00080000u -#define WT_SESSION_LOGGING_INMEM 0x00100000u -#define WT_SESSION_NO_DATA_HANDLES 0x00200000u -#define WT_SESSION_NO_LOGGING 0x00400000u -#define WT_SESSION_NO_RECONCILE 0x00800000u -#define WT_SESSION_NO_SCHEMA_LOCK 0x01000000u -#define WT_SESSION_QUIET_CORRUPT_FILE 0x02000000u -#define WT_SESSION_READ_WONT_NEED 0x04000000u -#define WT_SESSION_RESOLVING_TXN 0x08000000u -#define WT_SESSION_ROLLBACK_TO_STABLE 0x10000000u -#define WT_SESSION_SCHEMA_TXN 0x20000000u -#define WT_SESSION_SERVER_ASYNC 0x40000000u +#define WT_SESSION_IGNORE_CACHE_SIZE 0x00000010u +#define WT_SESSION_INSTANTIATE_PREPARE 0x00000020u +#define WT_SESSION_INTERNAL 0x00000040u +#define WT_SESSION_LOCKED_CHECKPOINT 0x00000080u +#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x00000100u +#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x00000200u +#define WT_SESSION_LOCKED_HOTBACKUP_READ 0x00000400u +#define WT_SESSION_LOCKED_HOTBACKUP_WRITE 0x00000800u +#define WT_SESSION_LOCKED_METADATA 0x00001000u +#define WT_SESSION_LOCKED_PASS 0x00002000u +#define WT_SESSION_LOCKED_SCHEMA 0x00004000u +#define WT_SESSION_LOCKED_SLOT 0x00008000u +#define WT_SESSION_LOCKED_TABLE_READ 0x00010000u +#define WT_SESSION_LOCKED_TABLE_WRITE 0x00020000u +#define WT_SESSION_LOCKED_TURTLE 0x00040000u +#define WT_SESSION_LOGGING_INMEM 0x00080000u +#define WT_SESSION_NO_DATA_HANDLES 0x00100000u +#define WT_SESSION_NO_LOGGING 0x00200000u +#define WT_SESSION_NO_RECONCILE 0x00400000u +#define WT_SESSION_NO_SCHEMA_LOCK 0x00800000u +#define WT_SESSION_QUIET_CORRUPT_FILE 0x01000000u +#define WT_SESSION_READ_WONT_NEED 0x02000000u +#define WT_SESSION_RESOLVING_TXN 0x04000000u +#define WT_SESSION_ROLLBACK_TO_STABLE 0x08000000u +#define WT_SESSION_SCHEMA_TXN 0x10000000u +#define WT_SESSION_SERVER_ASYNC 0x20000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 11b966a0e55..6e070b9c1c6 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -480,6 +480,8 @@ struct __wt_connection_stats { int64_t cond_auto_wait_skipped; int64_t time_travel; int64_t file_open; + int64_t buckets_dh; + int64_t buckets; int64_t memory_allocation; int64_t memory_free; int64_t memory_grow; @@ -747,6 +749,12 @@ struct __wt_connection_stats { int64_t txn_hs_ckpt_duration; int64_t txn_checkpoint_time_max; int64_t txn_checkpoint_time_min; + int64_t txn_checkpoint_handle_duration; + int64_t txn_checkpoint_handle_duration_apply; + int64_t txn_checkpoint_handle_duration_skip; + int64_t txn_checkpoint_handle_applied; + int64_t txn_checkpoint_handle_skipped; + int64_t txn_checkpoint_handle_walked; int64_t txn_checkpoint_time_recent; int64_t txn_checkpoint_prep_running; int64_t txn_checkpoint_prep_max; diff --git a/src/third_party/wiredtiger/src/include/time.i b/src/third_party/wiredtiger/src/include/time.i index a218e3cb97b..80ae3048a8a 100644 --- a/src/third_party/wiredtiger/src/include/time.i +++ b/src/third_party/wiredtiger/src/include/time.i @@ -162,7 +162,7 @@ __wt_op_timer_start(WT_SESSION_IMPL *session) { uint64_t timeout_us; - if (F_ISSET(session, WT_SESSION_HS_CURSOR)) + if (session->hs_cursor != NULL) return; /* Timer can be configured per-transaction, and defaults to per-connection. */ @@ -183,7 +183,7 @@ __wt_op_timer_start(WT_SESSION_IMPL *session) static inline void __wt_op_timer_stop(WT_SESSION_IMPL *session) { - if (F_ISSET(session, WT_SESSION_HS_CURSOR)) + if (session->hs_cursor != NULL) return; session->operation_start_us = session->operation_timeout_us = 0; @@ -198,7 +198,7 @@ __wt_op_timer_fired(WT_SESSION_IMPL *session) { uint64_t diff, now; - if (F_ISSET(session, WT_SESSION_HS_CURSOR)) + if (session->hs_cursor != NULL) return (false); if (session->operation_start_us == 0 || session->operation_timeout_us == 0) return (false); diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index ae3f8f94350..bc640aed73b 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -76,11 +76,12 @@ typedef enum { (s)->txn->isolation = saved_txn_iso; \ WT_ASSERT((s), (s)->txn->forced_iso > 0); \ (s)->txn->forced_iso--; \ - WT_ASSERT((s), txn_shared->id == saved_txn_shared.id && \ + WT_ASSERT((s), \ + txn_shared->id == saved_txn_shared.id && \ (txn_shared->metadata_pinned == saved_txn_shared.metadata_pinned || \ - saved_txn_shared.metadata_pinned == WT_TXN_NONE) && \ + saved_txn_shared.metadata_pinned == WT_TXN_NONE) && \ (txn_shared->pinned_id == saved_txn_shared.pinned_id || \ - saved_txn_shared.pinned_id == WT_TXN_NONE)); \ + saved_txn_shared.pinned_id == WT_TXN_NONE)); \ txn_shared->metadata_pinned = saved_txn_shared.metadata_pinned; \ txn_shared->pinned_id = saved_txn_shared.pinned_id; \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 450c7213e43..0d98eb33759 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -370,9 +370,8 @@ __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) if (F_ISSET(txn, WT_TXN_READONLY)) { if (F_ISSET(txn, WT_TXN_IGNORE_PREPARE)) - WT_RET_MSG(session, ENOTSUP, - "Transactions with ignore_prepare=true" - " cannot perform updates"); + WT_RET_MSG( + session, ENOTSUP, "Transactions with ignore_prepare=true cannot perform updates"); WT_RET_MSG(session, WT_ROLLBACK, "Attempt to update in a read-only transaction"); } @@ -852,7 +851,7 @@ __wt_txn_read_upd_list( * If the cursor is configured to ignore tombstones, copy the timestamps from the tombstones * to the stop time window of the update value being returned to the caller. Caller can * process the stop time window to decide if there was a tombstone on the update chain. If - * the time window already has a stop time set then we must've seen a tombstone prior to + * the time window already has a stop time set then we must have seen a tombstone prior to * ours in the update list, and therefore don't need to do this again. */ if (type == WT_UPDATE_TOMBSTONE && F_ISSET(&cbt->iface, WT_CURSTD_IGNORE_TOMBSTONE) && @@ -1006,8 +1005,9 @@ retry: */ if (prepare_upd != NULL) { WT_ASSERT(session, F_ISSET(prepare_upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS)); - if (retry && (prepare_upd->txnid == WT_TXN_ABORTED || - prepare_upd->prepare_state == WT_PREPARE_RESOLVED)) { + if (retry && + (prepare_upd->txnid == WT_TXN_ABORTED || + prepare_upd->prepare_state == WT_PREPARE_RESOLVED)) { retry = false; /* Clean out any stale value before performing the retry. */ __wt_upd_value_clear(cbt->upd_value); @@ -1045,8 +1045,7 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[]) WT_ASSERT(session, !F_ISSET(txn, WT_TXN_RUNNING)); - if (cfg != NULL) - WT_RET(__wt_txn_config(session, cfg)); + WT_RET(__wt_txn_config(session, cfg)); /* Allocate a snapshot if required. */ if (txn->isolation == WT_ISO_SNAPSHOT) { @@ -1211,14 +1210,11 @@ __wt_txn_search_check(WT_SESSION_IMPL *session) if (!F_ISSET(S2C(session), WT_CONN_RECOVERING) && FLD_ISSET(btree->assert_flags, WT_ASSERT_READ_TS_ALWAYS) && !F_ISSET(txn, WT_TXN_SHARED_TS_READ)) - WT_RET_MSG(session, EINVAL, - "read_timestamp required and " - "none set on this transaction"); + WT_RET_MSG(session, EINVAL, "read_timestamp required and none set on this transaction"); if (FLD_ISSET(btree->assert_flags, WT_ASSERT_READ_TS_NEVER) && F_ISSET(txn, WT_TXN_SHARED_TS_READ)) - WT_RET_MSG(session, EINVAL, - "no read_timestamp required and " - "timestamp set on this transaction"); + WT_RET_MSG( + session, EINVAL, "no read_timestamp required and timestamp set on this transaction"); return (0); } diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 977ad90cdc9..f65dbc350b0 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1766,12 +1766,12 @@ struct __wt_session { * \c "read-committed"\, \c "snapshot"; default empty.} * @config{name, name of the transaction for tracing and debugging., a string; default * empty.} - * @config{operation_timeout_ms, when non-zero\, a requested limit on the number of elapsed - * real time milliseconds taken to complete database operations in this transaction. Time - * is measured from the start of each WiredTiger API call. There is no guarantee any - * operation will not take longer than this amount of time. If WiredTiger notices the limit - * has been exceeded\, an operation may return a WT_ROLLBACK error. Default is to have no - * limit., an integer greater than or equal to 1; default \c 0.} + * @config{operation_timeout_ms, when non-zero\, a requested limit on the time taken to + * complete operations in this transaction. Time is measured in real time milliseconds from + * the start of each WiredTiger API call. There is no guarantee any operation will not take + * longer than this amount of time. If WiredTiger notices the limit has been exceeded\, an + * operation may return a WT_ROLLBACK error. Default is to have no limit., an integer + * greater than or equal to 1; default \c 0.} * @config{priority, priority of the transaction for resolving conflicts. Transactions with * higher values are less likely to abort., an integer between -100 and 100; default \c 0.} * @config{read_timestamp, read using the specified timestamp. The supplied value must not @@ -1820,6 +1820,12 @@ struct __wt_session { * supplied value must not be older than the commit timestamp set for the current * transaction. The value must also not be older than the current stable timestamp. See * @ref transaction_timestamps., a string; default empty.} + * @config{operation_timeout_ms, when non-zero\, a requested limit on the time taken to + * complete operations in this transaction. Time is measured in real time milliseconds from + * the start of each WiredTiger API call. There is no guarantee any operation will not take + * longer than this amount of time. If WiredTiger notices the limit has been exceeded\, an + * operation may return a WT_ROLLBACK error. Default is to have no limit., an integer + * greater than or equal to 1; default \c 0.} * @config{sync, override whether to sync log records when the transaction commits\, * inherited from ::wiredtiger_open \c transaction_sync. The \c background setting * initiates a background synchronization intended to be used with a later call to @@ -1869,7 +1875,14 @@ struct __wt_session { * @snippet ex_all.c transaction commit/rollback * * @param session the session handle - * @configempty{WT_SESSION.rollback_transaction, see dist/api_data.py} + * @configstart{WT_SESSION.rollback_transaction, see dist/api_data.py} + * @config{operation_timeout_ms, when non-zero\, a requested limit on the time taken to + * complete operations in this transaction. Time is measured in real time milliseconds from + * the start of each WiredTiger API call. There is no guarantee any operation will not take + * longer than this amount of time. If WiredTiger notices the limit has been exceeded\, an + * operation may return a WT_ROLLBACK error. Default is to have no limit., an integer + * greater than or equal to 1; default \c 0.} + * @configend * @errors */ int __F(rollback_transaction)(WT_SESSION *session, const char *config); @@ -2143,15 +2156,6 @@ struct __wt_connection { * @config{cache_max_wait_ms, the maximum number of milliseconds an application thread will * wait for space to be available in cache before giving up. Default will wait forever., an * integer greater than or equal to 0; default \c 0.} - * @config{cache_overflow = (, cache overflow configuration options., a set of related - * configuration options defined below.} - * @config{ file_max, The - * maximum number of bytes that WiredTiger is allowed to use for its cache overflow - * mechanism. If the cache overflow file exceeds this size\, a panic will be triggered. - * The default value means that the cache overflow file is unbounded and may use as much - * space as the filesystem will accommodate. The minimum non-zero setting is 100MB., an - * integer greater than or equal to 0; default \c 0.} - * @config{ ),,} * @config{cache_overhead, assume the heap allocator overhead is the specified percentage\, * and adjust the cache usage by that amount (for example\, if there is 10GB of data in * cache\, a percentage of 10 means WiredTiger treats this as 11GB). This value is @@ -2766,14 +2770,6 @@ struct __wt_connection { * @config{cache_max_wait_ms, the maximum number of milliseconds an application thread will wait for * space to be available in cache before giving up. Default will wait forever., an integer greater * than or equal to 0; default \c 0.} - * @config{cache_overflow = (, cache overflow configuration options., a set of related configuration - * options defined below.} - * @config{ file_max, The maximum number of bytes - * that WiredTiger is allowed to use for its cache overflow mechanism. If the cache overflow file - * exceeds this size\, a panic will be triggered. The default value means that the cache overflow - * file is unbounded and may use as much space as the filesystem will accommodate. The minimum - * non-zero setting is 100MB., an integer greater than or equal to 0; default \c 0.} - * @config{ ),,} * @config{cache_overhead, assume the heap allocator overhead is the specified percentage\, and * adjust the cache usage by that amount (for example\, if there is 10GB of data in cache\, a * percentage of 10 means WiredTiger treats this as 11GB). This value is configurable because @@ -2954,6 +2950,16 @@ struct __wt_connection { * @config{ close_scan_interval, interval in seconds at which to check for * files that are inactive and close them., an integer between 1 and 100000; default \c 10.} * @config{ ),,} + * @config{hash = (, manage resources around hash bucket arrays. All values must be a power of two. + * Note that setting large values can significantly increase memory usage inside WiredTiger., a set + * of related configuration options defined below.} + * @config{ buckets, + * configure the number of hash buckets for most system hash arrays., an integer between 64 and + * 65536; default \c 512.} + * @config{ dhandle_buckets, configure the number of + * hash buckets for hash arrays relating to data handles., an integer between 64 and 65536; default + * \c 512.} + * @config{ ),,} * @config{history_store = (, history store configuration options., a set of related configuration * options defined below.} * @config{ file_max, The maximum number of bytes @@ -5395,692 +5401,717 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_TIME_TRAVEL 1183 /*! connection: files currently open */ #define WT_STAT_CONN_FILE_OPEN 1184 +/*! connection: hash bucket array size for data handles */ +#define WT_STAT_CONN_BUCKETS_DH 1185 +/*! connection: hash bucket array size general */ +#define WT_STAT_CONN_BUCKETS 1186 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1185 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1187 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1186 +#define WT_STAT_CONN_MEMORY_FREE 1188 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1187 +#define WT_STAT_CONN_MEMORY_GROW 1189 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1188 +#define WT_STAT_CONN_COND_WAIT 1190 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1189 +#define WT_STAT_CONN_RWLOCK_READ 1191 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1190 +#define WT_STAT_CONN_RWLOCK_WRITE 1192 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1191 +#define WT_STAT_CONN_FSYNC_IO 1193 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1192 +#define WT_STAT_CONN_READ_IO 1194 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1193 +#define WT_STAT_CONN_WRITE_IO 1195 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1194 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1196 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1195 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1197 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1196 +#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1198 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1197 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1199 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1198 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1200 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1199 +#define WT_STAT_CONN_CURSOR_CACHE 1201 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1200 +#define WT_STAT_CONN_CURSOR_CREATE 1202 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1201 +#define WT_STAT_CONN_CURSOR_INSERT 1203 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1202 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1204 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1203 +#define WT_STAT_CONN_CURSOR_MODIFY 1205 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1204 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1206 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1205 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1207 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1206 +#define WT_STAT_CONN_CURSOR_NEXT 1208 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1207 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1209 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1208 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1210 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1209 +#define WT_STAT_CONN_CURSOR_RESTART 1211 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1210 +#define WT_STAT_CONN_CURSOR_PREV 1212 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1211 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1213 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone in rollback to stable */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE_RTS 1212 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE_RTS 1214 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1213 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1215 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1214 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1216 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1215 +#define WT_STAT_CONN_CURSOR_REMOVE 1217 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1216 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1218 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1217 +#define WT_STAT_CONN_CURSOR_RESERVE 1219 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1218 +#define WT_STAT_CONN_CURSOR_RESET 1220 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1219 +#define WT_STAT_CONN_CURSOR_SEARCH 1221 /*! cursor: cursor search history store calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_HS 1220 +#define WT_STAT_CONN_CURSOR_SEARCH_HS 1222 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1221 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1223 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1222 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1224 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1223 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1225 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1224 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1226 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1225 +#define WT_STAT_CONN_CURSOR_SWEEP 1227 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1226 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1228 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1227 +#define WT_STAT_CONN_CURSOR_UPDATE 1229 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1228 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1230 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1229 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1231 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1230 +#define WT_STAT_CONN_CURSOR_REOPEN 1232 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1231 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1233 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1232 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1234 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1233 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1235 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1234 +#define WT_STAT_CONN_DH_SWEEP_REF 1236 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1235 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1237 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1236 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1238 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1237 +#define WT_STAT_CONN_DH_SWEEP_TOD 1239 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1238 +#define WT_STAT_CONN_DH_SWEEPS 1240 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1239 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1241 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1240 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1242 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1241 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1243 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1242 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1244 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1243 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1245 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1244 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1246 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1245 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1247 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1246 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1248 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1247 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1249 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1248 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1250 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1249 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1251 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1250 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1252 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1251 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1253 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1252 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1254 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1253 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1255 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1254 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1256 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1255 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1257 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1256 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1258 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1257 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1259 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1258 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1260 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1259 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1261 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1260 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1262 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1261 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1263 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1262 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1264 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1263 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1265 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1264 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1266 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1265 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1267 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1266 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1268 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1267 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1269 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1268 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1270 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1269 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1271 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1270 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1272 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1271 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1273 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1272 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1274 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1273 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1275 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1274 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1276 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1275 +#define WT_STAT_CONN_LOG_FLUSH 1277 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1276 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1278 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1277 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1279 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1278 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1280 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1279 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1281 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1280 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1282 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1281 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1283 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1282 +#define WT_STAT_CONN_LOG_SCANS 1284 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1283 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1285 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1284 +#define WT_STAT_CONN_LOG_WRITE_LSN 1286 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1285 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1287 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1286 +#define WT_STAT_CONN_LOG_SYNC 1288 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1287 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1289 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1288 +#define WT_STAT_CONN_LOG_SYNC_DIR 1290 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1289 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1291 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1290 +#define WT_STAT_CONN_LOG_WRITES 1292 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1291 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1293 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1292 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1294 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1293 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1295 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1294 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1296 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1295 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1297 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1296 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1298 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1297 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1299 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1298 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1300 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1299 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1301 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1300 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1302 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1301 +#define WT_STAT_CONN_LOG_SLOT_RACES 1303 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1302 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1304 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1303 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1305 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1304 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1306 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1305 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1307 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1306 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1308 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1307 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1309 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1308 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1310 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1309 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1311 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1310 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1312 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1311 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1313 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1312 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1314 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1313 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1315 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1314 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1316 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1315 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1317 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1316 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1318 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1317 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1319 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1318 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1320 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1319 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1321 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1320 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1322 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1321 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1323 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1322 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1324 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1323 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1325 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1324 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1326 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1325 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1327 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1326 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1328 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1327 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1329 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1328 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1330 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1329 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1331 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1330 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1332 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1331 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1333 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1332 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1334 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1333 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1335 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1334 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1336 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1335 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1337 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1336 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1338 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1337 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1339 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1338 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1340 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1339 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1341 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1340 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1342 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1341 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1343 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1342 +#define WT_STAT_CONN_REC_PAGES 1344 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1343 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1345 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1344 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1346 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1345 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1347 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1346 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1348 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1347 +#define WT_STAT_CONN_REC_PAGE_DELETE 1349 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1348 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1350 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1349 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1351 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1350 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1352 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1351 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1353 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1352 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1354 /*! * reconciliation: pages written including an aggregated oldest start * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TXN 1353 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TXN 1355 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1354 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1356 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1355 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1357 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1356 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1358 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1357 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1359 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1358 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1360 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1359 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1361 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1360 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1362 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1361 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1363 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1362 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1364 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1363 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1365 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1364 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1366 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1365 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1367 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1366 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1368 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1367 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1369 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1368 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1370 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1369 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1371 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1370 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1372 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1371 +#define WT_STAT_CONN_SESSION_OPEN 1373 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1372 +#define WT_STAT_CONN_SESSION_QUERY_TS 1374 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1373 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1375 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1374 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1376 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1375 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1377 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1376 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1378 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1377 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1379 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1378 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1380 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1379 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1381 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1380 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1382 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1381 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1383 /*! session: table import failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1382 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1384 /*! session: table import successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1383 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1385 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1384 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1386 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1385 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1387 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1386 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1388 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1387 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1389 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1388 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1390 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1389 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1391 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1390 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1392 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1391 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1393 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1392 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1394 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1393 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1395 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1394 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1396 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1395 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1397 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1396 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1398 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1397 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1399 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1398 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1400 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1399 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1401 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1400 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1402 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1401 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1403 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1402 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1404 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1403 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1405 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1404 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1406 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1405 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1407 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1406 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1408 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1407 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1409 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1408 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1410 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1409 +#define WT_STAT_CONN_PAGE_SLEEP 1411 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1410 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1412 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1411 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1413 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1412 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1414 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1413 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1415 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1414 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1416 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1415 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1417 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1416 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1418 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1417 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1419 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1418 +#define WT_STAT_CONN_TXN_PREPARE 1420 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1419 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1421 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1420 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1422 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1421 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1423 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1422 +#define WT_STAT_CONN_TXN_QUERY_TS 1424 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1423 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1425 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1424 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1426 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1425 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1427 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1426 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1428 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1427 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1429 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1428 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1430 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1429 +#define WT_STAT_CONN_TXN_RTS 1431 /*! * transaction: rollback to stable hs records with stop timestamps older * than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1430 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1432 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1431 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1433 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1432 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1434 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1433 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1435 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1434 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1436 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1435 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1437 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1436 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1438 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1437 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1439 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1438 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1440 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1439 +#define WT_STAT_CONN_TXN_SET_TS 1441 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1440 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1442 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1441 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1443 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1442 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1444 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1443 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1445 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1444 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1446 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1445 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1447 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1446 +#define WT_STAT_CONN_TXN_BEGIN 1448 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1447 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1449 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1448 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1450 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1449 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1451 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1450 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1452 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1451 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1453 +/*! + * transaction: transaction checkpoint most recent duration for gathering + * all handles (usecs) + */ +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1454 +/*! + * transaction: transaction checkpoint most recent duration for gathering + * applied handles (usecs) + */ +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1455 +/*! + * transaction: transaction checkpoint most recent duration for gathering + * skipped handles (usecs) + */ +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1456 +/*! transaction: transaction checkpoint most recent handles applied */ +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1457 +/*! transaction: transaction checkpoint most recent handles skipped */ +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1458 +/*! transaction: transaction checkpoint most recent handles walked */ +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1459 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1452 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1460 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1453 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1461 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1454 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1462 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1455 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1463 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1456 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1464 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1457 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1465 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1458 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1466 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1459 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1467 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1460 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1468 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1461 +#define WT_STAT_CONN_TXN_CHECKPOINT 1469 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1462 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1470 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1463 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1471 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1464 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1472 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1465 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1473 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1466 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1474 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1467 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1475 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1468 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1476 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1469 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1477 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1470 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1478 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1471 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1479 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1472 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1480 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1473 +#define WT_STAT_CONN_TXN_SYNC 1481 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1474 +#define WT_STAT_CONN_TXN_COMMIT 1482 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1475 +#define WT_STAT_CONN_TXN_ROLLBACK 1483 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1476 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1484 /*! * @} diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index e8348fb9933..69b807eab5c 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -920,9 +920,8 @@ __log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, WT_LSN *ls */ if (desc->version > WT_LOG_VERSION) WT_ERR_MSG(session, WT_ERROR, - "unsupported WiredTiger file version: this build" - " only supports versions up to %d," - " and the file is version %" PRIu16, + "unsupported WiredTiger file version: this build only supports versions up to %d, and " + "the file is version %" PRIu16, WT_LOG_VERSION, desc->version); /* @@ -930,19 +929,17 @@ __log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, WT_LSN *ls * maximum. */ if (conn->req_max_major != WT_CONN_COMPAT_NONE && desc->version > conn->log_req_max) - WT_ERR_MSG(session, WT_ERROR, WT_COMPAT_MSG_PREFIX - "unsupported WiredTiger file version: this build" - " requires a maximum version of %" PRIu16 - "," - " and the file is version %" PRIu16, + WT_ERR_MSG(session, WT_ERROR, + WT_COMPAT_MSG_PREFIX + "unsupported WiredTiger file version: this build requires a maximum version of %" PRIu16 + ", and the file is version %" PRIu16, conn->log_req_max, desc->version); if (conn->req_min_major != WT_CONN_COMPAT_NONE && desc->version < conn->log_req_min) - WT_ERR_MSG(session, WT_ERROR, WT_COMPAT_MSG_PREFIX - "unsupported WiredTiger file version: this build" - " requires a minimum version of %" PRIu16 - "," - " and the file is version %" PRIu16, + WT_ERR_MSG(session, WT_ERROR, + WT_COMPAT_MSG_PREFIX + "unsupported WiredTiger file version: this build requires a minimum version of %" PRIu16 + ", and the file is version %" PRIu16, conn->log_req_min, desc->version); /* @@ -1030,8 +1027,9 @@ __log_record_verify( } for (i = 0; i < sizeof(logrec.unused); i++) if (logrec.unused[i] != 0) { - WT_RET(__wt_msg(session, "%s: log record at position %" PRIu32 - " has unused[%" WT_SIZET_FMT "] corruption 0x%" PRIx8, + WT_RET(__wt_msg(session, + "%s: log record at position %" PRIu32 " has unused[%" WT_SIZET_FMT + "] corruption 0x%" PRIx8, log_fh->name, offset, i, logrec.unused[i])); *corrupt = true; } @@ -2046,7 +2044,7 @@ __log_salvage_message( int __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, int (*func)(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN *next_lsnp, - void *cookie, int firstrecord), + void *cookie, int firstrecord), void *cookie) { WT_CONNECTION_IMPL *conn; @@ -2235,9 +2233,7 @@ advance: if (LF_ISSET(WT_LOGSCAN_RECOVER) && version == WT_LOG_VERSION_SYSTEM && WT_IS_ZERO_LSN(&prev_lsn)) { __wt_verbose(session, WT_VERB_LOG, - "log_scan: Stopping, no system " - "record detected in %s.", - log_fh->name); + "log_scan: Stopping, no system record detected in %s.", log_fh->name); break; } WT_ERR(__wt_filesize(session, log_fh, &log_size)); @@ -2423,13 +2419,11 @@ err: if (ret != 0 && firstrecord && LF_ISSET(WT_LOGSCAN_RECOVER | WT_LOGSCAN_RECOVER_METADATA)) { __wt_err(session, ret, "WiredTiger is unable to read the recovery log"); __wt_err(session, ret, - "This may be due to the log" - " files being encrypted, being from an older" - " version or due to corruption on disk"); + "This may be due to the log files being encrypted, being from an older version or due to " + "corruption on disk"); __wt_err(session, ret, - "You should confirm that you have" - " opened the database with the correct options including" - " all encryption and compression options"); + "You should confirm that you have opened the database with the correct options including " + "all encryption and compression options"); } WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); @@ -2600,9 +2594,8 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, ui conn = S2C(session); log = conn->log; if (record->size > UINT32_MAX) - WT_RET_MSG(session, EFBIG, "Log record size of %" WT_SIZET_FMT - " exceeds the maximum " - "supported size of %" PRIu32, + WT_RET_MSG(session, EFBIG, + "Log record size of %" WT_SIZET_FMT " exceeds the maximum supported size of %" PRIu32, record->size, UINT32_MAX); WT_INIT_LSN(&lsn); myslot.slot = NULL; diff --git a/src/third_party/wiredtiger/src/log/log_sys.c b/src/third_party/wiredtiger/src/log/log_sys.c index 0d0348499e9..73ca8cc2ada 100644 --- a/src/third_party/wiredtiger/src/log/log_sys.c +++ b/src/third_party/wiredtiger/src/log/log_sys.c @@ -116,14 +116,14 @@ __wt_verbose_dump_log(WT_SESSION_IMPL *session) WT_RET(__wt_msg(session, "Pre-allocate files: %s", conn->log_prealloc > 0 ? "yes" : "no")); WT_RET(__wt_msg(session, "Logging directory: %s", conn->log_path)); WT_RET(__wt_msg(session, "Logging maximum file size: %" PRId64, (int64_t)conn->log_file_max)); - WT_RET( - __wt_msg(session, "Log sync setting: %s", !FLD_ISSET(conn->txn_logsync, WT_LOG_SYNC_ENABLED) ? - "none" : - FLD_ISSET(conn->txn_logsync, WT_LOG_DSYNC) ? - "dsync" : - FLD_ISSET(conn->txn_logsync, WT_LOG_FLUSH) ? - "write to OS" : - FLD_ISSET(conn->txn_logsync, WT_LOG_FSYNC) ? "fsync to disk" : "unknown sync setting")); + WT_RET(__wt_msg(session, "Log sync setting: %s", + !FLD_ISSET(conn->txn_logsync, WT_LOG_SYNC_ENABLED) ? + "none" : + FLD_ISSET(conn->txn_logsync, WT_LOG_DSYNC) ? + "dsync" : + FLD_ISSET(conn->txn_logsync, WT_LOG_FLUSH) ? + "write to OS" : + FLD_ISSET(conn->txn_logsync, WT_LOG_FSYNC) ? "fsync to disk" : "unknown sync setting")); WT_RET(__wt_msg(session, "Log record allocation alignment: %" PRIu32, log->allocsize)); WT_RET(__wt_msg(session, "Current log file number: %" PRIu32, log->fileid)); WT_RET(__wt_msg(session, "Current log version number: %" PRIu16, log->log_version)); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index 0f048ba2e1b..815cafd8047 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -107,7 +107,7 @@ __clsm_enter_update(WT_CURSOR_LSM *clsm) WT_ASSERT(session, F_ISSET(session->txn, WT_TXN_HAS_ID)); have_primary = (primary != NULL && primary_chunk != NULL && (primary_chunk->switch_txn == WT_TXN_NONE || - WT_TXNID_LT(session->txn->id, primary_chunk->switch_txn))); + WT_TXNID_LT(session->txn->id, primary_chunk->switch_txn))); } /* @@ -229,7 +229,7 @@ __clsm_enter(WT_CURSOR_LSM *clsm, bool reset, bool update) if ((!update || txn->isolation != WT_ISO_SNAPSHOT || F_ISSET(clsm, WT_CLSM_OPEN_SNAPSHOT)) && ((update && clsm->primary_chunk != NULL) || - (!update && F_ISSET(clsm, WT_CLSM_OPEN_READ)))) + (!update && F_ISSET(clsm, WT_CLSM_OPEN_READ)))) break; open: @@ -682,14 +682,14 @@ err: if (lsm_tree->custom_generation == 0 || chunk->generation < lsm_tree->custom_generation) { checkpoint = ((WT_CURSOR_BTREE *)cursor)->dhandle->checkpoint; - WT_ASSERT(session, (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && !chunk->empty) ? - checkpoint != NULL : - checkpoint == NULL); + WT_ASSERT(session, + (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && !chunk->empty) ? checkpoint != NULL : + checkpoint == NULL); } /* Make sure the Bloom config matches. */ - WT_ASSERT( - session, (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM) && !F_ISSET(clsm, WT_CLSM_MERGE)) ? + WT_ASSERT(session, + (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM) && !F_ISSET(clsm, WT_CLSM_MERGE)) ? clsm->chunks[i]->bloom != NULL : clsm->chunks[i]->bloom == NULL); } @@ -1386,9 +1386,10 @@ __clsm_put(WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm, const WT_ITEM *key, co lsm_tree = clsm->lsm_tree; - WT_ASSERT(session, F_ISSET(session->txn, WT_TXN_HAS_ID) && clsm->primary_chunk != NULL && + WT_ASSERT(session, + F_ISSET(session->txn, WT_TXN_HAS_ID) && clsm->primary_chunk != NULL && (clsm->primary_chunk->switch_txn == WT_TXN_NONE || - WT_TXNID_LE(session->txn->id, clsm->primary_chunk->switch_txn))); + WT_TXNID_LE(session->txn->id, clsm->primary_chunk->switch_txn))); /* * Clear the existing cursor position. Don't clear the primary cursor: we're about to use it diff --git a/src/third_party/wiredtiger/src/lsm/lsm_manager.c b/src/third_party/wiredtiger/src/lsm/lsm_manager.c index 13246d2c6a4..707ef28c086 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_manager.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_manager.c @@ -261,7 +261,6 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) WT_DECL_RET; WT_LSM_MANAGER *manager; WT_LSM_WORK_UNIT *current; - WT_SESSION *wt_session; uint64_t removed; uint32_t i; @@ -303,10 +302,8 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) } /* Close all LSM worker sessions. */ - for (i = 0; i < WT_LSM_MAX_WORKERS; i++) { - wt_session = &manager->lsm_worker_cookies[i].session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); - } + for (i = 0; i < WT_LSM_MAX_WORKERS; i++) + WT_TRET(__wt_session_close_internal(manager->lsm_worker_cookies[i].session)); } WT_STAT_CONN_INCRV(session, lsm_work_units_discarded, removed); @@ -391,17 +388,15 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session) else if ((!lsm_tree->modified && lsm_tree->nchunks > 1) || (lsm_tree->queue_ref == 0 && lsm_tree->nchunks > 1) || (lsm_tree->merge_aggressiveness > WT_LSM_AGGRESSIVE_THRESHOLD && - !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACTING)) || + !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACTING)) || idlems > fillms) { WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_SWITCH, 0, lsm_tree)); WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_DROP, 0, lsm_tree)); WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_FLUSH, 0, lsm_tree)); WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_BLOOM, 0, lsm_tree)); __wt_verbose(session, WT_VERB_LSM_MANAGER, - "MGR %s: queue %" PRIu32 - " mod %d " - "nchunks %" PRIu32 " flags %#" PRIx32 " aggressive %" PRIu32 " idlems %" PRIu64 - " fillms %" PRIu64, + "MGR %s: queue %" PRIu32 " mod %d nchunks %" PRIu32 " flags %#" PRIx32 + " aggressive %" PRIu32 " idlems %" PRIu64 " fillms %" PRIu64, lsm_tree->name, lsm_tree->queue_ref, lsm_tree->modified, lsm_tree->nchunks, lsm_tree->flags, lsm_tree->merge_aggressiveness, idlems, fillms); WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_MERGE, 0, lsm_tree)); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_merge.c b/src/third_party/wiredtiger/src/lsm/lsm_merge.c index ac586eb9e05..e8fce20860f 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_merge.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_merge.c @@ -113,10 +113,8 @@ __lsm_merge_aggressive_update(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (new_aggressive > lsm_tree->merge_aggressiveness) { __wt_verbose(session, WT_VERB_LSM, - "LSM merge %s got aggressive " - "(old %" PRIu32 " new %" PRIu32 - "), " - "merge_min %u, %" PRIu64 " / %" PRIu64, + "LSM merge %s got aggressive (old %" PRIu32 " new %" PRIu32 "), merge_min %u, %" PRIu64 + " / %" PRIu64, lsm_tree->name, lsm_tree->merge_aggressiveness, new_aggressive, lsm_tree->merge_min, msec_since_last_merge, lsm_tree->chunk_fill_ms); lsm_tree->merge_aggressiveness = new_aggressive; @@ -243,8 +241,9 @@ retry_find: * small chunks in the middle. */ if ((chunk_size += chunk->size) > lsm_tree->chunk_max) - if (nchunks < merge_min || (chunk->generation > youngest->generation && - chunk_size - youngest->size > lsm_tree->chunk_max)) + if (nchunks < merge_min || + (chunk->generation > youngest->generation && + chunk_size - youngest->size > lsm_tree->chunk_max)) break; /* Track chunk generations seen while looking for a merge */ @@ -267,7 +266,7 @@ retry_find: */ if (chunk_size > lsm_tree->chunk_max || (nchunks == merge_max && start_chunk > 0 && - chunk->generation == lsm_tree->chunk[start_chunk - 1]->generation)) { + chunk->generation == lsm_tree->chunk[start_chunk - 1]->generation)) { /* * Try again with smaller range. Unfortunately all the intermediate state will be reset. * Since there's no easy way to restore youngest_gen and oldest_gen. @@ -374,13 +373,13 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) * in the conditional. Avoid the loop in the normal path. */ if (WT_VERBOSE_ISSET(session, WT_VERB_LSM)) { - __wt_verbose(session, WT_VERB_LSM, "Merging %s chunks %u-%u into %u (%" PRIu64 - " records)" - ", generation %" PRIu32, + __wt_verbose(session, WT_VERB_LSM, + "Merging %s chunks %u-%u into %u (%" PRIu64 " records), generation %" PRIu32, lsm_tree->name, start_chunk, end_chunk, dest_id, record_count, generation); for (verb = start_chunk; verb < end_chunk + 1; verb++) - __wt_verbose(session, WT_VERB_LSM, "Merging %s: Chunk[%u] id %" PRIu32 ", gen: %" PRIu32 - ", size: %" PRIu64 ", records: %" PRIu64, + __wt_verbose(session, WT_VERB_LSM, + "Merging %s: Chunk[%u] id %" PRIu32 ", gen: %" PRIu32 ", size: %" PRIu64 + ", records: %" PRIu64, lsm_tree->name, verb, lsm_tree->chunk[verb]->id, lsm_tree->chunk[verb]->generation, lsm_tree->chunk[verb]->size, lsm_tree->chunk[verb]->count); } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_meta.c b/src/third_party/wiredtiger/src/lsm/lsm_meta.c index cfcb23de202..cf5ba61bdbf 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_meta.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_meta.c @@ -199,8 +199,8 @@ __lsm_meta_read_v1(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char * if (FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OFF) && FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST)) WT_ERR_MSG(session, EINVAL, - "Bloom filters can only be created on newest and oldest " - "chunks if bloom filters are enabled"); + "Bloom filters can only be created on newest and oldest chunks if bloom filters are " + "enabled"); WT_ERR(__wt_config_getones(session, lsmconf, "lsm.bloom_bit_count", &cv)); lsm_tree->bloom_bit_count = (uint32_t)cv.val; @@ -228,8 +228,8 @@ __lsm_meta_read_v1(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char * if (lsm_tree->chunk_size > lsm_tree->chunk_max) WT_ERR_MSG(session, EINVAL, - "Chunk size (chunk_size) must be smaller than or equal to " - "the maximum chunk size (chunk_max)"); + "Chunk size (chunk_size) must be smaller than or equal to the maximum chunk size " + "(chunk_max)"); WT_ERR(__wt_config_getones(session, lsmconf, "lsm.merge_max", &cv)); lsm_tree->merge_max = (uint32_t)cv.val; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 6e71c0434c5..236120c29f3 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -439,9 +439,9 @@ __lsm_tree_open_check(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) required = WT_LSM_TREE_MINIMUM_SIZE(lsm_tree->chunk_size, lsm_tree->merge_max, maxleafpage); if (conn->cache_size < required) - WT_RET_MSG(session, EINVAL, "LSM cache size %" PRIu64 " (%" PRIu64 - "MB) too small, " - "must be at least %" PRIu64 " (%" PRIu64 "MB)", + WT_RET_MSG(session, EINVAL, + "LSM cache size %" PRIu64 " (%" PRIu64 "MB) too small, must be at least %" PRIu64 + " (%" PRIu64 "MB)", conn->cache_size, conn->cache_size / WT_MEGABYTE, required, (required + (WT_MEGABYTE - 1)) / WT_MEGABYTE); return (0); @@ -1083,8 +1083,9 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) * There is no work to do if there is only a single chunk in the tree and it has a bloom filter * or is configured to never have a bloom filter. */ - if (lsm_tree->nchunks == 1 && (!FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST) || - F_ISSET(lsm_tree->chunk[0], WT_LSM_CHUNK_BLOOM))) { + if (lsm_tree->nchunks == 1 && + (!FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST) || + F_ISSET(lsm_tree->chunk[0], WT_LSM_CHUNK_BLOOM))) { __wt_lsm_tree_release(session, lsm_tree); return (0); } @@ -1174,9 +1175,8 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) if (flushing) { WT_ASSERT(session, chunk != NULL); if (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) { - __wt_verbose(session, WT_VERB_LSM, "Compact flush done %s chunk %" PRIu32 - ". " - "Start compacting progress %" PRIu64, + __wt_verbose(session, WT_VERB_LSM, + "Compact flush done %s chunk %" PRIu32 ". Start compacting progress %" PRIu64, name, chunk->id, lsm_tree->merge_progressing); (void)__wt_atomic_sub32(&chunk->refcnt, 1); flushing = ref = false; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index ed4a5b43c4b..3d8c48f901a 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -572,8 +572,7 @@ __lsm_bloom_create( WT_ERR_NOTFOUND_OK(__wt_bloom_get(bloom, &key), false); __wt_verbose(session, WT_VERB_LSM, - "LSM worker created bloom filter %s. " - "Expected %" PRIu64 " items, got %" PRIu64, + "LSM worker created bloom filter %s. Expected %" PRIu64 " items, got %" PRIu64, chunk->bloom_uri, chunk->count, insert_count); /* Ensure the bloom filter is in the metadata. */ diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index cb536467878..c555279fca6 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -11,7 +11,6 @@ static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *); static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **); static int __ckpt_load(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *); -static int __ckpt_load_blk_mods(WT_SESSION_IMPL *, const char *, WT_CKPT *); static int __ckpt_named(WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *); static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *, bool); static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *); @@ -73,6 +72,16 @@ __ckpt_load_blk_mods(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt blk_mod->nbits = (uint64_t)b.val; WT_RET(__wt_config_subgets(session, &v, "offset", &b)); blk_mod->offset = (uint64_t)b.val; + /* + * The rename configuration string component was added later. So don't error if we don't + * find it in the string. If we don't have it, we're not doing a rename. + */ + ret = __wt_config_subgets(session, &v, "rename", &b); + WT_RET_NOTFOUND_OK(ret); + if (ret == 0 && b.val) + F_SET(blk_mod, WT_BLOCK_MODS_RENAME); + else + F_CLR(blk_mod, WT_BLOCK_MODS_RENAME); ret = __wt_config_subgets(session, &v, "blocks", &b); WT_RET_NOTFOUND_OK(ret); if (ret != WT_NOTFOUND) { @@ -355,9 +364,8 @@ __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT * */ WT_ERR(__wt_config_gets(session, filecfg, "encryption", &cval)); WT_ERR(__wt_buf_fmt(session, b, - "encryption=%.*s," - "block_metadata_encrypted=%s,block_metadata=[%.*s]", - (int)cval.len, cval.str, kencryptor == NULL ? "false" : "true", (int)metadata_len, metadata)); + "encryption=%.*s,block_metadata_encrypted=%s,block_metadata=[%.*s]", (int)cval.len, cval.str, + kencryptor == NULL ? "false" : "true", (int)metadata_len, metadata)); WT_ERR(__wt_strndup(session, b->data, b->size, &ckpt->block_metadata)); err: @@ -388,7 +396,7 @@ __ckpt_compare_order(const void *a, const void *b) * information. */ static int -__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt) +__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt, bool rename) { WT_BLKINCR *blk; WT_BLOCK_MODS *blk_mod; @@ -426,6 +434,10 @@ __ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt) setup = true; } + /* If we are keeping or setting up an entry on a rename, set the flag. */ + if (rename && (!free || setup)) + F_SET(blk_mod, WT_BLOCK_MODS_RENAME); + /* Free any old information if we need to do so. */ if (free && F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) { __wt_free(session, blk_mod->id_str); @@ -450,6 +462,32 @@ __ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt) } /* + * __wt_meta_blk_mods_load -- + * Load the block mods for a given checkpoint and set up all the information to store. + */ +int +__wt_meta_blk_mods_load(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt, bool rename) +{ + /* + * Load most recent checkpoint backup blocks to this checkpoint. + */ + WT_RET(__ckpt_load_blk_mods(session, config, ckpt)); + + WT_RET(__wt_meta_block_metadata(session, config, ckpt)); + + /* + * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of the + * checkpoint's modified blocks from the block manager. + */ + F_SET(ckpt, WT_CKPT_ADD); + if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) { + F_SET(ckpt, WT_CKPT_BLOCK_MODS); + WT_RET(__ckpt_valid_blk_mods(session, ckpt, rename)); + } + return (0); +} + +/* * __wt_meta_ckptlist_get -- * Load all available checkpoint information for a file. */ @@ -523,22 +561,7 @@ __wt_meta_ckptlist_get( __wt_atomic_cas64(&conn->ckpt_most_recent, most_recent, ckpt->sec)) break; } - /* - * Load most recent checkpoint backup blocks to this checkpoint. - */ - WT_ERR(__ckpt_load_blk_mods(session, config, ckpt)); - - WT_ERR(__wt_meta_block_metadata(session, config, ckpt)); - - /* - * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of - * the checkpoint's modified blocks from the block manager. - */ - F_SET(ckpt, WT_CKPT_ADD); - if (F_ISSET(conn, WT_CONN_INCR_BACKUP)) { - F_SET(ckpt, WT_CKPT_BLOCK_MODS); - WT_ERR(__ckpt_valid_blk_mods(session, ckpt)); - } + WT_ERR(__wt_meta_blk_mods_load(session, config, ckpt, false)); } /* Return the array to our caller. */ @@ -764,19 +787,19 @@ __wt_meta_ckptlist_to_meta(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM } /* - * __ckpt_blkmod_to_meta -- + * __wt_ckpt_blkmod_to_meta -- * Add in any modification block string needed, including an empty one. */ -static int -__ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) +int +__wt_ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) { WT_BLOCK_MODS *blk; WT_ITEM bitstring; u_int i; - bool valid; + bool skip_rename, valid; WT_CLEAR(bitstring); - valid = false; + skip_rename = valid = false; for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk) if (F_ISSET(blk, WT_BLOCK_MODS_VALID)) valid = true; @@ -796,11 +819,22 @@ __ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk) { if (!F_ISSET(blk, WT_BLOCK_MODS_VALID)) continue; + + /* + * Occasionally skip including the rename string at all when it's not necessary for + * correctness, that lets us simulate what is generated in the config string by earlier + * versions of WiredTiger + */ + if (FLD_ISSET(S2C(session)->timing_stress_flags, WT_TIMING_STRESS_BACKUP_RENAME) && + !F_ISSET(blk, WT_BLOCK_MODS_RENAME) && __wt_random(&session->rnd) % 10 == 0) + skip_rename = true; + WT_RET(__wt_raw_to_hex(session, blk->bitstring.data, blk->bitstring.size, &bitstring)); WT_RET(__wt_buf_catfmt(session, buf, "%s\"%s\"=(id=%" PRIu32 ",granularity=%" PRIu64 ",nbits=%" PRIu64 ",offset=%" PRIu64 - ",blocks=%.*s)", + "%s,blocks=%.*s)", i == 0 ? "" : ",", blk->id_str, i, blk->granularity, blk->nbits, blk->offset, + skip_rename ? "" : F_ISSET(blk, WT_BLOCK_MODS_RENAME) ? ",rename=1" : ",rename=0", (int)bitstring.size, (char *)bitstring.data)); /* The hex string length should match the appropriate number of bits. */ WT_ASSERT(session, (blk->nbits >> 2) <= bitstring.size); @@ -829,7 +863,7 @@ __wt_meta_ckptlist_set( /* Add backup block modifications for any added checkpoint. */ WT_CKPT_FOREACH (ckptbase, ckpt) if (F_ISSET(ckpt, WT_CKPT_ADD)) - WT_ERR(__ckpt_blkmod_to_meta(session, buf, ckpt)); + WT_ERR(__wt_ckpt_blkmod_to_meta(session, buf, ckpt)); has_lsn = ckptlsn != NULL; if (ckptlsn != NULL) @@ -954,9 +988,8 @@ __ckpt_version_chk(WT_SESSION_IMPL *session, const char *fname, const char *conf (majorv == WT_BTREE_MAJOR_VERSION_MIN && minorv < WT_BTREE_MINOR_VERSION_MIN) || (majorv == WT_BTREE_MAJOR_VERSION_MAX && minorv > WT_BTREE_MINOR_VERSION_MAX)) WT_RET_MSG(session, EACCES, - "%s is an unsupported WiredTiger source file version %d.%d" - "; this WiredTiger build only supports versions from %d.%d " - "to %d.%d", + "%s is an unsupported WiredTiger source file version %d.%d; this WiredTiger build only " + "supports versions from %d.%d to %d.%d", fname, majorv, minorv, WT_BTREE_MAJOR_VERSION_MIN, WT_BTREE_MINOR_VERSION_MIN, WT_BTREE_MAJOR_VERSION_MAX, WT_BTREE_MINOR_VERSION_MAX); return (0); diff --git a/src/third_party/wiredtiger/src/meta/meta_track.c b/src/third_party/wiredtiger/src/meta/meta_track.c index a569e132d75..10d2904dbf7 100644 --- a/src/third_party/wiredtiger/src/meta/meta_track.c +++ b/src/third_party/wiredtiger/src/meta/meta_track.c @@ -536,14 +536,12 @@ __wt_meta_track_destroy(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION *wt_session; conn = S2C(session); /* Close the session used for metadata checkpoints. */ if (conn->meta_ckpt_session != NULL) { - wt_session = &conn->meta_ckpt_session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(conn->meta_ckpt_session)); conn->meta_ckpt_session = NULL; } diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 00427acae80..0ed6257060a 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -232,7 +232,13 @@ __wt_turtle_init(WT_SESSION_IMPL *session) * Discard any turtle setup file left-over from previous runs. This doesn't matter for * correctness, it's just cleaning up random files. */ - WT_RET(__wt_remove_if_exists(session, WT_METADATA_TURTLE_SET, false)); + + if ((ret = __wt_remove_if_exists(session, WT_METADATA_TURTLE_SET, false)) != 0) { + /* If we're a readonly database, we can skip discarding the leftover file. */ + if (ret == EACCES) + ret = 0; + WT_RET(ret); + } /* * If we found a corrupted turtle file, then delete it and create a new. We could die after @@ -277,17 +283,13 @@ __wt_turtle_init(WT_SESSION_IMPL *session) * incremental backup file and a destination database that incorrectly ran recovery. */ if (exist_incr && !exist_isrc) - WT_RET_MSG(session, EINVAL, - "Incremental backup after running recovery " - "is not allowed"); + WT_RET_MSG(session, EINVAL, "Incremental backup after running recovery is not allowed"); /* * If we have a backup file and metadata and turtle files, we want to recreate the metadata * from the backup. */ if (exist_backup) { - WT_RET(__wt_msg(session, - "Both %s and %s exist; recreating metadata from " - "backup", + WT_RET(__wt_msg(session, "Both %s and %s exist; recreating metadata from backup", WT_METADATA_TURTLE, WT_METADATA_BACKUP)); WT_RET(__wt_remove_if_exists(session, WT_METAFILE, false)); WT_RET(__wt_remove_if_exists(session, WT_METADATA_TURTLE, false)); diff --git a/src/third_party/wiredtiger/src/os_common/filename.c b/src/third_party/wiredtiger/src/os_common/filename.c index f91d93c1ebd..3c777281613 100644 --- a/src/third_party/wiredtiger/src/os_common/filename.c +++ b/src/third_party/wiredtiger/src/os_common/filename.c @@ -73,7 +73,7 @@ __wt_filename_construct(WT_SESSION_IMPL *session, const char *path, const char * /* * __wt_remove_if_exists -- - * Remove a file if it exists. + * Remove a file if it exists and return error if WT_CONN_READONLY is set. */ int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) @@ -81,8 +81,12 @@ __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) bool exist; WT_RET(__wt_fs_exist(session, name, &exist)); - if (exist) + if (exist) { + /* Modifications are not allowed on a read only database. */ + if (F_ISSET(S2C(session), WT_CONN_READONLY)) + return (EACCES); WT_RET(__wt_fs_remove(session, name, durable)); + } return (0); } diff --git a/src/third_party/wiredtiger/src/os_common/os_alloc.c b/src/third_party/wiredtiger/src/os_common/os_alloc.c index d36897729c2..779d55620a4 100644 --- a/src/third_party/wiredtiger/src/os_common/os_alloc.c +++ b/src/third_party/wiredtiger/src/os_common/os_alloc.c @@ -106,7 +106,8 @@ __realloc_func(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t byt */ p = *(void **)retp; bytes_allocated = (bytes_allocated_ret == NULL) ? 0 : *bytes_allocated_ret; - WT_ASSERT(session, (p == NULL && bytes_allocated == 0) || + WT_ASSERT(session, + (p == NULL && bytes_allocated == 0) || (p != NULL && (bytes_allocated_ret == NULL || bytes_allocated != 0))); WT_ASSERT(session, bytes_to_allocate != 0); WT_ASSERT(session, bytes_allocated < bytes_to_allocate); @@ -184,7 +185,8 @@ __wt_realloc_aligned( */ p = *(void **)retp; bytes_allocated = (bytes_allocated_ret == NULL) ? 0 : *bytes_allocated_ret; - WT_ASSERT(session, (p == NULL && bytes_allocated == 0) || + WT_ASSERT(session, + (p == NULL && bytes_allocated == 0) || (p != NULL && (bytes_allocated_ret == NULL || bytes_allocated != 0))); WT_ASSERT(session, bytes_to_allocate != 0); WT_ASSERT(session, bytes_allocated < bytes_to_allocate); diff --git a/src/third_party/wiredtiger/src/os_common/os_fhandle.c b/src/third_party/wiredtiger/src/os_common/os_fhandle.c index 91702db3c59..3cd22b65260 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fhandle.c +++ b/src/third_party/wiredtiger/src/os_common/os_fhandle.c @@ -58,7 +58,7 @@ __wt_handle_is_open(WT_SESSION_IMPL *session, const char *name) found = false; hash = __wt_hash_city64(name, strlen(name)); - bucket = hash % WT_HASH_ARRAY_SIZE; + bucket = hash & (conn->hash_size - 1); __wt_spin_lock(session, &conn->fh_lock); @@ -92,7 +92,7 @@ __handle_search(WT_SESSION_IMPL *session, const char *name, WT_FH *newfh, WT_FH found = false; hash = __wt_hash_city64(name, strlen(name)); - bucket = hash % WT_HASH_ARRAY_SIZE; + bucket = hash & (conn->hash_size - 1); __wt_spin_lock(session, &conn->fh_lock); @@ -293,7 +293,7 @@ __handle_close(WT_SESSION_IMPL *session, WT_FH *fh, bool locked) } /* Remove from the list. */ - bucket = fh->name_hash % WT_HASH_ARRAY_SIZE; + bucket = fh->name_hash & (conn->hash_size - 1); WT_FILE_HANDLE_REMOVE(conn, fh, bucket); (void)__wt_atomic_sub32(&conn->open_file_count, 1); diff --git a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c index 3c41b5f3a6f..8c62a47c651 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c +++ b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c @@ -14,7 +14,7 @@ typedef struct { WT_FILE_SYSTEM iface; - TAILQ_HEAD(__wt_fhhash_inmem, __wt_file_handle_inmem) fhhash[WT_HASH_ARRAY_SIZE]; + TAILQ_HEAD(__wt_fhhash_inmem, __wt_file_handle_inmem) * fhhash; TAILQ_HEAD(__wt_fh_inmem_qh, __wt_file_handle_inmem) fhqh; WT_SPINLOCK lock; @@ -27,7 +27,7 @@ static int __im_file_size(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); * Return a matching handle, if one exists. */ static WT_FILE_HANDLE_INMEM * -__im_handle_search(WT_FILE_SYSTEM *file_system, const char *name) +__im_handle_search(WT_SESSION_IMPL *session, WT_FILE_SYSTEM *file_system, const char *name) { WT_FILE_HANDLE_INMEM *im_fh; WT_FILE_SYSTEM_INMEM *im_fs; @@ -36,7 +36,7 @@ __im_handle_search(WT_FILE_SYSTEM *file_system, const char *name) im_fs = (WT_FILE_SYSTEM_INMEM *)file_system; hash = __wt_hash_city64(name, strlen(name)); - bucket = hash % WT_HASH_ARRAY_SIZE; + bucket = hash & (S2C(session)->hash_size - 1); TAILQ_FOREACH (im_fh, &im_fs->fhhash[bucket], hashq) if (strcmp(im_fh->iface.name, name) == 0) break; @@ -64,7 +64,7 @@ __im_handle_remove( return (__wt_set_return(session, EBUSY)); } - bucket = im_fh->name_hash % WT_HASH_ARRAY_SIZE; + bucket = im_fh->name_hash & (S2C(session)->hash_size - 1); WT_FILE_HANDLE_REMOVE(im_fs, im_fh, bucket); /* Clean up private information. */ @@ -174,7 +174,7 @@ __im_fs_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *n __wt_spin_lock(session, &im_fs->lock); - *existp = __im_handle_search(file_system, name) != NULL; + *existp = __im_handle_search(session, file_system, name) != NULL; __wt_spin_unlock(session, &im_fs->lock); return (0); @@ -201,7 +201,7 @@ __im_fs_remove( __wt_spin_lock(session, &im_fs->lock); ret = ENOENT; - if ((im_fh = __im_handle_search(file_system, name)) != NULL) + if ((im_fh = __im_handle_search(session, file_system, name)) != NULL) ret = __im_handle_remove(session, file_system, im_fh, false); __wt_spin_unlock(session, &im_fs->lock); @@ -231,15 +231,15 @@ __im_fs_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char * __wt_spin_lock(session, &im_fs->lock); ret = ENOENT; - if ((im_fh = __im_handle_search(file_system, from)) != NULL) { + if ((im_fh = __im_handle_search(session, file_system, from)) != NULL) { WT_ERR(__wt_strdup(session, to, ©)); __wt_free(session, im_fh->iface.name); im_fh->iface.name = copy; - bucket = im_fh->name_hash % WT_HASH_ARRAY_SIZE; + bucket = im_fh->name_hash & (S2C(session)->hash_size - 1); WT_FILE_HANDLE_REMOVE(im_fs, im_fh, bucket); im_fh->name_hash = __wt_hash_city64(to, strlen(to)); - bucket = im_fh->name_hash % WT_HASH_ARRAY_SIZE; + bucket = im_fh->name_hash & (S2C(session)->hash_size - 1); WT_FILE_HANDLE_INSERT(im_fs, im_fh, bucket); } @@ -266,7 +266,7 @@ __im_fs_size(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *na __wt_spin_lock(session, &im_fs->lock); /* Search for the handle, then get its size. */ - if ((im_fh = __im_handle_search(file_system, name)) == NULL) + if ((im_fh = __im_handle_search(session, file_system, name)) == NULL) ret = __wt_set_return(session, ENOENT); else *sizep = (wt_off_t)im_fh->buf.size; @@ -343,9 +343,8 @@ __im_file_read( __wt_spin_unlock(session, &im_fs->lock); if (ret == 0) return (0); - WT_RET_MSG(session, WT_ERROR, "%s: handle-read: failed to read %" WT_SIZET_FMT - " bytes at " - "offset %" WT_SIZET_FMT, + WT_RET_MSG(session, WT_ERROR, + "%s: handle-read: failed to read %" WT_SIZET_FMT " bytes at offset %" WT_SIZET_FMT, file_handle->name, len, off); } @@ -416,9 +415,8 @@ err: __wt_spin_unlock(session, &im_fs->lock); if (ret == 0) return (0); - WT_RET_MSG(session, ret, "%s: handle-write: failed to write %" WT_SIZET_FMT - " bytes at " - "offset %" WT_SIZET_FMT, + WT_RET_MSG(session, ret, + "%s: handle-write: failed to write %" WT_SIZET_FMT " bytes at offset %" WT_SIZET_FMT, file_handle->name, len, off); } @@ -449,7 +447,7 @@ __im_file_open(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char * * First search the file queue, if we find it, assert there's only a single reference, in-memory * only supports a single handle on any file, for now. */ - im_fh = __im_handle_search(file_system, name); + im_fh = __im_handle_search(session, file_system, name); if (im_fh != NULL) { if (im_fh->ref != 0) @@ -475,7 +473,7 @@ __im_file_open(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char * im_fh->ref = 1; hash = __wt_hash_city64(name, strlen(name)); - bucket = hash % WT_HASH_ARRAY_SIZE; + bucket = hash & (S2C(session)->hash_size - 1); im_fh->name_hash = hash; WT_FILE_HANDLE_INSERT(im_fs, im_fh, bucket); @@ -519,6 +517,7 @@ __im_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session) WT_TAILQ_SAFE_REMOVE_END __wt_spin_destroy(session, &im_fs->lock); + __wt_free(session, im_fs->fhhash); __wt_free(session, im_fs); return (ret); @@ -534,13 +533,14 @@ __wt_os_inmemory(WT_SESSION_IMPL *session) WT_DECL_RET; WT_FILE_SYSTEM *file_system; WT_FILE_SYSTEM_INMEM *im_fs; - u_int i; + uint64_t i; WT_RET(__wt_calloc_one(session, &im_fs)); + WT_ERR(__wt_calloc_def(session, S2C(session)->hash_size, &im_fs->fhhash)); /* Initialize private information. */ TAILQ_INIT(&im_fs->fhqh); - for (i = 0; i < WT_HASH_ARRAY_SIZE; i++) + for (i = 0; i < S2C(session)->hash_size; i++) TAILQ_INIT(&im_fs->fhhash[i]); WT_ERR(__wt_spin_init(session, &im_fs->lock, "in-memory I/O")); @@ -562,6 +562,7 @@ __wt_os_inmemory(WT_SESSION_IMPL *session) return (0); err: + __wt_free(session, im_fs->fhhash); __wt_free(session, im_fs); return (ret); } diff --git a/src/third_party/wiredtiger/src/os_posix/os_fs.c b/src/third_party/wiredtiger/src/os_posix/os_fs.c index ab286c43d2e..2500e3d2865 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fs.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fs.c @@ -78,9 +78,7 @@ __posix_sync(WT_SESSION_IMPL *session, int fd, const char *name, const char *fun * fdatasync or fsync. */ ff_status = FF_IGNORE; - __wt_err(session, ret, - "fcntl(F_FULLFSYNC) failed, falling back to fdatasync " - "or fsync"); + __wt_err(session, ret, "fcntl(F_FULLFSYNC) failed, falling back to fdatasync or fsync"); break; case FF_IGNORE: break; @@ -262,9 +260,9 @@ __posix_fs_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const cha fp = strrchr(from, '/'); tp = strrchr(to, '/'); - same_directory = - (fp == NULL && tp == NULL) || (fp != NULL && tp != NULL && fp - from == tp - to && - memcmp(from, to, (size_t)(fp - from)) == 0); + same_directory = (fp == NULL && tp == NULL) || + (fp != NULL && tp != NULL && fp - from == tp - to && + memcmp(from, to, (size_t)(fp - from)) == 0); if (!same_directory) WT_RET(__posix_directory_sync(session, to)); @@ -415,8 +413,8 @@ __posix_file_read( file_handle->name, pfh->fd, offset, len); /* Assert direct I/O is aligned and a multiple of the alignment. */ - WT_ASSERT( - session, !pfh->direct_io || S2C(session)->buffer_alignment == 0 || + WT_ASSERT(session, + !pfh->direct_io || S2C(session)->buffer_alignment == 0 || (!((uintptr_t)buf & (uintptr_t)(S2C(session)->buffer_alignment - 1)) && len >= S2C(session)->buffer_alignment && len % S2C(session)->buffer_alignment == 0)); @@ -600,8 +598,8 @@ __posix_file_write( file_handle->name, pfh->fd, offset, len); /* Assert direct I/O is aligned and a multiple of the alignment. */ - WT_ASSERT( - session, !pfh->direct_io || S2C(session)->buffer_alignment == 0 || + WT_ASSERT(session, + !pfh->direct_io || S2C(session)->buffer_alignment == 0 || (!((uintptr_t)buf & (uintptr_t)(S2C(session)->buffer_alignment - 1)) && len >= S2C(session)->buffer_alignment && len % S2C(session)->buffer_alignment == 0)); @@ -800,8 +798,8 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const cha WT_SYSCALL_RETRY(((pfh->fd = open(name, f, mode)) == -1 ? -1 : 0), ret); if (ret != 0) WT_ERR_MSG(session, ret, - pfh->direct_io ? "%s: handle-open: open: failed with direct I/O configured, " - "some filesystem types do not support direct I/O" : + pfh->direct_io ? "%s: handle-open: open: failed with direct I/O configured, some " + "filesystem types do not support direct I/O" : "%s: handle-open: open", name); diff --git a/src/third_party/wiredtiger/src/os_win/os_fs.c b/src/third_party/wiredtiger/src/os_win/os_fs.c index 276aae62796..9463252a6bb 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fs.c +++ b/src/third_party/wiredtiger/src/os_win/os_fs.c @@ -268,8 +268,8 @@ __win_file_read( nr = 0; /* Assert direct I/O is aligned and a multiple of the alignment. */ - WT_ASSERT( - session, !win_fh->direct_io || S2C(session)->buffer_alignment == 0 || + WT_ASSERT(session, + !win_fh->direct_io || S2C(session)->buffer_alignment == 0 || (!((uintptr_t)buf & (uintptr_t)(S2C(session)->buffer_alignment - 1)) && len >= S2C(session)->buffer_alignment && len % S2C(session)->buffer_alignment == 0)); @@ -283,8 +283,7 @@ __win_file_read( windows_error = __wt_getlasterror(); ret = __wt_map_windows_error(windows_error); __wt_err(session, ret, - "%s: handle-read: ReadFile: failed to read %lu " - "bytes at offset %" PRIuMAX ": %s", + "%s: handle-read: ReadFile: failed to read %lu bytes at offset %" PRIuMAX ": %s", file_handle->name, chunk, (uintmax_t)offset, __wt_formatmessage(session, windows_error)); return (ret); @@ -416,8 +415,8 @@ __win_file_write( nw = 0; /* Assert direct I/O is aligned and a multiple of the alignment. */ - WT_ASSERT( - session, !win_fh->direct_io || S2C(session)->buffer_alignment == 0 || + WT_ASSERT(session, + !win_fh->direct_io || S2C(session)->buffer_alignment == 0 || (!((uintptr_t)buf & (uintptr_t)(S2C(session)->buffer_alignment - 1)) && len >= S2C(session)->buffer_alignment && len % S2C(session)->buffer_alignment == 0)); @@ -431,8 +430,7 @@ __win_file_write( windows_error = __wt_getlasterror(); ret = __wt_map_windows_error(windows_error); __wt_err(session, ret, - "%s: handle-write: WriteFile: failed to write %lu " - "bytes at offset %" PRIuMAX ": %s", + "%s: handle-write: WriteFile: failed to write %lu bytes at offset %" PRIuMAX ": %s", file_handle->name, chunk, (uintmax_t)offset, __wt_formatmessage(session, windows_error)); return (ret); @@ -530,10 +528,10 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char windows_error = __wt_getlasterror(); ret = __wt_map_windows_error(windows_error); __wt_err(session, ret, - win_fh->direct_io ? "%s: handle-open: CreateFileW: failed with direct " - "I/O configured, some filesystem types do not " - "support direct I/O: %s" : - "%s: handle-open: CreateFileW: %s", + win_fh->direct_io ? + "%s: handle-open: CreateFileW: failed with direct I/O configured, some filesystem " + "types do not support direct I/O: %s" : + "%s: handle-open: CreateFileW: %s", name, __wt_formatmessage(session, windows_error)); WT_ERR(ret); } diff --git a/src/third_party/wiredtiger/src/os_win/os_winerr.c b/src/third_party/wiredtiger/src/os_win/os_winerr.c index cef80d8ad21..56869f0432b 100644 --- a/src/third_party/wiredtiger/src/os_win/os_winerr.c +++ b/src/third_party/wiredtiger/src/os_win/os_winerr.c @@ -38,26 +38,59 @@ __wt_map_windows_error(DWORD windows_error) int windows_error; int posix_error; } list[] = { - {ERROR_ACCESS_DENIED, EACCES}, {ERROR_ALREADY_EXISTS, EEXIST}, {ERROR_ARENA_TRASHED, EFAULT}, - {ERROR_BAD_COMMAND, EFAULT}, {ERROR_BAD_ENVIRONMENT, EFAULT}, {ERROR_BAD_FORMAT, EFAULT}, - {ERROR_BAD_NETPATH, ENOENT}, {ERROR_BAD_NET_NAME, ENOENT}, {ERROR_BAD_PATHNAME, ENOENT}, - {ERROR_BROKEN_PIPE, EPIPE}, {ERROR_CANNOT_MAKE, EACCES}, {ERROR_CHILD_NOT_COMPLETE, ECHILD}, - {ERROR_CURRENT_DIRECTORY, EACCES}, {ERROR_DIRECT_ACCESS_HANDLE, EBADF}, - {ERROR_DIR_NOT_EMPTY, ENOTEMPTY}, {ERROR_DISK_FULL, ENOSPC}, {ERROR_DRIVE_LOCKED, EACCES}, - {ERROR_FAIL_I24, EACCES}, {ERROR_FILENAME_EXCED_RANGE, ENOENT}, {ERROR_FILE_EXISTS, EEXIST}, - {ERROR_FILE_NOT_FOUND, ENOENT}, {ERROR_GEN_FAILURE, EFAULT}, {ERROR_INVALID_ACCESS, EACCES}, - {ERROR_INVALID_BLOCK, EFAULT}, {ERROR_INVALID_DATA, EFAULT}, {ERROR_INVALID_DRIVE, ENOENT}, - {ERROR_INVALID_FUNCTION, EINVAL}, {ERROR_INVALID_HANDLE, EBADF}, - {ERROR_INVALID_PARAMETER, EINVAL}, {ERROR_INVALID_TARGET_HANDLE, EBADF}, - {ERROR_LOCK_FAILED, EBUSY}, {ERROR_LOCK_VIOLATION, EBUSY}, {ERROR_MAX_THRDS_REACHED, EAGAIN}, - {ERROR_NEGATIVE_SEEK, EINVAL}, {ERROR_NESTING_NOT_ALLOWED, EAGAIN}, - {ERROR_NETWORK_ACCESS_DENIED, EACCES}, {ERROR_NOT_ENOUGH_MEMORY, ENOMEM}, - {ERROR_NOT_ENOUGH_QUOTA, ENOMEM}, {ERROR_NOT_LOCKED, EACCES}, {ERROR_NOT_READY, EBUSY}, - {ERROR_NOT_SAME_DEVICE, EXDEV}, {ERROR_NO_DATA, EPIPE}, {ERROR_NO_MORE_FILES, EMFILE}, - {ERROR_NO_PROC_SLOTS, EAGAIN}, {ERROR_PATH_NOT_FOUND, ENOENT}, {ERROR_READ_FAULT, EFAULT}, - {ERROR_RETRY, EINTR}, {ERROR_SEEK_ON_DEVICE, EACCES}, {ERROR_SHARING_VIOLATION, EBUSY}, - {ERROR_TOO_MANY_OPEN_FILES, EMFILE}, {ERROR_WAIT_NO_CHILDREN, ECHILD}, - {ERROR_WRITE_FAULT, EFAULT}, {ERROR_WRITE_PROTECT, EACCES}, + {ERROR_ACCESS_DENIED, EACCES}, + {ERROR_ALREADY_EXISTS, EEXIST}, + {ERROR_ARENA_TRASHED, EFAULT}, + {ERROR_BAD_COMMAND, EFAULT}, + {ERROR_BAD_ENVIRONMENT, EFAULT}, + {ERROR_BAD_FORMAT, EFAULT}, + {ERROR_BAD_NETPATH, ENOENT}, + {ERROR_BAD_NET_NAME, ENOENT}, + {ERROR_BAD_PATHNAME, ENOENT}, + {ERROR_BROKEN_PIPE, EPIPE}, + {ERROR_CANNOT_MAKE, EACCES}, + {ERROR_CHILD_NOT_COMPLETE, ECHILD}, + {ERROR_CURRENT_DIRECTORY, EACCES}, + {ERROR_DIRECT_ACCESS_HANDLE, EBADF}, + {ERROR_DIR_NOT_EMPTY, ENOTEMPTY}, + {ERROR_DISK_FULL, ENOSPC}, + {ERROR_DRIVE_LOCKED, EACCES}, + {ERROR_FAIL_I24, EACCES}, + {ERROR_FILENAME_EXCED_RANGE, ENOENT}, + {ERROR_FILE_EXISTS, EEXIST}, + {ERROR_FILE_NOT_FOUND, ENOENT}, + {ERROR_GEN_FAILURE, EFAULT}, + {ERROR_INVALID_ACCESS, EACCES}, + {ERROR_INVALID_BLOCK, EFAULT}, + {ERROR_INVALID_DATA, EFAULT}, + {ERROR_INVALID_DRIVE, ENOENT}, + {ERROR_INVALID_FUNCTION, EINVAL}, + {ERROR_INVALID_HANDLE, EBADF}, + {ERROR_INVALID_PARAMETER, EINVAL}, + {ERROR_INVALID_TARGET_HANDLE, EBADF}, + {ERROR_LOCK_FAILED, EBUSY}, + {ERROR_LOCK_VIOLATION, EBUSY}, + {ERROR_MAX_THRDS_REACHED, EAGAIN}, + {ERROR_NEGATIVE_SEEK, EINVAL}, + {ERROR_NESTING_NOT_ALLOWED, EAGAIN}, + {ERROR_NETWORK_ACCESS_DENIED, EACCES}, + {ERROR_NOT_ENOUGH_MEMORY, ENOMEM}, + {ERROR_NOT_ENOUGH_QUOTA, ENOMEM}, + {ERROR_NOT_LOCKED, EACCES}, + {ERROR_NOT_READY, EBUSY}, + {ERROR_NOT_SAME_DEVICE, EXDEV}, + {ERROR_NO_DATA, EPIPE}, + {ERROR_NO_MORE_FILES, EMFILE}, + {ERROR_NO_PROC_SLOTS, EAGAIN}, + {ERROR_PATH_NOT_FOUND, ENOENT}, + {ERROR_READ_FAULT, EFAULT}, + {ERROR_RETRY, EINTR}, + {ERROR_SEEK_ON_DEVICE, EACCES}, + {ERROR_SHARING_VIOLATION, EBUSY}, + {ERROR_TOO_MANY_OPEN_FILES, EMFILE}, + {ERROR_WAIT_NO_CHILDREN, ECHILD}, + {ERROR_WRITE_FAULT, EFAULT}, + {ERROR_WRITE_PROTECT, EACCES}, }; int i; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index dc4487321dc..a9eb3443fe0 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -816,8 +816,8 @@ compare: if (rle != 0) { if (WT_TIME_WINDOWS_EQUAL(&tw, &last.tw) && ((deleted && last.deleted) || - (!deleted && !last.deleted && last.value->size == size && - memcmp(last.value->data, data, size) == 0))) { + (!deleted && !last.deleted && last.value->size == size && + memcmp(last.value->data, data, size) == 0))) { /* The time window for deleted keys must be empty. */ WT_ASSERT( session, (!deleted && !last.deleted) || WT_TIME_WINDOW_IS_EMPTY(&last.tw)); @@ -960,8 +960,8 @@ compare: if (rle != 0) { if (WT_TIME_WINDOWS_EQUAL(&last.tw, &tw) && ((deleted && last.deleted) || - (!deleted && !last.deleted && size != 0 && last.value->size == size && - memcmp(last.value->data, data, size) == 0))) { + (!deleted && !last.deleted && size != 0 && last.value->size == size && + memcmp(last.value->data, data, size) == 0))) { /* * The time window for deleted keys must be empty. */ @@ -994,10 +994,10 @@ compare: last.deleted = deleted; rle = 1; - /* - * Move to the next record. It's not a simple increment because if it's the maximum record, - * incrementing it wraps to 0 and this turns into an infinite loop. - */ + /* + * Move to the next record. It's not a simple increment because if it's the maximum + * record, incrementing it wraps to 0 and this turns into an infinite loop. + */ next: if (src_recno == UINT64_MAX) break; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index 2a050333d18..1e185469c99 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -720,7 +720,7 @@ __wt_rec_row_leaf( WT_UPDATE *upd; WT_UPDATE_SELECT upd_select; uint64_t slvg_skip; - uint32_t i, session_flags; + uint32_t i; bool dictionary, key_onpage_ovfl, ovfl_key; void *copy; @@ -918,9 +918,9 @@ __wt_rec_row_leaf( * ever need to blow away history store content, so we can skip this. */ if (!F_ISSET(session, WT_SESSION_NO_DATA_HANDLES)) { - WT_ERR(__wt_hs_cursor_open(session, &session_flags)); + WT_ERR(__wt_hs_cursor_open(session)); WT_ERR(__wt_hs_delete_key_from_ts(session, btree->id, tmpkey, WT_TS_NONE)); - WT_ERR(__wt_hs_cursor_close(session, session_flags)); + WT_ERR(__wt_hs_cursor_close(session)); WT_STAT_CONN_INCR(session, cache_hs_key_truncate_onpage_removal); } } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index 8e11c5edad4..044568b723f 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -33,7 +33,8 @@ __rec_update_save(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, voi /* If nothing is committed, we must restore the update chain. */ WT_ASSERT(session, onpage_upd != NULL || supd_restore); /* We can only write a standard update or a modify to the data store. */ - WT_ASSERT(session, onpage_upd == NULL || onpage_upd->type == WT_UPDATE_STANDARD || + WT_ASSERT(session, + onpage_upd == NULL || onpage_upd->type == WT_UPDATE_STANDARD || onpage_upd->type == WT_UPDATE_MODIFY); WT_RET(__wt_realloc_def(session, &r->supd_allocated, r->supd_next + 1, &r->supd)); @@ -231,7 +232,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v WT_DECL_RET; WT_PAGE *page; WT_TIME_WINDOW *select_tw; - WT_UPDATE *first_txn_upd, *first_upd, *upd, *last_upd, *tombstone; + WT_UPDATE *first_txn_upd, *first_upd, *upd, *last_upd, *same_txn_valid_upd, *tombstone; wt_timestamp_t max_ts; size_t upd_memsize; uint64_t max_txn, txnid; @@ -248,7 +249,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v WT_TIME_WINDOW_INIT(select_tw); page = r->page; - first_txn_upd = upd = last_upd = tombstone = NULL; + first_txn_upd = upd = last_upd = same_txn_valid_upd = tombstone = NULL; upd_memsize = 0; max_ts = WT_TS_NONE; max_txn = WT_TXN_NONE; @@ -331,8 +332,9 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v * Otherwise, it removes the key. */ WT_ASSERT(session, - F_ISSET(r, WT_REC_EVICT) || (F_ISSET(r, WT_REC_VISIBILITY_ERR) && - F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS))); + F_ISSET(r, WT_REC_EVICT) || + (F_ISSET(r, WT_REC_VISIBILITY_ERR) && + F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS))); WT_ASSERT(session, upd->prepare_state == WT_PREPARE_INPROGRESS); } } @@ -362,8 +364,8 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v * The checkpoint transaction is special. Make sure we never write metadata updates from a * checkpoint in a concurrent session. */ - WT_ASSERT(session, !WT_IS_METADATA(session->dhandle) || upd == NULL || - upd->txnid == WT_TXN_NONE || + WT_ASSERT(session, + !WT_IS_METADATA(session->dhandle) || upd == NULL || upd->txnid == WT_TXN_NONE || upd->txnid != S2C(session)->txn_global.checkpoint_txn_shared.id || WT_SESSION_IS_CHECKPOINT(session)); @@ -420,20 +422,77 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v /* Find the update this tombstone applies to. */ if (!__wt_txn_upd_visible_all(session, upd)) { - while (upd->next != NULL && upd->next->txnid == WT_TXN_ABORTED) - upd = upd->next; + /* + * Loop until a valid update from a different transaction is found in the update + * list. + */ + while (upd->next != NULL) { + if (upd->next->txnid == WT_TXN_ABORTED) + upd = upd->next; + else if (upd->next->txnid != WT_TXN_NONE && + tombstone->txnid == upd->next->txnid) { + upd = upd->next; + /* Save the latest update from the same transaction. */ + if (same_txn_valid_upd == NULL) + same_txn_valid_upd = upd; + } else + break; + } + WT_ASSERT(session, upd->next == NULL || upd->next->txnid != WT_TXN_ABORTED); - if (upd->next == NULL) - last_upd = upd; upd_select->upd = upd = upd->next; + + /* + * If there is no on-disk update and any valid update from a different transaction + * is not found in the update list, write the same transaction update itself to disk + * to avoid blocking the eviction. + */ + if (vpack == NULL && upd == NULL) + upd_select->upd = upd = same_txn_valid_upd; + else if (upd != NULL && upd->type == WT_UPDATE_TOMBSTONE) { + /* + * The selected update from a different transaction is also a tombstone, use the + * update from the same transaction as the selected update. + */ + WT_ASSERT(session, + same_txn_valid_upd != NULL && + same_txn_valid_upd->type != WT_UPDATE_TOMBSTONE); + upd_select->upd = upd = same_txn_valid_upd; + } else if (same_txn_valid_upd != NULL && vpack != NULL && + WT_TIME_WINDOW_HAS_STOP(&vpack->tw)) { + /* + * The on-disk version has a valid stop timestamp, use the update from the same + * transaction as the selected update. + */ + WT_ASSERT(session, same_txn_valid_upd->type != WT_UPDATE_TOMBSTONE); + upd_select->upd = upd = same_txn_valid_upd; + } } } if (upd != NULL) /* The beginning of the validity window is the selected update's time point. */ WT_TIME_WINDOW_SET_START(select_tw, upd); else if (select_tw->stop_ts != WT_TS_NONE || select_tw->stop_txn != WT_TXN_NONE) { - /* If we only have a tombstone in the update list, we must have an ondisk value. */ - WT_ASSERT(session, vpack != NULL && tombstone != NULL && last_upd->next == NULL); + /* + * We only have a tombstone on the update list or all the updates are from the same + * transaction. + */ + WT_ASSERT(session, tombstone != NULL); + + /* We must have an ondisk value and it can't be a prepared update. */ + WT_ASSERT(session, vpack != NULL && vpack->type != WT_CELL_DEL && !vpack->tw.prepare); + + /* Move the pointer to the last update on the update chain. */ + for (last_upd = tombstone; last_upd->next != NULL; last_upd = last_upd->next) + /* + * Tombstone is the only non-aborted update on the update chain or all the updates + * are from the same transaction. + */ + WT_ASSERT(session, + last_upd->next->txnid == WT_TXN_ABORTED || + (last_upd->next->txnid == tombstone->txnid && + last_upd->next->start_ts == tombstone->start_ts)); + /* * It's possible to have a tombstone as the only update in the update list. If we * reconciled before with only a single update and then read the page back into cache, @@ -443,28 +502,46 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v * In this case, we should leave the selected update unset to indicate that we want to * keep the same on-disk value but set the stop time point to indicate that the validity * window ends when this tombstone started. + * + * FIXME-WT-6557: no need to check this after WT-6557 is done as the tombstone will be + * freed when it is written to the disk image in the previous eviction. */ - WT_ERR(__rec_append_orig_value(session, page, tombstone, vpack)); + if (!F_ISSET(tombstone, WT_UPDATE_RESTORED_FROM_DS | WT_UPDATE_RESTORED_FROM_HS)) { + WT_ERR(__rec_append_orig_value(session, page, tombstone, vpack)); - /* - * We may have updated the global transaction concurrently and the tombstone is now - * globally visible. In this case, the on page value is not appended. Check that. - */ - if (last_upd->next != NULL) { - WT_ASSERT(session, last_upd->next->txnid == vpack->tw.start_txn && - last_upd->next->start_ts == vpack->tw.start_ts && - last_upd->next->type == WT_UPDATE_STANDARD && last_upd->next->next == NULL); - upd_select->upd = last_upd->next; - WT_TIME_WINDOW_SET_START(select_tw, last_upd->next); - } else { /* - * If the tombstone is aborted concurrently, we should still have appended the - * onpage value. + * We may have updated the global transaction concurrently and the tombstone is now + * globally visible. In this case, the on page value is not appended. Verify that. */ - WT_ASSERT(session, tombstone->txnid != WT_TXN_ABORTED && - __wt_txn_upd_visible_all(session, tombstone) && upd_select->upd == NULL); - upd_select->upd = tombstone; - } + if (last_upd->next != NULL) { + WT_ASSERT(session, + last_upd->next->txnid == vpack->tw.start_txn && + last_upd->next->start_ts == vpack->tw.start_ts && + last_upd->next->type == WT_UPDATE_STANDARD && last_upd->next->next == NULL); + upd_select->upd = last_upd->next; + WT_TIME_WINDOW_SET_START(select_tw, last_upd->next); + } else { + /* + * It's possible that onpage value is not appended if the tombstone becomes + * globally visible because the oldest transaction id or the oldest timestamp is + * moved concurrently. + * + * If the tombstone is aborted concurrently, we should still have appended the + * onpage value. + */ + WT_ASSERT(session, + tombstone->txnid != WT_TXN_ABORTED && + __wt_txn_upd_visible_all(session, tombstone) && upd_select->upd == NULL); + upd_select->upd = tombstone; + } + } else + /* + * If the tombstone is restored from the disk or history store, it must have already + * been written to the disk image in the previous eviction. + */ + WT_ASSERT(session, + upd_select->upd == NULL && vpack->tw.durable_stop_ts == tombstone->durable_ts && + vpack->tw.stop_txn == tombstone->txnid); } } @@ -520,7 +597,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v */ supd_restore = F_ISSET(r, WT_REC_EVICT) && (has_newer_updates || F_ISSET(S2C(session), WT_CONN_IN_MEMORY) || - page->type == WT_PAGE_COL_FIX); + page->type == WT_PAGE_COL_FIX); if (supd_restore) r->cache_write_restore = true; WT_ERR(__rec_update_save(session, r, ins, ripcip, diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index d2242a9b2a0..929728740dc 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -48,7 +48,8 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage * doesn't apply to checkpoints: there are (rare) cases where we write data at read-uncommitted * isolation. */ - WT_ASSERT(session, !LF_ISSET(WT_REC_EVICT) || LF_ISSET(WT_REC_VISIBLE_ALL) || + WT_ASSERT(session, + !LF_ISSET(WT_REC_EVICT) || LF_ISSET(WT_REC_VISIBLE_ALL) || F_ISSET(session->txn, WT_TXN_HAS_SNAPSHOT)); /* It's an error to be called with a clean page. */ @@ -318,7 +319,8 @@ __rec_write_page_status(WT_SESSION_IMPL *session, WT_RECONCILE *r) * eviction case. Otherwise, we must be reconciling a fixed length column store page (which * does not allow history store content). */ - WT_ASSERT(session, !F_ISSET(r, WT_REC_EVICT) || + WT_ASSERT(session, + !F_ISSET(r, WT_REC_EVICT) || (F_ISSET(r, WT_REC_HS | WT_REC_IN_MEMORY) || page->type == WT_PAGE_COL_FIX)); } else { /* @@ -1848,7 +1850,8 @@ copy_image: * The I/O routines verify all disk images we write, but there are paths in reconciliation that * don't do I/O. Verify those images, too. */ - WT_ASSERT(session, verify_image == false || + WT_ASSERT(session, + verify_image == false || __wt_verify_dsk_image( session, "[reconcile-image]", chunk->image.data, 0, &multi->addr, true) == 0); #endif @@ -2027,7 +2030,7 @@ __rec_split_dump_keys(WT_SESSION_IMPL *session, WT_RECONCILE *r) for (multi = r->multi, i = 0; i < r->multi_next; ++multi, ++i) __wt_verbose(session, WT_VERB_SPLIT, "starting key %s", __wt_buf_set_printable( - session, WT_IKEY_DATA(multi->key.ikey), multi->key.ikey->size, tkey)); + session, WT_IKEY_DATA(multi->key.ikey), multi->key.ikey->size, tkey)); __wt_scr_free(session, &tkey); } else for (multi = r->multi, i = 0; i < r->multi_next; ++multi, ++i) @@ -2148,20 +2151,21 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) mod->rec_result = WT_PM_REC_EMPTY; break; case 1: /* 1-for-1 page swap */ - /* - * Because WiredTiger's pages grow without splitting, we're replacing a single page with - * another single page most of the time. - * - * If in-memory, or saving/restoring changes for this page and there's only one block, - * there's nothing to write. Set up a single block as if to split, then use that disk - * image to rewrite the page in memory. This is separate from simple replacements where - * eviction has decided to retain the page in memory because the latter can't handle - * update lists and splits can. - */ + /* + * Because WiredTiger's pages grow without splitting, we're replacing a single page with + * another single page most of the time. + * + * If in-memory, or saving/restoring changes for this page and there's only one block, + * there's nothing to write. Set up a single block as if to split, then use that disk image + * to rewrite the page in memory. This is separate from simple replacements where eviction + * has decided to retain the page in memory because the latter can't handle update lists and + * splits can. + */ if (F_ISSET(r, WT_REC_IN_MEMORY) || r->multi->supd_restore) { - WT_ASSERT(session, F_ISSET(r, WT_REC_IN_MEMORY) || + WT_ASSERT(session, + F_ISSET(r, WT_REC_IN_MEMORY) || (F_ISSET(r, WT_REC_EVICT) && (r->leave_dirty || page->type == WT_PAGE_COL_FIX) && - r->multi->supd_entries != 0)); + r->multi->supd_entries != 0)); goto split; } @@ -2192,10 +2196,10 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) if (WT_VERBOSE_ISSET(session, WT_VERB_SPLIT)) WT_RET(__rec_split_dump_keys(session, r)); - /* - * The reuse flag was set in some cases, but we have to clear it, otherwise on subsequent - * reconciliation we would fail to remove blocks that are being discarded. - */ + /* + * The reuse flag was set in some cases, but we have to clear it, otherwise on subsequent + * reconciliation we would fail to remove blocks that are being discarded. + */ split: for (multi = r->multi, i = 0; i < r->multi_next; ++multi, ++i) multi->addr.reuse = 0; @@ -2263,7 +2267,7 @@ __rec_hs_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r) { WT_DECL_RET; WT_MULTI *multi; - uint32_t i, session_flags; + uint32_t i; /* Check if there's work to do. */ for (multi = r->multi, i = 0; i < r->multi_next; ++multi, ++i) @@ -2272,7 +2276,7 @@ __rec_hs_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r) if (i == r->multi_next) return (0); - WT_RET(__wt_hs_cursor_open(session, &session_flags)); + WT_RET(__wt_hs_cursor_open(session)); for (multi = r->multi, i = 0; i < r->multi_next; ++multi, ++i) if (multi->supd != NULL) { @@ -2285,7 +2289,7 @@ __rec_hs_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r) } err: - WT_TRET(__wt_hs_cursor_close(session, session_flags)); + WT_TRET(__wt_hs_cursor_close(session)); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index ef1c5d96aba..093d072a611 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -36,9 +36,8 @@ __wt_direct_io_size_check( align = (int64_t)conn->buffer_alignment; if (align != 0 && (cval.val < align || cval.val % align != 0)) WT_RET_MSG(session, EINVAL, - "when direct I/O is configured, the %s size must " - "be at least as large as the buffer alignment as " - "well as a multiple of the buffer alignment", + "when direct I/O is configured, the %s size must be at least as large as the buffer " + "alignment as well as a multiple of the buffer alignment", config_name); } *allocsizep = (uint32_t)cval.val; @@ -378,10 +377,8 @@ __create_index(WT_SESSION_IMPL *session, const char *name, bool exclusive, const WT_PREFIX_SKIP_REQUIRED(session, tablename, "index:"); idxname = strchr(tablename, ':'); if (idxname == NULL) - WT_RET_MSG(session, EINVAL, - "Invalid index name, " - "should be <table name>:<index name>: %s", - name); + WT_RET_MSG( + session, EINVAL, "Invalid index name, should be <table name>:<index name>: %s", name); /* * Note: it would be better to keep the table exclusive here, while changing its indexes. We @@ -470,8 +467,7 @@ __create_index(WT_SESSION_IMPL *session, const char *name, bool exclusive, const if (__wt_config_subgetraw(session, &icols, &ckey, &cval) == 0) { if (have_extractor) WT_ERR_MSG(session, EINVAL, - "an index with a custom extractor may not " - "include primary key columns"); + "an index with a custom extractor may not include primary key columns"); continue; } WT_ERR(__wt_buf_catfmt(session, &extra_cols, "%.*s,", (int)ckey.len, ckey.str)); @@ -495,9 +491,8 @@ __create_index(WT_SESSION_IMPL *session, const char *name, bool exclusive, const /* Check for a record number index key, which makes no sense. */ WT_ERR(__wt_config_getones(session, fmt.data, "key_format", &cval)); if (cval.len == 1 && cval.str[0] == 'r') - WT_ERR_MSG(session, EINVAL, - "column-store index may not use the record number as its " - "index key"); + WT_ERR_MSG( + session, EINVAL, "column-store index may not use the record number as its index key"); WT_ERR(__wt_buf_catfmt(session, &fmt, ",index_key_columns=%u", npublic_cols)); @@ -625,9 +620,7 @@ __create_data_source( */ if (__wt_config_getones_none(session, config, "collator", &cval) != WT_NOTFOUND && cval.len != 0) - WT_RET_MSG(session, EINVAL, - "WT_DATA_SOURCE objects do not support WT_COLLATOR " - "ordering"); + WT_RET_MSG(session, EINVAL, "WT_DATA_SOURCE objects do not support WT_COLLATOR ordering"); return (dsrc->create(dsrc, &session->iface, uri, (WT_CONFIG_ARG *)cfg)); } diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 26d5df5bb4f..8f924041e08 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -29,9 +29,7 @@ __wt_schema_get_table_uri( if (!ok_incomplete && !table->cg_complete) { WT_ERR(__wt_session_release_dhandle(session)); ret = __wt_set_return(session, EINVAL); - WT_ERR_MSG(session, ret, - "'%s' cannot be used " - "until all column groups are created", + WT_ERR_MSG(session, ret, "'%s' cannot be used until all column groups are created", table->iface.name); } *tablep = table; diff --git a/src/third_party/wiredtiger/src/schema/schema_open.c b/src/third_party/wiredtiger/src/schema/schema_open.c index 02afbcd6b02..79a07f95912 100644 --- a/src/third_party/wiredtiger/src/schema/schema_open.c +++ b/src/third_party/wiredtiger/src/schema/schema_open.c @@ -372,9 +372,9 @@ __wt_schema_open_index( { WT_DECL_RET; - WT_WITH_TABLE_WRITE_LOCK( - session, WT_WITH_TXN_ISOLATION(session, WT_ISO_READ_UNCOMMITTED, - ret = __schema_open_index(session, table, idxname, len, indexp))); + WT_WITH_TABLE_WRITE_LOCK(session, + WT_WITH_TXN_ISOLATION(session, WT_ISO_READ_UNCOMMITTED, + ret = __schema_open_index(session, table, idxname, len, indexp))); return (ret); } @@ -555,8 +555,9 @@ __wt_schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) { WT_DECL_RET; - WT_WITH_TABLE_WRITE_LOCK(session, WT_WITH_TXN_ISOLATION(session, WT_ISO_READ_UNCOMMITTED, - ret = __schema_open_table(session, cfg))); + WT_WITH_TABLE_WRITE_LOCK(session, + WT_WITH_TXN_ISOLATION( + session, WT_ISO_READ_UNCOMMITTED, ret = __schema_open_table(session, cfg))); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_plan.c b/src/third_party/wiredtiger/src/schema/schema_plan.c index 5bed57b429c..153a7ba1944 100644 --- a/src/third_party/wiredtiger/src/schema/schema_plan.c +++ b/src/third_party/wiredtiger/src/schema/schema_plan.c @@ -108,8 +108,7 @@ __wt_schema_colcheck(WT_SESSION_IMPL *session, const char *key_format, const cha if (ncols != 0 && ncols != kcols + vcols) WT_RET_MSG(session, EINVAL, - "Number of columns in '%.*s' " - "does not match key format '%s' plus value format '%s'", + "Number of columns in '%.*s' does not match key format '%s' plus value format '%s'", (int)colconf->len, colconf->str, key_format, value_format); if (kcolsp != NULL) @@ -146,9 +145,7 @@ __wt_table_check(WT_SESSION_IMPL *session, WT_TABLE *table) coltype = 0; while ((ret = __wt_config_next(&conf, &k, &v)) == 0) { if (__find_next_col(session, table, &k, &cg, &col, &coltype) != 0) - WT_RET_MSG(session, EINVAL, - "Column '%.*s' in '%s' does not appear in a " - "column group", + WT_RET_MSG(session, EINVAL, "Column '%.*s' in '%s' does not appear in a column group", (int)k.len, k.str, table->iface.name); /* * Column groups can't store key columns in their value: @@ -334,9 +331,7 @@ __wt_struct_reformat(WT_SESSION_IMPL *session, WT_TABLE *table, const char *colu if ((ret = __find_column_format(session, table, &k, value_only, &pv)) != 0) { if (value_only && ret == EINVAL) WT_RET_MSG(session, EINVAL, - "A column group cannot store key column " - "'%.*s' in its value", - (int)k.len, k.str); + "A column group cannot store key column '%.*s' in its value", (int)k.len, k.str); WT_RET_MSG(session, EINVAL, "Column '%.*s' not found", (int)k.len, k.str); } diff --git a/src/third_party/wiredtiger/src/schema/schema_project.c b/src/third_party/wiredtiger/src/schema/schema_project.c index 54f60e3be71..d3201d20d5c 100644 --- a/src/third_party/wiredtiger/src/schema/schema_project.c +++ b/src/third_party/wiredtiger/src/schema/schema_project.c @@ -100,7 +100,7 @@ __wt_schema_project_in(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *pro case WT_PROJ_NEXT: WT_RET(__pack_next(&pack, &pv)); WT_PACK_GET(session, pv, ap); - /* FALLTHROUGH */ + /* FALLTHROUGH */ case WT_PROJ_REUSE: /* Read the item we're about to overwrite. */ @@ -301,7 +301,7 @@ __wt_schema_project_slice(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char * case WT_PROJ_NEXT: WT_RET(__pack_next(&vpack, &vpv)); WT_RET(__unpack_read(session, &vpv, &vp, (size_t)(vend - vp))); - /* FALLTHROUGH */ + /* FALLTHROUGH */ case WT_PROJ_REUSE: if (skip) diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index 3dfc99493c4..53c006ed226 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -9,14 +9,40 @@ #include "wt_internal.h" /* + * __rename_blkmod -- + * Reset the incremental backup information for a rename. + */ +static int +__rename_blkmod(WT_SESSION_IMPL *session, const char *oldvalue, WT_ITEM *buf) +{ + WT_CKPT ckpt; + WT_DECL_RET; + + WT_CLEAR(ckpt); + /* + * Replace the old file entries with new file entries. We need to recreate the incremental + * backup information to indicate copying the entire file in its bitmap. + */ + /* First load any existing backup information into a temp checkpoint structure. */ + WT_RET(__wt_meta_blk_mods_load(session, oldvalue, &ckpt, true)); + + /* Take the checkpoint structure and generate the metadata string. */ + ret = __wt_ckpt_blkmod_to_meta(session, buf, &ckpt); + __wt_meta_checkpoint_free(session, &ckpt); + return (ret); +} + +/* * __rename_file -- * WT_SESSION::rename for a file. */ static int __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) { + WT_DECL_ITEM(buf); WT_DECL_RET; char *newvalue, *oldvalue; + const char *filecfg[3] = {NULL, NULL, NULL}; const char *filename, *newfile; bool exist; @@ -33,6 +59,7 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) WT_WITH_HANDLE_LIST_WRITE_LOCK( session, ret = __wt_conn_dhandle_close_all(session, uri, true, false)); WT_ERR(ret); + WT_ERR(__wt_scr_alloc(session, 1024, &buf)); /* * First, check if the file being renamed exists in the system. Doing this check first matches @@ -54,13 +81,20 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) default: WT_ERR(ret); } + __wt_free(session, newvalue); WT_ERR(__wt_fs_exist(session, newfile, &exist)); if (exist) WT_ERR_MSG(session, EEXIST, "%s", newfile); - /* Replace the old file entries with new file entries. */ WT_ERR(__wt_metadata_remove(session, uri)); - WT_ERR(__wt_metadata_insert(session, newuri, oldvalue)); + filecfg[0] = oldvalue; + if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) { + WT_ERR(__rename_blkmod(session, oldvalue, buf)); + filecfg[1] = buf->mem; + } else + filecfg[1] = NULL; + WT_ERR(__wt_config_collapse(session, filecfg, &newvalue)); + WT_ERR(__wt_metadata_insert(session, newuri, newvalue)); /* Rename the underlying file. */ WT_ERR(__wt_fs_rename(session, filename, newfile, false)); @@ -68,6 +102,7 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) WT_ERR(__wt_meta_track_fileop(session, uri, newuri)); err: + __wt_scr_free(session, &buf); __wt_free(session, newvalue); __wt_free(session, oldvalue); return (ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_util.c b/src/third_party/wiredtiger/src/schema/schema_util.c index 96a7537e980..d5655f21a7c 100644 --- a/src/third_party/wiredtiger/src/schema/schema_util.c +++ b/src/third_party/wiredtiger/src/schema/schema_util.c @@ -100,12 +100,8 @@ __wt_schema_internal_session(WT_SESSION_IMPL *session, WT_SESSION_IMPL **int_ses int __wt_schema_session_release(WT_SESSION_IMPL *session, WT_SESSION_IMPL *int_session) { - WT_SESSION *wt_session; - - if (session != int_session) { - wt_session = &int_session->iface; - WT_RET(wt_session->close(wt_session, NULL)); - } + if (session != int_session) + WT_RET(__wt_session_close_internal(int_session)); return (0); } @@ -121,9 +117,7 @@ __str_name_check(WT_SESSION_IMPL *session, const char *name, bool skip_wt) if (!skip_wt && WT_PREFIX_MATCH(name, "WiredTiger")) WT_RET_MSG(session, EINVAL, - "%s: the \"WiredTiger\" name space may not be " - "used by applications", - name); + "%s: the \"WiredTiger\" name space may not be used by applications", name); /* * Disallow JSON quoting characters -- the config string parsing code supports quoted strings, @@ -131,9 +125,7 @@ __str_name_check(WT_SESSION_IMPL *session, const char *name, bool skip_wt) */ if (strpbrk(name, "{},:[]\\\"'") != NULL) WT_RET_MSG(session, EINVAL, - "%s: WiredTiger objects should not include grouping " - "characters in their names", - name); + "%s: WiredTiger objects should not include grouping characters in their names", name); return (0); } diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 895c555fc44..ac7cef167ff 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -82,7 +82,7 @@ __wt_session_cursor_cache_sweep(WT_SESSION_IMPL *session) for (i = 0; i < WT_SESSION_CURSOR_SWEEP_MAX && productive; i++) { ++nbuckets; cached_list = &session->cursor_cache[position]; - position = (position + 1) % WT_HASH_ARRAY_SIZE; + position = (position + 1) & (S2C(session)->hash_size - 1); TAILQ_FOREACH_SAFE(cursor, cached_list, q, cursor_tmp) { /* @@ -133,9 +133,10 @@ __wt_session_copy_values(WT_SESSION_IMPL *session) * checkpoint. */ WT_TXN_SHARED *txn_shared = WT_SESSION_TXN_SHARED(session); - WT_ASSERT(session, txn_shared->pinned_id != WT_TXN_NONE || + WT_ASSERT(session, + txn_shared->pinned_id != WT_TXN_NONE || (WT_PREFIX_MATCH(cursor->uri, "file:") && - F_ISSET((WT_CURSOR_BTREE *)cursor, WT_CBT_NO_TXN))); + F_ISSET((WT_CURSOR_BTREE *)cursor, WT_CBT_NO_TXN))); #endif WT_RET(__cursor_localvalue(cursor)); } @@ -244,9 +245,9 @@ static int __session_close_cached_cursors(WT_SESSION_IMPL *session) { WT_DECL_RET; - int i; + uint64_t i; - for (i = 0; i < WT_HASH_ARRAY_SIZE; i++) + for (i = 0; i < S2C(session)->hash_size; i++) WT_TRET(__session_close_cursors(session, &session->cursor_cache[i])); return (ret); } @@ -258,22 +259,39 @@ __session_close_cached_cursors(WT_SESSION_IMPL *session) static int __session_close(WT_SESSION *wt_session, const char *config) { - WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_SESSION_IMPL *session; - conn = (WT_CONNECTION_IMPL *)wt_session->connection; session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL_PREPARE_ALLOWED(session, close, config, cfg); WT_UNUSED(cfg); + WT_ERR(__wt_session_close_internal(session)); + session = NULL; + +err: + API_END_RET_NOTFOUND_MAP(session, ret); +} + +/* + * __wt_session_close_internal -- + * Internal function of WT_SESSION->close method. + */ +int +__wt_session_close_internal(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + + conn = S2C(session); + /* Close all open cursors while the cursor cache is disabled. */ F_CLR(session, WT_SESSION_CACHE_CURSORS); /* Rollback any active transaction. */ if (F_ISSET(session->txn, WT_TXN_RUNNING)) - WT_TRET(__session_rollback_transaction(wt_session, NULL)); + WT_TRET(__session_rollback_transaction((WT_SESSION *)session, NULL)); /* * Also release any pinned transaction ID from a non-transactional operation. @@ -347,11 +365,7 @@ __session_close(WT_SESSION *wt_session, const char *config) __wt_spin_unlock(session, &conn->api_lock); - /* We no longer have a session, don't try to update it. */ - session = NULL; - -err: - API_END_RET_NOTFOUND_MAP(session, ret); + return (ret); } /* @@ -552,8 +566,7 @@ __session_open_cursor(WT_SESSION *wt_session, const char *uri, WT_CURSOR *to_dup if (!statjoin) { if ((to_dup == NULL && uri == NULL) || (to_dup != NULL && uri != NULL)) WT_ERR_MSG(session, EINVAL, - "should be passed either a URI or a cursor to " - "duplicate, but not both"); + "should be passed either a URI or a cursor to duplicate, but not both"); if ((ret = __wt_cursor_cache_get(session, uri, to_dup, cfg, &cursor)) == 0) goto done; @@ -867,8 +880,9 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, - WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_bt_rebalance, NULL, - cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_REBALANCE))); + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_schema_worker( + session, uri, __wt_bt_rebalance, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_REBALANCE))); err: if (ret != 0) @@ -918,8 +932,8 @@ __session_rename(WT_SESSION *wt_session, const char *uri, const char *newuri, co WT_ERR(__wt_str_name_check(session, newuri)); WT_WITH_CHECKPOINT_LOCK(session, - WT_WITH_SCHEMA_LOCK(session, WT_WITH_TABLE_WRITE_LOCK(session, - ret = __wt_schema_rename(session, uri, newuri, cfg)))); + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, ret = __wt_schema_rename(session, uri, newuri, cfg)))); err: if (ret != 0) WT_STAT_CONN_INCR(session, session_table_rename_fail); @@ -1014,21 +1028,22 @@ __session_drop(WT_SESSION *wt_session, const char *uri, const char *config) */ if (checkpoint_wait) { if (lock_wait) - WT_WITH_CHECKPOINT_LOCK( - session, WT_WITH_SCHEMA_LOCK(session, WT_WITH_TABLE_WRITE_LOCK(session, - ret = __wt_schema_drop(session, uri, cfg)))); + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, ret = __wt_schema_drop(session, uri, cfg)))); else - WT_WITH_CHECKPOINT_LOCK_NOWAIT( - session, ret, WT_WITH_SCHEMA_LOCK_NOWAIT( - session, ret, WT_WITH_TABLE_WRITE_LOCK_NOWAIT(session, ret, - ret = __wt_schema_drop(session, uri, cfg)))); + WT_WITH_CHECKPOINT_LOCK_NOWAIT(session, ret, + WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, + WT_WITH_TABLE_WRITE_LOCK_NOWAIT( + session, ret, ret = __wt_schema_drop(session, uri, cfg)))); } else { if (lock_wait) WT_WITH_SCHEMA_LOCK(session, WT_WITH_TABLE_WRITE_LOCK(session, ret = __wt_schema_drop(session, uri, cfg))); else - WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, WT_WITH_TABLE_WRITE_LOCK_NOWAIT(session, ret, - ret = __wt_schema_drop(session, uri, cfg))); + WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, + WT_WITH_TABLE_WRITE_LOCK_NOWAIT( + session, ret, ret = __wt_schema_drop(session, uri, cfg))); } err: @@ -1177,9 +1192,7 @@ __session_join( WT_ERR_MSG(session, EINVAL, "requires reference cursor be positioned"); cjoin = (WT_CURSOR_JOIN *)join_cursor; if (cjoin->table != table) - WT_ERR_MSG(session, EINVAL, - "table for join cursor does not match table for " - "ref_cursor"); + WT_ERR_MSG(session, EINVAL, "table for join cursor does not match table for ref_cursor"); if (F_ISSET(ref_cursor, WT_CURSTD_JOINED)) WT_ERR_MSG(session, EINVAL, "cursor already used in a join"); @@ -1230,8 +1243,8 @@ __session_join( if (nested && (count != 0 || range != WT_CURJOIN_END_EQ || LF_ISSET(WT_CURJOIN_ENTRY_BLOOM))) WT_ERR_MSG(session, EINVAL, - "joining a nested join cursor is incompatible with " - "setting \"strategy\", \"compare\" or \"count\""); + "joining a nested join cursor is incompatible with setting \"strategy\", \"compare\" or " + "\"count\""); WT_ERR(__wt_curjoin_join( session, cjoin, idx, ref_cursor, flags, range, count, bloom_bit_count, bloom_hash_count)); @@ -1267,9 +1280,10 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK( - session, WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_salvage, - NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_SALVAGE))); + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_schema_worker( + session, uri, __wt_salvage, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_SALVAGE))); err: if (ret != 0) @@ -1353,9 +1367,8 @@ __wt_session_range_truncate( if (start != NULL && stop != NULL && start->compare != NULL) { WT_ERR(start->compare(start, stop, &cmp)); if (cmp > 0) - WT_ERR_MSG(session, EINVAL, - "the start cursor position is after the stop " - "cursor position"); + WT_ERR_MSG( + session, EINVAL, "the start cursor position is after the stop cursor position"); } /* @@ -1450,8 +1463,7 @@ __session_truncate( if ((uri == NULL && start == NULL && stop == NULL) || (uri != NULL && !WT_PREFIX_MATCH(uri, "log:") && (start != NULL || stop != NULL))) WT_ERR_MSG(session, EINVAL, - "the truncate method should be passed either a URI or " - "start/stop cursors, but not both"); + "the truncate method should be passed either a URI or start/stop cursors, but not both"); if (uri != NULL) { /* Disallow objects in the WiredTiger name space. */ @@ -1463,8 +1475,7 @@ __session_truncate( */ if (strcmp(uri, "log:") != 0) WT_ERR_MSG(session, EINVAL, - "the truncate method should not specify any" - "target after the log: URI prefix"); + "the truncate method should not specify any target after the log: URI prefix"); WT_ERR(__wt_log_truncate_files(session, start, false)); } else if (WT_PREFIX_MATCH(uri, "file:")) WT_ERR(__wt_session_range_truncate(session, uri, start, stop)); @@ -1530,9 +1541,10 @@ __session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config) WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK( - session, WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_upgrade, - NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_UPGRADE))); + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_schema_worker( + session, uri, __wt_upgrade, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_UPGRADE))); err: API_END_RET_NOTFOUND_MAP(session, ret); @@ -1574,9 +1586,10 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK( - session, WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_verify, - NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_VERIFY))); + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_schema_worker( + session, uri, __wt_verify, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_VERIFY))); WT_ERR(ret); err: if (ret != 0) @@ -2060,9 +2073,8 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const if (!session_ret->active) break; if (i == conn->session_size) - WT_ERR_MSG(session, WT_ERROR, "out of sessions, configured for %" PRIu32 - " (including " - "internal sessions)", + WT_ERR_MSG(session, WT_ERROR, + "out of sessions, configured for %" PRIu32 " (including internal sessions)", conn->session_size); /* @@ -2093,16 +2105,16 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const * array as well. */ if (session_ret->cursor_cache == NULL) - WT_ERR(__wt_calloc_def(session, WT_HASH_ARRAY_SIZE, &session_ret->cursor_cache)); + WT_ERR(__wt_calloc_def(session, conn->hash_size, &session_ret->cursor_cache)); if (session_ret->dhhash == NULL) - WT_ERR(__wt_calloc_def(session, WT_HASH_ARRAY_SIZE, &session_ret->dhhash)); + WT_ERR(__wt_calloc_def(session, conn->dh_hash_size, &session_ret->dhhash)); /* Initialize the dhandle hash array. */ - for (i = 0; i < WT_HASH_ARRAY_SIZE; i++) + for (i = 0; i < (uint32_t)conn->dh_hash_size; i++) TAILQ_INIT(&session_ret->dhhash[i]); /* Initialize the cursor cache hash buckets and sweep trigger. */ - for (i = 0; i < WT_HASH_ARRAY_SIZE; i++) + for (i = 0; i < (uint32_t)conn->hash_size; i++) TAILQ_INIT(&session_ret->cursor_cache[i]); session_ret->cursor_sweep_countdown = WT_SESSION_CURSOR_SWEEP_COUNTDOWN; @@ -2121,8 +2133,14 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const session_ret->nhazard = 0; } - /* Cache the offset of this session's statistics bucket. */ - session_ret->stat_bucket = WT_STATS_SLOT_ID(session); + /* + * Cache the offset of this session's statistics bucket. It's important we pass the correct + * session to the hash define here or we'll calculate the stat bucket with the wrong session id. + */ + session_ret->stat_bucket = WT_STATS_SLOT_ID(session_ret); + + /* Safety check to make sure we're doing the right thing. */ + WT_ASSERT(session, session_ret->stat_bucket == session_ret->id % WT_COUNTER_SLOTS); /* Allocate the buffer for operation tracking */ if (F_ISSET(conn, WT_CONN_OPTRACK)) { @@ -2169,7 +2187,6 @@ __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, con bool open_metadata, WT_SESSION_IMPL **sessionp) { WT_DECL_RET; - WT_SESSION *wt_session; WT_SESSION_IMPL *session; *sessionp = NULL; @@ -2187,8 +2204,7 @@ __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, con if (open_metadata) { WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)); if ((ret = __wt_metadata_cursor(session, NULL)) != 0) { - wt_session = &session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_TRET(__wt_session_close_internal(session)); return (ret); } } diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index 1a92a1f985e..3184cdd579d 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -295,9 +295,7 @@ __compact_worker(WT_SESSION_IMPL *session) */ if (ret == EBUSY) { if (__wt_cache_stuck(session)) { - WT_ERR_MSG(session, EBUSY, - "compaction halted by eviction " - "pressure"); + WT_ERR_MSG(session, EBUSY, "compaction halted by eviction pressure"); } ret = 0; another_pass = true; @@ -388,8 +386,9 @@ __wt_session_compact(WT_SESSION *wt_session, const char *uri, const char *config * table, so acquire the table lock in write mode. */ WT_WITH_SCHEMA_LOCK(session, - WT_WITH_TABLE_WRITE_LOCK(session, ret = __wt_schema_worker(session, uri, - __compact_handle_append, __compact_uri_analyze, cfg, 0))); + WT_WITH_TABLE_WRITE_LOCK(session, + ret = __wt_schema_worker( + session, uri, __compact_handle_append, __compact_uri_analyze, cfg, 0))); WT_ERR(ret); if (session->compact->lsm_count != 0) diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 8ec514caf46..69656bb02a4 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -23,7 +23,7 @@ __session_add_dhandle(WT_SESSION_IMPL *session) dhandle_cache->dhandle = session->dhandle; - bucket = dhandle_cache->dhandle->name_hash % WT_HASH_ARRAY_SIZE; + bucket = dhandle_cache->dhandle->name_hash & (S2C(session)->dh_hash_size - 1); TAILQ_INSERT_HEAD(&session->dhandles, dhandle_cache, q); TAILQ_INSERT_HEAD(&session->dhhash[bucket], dhandle_cache, hashq); @@ -39,7 +39,7 @@ __session_discard_dhandle(WT_SESSION_IMPL *session, WT_DATA_HANDLE_CACHE *dhandl { uint64_t bucket; - bucket = dhandle_cache->dhandle->name_hash % WT_HASH_ARRAY_SIZE; + bucket = dhandle_cache->dhandle->name_hash & (S2C(session)->dh_hash_size - 1); TAILQ_REMOVE(&session->dhandles, dhandle_cache, q); TAILQ_REMOVE(&session->dhhash[bucket], dhandle_cache, hashq); @@ -61,7 +61,7 @@ __session_find_dhandle(WT_SESSION_IMPL *session, const char *uri, const char *ch dhandle = NULL; - bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; + bucket = __wt_hash_city64(uri, strlen(uri)) & (S2C(session)->dh_hash_size - 1); retry: TAILQ_FOREACH (dhandle_cache, &session->dhhash[bucket], hashq) { dhandle = dhandle_cache->dhandle; @@ -387,8 +387,8 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) if (dhandle != session->dhandle && dhandle->session_inuse == 0 && (WT_DHANDLE_INACTIVE(dhandle) || - (dhandle->timeofdeath != 0 && now - dhandle->timeofdeath > conn->sweep_idle_time) || - empty_btree)) { + (dhandle->timeofdeath != 0 && now - dhandle->timeofdeath > conn->sweep_idle_time) || + empty_btree)) { WT_STAT_CONN_INCR(session, dh_session_handles); WT_ASSERT(session, !WT_IS_METADATA(dhandle)); __session_discard_dhandle(session, dhandle_cache); @@ -526,7 +526,8 @@ __wt_session_get_dhandle(WT_SESSION_IMPL *session, const char *uri, const char * WT_ASSERT(session, !F_ISSET(dhandle, WT_DHANDLE_DEAD)); WT_ASSERT(session, LF_ISSET(WT_DHANDLE_LOCK_ONLY) || F_ISSET(dhandle, WT_DHANDLE_OPEN)); - WT_ASSERT(session, LF_ISSET(WT_DHANDLE_EXCLUSIVE) == F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) || + WT_ASSERT(session, + LF_ISSET(WT_DHANDLE_EXCLUSIVE) == F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) || dhandle->excl_ref > 1); return (0); diff --git a/src/third_party/wiredtiger/src/support/crypto.c b/src/third_party/wiredtiger/src/support/crypto.c index bc324921d52..7357fd0ff68 100644 --- a/src/third_party/wiredtiger/src/support/crypto.c +++ b/src/third_party/wiredtiger/src/support/crypto.c @@ -27,9 +27,8 @@ __wt_decrypt( #endif if (encrypt_len > in->size) - WT_RET_MSG(session, WT_ERROR, - "corrupted encrypted item: padded size less than " - "actual size"); + WT_RET_MSG( + session, WT_ERROR, "corrupted encrypted item: padded size less than actual size"); /* * We're allocating the number of bytes we're expecting from decryption plus the unencrypted * header. diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index b8e43ae5a7b..629990b8830 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -271,9 +271,8 @@ __eventv(WT_SESSION_IMPL *session, bool msg_event, int error, const char *func, * code, it's a recursive call. */ if (ret == 0 && remain == 0) - __wt_err(session, ENOMEM, - "error or message truncated: internal WiredTiger buffer " - "too small"); + __wt_err( + session, ENOMEM, "error or message truncated: internal WiredTiger buffer too small"); if (ret != 0) { err: diff --git a/src/third_party/wiredtiger/src/support/global.c b/src/third_party/wiredtiger/src/support/global.c index e8bb49f1a69..9463f39f4d1 100644 --- a/src/third_party/wiredtiger/src/support/global.c +++ b/src/third_party/wiredtiger/src/support/global.c @@ -35,9 +35,7 @@ __endian_check(void) e = "little-endian"; #endif fprintf(stderr, - "This is a %s build of the WiredTiger data engine, incompatible " - "with this system\n", - e); + "This is a %s build of the WiredTiger data engine, incompatible with this system\n", e); return (EINVAL); } diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index 336ee99c27c..f1e552b8165 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -66,7 +66,7 @@ __wt_hazard_set_func(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp , const char *func, int line #endif - ) +) { WT_HAZARD *hp; uint8_t current_state; @@ -89,7 +89,8 @@ __wt_hazard_set_func(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp /* If we have filled the current hazard pointer array, grow it. */ if (session->nhazard >= session->hazard_size) { - WT_ASSERT(session, session->nhazard == session->hazard_size && + WT_ASSERT(session, + session->nhazard == session->hazard_size && session->hazard_inuse == session->hazard_size); WT_RET(hazard_grow(session)); } @@ -98,11 +99,13 @@ __wt_hazard_set_func(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp * If there are no available hazard pointer slots, make another one visible. */ if (session->nhazard >= session->hazard_inuse) { - WT_ASSERT(session, session->nhazard == session->hazard_inuse && + WT_ASSERT(session, + session->nhazard == session->hazard_inuse && session->hazard_inuse < session->hazard_size); hp = &session->hazard[session->hazard_inuse++]; } else { - WT_ASSERT(session, session->nhazard < session->hazard_inuse && + WT_ASSERT(session, + session->nhazard < session->hazard_inuse && session->hazard_inuse <= session->hazard_size); /* @@ -258,9 +261,7 @@ __wt_hazard_close(WT_SESSION_IMPL *session) } if (session->nhazard != 0) - __wt_errx(session, - "session %p: close hazard pointer table: count didn't " - "match entries", + __wt_errx(session, "session %p: close hazard pointer table: count didn't match entries", (void *)session); } @@ -390,8 +391,7 @@ __wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor) __wt_sleep(0, 10000); } __wt_errx(session, - "hazard pointer reference to discarded object: " - "(%p: session %p name %s: %s, line %d)", + "hazard pointer reference to discarded object: (%p: session %p name %s: %s, line %d)", (void *)hp->ref, (void *)s, s->name == NULL ? "UNKNOWN" : s->name, hp->func, hp->line); return (false); } diff --git a/src/third_party/wiredtiger/src/support/huffman.c b/src/third_party/wiredtiger/src/support/huffman.c index 2bd9b6daea6..33595015ad9 100644 --- a/src/third_party/wiredtiger/src/support/huffman.c +++ b/src/third_party/wiredtiger/src/support/huffman.c @@ -210,7 +210,7 @@ set_codes(WT_FREQTREE_NODE *node, WT_HUFFMAN_CODE *codes, uint16_t pattern, uint */ if (len < MAX_CODE_LENGTH && ((half = (uint16_t)(1 << (remaining - 1))) < node->left->weight || - half < node->right->weight)) { + half < node->right->weight)) { pattern = (uint16_t)(pattern << remaining); len = MAX_CODE_LENGTH; } @@ -315,9 +315,7 @@ __wt_huffman_open( * expected to appear in the input). Validate the symbols are within range. */ if (numbytes != 1 && numbytes != 2) - WT_ERR_MSG(session, EINVAL, - "illegal number of symbol bytes specified for a huffman " - "table"); + WT_ERR_MSG(session, EINVAL, "illegal number of symbol bytes specified for a huffman table"); if (symcnt == 0) WT_ERR_MSG(session, EINVAL, "illegal number of symbols specified for a huffman table"); @@ -331,14 +329,12 @@ __wt_huffman_open( __wt_qsort(sym, symcnt, sizeof(INDEXED_SYMBOL), indexed_symbol_compare); for (i = 0; i < symcnt; ++i) { if (i > 0 && sym[i].symbol == sym[i - 1].symbol) - WT_ERR_MSG(session, EINVAL, "duplicate symbol %" PRIu32 " (%#" PRIx32 - ") " - "specified in a huffman table", + WT_ERR_MSG(session, EINVAL, + "duplicate symbol %" PRIu32 " (%#" PRIx32 ") specified in a huffman table", sym[i].symbol, sym[i].symbol); if (sym[i].symbol > huffman->numSymbols) - WT_ERR_MSG(session, EINVAL, "out-of-range symbol %" PRIu32 " (%#" PRIx32 - ") " - "specified in a huffman table", + WT_ERR_MSG(session, EINVAL, + "out-of-range symbol %" PRIu32 " (%#" PRIx32 ") specified in a huffman table", sym[i].symbol, sym[i].symbol); } @@ -472,9 +468,7 @@ __wt_huffman_open( indexed_freqs[i].frequency, huffman->codes[symbol].length); } printf( - "weighted length of all codes (the smaller the better): " - "%" PRIu32 "\n", - weighted_length); + "weighted length of all codes (the smaller the better): %" PRIu32 "\n", weighted_length); } #endif @@ -533,10 +527,8 @@ __wt_print_huffman_code(void *huffman_arg, uint16_t symbol) printf("symbol %" PRIu16 " not defined -- 0 frequency\n", symbol); else /* should print code as binary */ - printf("%" PRIu16 - " -> code pattern " - "%" PRIx16 ", length %" PRIu8 "\n", - symbol, code.pattern, code.length); + printf("%" PRIu16 " -> code pattern %" PRIx16 ", length %" PRIu8 "\n", symbol, + code.pattern, code.length); } } #endif diff --git a/src/third_party/wiredtiger/src/support/modify.c b/src/third_party/wiredtiger/src/support/modify.c index 0d22afd615d..e5106699dda 100644 --- a/src/third_party/wiredtiger/src/support/modify.c +++ b/src/third_party/wiredtiger/src/support/modify.c @@ -198,10 +198,12 @@ __modify_apply_one(WT_SESSION_IMPL *session, WT_ITEM *value, WT_MODIFY *modify, } else { /* Shrink or grow */ /* Move trailing data forward/backward to its new location. */ from = (const uint8_t *)value->data + (offset + size); - WT_ASSERT(session, WT_DATA_IN_ITEM(value) && + WT_ASSERT(session, + WT_DATA_IN_ITEM(value) && from + (value->size - (offset + size)) <= (uint8_t *)value->mem + value->memsize); to = (uint8_t *)value->data + (offset + data_size); - WT_ASSERT(session, WT_DATA_IN_ITEM(value) && + WT_ASSERT(session, + WT_DATA_IN_ITEM(value) && to + (value->size - (offset + size)) <= (uint8_t *)value->mem + value->memsize); memmove(to, from, value->size - (offset + size)); diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index 67e0a6576f8..19b2b57e24e 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -333,9 +333,7 @@ __wt_scr_discard(WT_SESSION_IMPL *session) continue; if (F_ISSET(*bufp, WT_ITEM_INUSE)) #ifdef HAVE_DIAGNOSTIC - __wt_errx(session, - "scratch buffer allocated and never discarded" - ": %s: %d", + __wt_errx(session, "scratch buffer allocated and never discarded: %s: %d", session->scratch_track[bufp - session->scratch].func, session->scratch_track[bufp - session->scratch].line); #else diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index fce599e6633..a893ccc3865 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -3,104 +3,167 @@ #include "wt_internal.h" static const char *const __stats_dsrc_desc[] = { - "LSM: bloom filter false positives", "LSM: bloom filter hits", "LSM: bloom filter misses", - "LSM: bloom filter pages evicted from cache", "LSM: bloom filter pages read into cache", - "LSM: bloom filters in the LSM tree", "LSM: chunks in the LSM tree", + "LSM: bloom filter false positives", + "LSM: bloom filter hits", + "LSM: bloom filter misses", + "LSM: bloom filter pages evicted from cache", + "LSM: bloom filter pages read into cache", + "LSM: bloom filters in the LSM tree", + "LSM: chunks in the LSM tree", "LSM: highest merge generation in the LSM tree", "LSM: queries that could have benefited from a Bloom filter that did not exist", - "LSM: sleep for LSM checkpoint throttle", "LSM: sleep for LSM merge throttle", - "LSM: total size of bloom filters", "block-manager: allocations requiring file extension", - "block-manager: blocks allocated", "block-manager: blocks freed", - "block-manager: checkpoint size", "block-manager: file allocation unit size", - "block-manager: file bytes available for reuse", "block-manager: file magic number", - "block-manager: file major version number", "block-manager: file size in bytes", - "block-manager: minor version number", "btree: btree checkpoint generation", - "btree: btree clean tree checkpoint expiration time", "btree: column-store fixed-size leaf pages", - "btree: column-store internal pages", "btree: column-store variable-size RLE encoded values", + "LSM: sleep for LSM checkpoint throttle", + "LSM: sleep for LSM merge throttle", + "LSM: total size of bloom filters", + "block-manager: allocations requiring file extension", + "block-manager: blocks allocated", + "block-manager: blocks freed", + "block-manager: checkpoint size", + "block-manager: file allocation unit size", + "block-manager: file bytes available for reuse", + "block-manager: file magic number", + "block-manager: file major version number", + "block-manager: file size in bytes", + "block-manager: minor version number", + "btree: btree checkpoint generation", + "btree: btree clean tree checkpoint expiration time", + "btree: column-store fixed-size leaf pages", + "btree: column-store internal pages", + "btree: column-store variable-size RLE encoded values", "btree: column-store variable-size deleted values", - "btree: column-store variable-size leaf pages", "btree: fixed-record size", - "btree: maximum internal page key size", "btree: maximum internal page size", - "btree: maximum leaf page key size", "btree: maximum leaf page size", - "btree: maximum leaf page value size", "btree: maximum tree depth", - "btree: number of key/value pairs", "btree: overflow pages", - "btree: pages rewritten by compaction", "btree: row-store empty values", - "btree: row-store internal pages", "btree: row-store leaf pages", - "cache: bytes currently in the cache", "cache: bytes dirty in the cache cumulative", - "cache: bytes read into cache", "cache: bytes written from cache", + "btree: column-store variable-size leaf pages", + "btree: fixed-record size", + "btree: maximum internal page key size", + "btree: maximum internal page size", + "btree: maximum leaf page key size", + "btree: maximum leaf page size", + "btree: maximum leaf page value size", + "btree: maximum tree depth", + "btree: number of key/value pairs", + "btree: overflow pages", + "btree: pages rewritten by compaction", + "btree: row-store empty values", + "btree: row-store internal pages", + "btree: row-store leaf pages", + "cache: bytes currently in the cache", + "cache: bytes dirty in the cache cumulative", + "cache: bytes read into cache", + "cache: bytes written from cache", "cache: checkpoint blocked page eviction", "cache: data source pages selected for eviction unable to be evicted", - "cache: eviction walk passes of a file", "cache: eviction walk target pages histogram - 0-9", + "cache: eviction walk passes of a file", + "cache: eviction walk target pages histogram - 0-9", "cache: eviction walk target pages histogram - 10-31", "cache: eviction walk target pages histogram - 128 and higher", "cache: eviction walk target pages histogram - 32-63", - "cache: eviction walk target pages histogram - 64-128", "cache: eviction walks abandoned", + "cache: eviction walk target pages histogram - 64-128", + "cache: eviction walks abandoned", "cache: eviction walks gave up because they restarted their walk twice", "cache: eviction walks gave up because they saw too many pages and found no candidates", "cache: eviction walks gave up because they saw too many pages and found too few candidates", - "cache: eviction walks reached end of tree", "cache: eviction walks started from root of tree", + "cache: eviction walks reached end of tree", + "cache: eviction walks started from root of tree", "cache: eviction walks started from saved location in tree", - "cache: hazard pointer blocked page eviction", "cache: history store table reads", - "cache: in-memory page passed criteria to be split", "cache: in-memory page splits", - "cache: internal pages evicted", "cache: internal pages split during eviction", - "cache: leaf pages split during eviction", "cache: modified pages evicted", - "cache: overflow pages read into cache", "cache: page split during eviction deepened the tree", - "cache: page written requiring history store records", "cache: pages read into cache", + "cache: hazard pointer blocked page eviction", + "cache: history store table reads", + "cache: in-memory page passed criteria to be split", + "cache: in-memory page splits", + "cache: internal pages evicted", + "cache: internal pages split during eviction", + "cache: leaf pages split during eviction", + "cache: modified pages evicted", + "cache: overflow pages read into cache", + "cache: page split during eviction deepened the tree", + "cache: page written requiring history store records", + "cache: pages read into cache", "cache: pages read into cache after truncate", "cache: pages read into cache after truncate in prepare state", - "cache: pages requested from the cache", "cache: pages seen by eviction walk", - "cache: pages written from cache", "cache: pages written requiring in-memory restoration", - "cache: tracked dirty bytes in the cache", "cache: unmodified pages evicted", + "cache: pages requested from the cache", + "cache: pages seen by eviction walk", + "cache: pages written from cache", + "cache: pages written requiring in-memory restoration", + "cache: tracked dirty bytes in the cache", + "cache: unmodified pages evicted", "cache_walk: Average difference between current eviction generation when the page was last " "considered", "cache_walk: Average on-disk page image size seen", "cache_walk: Average time in cache for pages that have been visited by the eviction server", "cache_walk: Average time in cache for pages that have not been visited by the eviction server", - "cache_walk: Clean pages currently in cache", "cache_walk: Current eviction generation", - "cache_walk: Dirty pages currently in cache", "cache_walk: Entries in the root page", - "cache_walk: Internal pages currently in cache", "cache_walk: Leaf pages currently in cache", + "cache_walk: Clean pages currently in cache", + "cache_walk: Current eviction generation", + "cache_walk: Dirty pages currently in cache", + "cache_walk: Entries in the root page", + "cache_walk: Internal pages currently in cache", + "cache_walk: Leaf pages currently in cache", "cache_walk: Maximum difference between current eviction generation when the page was last " "considered", - "cache_walk: Maximum page size seen", "cache_walk: Minimum on-disk page image size seen", + "cache_walk: Maximum page size seen", + "cache_walk: Minimum on-disk page image size seen", "cache_walk: Number of pages never visited by eviction server", "cache_walk: On-disk page image sizes smaller than a single allocation unit", "cache_walk: Pages created in memory and never written", "cache_walk: Pages currently queued for eviction", "cache_walk: Pages that could not be queued for eviction", - "cache_walk: Refs skipped during cache traversal", "cache_walk: Size of the root page", + "cache_walk: Refs skipped during cache traversal", + "cache_walk: Size of the root page", "cache_walk: Total number of pages currently in cache", - "checkpoint-cleanup: pages added for eviction", "checkpoint-cleanup: pages removed", - "checkpoint-cleanup: pages skipped during tree walk", "checkpoint-cleanup: pages visited", + "checkpoint-cleanup: pages added for eviction", + "checkpoint-cleanup: pages removed", + "checkpoint-cleanup: pages skipped during tree walk", + "checkpoint-cleanup: pages visited", "compression: compressed page maximum internal page size prior to compression", "compression: compressed page maximum leaf page size prior to compression ", - "compression: compressed pages read", "compression: compressed pages written", + "compression: compressed pages read", + "compression: compressed pages written", "compression: page written failed to compress", "compression: page written was too small to compress", "cursor: Total number of entries skipped by cursor next calls", "cursor: Total number of entries skipped by cursor prev calls", "cursor: Total number of entries skipped to position the history store cursor", - "cursor: bulk loaded cursor insert calls", "cursor: cache cursors reuse count", - "cursor: close calls that result in cache", "cursor: create calls", + "cursor: bulk loaded cursor insert calls", + "cursor: cache cursors reuse count", + "cursor: close calls that result in cache", + "cursor: create calls", "cursor: cursor next calls that skip greater than or equal to 100 entries", "cursor: cursor next calls that skip less than 100 entries", "cursor: cursor prev calls that skip greater than or equal to 100 entries", - "cursor: cursor prev calls that skip less than 100 entries", "cursor: insert calls", - "cursor: insert key and value bytes", "cursor: modify", - "cursor: modify key and value bytes affected", "cursor: modify value bytes modified", - "cursor: next calls", "cursor: open cursor count", "cursor: operation restarted", - "cursor: prev calls", "cursor: remove calls", "cursor: remove key bytes removed", - "cursor: reserve calls", "cursor: reset calls", "cursor: search calls", - "cursor: search history store calls", "cursor: search near calls", "cursor: truncate calls", - "cursor: update calls", "cursor: update key and value bytes", "cursor: update value size change", + "cursor: cursor prev calls that skip less than 100 entries", + "cursor: insert calls", + "cursor: insert key and value bytes", + "cursor: modify", + "cursor: modify key and value bytes affected", + "cursor: modify value bytes modified", + "cursor: next calls", + "cursor: open cursor count", + "cursor: operation restarted", + "cursor: prev calls", + "cursor: remove calls", + "cursor: remove key bytes removed", + "cursor: reserve calls", + "cursor: reset calls", + "cursor: search calls", + "cursor: search history store calls", + "cursor: search near calls", + "cursor: truncate calls", + "cursor: update calls", + "cursor: update key and value bytes", + "cursor: update value size change", "reconciliation: approximate byte size of timestamps in pages written", "reconciliation: approximate byte size of transaction IDs in pages written", - "reconciliation: dictionary matches", "reconciliation: fast-path pages deleted", + "reconciliation: dictionary matches", + "reconciliation: fast-path pages deleted", "reconciliation: internal page key bytes discarded using suffix compression", - "reconciliation: internal page multi-block writes", "reconciliation: internal-page overflow keys", + "reconciliation: internal page multi-block writes", + "reconciliation: internal-page overflow keys", "reconciliation: leaf page key bytes discarded using prefix compression", - "reconciliation: leaf page multi-block writes", "reconciliation: leaf-page overflow keys", - "reconciliation: maximum blocks required for a page", "reconciliation: overflow values written", - "reconciliation: page checksum matches", "reconciliation: page reconciliation calls", - "reconciliation: page reconciliation calls for eviction", "reconciliation: pages deleted", + "reconciliation: leaf page multi-block writes", + "reconciliation: leaf-page overflow keys", + "reconciliation: maximum blocks required for a page", + "reconciliation: overflow values written", + "reconciliation: page checksum matches", + "reconciliation: page reconciliation calls", + "reconciliation: page reconciliation calls for eviction", + "reconciliation: pages deleted", "reconciliation: pages written including an aggregated newest start durable timestamp ", "reconciliation: pages written including an aggregated newest stop durable timestamp ", "reconciliation: pages written including an aggregated newest stop timestamp ", @@ -121,8 +184,10 @@ static const char *const __stats_dsrc_desc[] = { "reconciliation: records written including a start transaction ID", "reconciliation: records written including a stop durable timestamp", "reconciliation: records written including a stop timestamp", - "reconciliation: records written including a stop transaction ID", "session: object compaction", - "transaction: race to read prepared update retry", "transaction: update conflicts", + "reconciliation: records written including a stop transaction ID", + "session: object compaction", + "transaction: race to read prepared update retry", + "transaction: update conflicts", }; int @@ -770,23 +835,40 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) } static const char *const __stats_connection_desc[] = { - "LSM: application work units currently queued", "LSM: merge work units currently queued", - "LSM: rows merged in an LSM tree", "LSM: sleep for LSM checkpoint throttle", - "LSM: sleep for LSM merge throttle", "LSM: switch work units currently queued", - "LSM: tree maintenance operations discarded", "LSM: tree maintenance operations executed", - "LSM: tree maintenance operations scheduled", "LSM: tree queue hit maximum", - "async: current work queue length", "async: maximum work queue length", - "async: number of allocation state races", "async: number of flush calls", + "LSM: application work units currently queued", + "LSM: merge work units currently queued", + "LSM: rows merged in an LSM tree", + "LSM: sleep for LSM checkpoint throttle", + "LSM: sleep for LSM merge throttle", + "LSM: switch work units currently queued", + "LSM: tree maintenance operations discarded", + "LSM: tree maintenance operations executed", + "LSM: tree maintenance operations scheduled", + "LSM: tree queue hit maximum", + "async: current work queue length", + "async: maximum work queue length", + "async: number of allocation state races", + "async: number of flush calls", "async: number of operation slots viewed for allocation", "async: number of times operation allocation failed", - "async: number of times worker found no work", "async: total allocations", - "async: total compact calls", "async: total insert calls", "async: total remove calls", - "async: total search calls", "async: total update calls", "block-manager: blocks pre-loaded", - "block-manager: blocks read", "block-manager: blocks written", "block-manager: bytes read", - "block-manager: bytes read via memory map API", "block-manager: bytes read via system call API", - "block-manager: bytes written", "block-manager: bytes written for checkpoint", + "async: number of times worker found no work", + "async: total allocations", + "async: total compact calls", + "async: total insert calls", + "async: total remove calls", + "async: total search calls", + "async: total update calls", + "block-manager: blocks pre-loaded", + "block-manager: blocks read", + "block-manager: blocks written", + "block-manager: bytes read", + "block-manager: bytes read via memory map API", + "block-manager: bytes read via system call API", + "block-manager: bytes written", + "block-manager: bytes written for checkpoint", "block-manager: bytes written via memory map API", - "block-manager: bytes written via system call API", "block-manager: mapped blocks read", + "block-manager: bytes written via system call API", + "block-manager: mapped blocks read", "block-manager: mapped bytes read", "block-manager: number of times the file was remapped because it changed size via fallocate or " "truncate", @@ -795,22 +877,29 @@ static const char *const __stats_connection_desc[] = { "cache: application threads page read from disk to cache time (usecs)", "cache: application threads page write from cache to disk count", "cache: application threads page write from cache to disk time (usecs)", - "cache: bytes allocated for updates", "cache: bytes belonging to page images in the cache", + "cache: bytes allocated for updates", + "cache: bytes belonging to page images in the cache", "cache: bytes belonging to the history store table in the cache", - "cache: bytes currently in the cache", "cache: bytes dirty in the cache cumulative", - "cache: bytes not belonging to page images in the cache", "cache: bytes read into cache", - "cache: bytes written from cache", "cache: cache overflow score", - "cache: checkpoint blocked page eviction", "cache: eviction calls to get a page", + "cache: bytes currently in the cache", + "cache: bytes dirty in the cache cumulative", + "cache: bytes not belonging to page images in the cache", + "cache: bytes read into cache", + "cache: bytes written from cache", + "cache: cache overflow score", + "cache: checkpoint blocked page eviction", + "cache: eviction calls to get a page", "cache: eviction calls to get a page found queue empty", "cache: eviction calls to get a page found queue empty after locking", - "cache: eviction currently operating in aggressive mode", "cache: eviction empty score", + "cache: eviction currently operating in aggressive mode", + "cache: eviction empty score", "cache: eviction passes of a file", "cache: eviction server candidate queue empty when topping up", "cache: eviction server candidate queue not empty when topping up", "cache: eviction server evicting pages", "cache: eviction server slept, because we did not make progress with eviction", "cache: eviction server unable to reach eviction goal", - "cache: eviction server waiting for a leaf page", "cache: eviction state", + "cache: eviction server waiting for a leaf page", + "cache: eviction state", "cache: eviction walk target pages histogram - 0-9", "cache: eviction walk target pages histogram - 10-31", "cache: eviction walk target pages histogram - 128 and higher", @@ -818,15 +907,20 @@ static const char *const __stats_connection_desc[] = { "cache: eviction walk target pages histogram - 64-128", "cache: eviction walk target strategy both clean and dirty pages", "cache: eviction walk target strategy only clean pages", - "cache: eviction walk target strategy only dirty pages", "cache: eviction walks abandoned", + "cache: eviction walk target strategy only dirty pages", + "cache: eviction walks abandoned", "cache: eviction walks gave up because they restarted their walk twice", "cache: eviction walks gave up because they saw too many pages and found no candidates", "cache: eviction walks gave up because they saw too many pages and found too few candidates", - "cache: eviction walks reached end of tree", "cache: eviction walks started from root of tree", + "cache: eviction walks reached end of tree", + "cache: eviction walks started from root of tree", "cache: eviction walks started from saved location in tree", - "cache: eviction worker thread active", "cache: eviction worker thread created", - "cache: eviction worker thread evicting pages", "cache: eviction worker thread removed", - "cache: eviction worker thread stable number", "cache: files with active eviction walks", + "cache: eviction worker thread active", + "cache: eviction worker thread created", + "cache: eviction worker thread evicting pages", + "cache: eviction worker thread removed", + "cache: eviction worker thread stable number", + "cache: files with active eviction walks", "cache: files with new eviction walks started", "cache: force re-tuning of eviction workers once in a while", "cache: forced eviction - history store pages failed to evict while session has history store " @@ -845,15 +939,20 @@ static const char *const __stats_connection_desc[] = { "cache: forced eviction - pages selected unable to be evicted time", "cache: forced eviction - session returned rollback error while force evicting due to being " "oldest", - "cache: hazard pointer blocked page eviction", "cache: hazard pointer check calls", - "cache: hazard pointer check entries walked", "cache: hazard pointer maximum array length", - "cache: history store score", "cache: history store table insert calls", + "cache: hazard pointer blocked page eviction", + "cache: hazard pointer check calls", + "cache: hazard pointer check entries walked", + "cache: hazard pointer maximum array length", + "cache: history store score", + "cache: history store table insert calls", "cache: history store table insert calls that returned restart", - "cache: history store table max on-disk size", "cache: history store table on-disk size", + "cache: history store table max on-disk size", + "cache: history store table on-disk size", "cache: history store table out-of-order resolved updates that lose their durable timestamp", "cache: history store table out-of-order updates that were fixed up by moving existing records", "cache: history store table out-of-order updates that were fixed up during insertion", - "cache: history store table reads", "cache: history store table reads missed", + "cache: history store table reads", + "cache: history store table reads missed", "cache: history store table reads requiring squashed modifies", "cache: history store table truncation by rollback to stable to remove an unstable update", "cache: history store table truncation by rollback to stable to remove an update", @@ -863,128 +962,204 @@ static const char *const __stats_connection_desc[] = { "the data page during reconciliation", "cache: history store table truncation to remove range of updates due to mixed timestamps", "cache: history store table writes requiring squashed modifies", - "cache: in-memory page passed criteria to be split", "cache: in-memory page splits", - "cache: internal pages evicted", "cache: internal pages queued for eviction", + "cache: in-memory page passed criteria to be split", + "cache: in-memory page splits", + "cache: internal pages evicted", + "cache: internal pages queued for eviction", "cache: internal pages seen by eviction walk", "cache: internal pages seen by eviction walk that are already queued", - "cache: internal pages split during eviction", "cache: leaf pages split during eviction", - "cache: maximum bytes configured", "cache: maximum page size at eviction", - "cache: modified pages evicted", "cache: modified pages evicted by application threads", - "cache: operations timed out waiting for space in cache", "cache: overflow pages read into cache", + "cache: internal pages split during eviction", + "cache: leaf pages split during eviction", + "cache: maximum bytes configured", + "cache: maximum page size at eviction", + "cache: modified pages evicted", + "cache: modified pages evicted by application threads", + "cache: operations timed out waiting for space in cache", + "cache: overflow pages read into cache", "cache: page split during eviction deepened the tree", - "cache: page written requiring history store records", "cache: pages currently held in the cache", - "cache: pages evicted by application threads", "cache: pages queued for eviction", - "cache: pages queued for eviction post lru sorting", "cache: pages queued for urgent eviction", - "cache: pages queued for urgent eviction during walk", "cache: pages read into cache", + "cache: page written requiring history store records", + "cache: pages currently held in the cache", + "cache: pages evicted by application threads", + "cache: pages queued for eviction", + "cache: pages queued for eviction post lru sorting", + "cache: pages queued for urgent eviction", + "cache: pages queued for urgent eviction during walk", + "cache: pages read into cache", "cache: pages read into cache after truncate", "cache: pages read into cache after truncate in prepare state", - "cache: pages requested from the cache", "cache: pages seen by eviction walk", + "cache: pages requested from the cache", + "cache: pages seen by eviction walk", "cache: pages seen by eviction walk that are already queued", "cache: pages selected for eviction unable to be evicted", "cache: pages selected for eviction unable to be evicted as the parent page has overflow items", "cache: pages selected for eviction unable to be evicted because of active children on an " "internal page", "cache: pages selected for eviction unable to be evicted because of failure in reconciliation", - "cache: pages walked for eviction", "cache: pages written from cache", - "cache: pages written requiring in-memory restoration", "cache: percentage overhead", + "cache: pages walked for eviction", + "cache: pages written from cache", + "cache: pages written requiring in-memory restoration", + "cache: percentage overhead", "cache: tracked bytes belonging to internal pages in the cache", "cache: tracked bytes belonging to leaf pages in the cache", - "cache: tracked dirty bytes in the cache", "cache: tracked dirty pages in the cache", - "cache: unmodified pages evicted", "capacity: background fsync file handles considered", - "capacity: background fsync file handles synced", "capacity: background fsync time (msecs)", - "capacity: bytes read", "capacity: bytes written for checkpoint", - "capacity: bytes written for eviction", "capacity: bytes written for log", - "capacity: bytes written total", "capacity: threshold to call fsync", + "cache: tracked dirty bytes in the cache", + "cache: tracked dirty pages in the cache", + "cache: unmodified pages evicted", + "capacity: background fsync file handles considered", + "capacity: background fsync file handles synced", + "capacity: background fsync time (msecs)", + "capacity: bytes read", + "capacity: bytes written for checkpoint", + "capacity: bytes written for eviction", + "capacity: bytes written for log", + "capacity: bytes written total", + "capacity: threshold to call fsync", "capacity: time waiting due to total capacity (usecs)", "capacity: time waiting during checkpoint (usecs)", - "capacity: time waiting during eviction (usecs)", "capacity: time waiting during logging (usecs)", - "capacity: time waiting during read (usecs)", "checkpoint-cleanup: pages added for eviction", - "checkpoint-cleanup: pages removed", "checkpoint-cleanup: pages skipped during tree walk", - "checkpoint-cleanup: pages visited", "connection: auto adjusting condition resets", + "capacity: time waiting during eviction (usecs)", + "capacity: time waiting during logging (usecs)", + "capacity: time waiting during read (usecs)", + "checkpoint-cleanup: pages added for eviction", + "checkpoint-cleanup: pages removed", + "checkpoint-cleanup: pages skipped during tree walk", + "checkpoint-cleanup: pages visited", + "connection: auto adjusting condition resets", "connection: auto adjusting condition wait calls", "connection: auto adjusting condition wait raced to update timeout and skipped updating", - "connection: detected system time went backwards", "connection: files currently open", - "connection: memory allocations", "connection: memory frees", "connection: memory re-allocations", + "connection: detected system time went backwards", + "connection: files currently open", + "connection: hash bucket array size for data handles", + "connection: hash bucket array size general", + "connection: memory allocations", + "connection: memory frees", + "connection: memory re-allocations", "connection: pthread mutex condition wait calls", "connection: pthread mutex shared lock read-lock calls", - "connection: pthread mutex shared lock write-lock calls", "connection: total fsync I/Os", - "connection: total read I/Os", "connection: total write I/Os", + "connection: pthread mutex shared lock write-lock calls", + "connection: total fsync I/Os", + "connection: total read I/Os", + "connection: total write I/Os", "cursor: Total number of entries skipped by cursor next calls", "cursor: Total number of entries skipped by cursor prev calls", "cursor: Total number of entries skipped to position the history store cursor", - "cursor: cached cursor count", "cursor: cursor bulk loaded cursor insert calls", - "cursor: cursor close calls that result in cache", "cursor: cursor create calls", - "cursor: cursor insert calls", "cursor: cursor insert key and value bytes", - "cursor: cursor modify calls", "cursor: cursor modify key and value bytes affected", - "cursor: cursor modify value bytes modified", "cursor: cursor next calls", + "cursor: cached cursor count", + "cursor: cursor bulk loaded cursor insert calls", + "cursor: cursor close calls that result in cache", + "cursor: cursor create calls", + "cursor: cursor insert calls", + "cursor: cursor insert key and value bytes", + "cursor: cursor modify calls", + "cursor: cursor modify key and value bytes affected", + "cursor: cursor modify value bytes modified", + "cursor: cursor next calls", "cursor: cursor next calls that skip greater than or equal to 100 entries", - "cursor: cursor next calls that skip less than 100 entries", "cursor: cursor operation restarted", + "cursor: cursor next calls that skip less than 100 entries", + "cursor: cursor operation restarted", "cursor: cursor prev calls", "cursor: cursor prev calls that skip due to a globally visible history store tombstone", "cursor: cursor prev calls that skip due to a globally visible history store tombstone in " "rollback to stable", "cursor: cursor prev calls that skip greater than or equal to 100 entries", - "cursor: cursor prev calls that skip less than 100 entries", "cursor: cursor remove calls", - "cursor: cursor remove key bytes removed", "cursor: cursor reserve calls", - "cursor: cursor reset calls", "cursor: cursor search calls", - "cursor: cursor search history store calls", "cursor: cursor search near calls", - "cursor: cursor sweep buckets", "cursor: cursor sweep cursors closed", - "cursor: cursor sweep cursors examined", "cursor: cursor sweeps", "cursor: cursor truncate calls", - "cursor: cursor update calls", "cursor: cursor update key and value bytes", - "cursor: cursor update value size change", "cursor: cursors reused from cache", - "cursor: open cursor count", "data-handle: connection data handle size", + "cursor: cursor prev calls that skip less than 100 entries", + "cursor: cursor remove calls", + "cursor: cursor remove key bytes removed", + "cursor: cursor reserve calls", + "cursor: cursor reset calls", + "cursor: cursor search calls", + "cursor: cursor search history store calls", + "cursor: cursor search near calls", + "cursor: cursor sweep buckets", + "cursor: cursor sweep cursors closed", + "cursor: cursor sweep cursors examined", + "cursor: cursor sweeps", + "cursor: cursor truncate calls", + "cursor: cursor update calls", + "cursor: cursor update key and value bytes", + "cursor: cursor update value size change", + "cursor: cursors reused from cache", + "cursor: open cursor count", + "data-handle: connection data handle size", "data-handle: connection data handles currently active", "data-handle: connection sweep candidate became referenced", "data-handle: connection sweep dhandles closed", "data-handle: connection sweep dhandles removed from hash list", - "data-handle: connection sweep time-of-death sets", "data-handle: connection sweeps", - "data-handle: session dhandles swept", "data-handle: session sweep attempts", + "data-handle: connection sweep time-of-death sets", + "data-handle: connection sweeps", + "data-handle: session dhandles swept", + "data-handle: session sweep attempts", "lock: checkpoint lock acquisitions", "lock: checkpoint lock application thread wait time (usecs)", "lock: checkpoint lock internal thread wait time (usecs)", "lock: dhandle lock application thread time waiting (usecs)", - "lock: dhandle lock internal thread time waiting (usecs)", "lock: dhandle read lock acquisitions", + "lock: dhandle lock internal thread time waiting (usecs)", + "lock: dhandle read lock acquisitions", "lock: dhandle write lock acquisitions", "lock: durable timestamp queue lock application thread time waiting (usecs)", "lock: durable timestamp queue lock internal thread time waiting (usecs)", "lock: durable timestamp queue read lock acquisitions", - "lock: durable timestamp queue write lock acquisitions", "lock: metadata lock acquisitions", + "lock: durable timestamp queue write lock acquisitions", + "lock: metadata lock acquisitions", "lock: metadata lock application thread wait time (usecs)", "lock: metadata lock internal thread wait time (usecs)", "lock: read timestamp queue lock application thread time waiting (usecs)", "lock: read timestamp queue lock internal thread time waiting (usecs)", "lock: read timestamp queue read lock acquisitions", - "lock: read timestamp queue write lock acquisitions", "lock: schema lock acquisitions", + "lock: read timestamp queue write lock acquisitions", + "lock: schema lock acquisitions", "lock: schema lock application thread wait time (usecs)", "lock: schema lock internal thread wait time (usecs)", "lock: table lock application thread time waiting for the table lock (usecs)", "lock: table lock internal thread time waiting for the table lock (usecs)", - "lock: table read lock acquisitions", "lock: table write lock acquisitions", + "lock: table read lock acquisitions", + "lock: table write lock acquisitions", "lock: txn global lock application thread time waiting (usecs)", "lock: txn global lock internal thread time waiting (usecs)", - "lock: txn global read lock acquisitions", "lock: txn global write lock acquisitions", - "log: busy returns attempting to switch slots", "log: force archive time sleeping (usecs)", - "log: log bytes of payload data", "log: log bytes written", "log: log files manually zero-filled", - "log: log flush operations", "log: log force write operations", - "log: log force write operations skipped", "log: log records compressed", - "log: log records not compressed", "log: log records too small to compress", - "log: log release advances write LSN", "log: log scan operations", - "log: log scan records requiring two reads", "log: log server thread advances write LSN", - "log: log server thread write LSN walk skipped", "log: log sync operations", - "log: log sync time duration (usecs)", "log: log sync_dir operations", - "log: log sync_dir time duration (usecs)", "log: log write operations", - "log: logging bytes consolidated", "log: maximum log file size", + "lock: txn global read lock acquisitions", + "lock: txn global write lock acquisitions", + "log: busy returns attempting to switch slots", + "log: force archive time sleeping (usecs)", + "log: log bytes of payload data", + "log: log bytes written", + "log: log files manually zero-filled", + "log: log flush operations", + "log: log force write operations", + "log: log force write operations skipped", + "log: log records compressed", + "log: log records not compressed", + "log: log records too small to compress", + "log: log release advances write LSN", + "log: log scan operations", + "log: log scan records requiring two reads", + "log: log server thread advances write LSN", + "log: log server thread write LSN walk skipped", + "log: log sync operations", + "log: log sync time duration (usecs)", + "log: log sync_dir operations", + "log: log sync_dir time duration (usecs)", + "log: log write operations", + "log: logging bytes consolidated", + "log: maximum log file size", "log: number of pre-allocated log files to create", - "log: pre-allocated log files not ready and missed", "log: pre-allocated log files prepared", - "log: pre-allocated log files used", "log: records processed by log scan", - "log: slot close lost race", "log: slot close unbuffered waits", "log: slot closures", - "log: slot join atomic update races", "log: slot join calls atomic updates raced", - "log: slot join calls did not yield", "log: slot join calls found active slot closed", - "log: slot join calls slept", "log: slot join calls yielded", - "log: slot join found active slot closed", "log: slot joins yield time (usecs)", - "log: slot transitions unable to find free slot", "log: slot unbuffered writes", - "log: total in-memory size of compressed records", "log: total log buffer size", - "log: total size of compressed records", "log: written slots coalesced", + "log: pre-allocated log files not ready and missed", + "log: pre-allocated log files prepared", + "log: pre-allocated log files used", + "log: records processed by log scan", + "log: slot close lost race", + "log: slot close unbuffered waits", + "log: slot closures", + "log: slot join atomic update races", + "log: slot join calls atomic updates raced", + "log: slot join calls did not yield", + "log: slot join calls found active slot closed", + "log: slot join calls slept", + "log: slot join calls yielded", + "log: slot join found active slot closed", + "log: slot joins yield time (usecs)", + "log: slot transitions unable to find free slot", + "log: slot unbuffered writes", + "log: total in-memory size of compressed records", + "log: total log buffer size", + "log: total size of compressed records", + "log: written slots coalesced", "log: yields waiting for previous log file close", "perf: file system read latency histogram (bucket 1) - 10-49ms", "perf: file system read latency histogram (bucket 2) - 50-99ms", @@ -1041,19 +1216,32 @@ static const char *const __stats_connection_desc[] = { "reconciliation: records written including a stop timestamp", "reconciliation: records written including a stop transaction ID", "reconciliation: split bytes currently awaiting free", - "reconciliation: split objects currently awaiting free", "session: open session count", - "session: session query timestamp calls", "session: table alter failed calls", - "session: table alter successful calls", "session: table alter unchanged and skipped", - "session: table compact failed calls", "session: table compact successful calls", - "session: table create failed calls", "session: table create successful calls", - "session: table drop failed calls", "session: table drop successful calls", - "session: table import failed calls", "session: table import successful calls", - "session: table rebalance failed calls", "session: table rebalance successful calls", - "session: table rename failed calls", "session: table rename successful calls", - "session: table salvage failed calls", "session: table salvage successful calls", - "session: table truncate failed calls", "session: table truncate successful calls", - "session: table verify failed calls", "session: table verify successful calls", - "thread-state: active filesystem fsync calls", "thread-state: active filesystem read calls", + "reconciliation: split objects currently awaiting free", + "session: open session count", + "session: session query timestamp calls", + "session: table alter failed calls", + "session: table alter successful calls", + "session: table alter unchanged and skipped", + "session: table compact failed calls", + "session: table compact successful calls", + "session: table create failed calls", + "session: table create successful calls", + "session: table drop failed calls", + "session: table drop successful calls", + "session: table import failed calls", + "session: table import successful calls", + "session: table rebalance failed calls", + "session: table rebalance successful calls", + "session: table rename failed calls", + "session: table rename successful calls", + "session: table salvage failed calls", + "session: table salvage successful calls", + "session: table truncate failed calls", + "session: table truncate successful calls", + "session: table verify failed calls", + "session: table verify successful calls", + "thread-state: active filesystem fsync calls", + "thread-state: active filesystem read calls", "thread-state: active filesystem write calls", "thread-yield: application thread time evicting (usecs)", "thread-yield: application thread time waiting for cache (usecs)", @@ -1063,42 +1251,59 @@ static const char *const __stats_connection_desc[] = { "thread-yield: get reference for page index and slot time sleeping (usecs)", "thread-yield: log server sync yielded for log write", "thread-yield: page access yielded due to prepare state change", - "thread-yield: page acquire busy blocked", "thread-yield: page acquire eviction blocked", - "thread-yield: page acquire locked blocked", "thread-yield: page acquire read blocked", + "thread-yield: page acquire busy blocked", + "thread-yield: page acquire eviction blocked", + "thread-yield: page acquire locked blocked", + "thread-yield: page acquire read blocked", "thread-yield: page acquire time sleeping (usecs)", "thread-yield: page delete rollback time sleeping for state change (usecs)", "thread-yield: page reconciliation yielded due to child modification", - "transaction: Number of prepared updates", "transaction: durable timestamp queue entries walked", + "transaction: Number of prepared updates", + "transaction: durable timestamp queue entries walked", "transaction: durable timestamp queue insert to empty", "transaction: durable timestamp queue inserts to head", "transaction: durable timestamp queue inserts total", - "transaction: durable timestamp queue length", "transaction: prepared transactions", + "transaction: durable timestamp queue length", + "transaction: prepared transactions", "transaction: prepared transactions committed", "transaction: prepared transactions currently active", - "transaction: prepared transactions rolled back", "transaction: query timestamp calls", + "transaction: prepared transactions rolled back", + "transaction: query timestamp calls", "transaction: race to read prepared update retry", "transaction: read timestamp queue entries walked", "transaction: read timestamp queue insert to empty", "transaction: read timestamp queue inserts to head", - "transaction: read timestamp queue inserts total", "transaction: read timestamp queue length", + "transaction: read timestamp queue inserts total", + "transaction: read timestamp queue length", "transaction: rollback to stable calls", "transaction: rollback to stable hs records with stop timestamps older than newer records", - "transaction: rollback to stable keys removed", "transaction: rollback to stable keys restored", + "transaction: rollback to stable keys removed", + "transaction: rollback to stable keys restored", "transaction: rollback to stable pages visited", "transaction: rollback to stable restored tombstones from history store", "transaction: rollback to stable sweeping history store keys", "transaction: rollback to stable tree walk skipping pages", "transaction: rollback to stable updates aborted", "transaction: rollback to stable updates removed from history store", - "transaction: set timestamp calls", "transaction: set timestamp durable calls", - "transaction: set timestamp durable updates", "transaction: set timestamp oldest calls", - "transaction: set timestamp oldest updates", "transaction: set timestamp stable calls", - "transaction: set timestamp stable updates", "transaction: transaction begins", + "transaction: set timestamp calls", + "transaction: set timestamp durable calls", + "transaction: set timestamp durable updates", + "transaction: set timestamp oldest calls", + "transaction: set timestamp oldest updates", + "transaction: set timestamp stable calls", + "transaction: set timestamp stable updates", + "transaction: transaction begins", "transaction: transaction checkpoint currently running", "transaction: transaction checkpoint generation", "transaction: transaction checkpoint history store file duration (usecs)", "transaction: transaction checkpoint max time (msecs)", "transaction: transaction checkpoint min time (msecs)", + "transaction: transaction checkpoint most recent duration for gathering all handles (usecs)", + "transaction: transaction checkpoint most recent duration for gathering applied handles (usecs)", + "transaction: transaction checkpoint most recent duration for gathering skipped handles (usecs)", + "transaction: transaction checkpoint most recent handles applied", + "transaction: transaction checkpoint most recent handles skipped", + "transaction: transaction checkpoint most recent handles walked", "transaction: transaction checkpoint most recent time (msecs)", "transaction: transaction checkpoint prepare currently running", "transaction: transaction checkpoint prepare max time (msecs)", @@ -1107,7 +1312,8 @@ static const char *const __stats_connection_desc[] = { "transaction: transaction checkpoint prepare total time (msecs)", "transaction: transaction checkpoint scrub dirty target", "transaction: transaction checkpoint scrub time (msecs)", - "transaction: transaction checkpoint total time (msecs)", "transaction: transaction checkpoints", + "transaction: transaction checkpoint total time (msecs)", + "transaction: transaction checkpoints", "transaction: transaction checkpoints skipped because database was clean", "transaction: transaction failures due to history store", "transaction: transaction fsync calls for checkpoint after allocating the transaction ID", @@ -1120,8 +1326,10 @@ static const char *const __stats_connection_desc[] = { "transaction: transaction range of timestamps pinned by the oldest active read timestamp", "transaction: transaction range of timestamps pinned by the oldest timestamp", "transaction: transaction read timestamp of the oldest active reader", - "transaction: transaction sync calls", "transaction: transactions committed", - "transaction: transactions rolled back", "transaction: update conflicts", + "transaction: transaction sync calls", + "transaction: transactions committed", + "transaction: transactions rolled back", + "transaction: update conflicts", }; int @@ -1347,6 +1555,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cond_auto_wait_skipped = 0; stats->time_travel = 0; /* not clearing file_open */ + /* not clearing buckets_dh */ + /* not clearing buckets */ stats->memory_allocation = 0; stats->memory_free = 0; stats->memory_grow = 0; @@ -1614,6 +1824,12 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->txn_hs_ckpt_duration = 0; /* not clearing txn_checkpoint_time_max */ /* not clearing txn_checkpoint_time_min */ + /* not clearing txn_checkpoint_handle_duration */ + /* not clearing txn_checkpoint_handle_duration_apply */ + /* not clearing txn_checkpoint_handle_duration_skip */ + stats->txn_checkpoint_handle_applied = 0; + stats->txn_checkpoint_handle_skipped = 0; + stats->txn_checkpoint_handle_walked = 0; /* not clearing txn_checkpoint_time_recent */ /* not clearing txn_checkpoint_prep_running */ /* not clearing txn_checkpoint_prep_max */ @@ -1861,6 +2077,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->cond_auto_wait_skipped += WT_STAT_READ(from, cond_auto_wait_skipped); to->time_travel += WT_STAT_READ(from, time_travel); to->file_open += WT_STAT_READ(from, file_open); + to->buckets_dh += WT_STAT_READ(from, buckets_dh); + to->buckets += WT_STAT_READ(from, buckets); to->memory_allocation += WT_STAT_READ(from, memory_allocation); to->memory_free += WT_STAT_READ(from, memory_free); to->memory_grow += WT_STAT_READ(from, memory_grow); @@ -2137,6 +2355,14 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->txn_hs_ckpt_duration += WT_STAT_READ(from, txn_hs_ckpt_duration); to->txn_checkpoint_time_max += WT_STAT_READ(from, txn_checkpoint_time_max); to->txn_checkpoint_time_min += WT_STAT_READ(from, txn_checkpoint_time_min); + to->txn_checkpoint_handle_duration += WT_STAT_READ(from, txn_checkpoint_handle_duration); + to->txn_checkpoint_handle_duration_apply += + WT_STAT_READ(from, txn_checkpoint_handle_duration_apply); + to->txn_checkpoint_handle_duration_skip += + WT_STAT_READ(from, txn_checkpoint_handle_duration_skip); + to->txn_checkpoint_handle_applied += WT_STAT_READ(from, txn_checkpoint_handle_applied); + to->txn_checkpoint_handle_skipped += WT_STAT_READ(from, txn_checkpoint_handle_skipped); + to->txn_checkpoint_handle_walked += WT_STAT_READ(from, txn_checkpoint_handle_walked); to->txn_checkpoint_time_recent += WT_STAT_READ(from, txn_checkpoint_time_recent); to->txn_checkpoint_prep_running += WT_STAT_READ(from, txn_checkpoint_prep_running); to->txn_checkpoint_prep_max += WT_STAT_READ(from, txn_checkpoint_prep_max); @@ -2166,8 +2392,10 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * } static const char *const __stats_join_desc[] = { - ": accesses to the main table", ": bloom filter false positives", - ": checks that conditions of membership are satisfied", ": items inserted into a bloom filter", + ": accesses to the main table", + ": bloom filter false positives", + ": checks that conditions of membership are satisfied", + ": items inserted into a bloom filter", ": items iterated", }; @@ -2215,9 +2443,12 @@ __wt_stat_join_aggregate(WT_JOIN_STATS **from, WT_JOIN_STATS *to) } static const char *const __stats_session_desc[] = { - "session: bytes read into cache", "session: bytes written from cache", - "session: dhandle lock wait time (usecs)", "session: page read from disk to cache time (usecs)", - "session: page write from cache to disk time (usecs)", "session: schema lock wait time (usecs)", + "session: bytes read into cache", + "session: bytes written from cache", + "session: dhandle lock wait time (usecs)", + "session: page read from disk to cache time (usecs)", + "session: page write from cache to disk time (usecs)", + "session: schema lock wait time (usecs)", "session: time waiting for cache (usecs)", }; diff --git a/src/third_party/wiredtiger/src/support/thread_group.c b/src/third_party/wiredtiger/src/support/thread_group.c index 2ba628f7d1f..fddf7941d35 100644 --- a/src/third_party/wiredtiger/src/support/thread_group.c +++ b/src/third_party/wiredtiger/src/support/thread_group.c @@ -47,7 +47,8 @@ err: * 2. When the connection is closing. * 3. When a shutdown has been requested via clearing the run flag. */ - WT_ASSERT(session, !F_ISSET(thread, WT_THREAD_RUN) || + WT_ASSERT(session, + !F_ISSET(thread, WT_THREAD_RUN) || F_ISSET(S2C(session), WT_CONN_CLOSING | WT_CONN_RECOVERING)); return (WT_THREAD_RET_VALUE); @@ -139,8 +140,9 @@ __thread_group_resize(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t conn = S2C(session); thread = NULL; - __wt_verbose(session, WT_VERB_THREAD_GROUP, "Resize thread group: %s, from min: %" PRIu32 - " -> %" PRIu32 " from max: %" PRIu32 " -> %" PRIu32, + __wt_verbose(session, WT_VERB_THREAD_GROUP, + "Resize thread group: %s, from min: %" PRIu32 " -> %" PRIu32 " from max: %" PRIu32 + " -> %" PRIu32, group->name, group->min, new_min, group->max, new_max); WT_ASSERT(session, @@ -150,8 +152,9 @@ __thread_group_resize(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t return (0); if (new_min > new_max) - WT_ERR_MSG(session, EINVAL, "Illegal thread group resize: %s, from min: %" PRIu32 - " -> %" PRIu32 " from max: %" PRIu32 " -> %" PRIu32, + WT_ERR_MSG(session, EINVAL, + "Illegal thread group resize: %s, from min: %" PRIu32 " -> %" PRIu32 " from max: %" PRIu32 + " -> %" PRIu32, group->name, group->min, new_min, group->max, new_max); /* @@ -175,7 +178,7 @@ __thread_group_resize(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t */ for (i = group->max; i < new_max; i++) { WT_ERR(__wt_calloc_one(session, &thread)); - /* Threads get their own session */ + /* Threads get their own session. */ session_flags = LF_ISSET(WT_THREAD_CAN_WAIT) ? WT_SESSION_CAN_WAIT : 0; WT_ERR( __wt_open_internal_session(conn, group->name, false, session_flags, &thread->session)); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 0c3ff78fa74..014b449c6f7 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -124,8 +124,8 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) txn_global = &S2C(session)->txn_global; txn_shared = WT_SESSION_TXN_SHARED(session); - WT_ASSERT(session, txn_shared->pinned_id == WT_TXN_NONE || - session->txn->isolation == WT_ISO_READ_UNCOMMITTED || + WT_ASSERT(session, + txn_shared->pinned_id == WT_TXN_NONE || session->txn->isolation == WT_ISO_READ_UNCOMMITTED || !__wt_txn_visible_all(session, txn_shared->pinned_id, WT_TS_NONE)); txn_shared->metadata_pinned = txn_shared->pinned_id = WT_TXN_NONE; @@ -406,7 +406,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) if ((oldest_id == prev_oldest_id || (!strict && WT_TXNID_LT(oldest_id, prev_oldest_id + 100))) && ((last_running == prev_last_running) || - (!strict && WT_TXNID_LT(last_running, prev_last_running + 100))) && + (!strict && WT_TXNID_LT(last_running, prev_last_running + 100))) && metadata_pinned == prev_metadata_pinned) return (0); @@ -444,9 +444,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) if (WT_VERBOSE_ISSET(session, WT_VERB_TRANSACTION) && current_id - oldest_id > 10000 && oldest_session != NULL) { __wt_verbose(session, WT_VERB_TRANSACTION, - "old snapshot %" PRIu64 " pinned in session %" PRIu32 - " [%s]" - " with snap_min %" PRIu64, + "old snapshot %" PRIu64 " pinned in session %" PRIu32 " [%s] with snap_min %" PRIu64, oldest_id, oldest_session->id, oldest_session->lastop, oldest_session->txn->snap_min); } } @@ -457,6 +455,33 @@ done: } /* + * __txn_config_operation_timeout -- + * Configure a transactions operation timeout duration. + */ +static int +__txn_config_operation_timeout(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_CONFIG_ITEM cval; + WT_TXN *txn; + + txn = session->txn; + + if (cfg == NULL) + return (0); + + /* Retrieve the maximum operation time, defaulting to the database-wide configuration. */ + WT_RET(__wt_config_gets(session, cfg, "operation_timeout_ms", &cval)); + + /* + * The default configuration value is 0, we can't tell if they're setting it back to 0 or, if + * the default was automatically passed in. + */ + if (cval.val != 0) + txn->operation_timeout_us = (uint64_t)(cval.val * WT_THOUSAND); + return (0); +} + +/* * __wt_txn_config -- * Configure a transaction. */ @@ -469,6 +494,9 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) txn = session->txn; + if (cfg == NULL) + return (0); + WT_RET(__wt_config_gets_def(session, cfg, "isolation", 0, &cval)); if (cval.len != 0) txn->isolation = WT_STRING_MATCH("snapshot", cval.str, cval.len) ? @@ -476,9 +504,7 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_STRING_MATCH("read-committed", cval.str, cval.len) ? WT_ISO_READ_COMMITTED : WT_ISO_READ_UNCOMMITTED; - /* Retrieve the maximum operation time, defaulting to the database-wide configuration. */ - WT_RET(__wt_config_gets(session, cfg, "operation_timeout_ms", &cval)); - txn->operation_timeout_us = (uint64_t)(cval.val * WT_THOUSAND); + WT_RET(__txn_config_operation_timeout(session, cfg)); /* * The default sync setting is inherited from the connection, but can be overridden by an @@ -723,8 +749,18 @@ __txn_append_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_ITEM * if (commit) goto done; + /* + * Set the flag to indicate that this update has been restored from history store for the + * rollback of a prepared transaction. + */ + F_SET(upd, WT_UPDATE_RESTORED_FROM_HS); total_size += size; + __wt_verbose(session, WT_VERB_TRANSACTION, + "update restored from history store (txnid: %" PRIu64 ", start_ts: %s, durable_ts: %s", + upd->txnid, __wt_timestamp_to_string(upd->start_ts, ts_string[0]), + __wt_timestamp_to_string(upd->durable_ts, ts_string[1])); + /* If the history store record has a valid stop time point, append it. */ if (hs_stop_durable_ts != WT_TS_MAX) { WT_ASSERT(session, hs_cbt->upd_value->tw.stop_ts != WT_TS_MAX); @@ -733,20 +769,20 @@ __txn_append_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_ITEM * tombstone->start_ts = hs_cbt->upd_value->tw.stop_ts; tombstone->txnid = hs_cbt->upd_value->tw.stop_txn; tombstone->next = upd; + /* + * Set the flag to indicate that this update has been restored from history store for the + * rollback of a prepared transaction. + */ + F_SET(tombstone, WT_UPDATE_RESTORED_FROM_HS); total_size += size; - } else - tombstone = upd; - __wt_verbose(session, WT_VERB_TRANSACTION, - "update restored from history store (txnid: %" PRIu64 ", start_ts: %s, durable_ts: %s", - upd->txnid, __wt_timestamp_to_string(upd->start_ts, ts_string[0]), - __wt_timestamp_to_string(upd->durable_ts, ts_string[1])); + __wt_verbose(session, WT_VERB_TRANSACTION, + "tombstone restored from history store (txnid: %" PRIu64 ", start_ts: %s, durable_ts: %s", + tombstone->txnid, __wt_timestamp_to_string(tombstone->start_ts, ts_string[0]), + __wt_timestamp_to_string(tombstone->durable_ts, ts_string[1])); - /* - * Set the flag to indicate that this update has been restored from history store for the - * rollback of a prepared transaction. - */ - F_SET(upd, WT_UPDATE_RESTORED_FROM_HS); + upd = tombstone; + } /* Walk to the end of the chain and we can only have prepared updates on the update chain. */ for (;; chain = chain->next) { @@ -758,15 +794,15 @@ __txn_append_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_ITEM * } /* Append the update to the end of the chain. */ - WT_PUBLISH(chain->next, tombstone); + WT_PUBLISH(chain->next, upd); *upd_appended = true; __wt_cache_page_inmem_incr(session, page, total_size); if (0) { err: - __wt_free(session, upd); - __wt_free(session, tombstone); + WT_ASSERT(session, tombstone == NULL || upd == tombstone); + __wt_free_update_list(session, &upd); } done: __wt_scr_free(session, &hs_key); @@ -914,13 +950,12 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_TXN *txn; WT_UPDATE *fix_upd, *tombstone, *upd; size_t not_used; - uint32_t hs_btree_id, session_flags; + uint32_t hs_btree_id; bool upd_appended; hs_cursor = NULL; txn = session->txn; fix_upd = tombstone = NULL; - session_flags = 0; upd_appended = false; WT_RET(__txn_search_prepared_op(session, op, cursorp, &upd)); @@ -953,7 +988,7 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, cbt = (WT_CURSOR_BTREE *)(*cursorp); hs_btree_id = S2BT(session)->id; /* Open a history store table cursor. */ - WT_ERR(__wt_hs_cursor_open(session, &session_flags)); + WT_ERR(__wt_hs_cursor_open(session)); hs_cursor = session->hs_cursor; /* @@ -976,8 +1011,9 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, * and instead write nothing. */ WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, ¬_used)); - WT_WITH_BTREE(session, op->btree, ret = __wt_row_modify(cbt, &cbt->iface.key, NULL, - tombstone, WT_UPDATE_INVALID, false)); + WT_WITH_BTREE(session, op->btree, + ret = + __wt_row_modify(cbt, &cbt->iface.key, NULL, tombstone, WT_UPDATE_INVALID, false)); WT_ERR(ret); tombstone = NULL; } else @@ -1044,7 +1080,7 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, err: if (hs_cursor != NULL) - WT_TRET(__wt_hs_cursor_close(session, session_flags)); + WT_TRET(__wt_hs_cursor_close(session)); if (!upd_appended) __wt_free(session, fix_upd); __wt_free(session, tombstone); @@ -1243,6 +1279,9 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || txn->mod_count == 0); + /* Configure the timeout for this commit operation. */ + WT_ERR(__txn_config_operation_timeout(session, cfg)); + /* * Clear the prepared round up flag if the transaction is not prepared. There is no rounding up * to do in that case. @@ -1513,8 +1552,8 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) */ if (!FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_DEBUG_MODE)) WT_RET_ASSERT(session, txn->logrec == NULL, EINVAL, - "A transaction should not have been assigned a log" - " record if WT_CONN_LOG_DEBUG mode is not enabled"); + "A transaction should not have been assigned a log record if WT_CONN_LOG_DEBUG mode is " + "not enabled"); /* Set the prepare timestamp. */ WT_RET(__wt_txn_set_timestamp(session, cfg)); @@ -1639,8 +1678,6 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) u_int i; bool prepare, readonly; - WT_UNUSED(cfg); - cursor = NULL; txn = session->txn; prepare = F_ISSET(txn, WT_TXN_PREPARE); @@ -1652,6 +1689,9 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) if (txn->notify != NULL) WT_TRET(txn->notify->notify(txn->notify, (WT_SESSION *)session, txn->id, 0)); + /* Configure the timeout for this rollback operation. */ + WT_RET(__txn_config_operation_timeout(session, cfg)); + /* * Resolving prepared updates is expensive. Sort prepared modifications so all updates for each * page within each file are done at the same time. @@ -1757,7 +1797,8 @@ __wt_txn_init(WT_SESSION_IMPL *session, WT_SESSION_IMPL *session_ret) txn->snapshot = txn->__snapshot; txn->id = WT_TXN_NONE; - WT_ASSERT(session, S2C(session_ret)->txn_global.txn_shared_list == NULL || + WT_ASSERT(session, + S2C(session_ret)->txn_global.txn_shared_list == NULL || WT_SESSION_TXN_SHARED(session_ret)->pinned_id == WT_TXN_NONE); /* @@ -1950,12 +1991,11 @@ __wt_txn_activity_drain(WT_SESSION_IMPL *session) * Shut down the global transaction state. */ int -__wt_txn_global_shutdown(WT_SESSION_IMPL *session, const char *config, const char **cfg) +__wt_txn_global_shutdown(WT_SESSION_IMPL *session, const char **cfg) { WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION *wt_session; WT_SESSION_IMPL *s; char ts_string[WT_TS_INT_STRING_SIZE]; const char *ckpt_cfg; @@ -1975,7 +2015,7 @@ __wt_txn_global_shutdown(WT_SESSION_IMPL *session, const char *config, const cha if (conn->txn_global.has_stable_timestamp) F_SET(conn, WT_CONN_CLOSING_TIMESTAMP); } - if (!F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_READONLY)) { + if (!F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_READONLY | WT_CONN_PANIC)) { /* * Perform rollback to stable to ensure that the stable version is written to disk on a * clean shutdown. @@ -1992,7 +2032,6 @@ __wt_txn_global_shutdown(WT_SESSION_IMPL *session, const char *config, const cha if (s != NULL) { const char *checkpoint_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_checkpoint), ckpt_cfg, NULL}; - wt_session = &s->iface; WT_TRET(__wt_txn_checkpoint(s, checkpoint_cfg, true)); /* @@ -2000,7 +2039,7 @@ __wt_txn_global_shutdown(WT_SESSION_IMPL *session, const char *config, const cha */ WT_WITH_DHANDLE(s, WT_SESSION_META_DHANDLE(s), __wt_tree_modify_set(s)); - WT_TRET(wt_session->close(wt_session, config)); + WT_TRET(__wt_session_close_internal(s)); } } @@ -2051,8 +2090,9 @@ __wt_txn_is_blocking(WT_SESSION_IMPL *session, bool conservative) * a transaction, we need to have considered splitting the page in the case that its updates are * on a single page. */ - if (conservative && (txn->mod_count < (10 + WT_REC_SPLIT_MIN_ITEMS_USE_MEM) || - F_ISSET(session, WT_SESSION_RESOLVING_TXN))) + if (conservative && + (txn->mod_count < (10 + WT_REC_SPLIT_MIN_ITEMS_USE_MEM) || + F_ISSET(session, WT_SESSION_RESOLVING_TXN))) return (0); /* @@ -2098,28 +2138,29 @@ __wt_verbose_dump_txn_one( * Dump the information of the passed transaction into a buffer, to be logged with an optional * error message. */ - WT_RET(__wt_snprintf(buf, - sizeof(buf), "transaction id: %" PRIu64 ", mod count: %u" - ", snap min: %" PRIu64 ", snap max: %" PRIu64 ", snapshot count: %u" - ", commit_timestamp: %s" - ", durable_timestamp: %s" - ", first_commit_timestamp: %s" - ", prepare_timestamp: %s" - ", pinned_durable_timestamp: %s" - ", read_timestamp: %s" - ", checkpoint LSN: [%" PRIu32 "][%" PRIu32 "]" - ", full checkpoint: %s" - ", rollback reason: %s" - ", flags: 0x%08" PRIx32 ", isolation: %s", - txn->id, txn->mod_count, txn->snap_min, txn->snap_max, txn->snapshot_count, - __wt_timestamp_to_string(txn->commit_timestamp, ts_string[0]), - __wt_timestamp_to_string(txn->durable_timestamp, ts_string[1]), - __wt_timestamp_to_string(txn->first_commit_timestamp, ts_string[2]), - __wt_timestamp_to_string(txn->prepare_timestamp, ts_string[3]), - __wt_timestamp_to_string(txn_shared->pinned_durable_timestamp, ts_string[4]), - __wt_timestamp_to_string(txn_shared->read_timestamp, ts_string[5]), txn->ckpt_lsn.l.file, - txn->ckpt_lsn.l.offset, txn->full_ckpt ? "true" : "false", - txn->rollback_reason == NULL ? "" : txn->rollback_reason, txn->flags, iso_tag)); + WT_RET( + __wt_snprintf(buf, sizeof(buf), + "transaction id: %" PRIu64 ", mod count: %u" + ", snap min: %" PRIu64 ", snap max: %" PRIu64 ", snapshot count: %u" + ", commit_timestamp: %s" + ", durable_timestamp: %s" + ", first_commit_timestamp: %s" + ", prepare_timestamp: %s" + ", pinned_durable_timestamp: %s" + ", read_timestamp: %s" + ", checkpoint LSN: [%" PRIu32 "][%" PRIu32 "]" + ", full checkpoint: %s" + ", rollback reason: %s" + ", flags: 0x%08" PRIx32 ", isolation: %s", + txn->id, txn->mod_count, txn->snap_min, txn->snap_max, txn->snapshot_count, + __wt_timestamp_to_string(txn->commit_timestamp, ts_string[0]), + __wt_timestamp_to_string(txn->durable_timestamp, ts_string[1]), + __wt_timestamp_to_string(txn->first_commit_timestamp, ts_string[2]), + __wt_timestamp_to_string(txn->prepare_timestamp, ts_string[3]), + __wt_timestamp_to_string(txn_shared->pinned_durable_timestamp, ts_string[4]), + __wt_timestamp_to_string(txn_shared->read_timestamp, ts_string[5]), txn->ckpt_lsn.l.file, + txn->ckpt_lsn.l.offset, txn->full_ckpt ? "true" : "false", + txn->rollback_reason == NULL ? "" : txn->rollback_reason, txn->flags, iso_tag)); /* * Log a message and return an error if error code and an optional error string has been passed. @@ -2258,13 +2299,14 @@ __wt_verbose_dump_update(WT_SESSION_IMPL *session, WT_UPDATE *upd) break; } - __wt_errx(session, "transaction id: %" PRIu64 - ", commit timestamp: %s" - ", durable timestamp: %s" - ", has next: %s" - ", size: %" PRIu32 - ", type: %s" - ", prepare state: %s", + __wt_errx(session, + "transaction id: %" PRIu64 + ", commit timestamp: %s" + ", durable timestamp: %s" + ", has next: %s" + ", size: %" PRIu32 + ", type: %s" + ", prepare state: %s", upd->txnid, __wt_timestamp_to_string(upd->start_ts, ts_string[0]), __wt_timestamp_to_string(upd->durable_ts, ts_string[1]), upd->next == NULL ? "no" : "yes", upd->size, upd_type, prepare_state); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 9b3d1bb51fb..59a10c36733 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -8,7 +8,7 @@ #include "wt_internal.h" -static void __checkpoint_timing_stress(WT_SESSION_IMPL *, bool); +static void __checkpoint_timing_stress(WT_SESSION_IMPL *, uint64_t, struct timespec *); static int __checkpoint_lock_dirty_tree(WT_SESSION_IMPL *, bool, bool, bool, const char *[]); static int __checkpoint_mark_skip(WT_SESSION_IMPL *, WT_CKPT *, bool); static int __checkpoint_presync(WT_SESSION_IMPL *, const char *[]); @@ -137,9 +137,7 @@ __checkpoint_apply_operation( } if (v.len != 0) - WT_ERR_MSG(session, EINVAL, - "invalid checkpoint target %.*s: URIs may require " - "quoting", + WT_ERR_MSG(session, EINVAL, "invalid checkpoint target %.*s: URIs may require quoting", (int)cval.len, (char *)cval.str); /* Some objects don't support named checkpoints. */ @@ -517,6 +515,7 @@ __checkpoint_fail_reset(WT_SESSION_IMPL *session) static int __checkpoint_prepare(WT_SESSION_IMPL *session, bool *trackingp, const char *cfg[]) { + struct timespec tsp; WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -546,6 +545,10 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, bool *trackingp, const char *cfg[ __wt_epoch(session, &conn->ckpt_prep_start); WT_RET(__wt_txn_begin(session, txn_cfg)); + /* Wait 1000 microseconds to simulate slowdown in checkpoint prepare. */ + tsp.tv_sec = 0; + tsp.tv_nsec = WT_MILLION; + __checkpoint_timing_stress(session, WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY, &tsp); original_snap_min = session->txn->snap_min; WT_DIAGNOSTIC_YIELD; @@ -585,7 +588,8 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, bool *trackingp, const char *cfg[ /* * Sanity check that the oldest ID hasn't moved on before we have cleared our entry. */ - WT_ASSERT(session, WT_TXNID_LE(txn_global->oldest_id, txn_shared->id) && + WT_ASSERT(session, + WT_TXNID_LE(txn_global->oldest_id, txn_shared->id) && WT_TXNID_LE(txn_global->oldest_id, txn_shared->pinned_id)); /* @@ -748,6 +752,7 @@ __txn_checkpoint_can_skip( static int __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) { + struct timespec tsp; WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *hs_dhandle; @@ -870,11 +875,14 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) if (full && logging) WT_ERR(__wt_txn_checkpoint_log(session, full, WT_TXN_LOG_CKPT_START, NULL)); - __checkpoint_timing_stress(session, false); + /* Add a ten second wait to simulate checkpoint slowness. */ + tsp.tv_sec = 10; + tsp.tv_nsec = 0; + __checkpoint_timing_stress(session, WT_TIMING_STRESS_CHECKPOINT_SLOW, &tsp); WT_ERR(__checkpoint_apply_to_dhandles(session, cfg, __checkpoint_tree_helper)); /* Wait prior to checkpointing the history store to simulate checkpoint slowness. */ - __checkpoint_timing_stress(session, true); + __checkpoint_timing_stress(session, WT_TIMING_STRESS_HS_CHECKPOINT_DELAY, &tsp); /* * Get a history store dhandle. If the history store file is opened for a special operation this @@ -1129,11 +1137,8 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[], bool waiting) */ #undef WT_CHECKPOINT_SESSION_FLAGS #define WT_CHECKPOINT_SESSION_FLAGS (WT_SESSION_CAN_WAIT | WT_SESSION_IGNORE_CACHE_SIZE) -#undef WT_CHECKPOINT_SESSION_FLAGS_OFF -#define WT_CHECKPOINT_SESSION_FLAGS_OFF (WT_SESSION_HS_CURSOR) - orig_flags = F_MASK(session, WT_CHECKPOINT_SESSION_FLAGS | WT_CHECKPOINT_SESSION_FLAGS_OFF); + orig_flags = F_MASK(session, WT_CHECKPOINT_SESSION_FLAGS); F_SET(session, WT_CHECKPOINT_SESSION_FLAGS); - F_CLR(session, WT_CHECKPOINT_SESSION_FLAGS_OFF); /* * Only one checkpoint can be active at a time, and checkpoints must run in the same order as @@ -1274,9 +1279,8 @@ __checkpoint_lock_dirty_tree_int(WT_SESSION_IMPL *session, bool is_checkpoint, b continue; } WT_RET_MSG(session, EBUSY, - "checkpoint %s blocked by hot backup: it would " - "delete an existing named checkpoint, and such " - "checkpoints cannot be deleted during a hot backup", + "checkpoint %s blocked by hot backup: it would delete an existing named checkpoint, " + "and such checkpoints cannot be deleted during a hot backup", ckpt->name); } /* @@ -1307,8 +1311,9 @@ __checkpoint_lock_dirty_tree_int(WT_SESSION_IMPL *session, bool is_checkpoint, b WT_CKPT_FOREACH (ckptbase, ckpt) { if (!F_ISSET(ckpt, WT_CKPT_DELETE)) continue; - WT_ASSERT(session, !WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT) || - conn->hot_backup_start == 0 || ckpt->sec > conn->hot_backup_start); + WT_ASSERT(session, + !WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT) || conn->hot_backup_start == 0 || + ckpt->sec > conn->hot_backup_start); /* * We can't delete checkpoints referenced by a cursor. WiredTiger checkpoints are * uniquely named and it's OK to have multiple in the system: clear the delete flag for @@ -1445,9 +1450,7 @@ __checkpoint_lock_dirty_tree( else if (WT_STRING_MATCH("to", k.str, k.len)) __drop_to(ckptbase, v.str, v.len); else - WT_ERR_MSG(session, EINVAL, - "unexpected value for checkpoint " - "key: %.*s", + WT_ERR_MSG(session, EINVAL, "unexpected value for checkpoint key: %.*s", (int)k.len, k.str); } WT_ERR_NOTFOUND_OK(ret, false); @@ -1536,8 +1539,8 @@ __checkpoint_mark_skip(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, bool force) name = (ckpt - 1)->name; if (ckpt > ckptbase + 1 && deleted < 2 && (strcmp(name, (ckpt - 2)->name) == 0 || - (WT_PREFIX_MATCH(name, WT_CHECKPOINT) && - WT_PREFIX_MATCH((ckpt - 2)->name, WT_CHECKPOINT)))) { + (WT_PREFIX_MATCH(name, WT_CHECKPOINT) && + WT_PREFIX_MATCH((ckpt - 2)->name, WT_CHECKPOINT)))) { F_SET(btree, WT_BTREE_SKIP_CKPT); /* * If there are potentially extra checkpoints to delete, we set the timer to recheck @@ -1887,7 +1890,7 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) */ if (btree->modified && !bulk && !__wt_btree_immediately_durable(session) && (S2C(session)->txn_global.has_stable_timestamp || - (!F_ISSET(S2C(session), WT_CONN_FILE_CLOSE_SYNC) && !metadata))) + (!F_ISSET(S2C(session), WT_CONN_FILE_CLOSE_SYNC) && !metadata))) return (__wt_set_return(session, EBUSY)); /* @@ -1915,12 +1918,12 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) /* * __checkpoint_timing_stress -- - * Optionally add a 10 second delay to a checkpoint to simulate a long running checkpoint for - * debug purposes. The reason for this option is finding operations that can block while waiting - * for a checkpoint to complete. + * Optionally add a delay to a checkpoint to simulate a long running checkpoint for debug + * purposes. The reason for this option is finding operations that can block while waiting for a + * checkpoint to complete. */ static void -__checkpoint_timing_stress(WT_SESSION_IMPL *session, bool history_store_stress) +__checkpoint_timing_stress(WT_SESSION_IMPL *session, uint64_t flag, struct timespec *tsp) { WT_CONNECTION_IMPL *conn; @@ -1931,9 +1934,6 @@ __checkpoint_timing_stress(WT_SESSION_IMPL *session, bool history_store_stress) * the session used is either of the two sessions set aside for internal checkpoints. */ if (conn->ckpt_session != session && conn->meta_ckpt_session != session && - ((FLD_ISSET(conn->timing_stress_flags, WT_TIMING_STRESS_CHECKPOINT_SLOW) && - !history_store_stress) || - (FLD_ISSET(conn->timing_stress_flags, WT_TIMING_STRESS_HS_CHECKPOINT_DELAY) && - history_store_stress))) - __wt_sleep(10, 0); + FLD_ISSET(conn->timing_stress_flags, flag)) + __wt_sleep((uint64_t)tsp->tv_sec, (uint64_t)tsp->tv_nsec / WT_THOUSAND); } diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index d541d8f48d5..b335d4ca3cb 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -554,7 +554,7 @@ __wt_txn_checkpoint_log(WT_SESSION_IMPL *session, bool full, uint32_t flags, WT_ */ if (conn->hot_backup_start == 0 && (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY) || - FLD_ISSET(conn->log_flags, WT_CONN_LOG_FORCE_DOWNGRADE)) && + FLD_ISSET(conn->log_flags, WT_CONN_LOG_FORCE_DOWNGRADE)) && txn->full_ckpt) __wt_log_ckpt(session, ckpt_lsn); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index bc60efa5d51..fd230dab529 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -282,8 +282,9 @@ done: return (0); err: - __wt_err(session, ret, "operation apply failed during recovery: operation type %" PRIu32 - " at LSN %" PRIu32 "/%" PRIu32, + __wt_err(session, ret, + "operation apply failed during recovery: operation type %" PRIu32 " at LSN %" PRIu32 + "/%" PRIu32, optype, lsnp->l.file, lsnp->l.offset); return (ret); } @@ -477,9 +478,8 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) if (r->files[fileid].uri != NULL) WT_RET_PANIC(r->session, WT_PANIC, - "metadata corruption: files %s and %s have the same " - "file ID %u", - uri, r->files[fileid].uri, fileid); + "metadata corruption: files %s and %s have the same file ID %u", uri, + r->files[fileid].uri, fileid); WT_RET(__wt_strdup(r->session, uri, &r->files[fileid].uri)); WT_RET(__wt_config_getones(r->session, config, "checkpoint_lsn", &cval)); /* If there is no checkpoint logged for the file, apply everything. */ @@ -851,7 +851,8 @@ done: * written. The rollback to stable operation should only rollback the latest page changes * solely based on the write generation numbers. */ - WT_ASSERT(session, conn->txn_global.has_stable_timestamp == false && + WT_ASSERT(session, + conn->txn_global.has_stable_timestamp == false && conn->txn_global.stable_timestamp == WT_TS_NONE); /* @@ -900,7 +901,7 @@ err: if (eviction_started) WT_TRET(__wt_evict_destroy(session)); - WT_TRET(session->iface.close(&session->iface, NULL)); + WT_TRET(__wt_session_close_internal(session)); F_CLR(conn, WT_CONN_RECOVERING); return (ret); diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 2c97ddf48c7..65ae870b8fe 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -35,7 +35,8 @@ __rollback_abort_newer_update(WT_SESSION_IMPL *session, WT_UPDATE *first_upd, * is not configured for key consistency check, the timestamps could be out of order * here. */ - WT_ASSERT(session, !FLD_ISSET(S2BT(session)->assert_flags, WT_ASSERT_COMMIT_TS_KEYS) || + WT_ASSERT(session, + !FLD_ISSET(S2BT(session)->assert_flags, WT_ASSERT_COMMIT_TS_KEYS) || upd == first_upd); first_upd = upd->next; @@ -165,7 +166,7 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW WT_UPDATE *hs_upd, *tombstone, *upd; wt_timestamp_t hs_durable_ts, hs_start_ts, hs_stop_durable_ts, newer_hs_durable_ts; uint64_t hs_counter, type_full; - uint32_t hs_btree_id, session_flags; + uint32_t hs_btree_id; uint8_t type; int cmp; char ts_string[4][WT_TS_INT_STRING_SIZE]; @@ -178,7 +179,6 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW hs_upd = tombstone = upd = NULL; hs_durable_ts = hs_start_ts = hs_stop_durable_ts = WT_TS_NONE; hs_btree_id = S2BT(session)->id; - session_flags = 0; WT_CLEAR(full_value); valid_update_found = false; #ifdef HAVE_DIAGNOSTIC @@ -200,7 +200,7 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW newer_hs_durable_ts = unpack->tw.durable_start_ts; /* Open a history store table cursor. */ - WT_ERR(__wt_hs_cursor_open(session, &session_flags)); + WT_ERR(__wt_hs_cursor_open(session)); hs_cursor = session->hs_cursor; cbt = (WT_CURSOR_BTREE *)hs_cursor; @@ -263,8 +263,9 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW * records newer than or equal to the onpage value if eviction runs concurrently with * checkpoint. In that case, don't verify the first record. */ - WT_ASSERT(session, hs_stop_durable_ts <= newer_hs_durable_ts || - hs_start_ts == hs_stop_durable_ts || first_record); + WT_ASSERT(session, + hs_stop_durable_ts <= newer_hs_durable_ts || hs_start_ts == hs_stop_durable_ts || + first_record); if (hs_stop_durable_ts < newer_hs_durable_ts) WT_STAT_CONN_INCR(session, txn_rts_hs_stop_older_than_newer_start); @@ -286,8 +287,8 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW /* Stop processing when we find a stable update according to the given timestamp. */ if (hs_durable_ts <= rollback_timestamp) { __wt_verbose(session, WT_VERB_RTS, - "history store update valid with start timestamp: %s, durable timestamp: %s, " - "stop timestamp: %s and stable timestamp: %s", + "history store update valid with start timestamp: %s, durable timestamp: %s, stop " + "timestamp: %s and stable timestamp: %s", __wt_timestamp_to_string(hs_start_ts, ts_string[0]), __wt_timestamp_to_string(hs_durable_ts, ts_string[1]), __wt_timestamp_to_string(hs_stop_durable_ts, ts_string[2]), @@ -331,8 +332,9 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW upd->txnid = cbt->upd_value->tw.start_txn; upd->durable_ts = cbt->upd_value->tw.durable_start_ts; upd->start_ts = cbt->upd_value->tw.start_ts; - __wt_verbose(session, WT_VERB_RTS, "update restored from history store (txnid: %" PRIu64 - ", start_ts: %s, durable_ts: %s", + __wt_verbose(session, WT_VERB_RTS, + "update restored from history store (txnid: %" PRIu64 + ", start_ts: %s, durable_ts: %s", upd->txnid, __wt_timestamp_to_string(upd->start_ts, ts_string[0]), __wt_timestamp_to_string(upd->durable_ts, ts_string[1])); @@ -395,7 +397,7 @@ err: __wt_scr_free(session, &hs_value); __wt_scr_free(session, &key); __wt_buf_free(session, &full_value); - WT_TRET(__wt_hs_cursor_close(session, session_flags)); + WT_TRET(__wt_hs_cursor_close(session)); return (ret); } @@ -421,10 +423,15 @@ __rollback_abort_row_ondisk_kv( __wt_row_leaf_value_cell(session, page, rip, NULL, vpack); prepared = vpack->tw.prepare; if (WT_IS_HS(S2BT(session))) { - if (vpack->tw.durable_stop_ts > rollback_timestamp) { + /* + * Abort the history store update with stop durable timestamp greater than the stable + * timestamp or the updates with max stop timestamp which implies that they are associated + * with prepared transactions. + */ + if (vpack->tw.durable_stop_ts > rollback_timestamp || vpack->tw.stop_ts == WT_TS_MAX) { __wt_verbose(session, WT_VERB_RTS, - "hs update aborted with start durable/commit timestamp: %s, %s, " - "stop durable/commit timestamp: %s, %s and stable timestamp: %s", + "hs update aborted with start durable/commit timestamp: %s, %s, stop durable/commit " + "timestamp: %s, %s and stable timestamp: %s", __wt_timestamp_to_string(vpack->tw.durable_start_ts, ts_string[0]), __wt_timestamp_to_string(vpack->tw.start_ts, ts_string[1]), __wt_timestamp_to_string(vpack->tw.durable_stop_ts, ts_string[2]), @@ -595,11 +602,11 @@ __rollback_abort_row_reconciled_page( if (mod->rec_result == WT_PM_REC_REPLACE && (mod->mod_replace.ta.newest_start_durable_ts > rollback_timestamp || - mod->mod_replace.ta.newest_stop_durable_ts > rollback_timestamp || - mod->mod_replace.ta.prepare)) { + mod->mod_replace.ta.newest_stop_durable_ts > rollback_timestamp || + mod->mod_replace.ta.prepare)) { __wt_verbose(session, WT_VERB_RTS, - "reconciled replace block page history store update removal on-disk with start " - "durable timestamp: %s, stop durable timestamp: %s and stable timestamp: %s", + "reconciled replace block page history store update removal on-disk with start durable " + "timestamp: %s, stop durable timestamp: %s and stable timestamp: %s", __wt_timestamp_to_string(mod->mod_replace.ta.newest_start_durable_ts, ts_string[0]), __wt_timestamp_to_string(mod->mod_replace.ta.newest_stop_durable_ts, ts_string[1]), __wt_timestamp_to_string(rollback_timestamp, ts_string[2])); @@ -622,9 +629,8 @@ __rollback_abort_row_reconciled_page( multi->addr.ta.newest_stop_durable_ts > rollback_timestamp || multi->addr.ta.prepare) { __wt_verbose(session, WT_VERB_RTS, - "reconciled multi block page history store update removal on-disk with " - "start durable timestamp: %s, stop durable timestamp: %s and stable " - "timestamp: %s", + "reconciled multi block page history store update removal on-disk with start " + "durable timestamp: %s, stop durable timestamp: %s and stable timestamp: %s", __wt_timestamp_to_string(multi->addr.ta.newest_start_durable_ts, ts_string[0]), __wt_timestamp_to_string(multi->addr.ta.newest_stop_durable_ts, ts_string[1]), __wt_timestamp_to_string(rollback_timestamp, ts_string[2])); @@ -696,6 +702,24 @@ __rollback_abort_newer_row_leaf( } /* + * __rollback_get_ref_max_durable_timestamp -- + * Returns the ref aggregated max durable timestamp. The max durable timestamp is calculated + * between both start and stop durable timestamps except for history store, because most of the + * history store updates have stop timestamp either greater or equal to the start timestamp + * except for the updates written for the prepared updates on the data store. To abort the + * updates with no stop timestamp, we must include the newest stop timestamp also into the + * calculation of maximum durable timestamp of the history store. + */ +static wt_timestamp_t +__rollback_get_ref_max_durable_timestamp(WT_SESSION_IMPL *session, WT_TIME_AGGREGATE *ta) +{ + if (WT_IS_HS(S2BT(session))) + return WT_MAX(ta->newest_stop_durable_ts, ta->newest_stop_ts); + else + return WT_MAX(ta->newest_start_durable_ts, ta->newest_stop_durable_ts); +} + +/* * __rollback_page_needs_abort -- * Check whether the page needs rollback. Return true if the page has modifications newer than * the given timestamp Otherwise return false. @@ -730,16 +754,15 @@ __rollback_page_needs_abort( */ if (mod != NULL && mod->rec_result == WT_PM_REC_REPLACE) { tag = "reconciled replace block"; - durable_ts = WT_MAX( - mod->mod_replace.ta.newest_start_durable_ts, mod->mod_replace.ta.newest_stop_durable_ts); + durable_ts = __rollback_get_ref_max_durable_timestamp(session, &mod->mod_replace.ta); prepared = mod->mod_replace.ta.prepare; result = (durable_ts > rollback_timestamp) || prepared; } else if (mod != NULL && mod->rec_result == WT_PM_REC_MULTIBLOCK) { tag = "reconciled multi block"; /* Calculate the max durable timestamp by traversing all multi addresses. */ for (multi = mod->mod_multi, i = 0; i < mod->mod_multi_entries; ++multi, ++i) { - durable_ts = WT_MAX(durable_ts, multi->addr.ta.newest_start_durable_ts); - durable_ts = WT_MAX(durable_ts, multi->addr.ta.newest_stop_durable_ts); + durable_ts = WT_MAX( + durable_ts, __rollback_get_ref_max_durable_timestamp(session, &multi->addr.ta)); if (multi->addr.ta.prepare) prepared = true; } @@ -748,12 +771,12 @@ __rollback_page_needs_abort( tag = "on page cell"; /* Check if the page is obsolete using the page disk address. */ __wt_cell_unpack_addr(session, ref->home->dsk, (WT_CELL *)addr, &vpack); - durable_ts = WT_MAX(vpack.ta.newest_start_durable_ts, vpack.ta.newest_stop_durable_ts); + durable_ts = __rollback_get_ref_max_durable_timestamp(session, &vpack.ta); prepared = vpack.ta.prepare; result = (durable_ts > rollback_timestamp) || prepared; } else if (addr != NULL) { tag = "address"; - durable_ts = WT_MAX(addr->ta.newest_start_durable_ts, addr->ta.newest_stop_durable_ts); + durable_ts = __rollback_get_ref_max_durable_timestamp(session, &addr->ta); prepared = addr->ta.prepare; result = (durable_ts > rollback_timestamp) || prepared; } @@ -987,19 +1010,18 @@ __rollback_to_stable_btree_hs_truncate(WT_SESSION_IMPL *session, uint32_t btree_ WT_UPDATE *hs_upd; wt_timestamp_t hs_start_ts; uint64_t hs_counter; - uint32_t hs_btree_id, session_flags; + uint32_t hs_btree_id; int exact; char ts_string[WT_TS_INT_STRING_SIZE]; hs_cursor = NULL; WT_CLEAR(key); hs_upd = NULL; - session_flags = 0; WT_RET(__wt_scr_alloc(session, 0, &hs_key)); /* Open a history store table cursor. */ - WT_ERR(__wt_hs_cursor_open(session, &session_flags)); + WT_ERR(__wt_hs_cursor_open(session)); hs_cursor = session->hs_cursor; cbt = (WT_CURSOR_BTREE *)hs_cursor; @@ -1048,7 +1070,7 @@ __rollback_to_stable_btree_hs_truncate(WT_SESSION_IMPL *session, uint32_t btree_ err: __wt_scr_free(session, &hs_key); __wt_free(session, hs_upd); - WT_TRET(__wt_hs_cursor_close(session, session_flags)); + WT_TRET(__wt_hs_cursor_close(session)); return (ret); } @@ -1064,7 +1086,7 @@ __rollback_to_stable_hs_final_pass(WT_SESSION_IMPL *session, wt_timestamp_t roll WT_CONFIG ckptconf; WT_CONFIG_ITEM cval, durableval, key; WT_DECL_RET; - wt_timestamp_t max_durable_ts, newest_start_durable_ts, newest_stop_durable_ts; + wt_timestamp_t max_durable_ts, newest_stop_durable_ts, newest_stop_ts; char *config; char ts_string[2][WT_TS_INT_STRING_SIZE]; @@ -1072,22 +1094,27 @@ __rollback_to_stable_hs_final_pass(WT_SESSION_IMPL *session, wt_timestamp_t roll WT_RET(__wt_metadata_search(session, WT_HS_URI, &config)); - /* Find out the max durable timestamp of the object from checkpoint. */ - newest_start_durable_ts = newest_stop_durable_ts = WT_TS_NONE; + /* + * Find out the max durable timestamp of the history store from checkpoint. Most of the history + * store updates have stop timestamp either greater or equal to the start timestamp except for + * the updates written for the prepared updates on the data store. To abort the updates with no + * stop timestamp, we must include the newest stop timestamp also into the calculation of + * maximum timestamp of the history store. + */ + newest_stop_durable_ts = newest_stop_ts = WT_TS_NONE; WT_ERR(__wt_config_getones(session, config, "checkpoint", &cval)); __wt_config_subinit(session, &ckptconf, &cval); for (; __wt_config_next(&ckptconf, &key, &cval) == 0;) { - ret = __wt_config_subgets(session, &cval, "newest_start_durable_ts", &durableval); - if (ret == 0) - newest_start_durable_ts = - WT_MAX(newest_start_durable_ts, (wt_timestamp_t)durableval.val); - WT_ERR_NOTFOUND_OK(ret, false); ret = __wt_config_subgets(session, &cval, "newest_stop_durable_ts", &durableval); if (ret == 0) newest_stop_durable_ts = WT_MAX(newest_stop_durable_ts, (wt_timestamp_t)durableval.val); WT_ERR_NOTFOUND_OK(ret, false); + ret = __wt_config_subgets(session, &cval, "newest_stop_ts", &durableval); + if (ret == 0) + newest_stop_ts = WT_MAX(newest_stop_ts, (wt_timestamp_t)durableval.val); + WT_ERR_NOTFOUND_OK(ret, false); } - max_durable_ts = WT_MAX(newest_start_durable_ts, newest_stop_durable_ts); + max_durable_ts = WT_MAX(newest_stop_ts, newest_stop_durable_ts); WT_ERR(__wt_session_get_dhandle(session, WT_HS_URI, NULL, NULL, 0)); /* @@ -1299,7 +1326,7 @@ __wt_rollback_to_stable(WT_SESSION_IMPL *session, const char *cfg[], bool no_ckp */ if (!F_ISSET(S2C(session), WT_CONN_IN_MEMORY) && !no_ckpt) WT_TRET(session->iface.checkpoint(&session->iface, "force=1")); - WT_TRET(session->iface.close(&session->iface, NULL)); + WT_TRET(__wt_session_close_internal(session)); return (ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index ba311cc93f0..c631b597f4d 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -422,8 +422,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) if (has_durable && (has_oldest || txn_global->has_oldest_timestamp) && oldest_ts > durable_ts) { __wt_readunlock(session, &txn_global->rwlock); WT_RET_MSG(session, EINVAL, - "set_timestamp: oldest timestamp %s must not be later than " - "durable timestamp %s", + "set_timestamp: oldest timestamp %s must not be later than durable timestamp %s", __wt_timestamp_to_string(oldest_ts, ts_string[0]), __wt_timestamp_to_string(durable_ts, ts_string[1])); } @@ -431,8 +430,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) if (has_durable && (has_stable || txn_global->has_stable_timestamp) && stable_ts > durable_ts) { __wt_readunlock(session, &txn_global->rwlock); WT_RET_MSG(session, EINVAL, - "set_timestamp: stable timestamp %s must not be later than " - "durable timestamp %s", + "set_timestamp: stable timestamp %s must not be later than durable timestamp %s", __wt_timestamp_to_string(stable_ts, ts_string[0]), __wt_timestamp_to_string(durable_ts, ts_string[1])); } @@ -444,8 +442,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) (has_stable || txn_global->has_stable_timestamp) && oldest_ts > stable_ts) { __wt_readunlock(session, &txn_global->rwlock); WT_RET_MSG(session, EINVAL, - "set_timestamp: oldest timestamp %s must not be later than " - "stable timestamp %s", + "set_timestamp: oldest timestamp %s must not be later than stable timestamp %s", __wt_timestamp_to_string(oldest_ts, ts_string[0]), __wt_timestamp_to_string(stable_ts, ts_string[1])); } @@ -537,9 +534,8 @@ __txn_assert_after_reads( if (tmp_timestamp >= ts) { __wt_readunlock(session, &txn_global->read_timestamp_rwlock); WT_RET_MSG(session, EINVAL, - "%s timestamp %s must be greater than the " - "latest active read timestamp %s ", - op, __wt_timestamp_to_string(ts, ts_string[0]), + "%s timestamp %s must be greater than the latest active read timestamp %s ", op, + __wt_timestamp_to_string(ts, ts_string[0]), __wt_timestamp_to_string(tmp_timestamp, ts_string[1])); } break; @@ -583,9 +579,7 @@ __wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t commit_ts if (txn->isolation != WT_ISO_SNAPSHOT) WT_RET_MSG(session, EINVAL, - "setting a commit_timestamp" - " requires a transaction running at snapshot" - " isolation"); + "setting a commit_timestamp requires a transaction running at snapshot isolation"); /* * Compare against the oldest and the stable timestamp. Return an error if the given timestamp @@ -604,16 +598,12 @@ __wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t commit_ts * timestamp. */ if (has_oldest_ts && commit_ts < oldest_ts) - WT_RET_MSG(session, EINVAL, - "commit timestamp %s is less than the oldest " - "timestamp %s", + WT_RET_MSG(session, EINVAL, "commit timestamp %s is less than the oldest timestamp %s", __wt_timestamp_to_string(commit_ts, ts_string[0]), __wt_timestamp_to_string(oldest_ts, ts_string[1])); if (has_stable_ts && commit_ts < stable_ts) - WT_RET_MSG(session, EINVAL, - "commit timestamp %s is less than the stable " - "timestamp %s", + WT_RET_MSG(session, EINVAL, "commit timestamp %s is less than the stable timestamp %s", __wt_timestamp_to_string(commit_ts, ts_string[0]), __wt_timestamp_to_string(stable_ts, ts_string[1])); @@ -623,8 +613,7 @@ __wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t commit_ts */ if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && commit_ts < txn->first_commit_timestamp) WT_RET_MSG(session, EINVAL, - "commit timestamp %s older than the first " - "commit timestamp %s for this transaction", + "commit timestamp %s older than the first commit timestamp %s for this transaction", __wt_timestamp_to_string(commit_ts, ts_string[0]), __wt_timestamp_to_string(txn->first_commit_timestamp, ts_string[1])); @@ -641,8 +630,7 @@ __wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t commit_ts if (txn->prepare_timestamp > commit_ts) { if (!F_ISSET(txn, WT_TXN_TS_ROUND_PREPARED)) WT_RET_MSG(session, EINVAL, - "commit timestamp %s is less than the " - "prepare timestamp %s for this transaction", + "commit timestamp %s is less than the prepare timestamp %s for this transaction", __wt_timestamp_to_string(commit_ts, ts_string[0]), __wt_timestamp_to_string(txn->prepare_timestamp, ts_string[1])); commit_ts = txn->prepare_timestamp; @@ -691,8 +679,7 @@ __wt_txn_set_durable_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t durable_ if (!F_ISSET(txn, WT_TXN_PREPARE)) WT_RET_MSG(session, EINVAL, - "durable timestamp should not be specified for " - "non-prepared transaction"); + "durable timestamp should not be specified for non-prepared transaction"); if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) WT_RET_MSG(session, EINVAL, "commit timestamp is needed before the durable timestamp"); @@ -725,8 +712,7 @@ __wt_txn_set_durable_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t durable_ /* Check if the durable timestamp is less than the commit timestamp. */ if (durable_ts < txn->commit_timestamp) WT_RET_MSG(session, EINVAL, - "durable timestamp %s is less than the commit timestamp %s " - "for this transaction", + "durable timestamp %s is less than the commit timestamp %s for this transaction", __wt_timestamp_to_string(durable_ts, ts_string[0]), __wt_timestamp_to_string(txn->commit_timestamp, ts_string[1])); @@ -760,8 +746,7 @@ __wt_txn_set_prepare_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t prepare_ if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) WT_RET_MSG(session, EINVAL, - "commit timestamp " - "should not have been set before the prepare timestamp"); + "commit timestamp should not have been set before the prepare timestamp"); WT_RET(__txn_assert_after_reads(session, "prepare", prepare_ts, &prev_shared)); @@ -781,16 +766,14 @@ __wt_txn_set_prepare_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t prepare_ WT_ASSERT(session, prev_shared == NULL); __wt_verbose(session, WT_VERB_TIMESTAMP, - "prepare timestamp %s rounded to oldest " - "timestamp %s", + "prepare timestamp %s rounded to oldest timestamp %s", __wt_timestamp_to_string(prepare_ts, ts_string[0]), __wt_timestamp_to_string(oldest_ts, ts_string[1])); prepare_ts = oldest_ts; } else WT_RET_MSG(session, EINVAL, - "prepare timestamp %s is older than the oldest " - "timestamp %s", + "prepare timestamp %s is older than the oldest timestamp %s", __wt_timestamp_to_string(prepare_ts, ts_string[0]), __wt_timestamp_to_string(oldest_ts, ts_string[1])); } @@ -825,15 +808,11 @@ __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t read_ts) txn->isolation = WT_ISO_SNAPSHOT; else if (txn->isolation != WT_ISO_SNAPSHOT) WT_RET_MSG(session, EINVAL, - "setting a read_timestamp" - " requires a transaction running at snapshot" - " isolation"); + "setting a read_timestamp requires a transaction running at snapshot isolation"); /* Read timestamps can't change once set. */ if (F_ISSET(txn, WT_TXN_SHARED_TS_READ)) - WT_RET_MSG(session, EINVAL, - "a read_timestamp" - " may only be set once per transaction"); + WT_RET_MSG(session, EINVAL, "a read_timestamp may only be set once per transaction"); /* * This code is not using the timestamp validate function to avoid a race between checking and @@ -860,9 +839,7 @@ __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t read_ts) * error message because that logs a MongoDB error, use an informational message to * provide the context instead. */ - WT_RET(__wt_msg(session, - "read timestamp " - "%s less than the oldest timestamp %s", + WT_RET(__wt_msg(session, "read timestamp %s less than the oldest timestamp %s", __wt_timestamp_to_string(read_ts, ts_string[0]), __wt_timestamp_to_string(ts_oldest, ts_string[1]))); return (EINVAL); @@ -878,8 +855,7 @@ __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t read_ts) */ if (did_roundup_to_oldest) __wt_verbose(session, WT_VERB_TIMESTAMP, - "read " - "timestamp %s : rounded to oldest timestamp %s", + "read timestamp %s : rounded to oldest timestamp %s", __wt_timestamp_to_string(read_ts, ts_string[0]), __wt_timestamp_to_string(ts_oldest, ts_string[1])); diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c index 482074d85fb..3aeb00c2e8b 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.c +++ b/src/third_party/wiredtiger/src/utilities/util_load.c @@ -74,9 +74,8 @@ util_load(WT_SESSION *session, int argc, char *argv[]) /* -a and -o are mutually exclusive. */ if (append && no_overwrite) - return (util_err(session, EINVAL, - "the -a (append) and -n (no-overwrite) flags are mutually " - "exclusive")); + return (util_err( + session, EINVAL, "the -a (append) and -n (no-overwrite) flags are mutually exclusive")); /* The remaining arguments are configuration uri/string pairs. */ if (argc != 0) { @@ -145,9 +144,7 @@ load_dump(WT_SESSION *session) * Check the append flag (it only applies to objects where the primary key is a record number). */ if (append && !WT_STREQ(cursor->key_format, "r")) { - fprintf(stderr, - "%s: %s: -a option illegal unless the primary key is a " - "record number\n", + fprintf(stderr, "%s: %s: -a option illegal unless the primary key is a record number\n", progname, uri); ret = 1; } else @@ -406,9 +403,7 @@ config_update(WT_SESSION *session, char **list) */ for (configp = cmdconfig; configp != NULL && *configp != NULL; configp += 2) if (strstr(configp[1], "key_format=") || strstr(configp[1], "value_format=")) - return (util_err(session, 0, - "an object's key or value format may not be " - "modified")); + return (util_err(session, 0, "an object's key or value format may not be modified")); /* * If there were command-line configuration pairs, walk the list of command-line URIs and find a @@ -422,15 +417,14 @@ config_update(WT_SESSION *session, char **list) switch (found) { case 0: return (util_err(session, 0, - "the command line object name %s was not matched " - "by any loaded object name", + "the command line object name %s was not matched by any loaded object name", *configp)); case 1: break; default: return (util_err(session, 0, - "the command line object name %s was not unique, " - "matching more than a single loaded object name", + "the command line object name %s was not unique, matching more than a single loaded " + "object name", *configp)); } } @@ -461,8 +455,8 @@ config_update(WT_SESSION *session, char **list) cfg[cnt++] = NULL; if ((ret = __wt_config_merge((WT_SESSION_IMPL *)session, cfg, - "filename=,id=," - "checkpoint=,checkpoint_backup_info=,checkpoint_lsn=,version=,source=,", + "filename=,id=,checkpoint=,checkpoint_backup_info=,checkpoint_lsn=,version=,source=" + ",", &p)) != 0) break; diff --git a/src/third_party/wiredtiger/src/utilities/util_loadtext.c b/src/third_party/wiredtiger/src/utilities/util_loadtext.c index 2ae1374a241..109b478bb15 100644 --- a/src/third_party/wiredtiger/src/utilities/util_loadtext.c +++ b/src/third_party/wiredtiger/src/utilities/util_loadtext.c @@ -82,8 +82,8 @@ text(WT_SESSION *session, const char *uri) if (!WT_STREQ(cursor->value_format, "S") || (!WT_STREQ(cursor->key_format, "S") && !WT_STREQ(cursor->key_format, "r"))) return (util_err(session, EINVAL, - "the loadtext command can only load objects configured " - "for record number or string keys, and string values")); + "the loadtext command can only load objects configured for record number or string keys, " + "and string values")); readkey = !WT_STREQ(cursor->key_format, "r"); /* Insert the records */ diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 3859c87cde7..3d83839962f 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -16,11 +16,14 @@ bool verbose = false; /* Verbose flag */ static const char *command; /* Command name */ +/* Give users a hint in the help output for if they're trying to read MongoDB data files */ +static const char *mongodb_config = "log=(enabled=true,path=journal,compressor=snappy)"; + #define READONLY "readonly=true" #define REC_ERROR "log=(recover=error)" #define REC_LOGOFF "log=(enabled=false)" #define REC_RECOVER "log=(recover=on)" -#define REC_SALVAGE "log=(recover=salvage)" +#define SALVAGE "salvage=true" static void usage(void) @@ -49,6 +52,7 @@ usage(void) fprintf(stderr, "WiredTiger Data Engine (version %d.%d)\n", WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR); + fprintf(stderr, "MongoDB wiredtiger_open configuration: \"%s\"\n", mongodb_config); util_usage(NULL, "global_options:", options); util_usage(NULL, "commands:", commands); } @@ -62,7 +66,7 @@ main(int argc, char *argv[]) size_t len; int ch, major_v, minor_v, tret, (*func)(WT_SESSION *, int, char *[]); char *p, *secretkey; - const char *cmd_config, *config, *p1, *p2, *p3, *readonly_config, *rec_config; + const char *cmd_config, *config, *p1, *p2, *p3, *readonly_config, *rec_config, *salvage_config; bool backward_compatible, logoff, meta_verify, readonly, recover, salvage; conn = NULL; @@ -79,13 +83,12 @@ main(int argc, char *argv[]) (void)wiredtiger_version(&major_v, &minor_v, NULL); if (major_v != WIREDTIGER_VERSION_MAJOR || minor_v != WIREDTIGER_VERSION_MINOR) { fprintf(stderr, - "%s: program build version %d.%d does not match " - "library build version %d.%d\n", - progname, WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR, major_v, minor_v); + "%s: program build version %d.%d does not match library build version %d.%d\n", progname, + WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR, major_v, minor_v); return (EXIT_FAILURE); } - cmd_config = config = readonly_config = secretkey = NULL; + cmd_config = config = readonly_config = salvage_config = secretkey = NULL; /* * We default to returning an error if recovery needs to be run. Generally we expect this to be * run after a clean shutdown. The printlog command disables logging entirely. If recovery is @@ -130,7 +133,7 @@ main(int argc, char *argv[]) readonly = true; break; case 'S': /* salvage */ - rec_config = REC_SALVAGE; + salvage_config = SALVAGE; salvage = true; break; case 'V': /* version */ @@ -267,6 +270,8 @@ open: len += strlen(cmd_config); if (readonly_config != NULL) len += strlen(readonly_config); + if (salvage_config != NULL) + len += strlen(salvage_config); if (secretkey != NULL) { len += strlen(secretkey) + 30; p1 = ",encryption=(secretkey="; @@ -278,9 +283,10 @@ open: (void)util_err(NULL, errno, NULL); goto err; } - if ((ret = __wt_snprintf(p, len, "error_prefix=wt,%s,%s,%s,%s%s%s%s", + if ((ret = __wt_snprintf(p, len, "error_prefix=wt,%s,%s,%s,%s,%s%s%s%s", config == NULL ? "" : config, cmd_config == NULL ? "" : cmd_config, - readonly_config == NULL ? "" : readonly_config, rec_config, p1, p2, p3)) != 0) { + readonly_config == NULL ? "" : readonly_config, rec_config, + salvage_config == NULL ? "" : salvage_config, p1, p2, p3)) != 0) { (void)util_err(NULL, ret, NULL); goto err; } diff --git a/src/third_party/wiredtiger/src/utilities/util_misc.c b/src/third_party/wiredtiger/src/utilities/util_misc.c index eb8eb5782fb..d0fff3519e9 100644 --- a/src/third_party/wiredtiger/src/utilities/util_misc.c +++ b/src/third_party/wiredtiger/src/utilities/util_misc.c @@ -116,7 +116,7 @@ util_str2num(WT_SESSION *session, const char *p, bool endnul, uint64_t *vp) */ if (endnul && endptr[0] != '\0') format: - return (util_err(session, EINVAL, "%s: invalid number", p)); + return (util_err(session, EINVAL, "%s: invalid number", p)); *vp = v; return (0); diff --git a/src/third_party/wiredtiger/src/utilities/util_read.c b/src/third_party/wiredtiger/src/utilities/util_read.c index 7c4d94cc124..46e913cb0fd 100644 --- a/src/third_party/wiredtiger/src/utilities/util_read.c +++ b/src/third_party/wiredtiger/src/utilities/util_read.c @@ -56,17 +56,14 @@ util_read(WT_SESSION *session, int argc, char *argv[]) */ if (!WT_STREQ(cursor->key_format, "r") && !WT_STREQ(cursor->key_format, "S")) { fprintf(stderr, - "%s: read command only possible when the key format is " - "a record number or string\n", + "%s: read command only possible when the key format is a record number or string\n", progname); return (1); } rkey = WT_STREQ(cursor->key_format, "r"); if (!WT_STREQ(cursor->value_format, "S")) { - fprintf(stderr, - "%s: read command only possible when the value format is " - "a string\n", - progname); + fprintf( + stderr, "%s: read command only possible when the value format is a string\n", progname); return (1); } diff --git a/src/third_party/wiredtiger/src/utilities/util_verify.c b/src/third_party/wiredtiger/src/utilities/util_verify.c index 34bd4f22fb8..ca4e0ae8389 100644 --- a/src/third_party/wiredtiger/src/utilities/util_verify.c +++ b/src/third_party/wiredtiger/src/utilities/util_verify.c @@ -45,10 +45,8 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) dump_layout = true; else if (WT_PREFIX_MATCH(__wt_optarg, "dump_offsets=")) { if (dump_offsets != NULL) { - fprintf(stderr, - "%s: only a single 'dump_offsets' " - "argument supported\n", - progname); + fprintf( + stderr, "%s: only a single 'dump_offsets' argument supported\n", progname); return (usage()); } dump_offsets = __wt_optarg + strlen("dump_offsets="); diff --git a/src/third_party/wiredtiger/src/utilities/util_write.c b/src/third_party/wiredtiger/src/utilities/util_write.c index fdb5f5a716c..d8156670b9e 100644 --- a/src/third_party/wiredtiger/src/utilities/util_write.c +++ b/src/third_party/wiredtiger/src/utilities/util_write.c @@ -77,17 +77,14 @@ util_write(WT_SESSION *session, int argc, char *argv[]) */ if (!WT_STREQ(cursor->key_format, "r") && !WT_STREQ(cursor->key_format, "S")) { fprintf(stderr, - "%s: write command only possible when the key format is " - "a record number or string\n", + "%s: write command only possible when the key format is a record number or string\n", progname); return (1); } rkey = WT_STREQ(cursor->key_format, "r"); if (!WT_STREQ(cursor->value_format, "S")) { - fprintf(stderr, - "%s: write command only possible when the value format is " - "a string\n", - progname); + fprintf( + stderr, "%s: write command only possible when the value format is a string\n", progname); return (1); } diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index 5831bb5eee1..b34add5fe51 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -282,9 +282,7 @@ verify_consistency(WT_SESSION *session, char *stable_timestamp) continue; else if (ret == WT_NOTFOUND || t_ret == WT_NOTFOUND) { (void)log_print_err( - "verify_consistency tables with different" - " amount of data", - EFAULT, 1); + "verify_consistency tables with different amount of data", EFAULT, 1); goto err; } diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 0166bddb45a..5785494487a 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -200,18 +200,15 @@ wt_connect(const char *config_open) */ if (g.sweep_stress) testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_cursors=false,statistics=(fast)," - "statistics_log=(json,wait=1),error_prefix=\"%s\"," - "file_manager=(close_handle_minimum=1,close_idle_time=1," - "close_scan_interval=1),log=(enabled),cache_size=1GB," - "timing_stress_for_test=(aggressive_sweep)%s%s%s", + "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),error_prefix=" + "\"%s\",file_manager=(close_handle_minimum=1,close_idle_time=1,close_scan_interval=1)," + "log=(enabled),cache_size=1GB,timing_stress_for_test=(aggressive_sweep)%s%s%s", progname, g.debug_mode ? DEBUG_MODE_CFG : "", config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); else testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_cursors=false,statistics=(fast)," - "statistics_log=(json,wait=1),error_prefix=\"%s\"" - "%s%s%s", + "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),error_prefix=" + "\"%s\"%s%s%s", progname, g.debug_mode ? DEBUG_MODE_CFG : "", config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); @@ -336,10 +333,8 @@ static int usage(void) { fprintf(stderr, - "usage: %s " - "[-C wiredtiger-config] [-c checkpoint] [-h home] [-k keys]\n\t" - "[-l log] [-n ops] [-r runs] [-T table-config] [-t f|r|v]\n\t" - "[-W workers]\n", + "usage: %s [-C wiredtiger-config] [-c checkpoint] [-h home] [-k keys]\n\t[-l log] [-n ops] " + "[-r runs] [-T table-config] [-t f|r|v]\n\t[-W workers]\n", progname); fprintf(stderr, "%s", "\t-C specify wiredtiger_open configuration arguments\n" diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index cf58cecc8a6..553738fd418 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -121,8 +121,7 @@ all_TESTS += test_wt3874_pad_byte_collator test_wt4105_large_doc_small_upd_SOURCES = wt4105_large_doc_small_upd/main.c noinst_PROGRAMS += test_wt4105_large_doc_small_upd -# Temporarily disabled (WT-5579) -# all_TESTS += test_wt4105_large_doc_small_upd +all_TESTS += test_wt4105_large_doc_small_upd test_wt4117_checksum_SOURCES = wt4117_checksum/main.c noinst_PROGRAMS += test_wt4117_checksum diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c index 792e10275b2..c9520f5621b 100644 --- a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c +++ b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c @@ -48,6 +48,8 @@ #define URI_FORMAT "table:t%d-%d" #define KEY_FORMAT "key-%d-%d" +#define CONN_CONFIG_COMMON "timing_stress_for_test=[backup_rename]" + static int verbose_level = 0; static uint64_t seed = 0; @@ -59,7 +61,7 @@ static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static bool slow_incremental = false; static bool do_drop = true; -static bool do_rename = false; +static bool do_rename = true; #define VERBOSE(level, fmt, ...) \ do { \ @@ -730,7 +732,7 @@ check_backup(const char *backup_home, const char *backup_check, TABLE_INFO *tinf buf, sizeof(buf), "rm -rf %s && cp -r %s %s", backup_check, backup_home, backup_check)); testutil_check(system(buf)); - testutil_check(wiredtiger_open(backup_check, NULL, NULL, &conn)); + testutil_check(wiredtiger_open(backup_check, NULL, CONN_CONFIG_COMMON, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); for (slot = 0; slot < tinfo->table_count; slot++) { @@ -818,8 +820,9 @@ main(int argc, char *argv[]) file_max = 200 + __wt_random(&rnd) % 1000; /* 200K to ~1M */ else file_max = 1000 + __wt_random(&rnd) % 20000; /* 1M to ~20M */ - testutil_check(__wt_snprintf(conf, sizeof(conf), - "create,%s,log=(enabled=true,file_max=%" PRIu32 "K)", backup_verbose, file_max)); + testutil_check( + __wt_snprintf(conf, sizeof(conf), "%s,create,%s,log=(enabled=true,file_max=%" PRIu32 "K)", + CONN_CONFIG_COMMON, backup_verbose, file_max)); VERBOSE(2, "wiredtiger config: %s\n", conf); testutil_check(wiredtiger_open(home, NULL, conf, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index 52161d089a8..10c12fbc68e 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -68,6 +68,12 @@ static bool inmem; #define ENV_CONFIG_REC "log=(recover=on)" /* + * A minimum width of 10, along with zero filling, means that all the keys sort according to their + * integer value, making each thread's key space distinct. + */ +#define KEY_FORMAT ("%010" PRIu64) + +/* * Maximum number of modifications that are allowed to perform cursor modify operation. */ #define MAX_MODIFY_ENTRIES 10 @@ -191,7 +197,7 @@ thread_run(void *arg) if (columnar_table) cursor->set_key(cursor, i); else { - testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, i)); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, i)); cursor->set_key(cursor, kname); } /* @@ -462,7 +468,7 @@ recover_and_verify(uint32_t nthreads) if (columnar_table) cursor->set_key(cursor, key); else { - testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, key)); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, key)); cursor->set_key(cursor, kname); } @@ -479,9 +485,7 @@ recover_and_verify(uint32_t nthreads) fatal = true; } else { if (!inmem) - printf( - "%s: deleted record" - " found with key %" PRIu64 "\n", + printf("%s: deleted record found with key %" PRIu64 "\n", fname[DELETE_RECORD_FILE_ID], key); absent++; middle = key; @@ -493,7 +497,7 @@ recover_and_verify(uint32_t nthreads) if (columnar_table) cursor->set_key(cursor, key); else { - testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, key)); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, key)); cursor->set_key(cursor, kname); } @@ -501,9 +505,7 @@ recover_and_verify(uint32_t nthreads) if (ret != WT_NOTFOUND) testutil_die(ret, "search"); if (!inmem) - printf( - "%s: no insert record" - " with key %" PRIu64 "\n", + printf("%s: no insert record with key %" PRIu64 "\n", fname[INSERT_RECORD_FILE_ID], key); absent++; middle = key; @@ -546,7 +548,7 @@ recover_and_verify(uint32_t nthreads) if (columnar_table) cursor->set_key(cursor, key); else { - testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, key)); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, key)); cursor->set_key(cursor, kname); } diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/main.c b/src/third_party/wiredtiger/test/csuite/random_directio/main.c index 09c8f1b3de3..5e21751781d 100644 --- a/src/third_party/wiredtiger/test/csuite/random_directio/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_directio/main.c @@ -153,7 +153,8 @@ static const char *const uri_rev = "table:rev"; * that has schema operations happens again at id 200, assuming frequency * set to 100. So it is a good test of schema operations 'in flight'. */ -#define SCHEMA_OP_FREQUENCY 100 +#define SCHEMA_FREQUENCY_DEFAULT 100 +static uint64_t schema_frequency; #define TEST_STREQ(expect, got, message) \ do { \ @@ -203,6 +204,8 @@ usage(void) fprintf(stderr, "usage: %s [options]\n", progname); fprintf(stderr, "options:\n"); fprintf(stderr, " %-20s%s\n", "-d data_size", "approximate size of keys and values [1000]"); + fprintf(stderr, " %-20s%s\n", "-f schema frequency", + "restart schema sequence every frequency period [100]"); fprintf(stderr, " %-20s%s\n", "-h home", "WiredTiger home directory [WT_TEST.directio]"); fprintf( stderr, " %-20s%s\n", "-i interval", "interval timeout between copy/recover cycles [3]"); @@ -243,7 +246,7 @@ usage(void) static bool has_schema_operation(uint64_t id, uint32_t offset) { - return (id >= offset && (id - offset) % SCHEMA_OP_FREQUENCY < 10); + return (id >= offset && (id - offset) % schema_frequency < 10); } /* @@ -355,7 +358,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o /* fprintf(stderr, "CREATE: %s\n", uri1); */ + testutil_check(session->log_printf(session, "CREATE: %s", uri1)); testutil_check(session->create(session, uri1, "key_format=S,value_format=S")); + testutil_check(session->log_printf(session, "CREATE: DONE %s", uri1)); break; case 1: /* Insert a value into the table. */ @@ -366,7 +371,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o testutil_check(session->open_cursor(session, uri1, NULL, NULL, &cursor)); cursor->set_key(cursor, uri1); cursor->set_value(cursor, uri1); + testutil_check(session->log_printf(session, "INSERT: %s", uri1)); testutil_check(cursor->insert(cursor)); + testutil_check(session->log_printf(session, "INSERT: DONE %s", uri1)); testutil_check(cursor->close(cursor)); break; case 2: @@ -378,7 +385,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o /* fprintf(stderr, "RENAME: %s->%s\n", uri1, uri2); */ + testutil_check(session->log_printf(session, "RENAME: %s->%s", uri1, uri2)); ret = session->rename(session, uri1, uri2, NULL); + testutil_check(session->log_printf(session, "RENAME: DONE %s->%s", uri1, uri2)); } break; case 3: @@ -391,7 +400,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o /* fprintf(stderr, "UPDATE: %s\n", uri2); */ + testutil_check(session->log_printf(session, "UPDATE: %s", uri2)); testutil_check(cursor->update(cursor)); + testutil_check(session->log_printf(session, "UPDATE: DONE %s", uri2)); testutil_check(cursor->close(cursor)); break; case 4: @@ -402,7 +413,9 @@ schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t o /* fprintf(stderr, "DROP: %s\n", uri1); */ + testutil_check(session->log_printf(session, "DROP: %s", uri1)); ret = session->drop(session, uri1, NULL); + testutil_check(session->log_printf(session, "DROP: DONE %s", uri1)); } } /* @@ -585,7 +598,8 @@ fill_db(uint32_t nth, uint32_t datasize, const char *method, uint32_t flags) testutil_check(wiredtiger_open(".", NULL, envconf, &conn)); datasize += 1; /* Add an extra byte for string termination */ - printf("Create %" PRIu32 " writer threads\n", nth); + printf( + "Create %" PRIu32 " writer threads. Schema frequency %" PRIu64 "\n", nth, schema_frequency); for (i = 0; i < nth; ++i) { td[i].conn = conn; td[i].data = dcalloc(datasize, 1); @@ -756,7 +770,7 @@ check_schema(WT_SESSION *session, uint64_t lastid, uint32_t threadid, uint32_t f * Make a copy of the database and verify its contents. */ static bool -check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) +check_db(uint32_t nth, uint32_t datasize, pid_t pid, bool directio, uint32_t flags) { WT_CONNECTION *conn; WT_CURSOR *cursor, *meta, *rev; @@ -765,7 +779,8 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) uint64_t gotid, id; uint64_t *lastid; uint32_t gotth, kvsize, th, threadmap; - char checkdir[4096], savedir[4096]; + int status; + char checkdir[4096], dbgdir[4096], savedir[4096]; char *gotkey, *gotvalue, *keybuf, *p; char **large_arr; @@ -778,6 +793,7 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) large_buf(large_arr[th], LARGE_WRITE_SIZE, th, true); } testutil_check(__wt_snprintf(checkdir, sizeof(checkdir), "%s.CHECK", home)); + testutil_check(__wt_snprintf(dbgdir, sizeof(savedir), "%s.DEBUG", home)); testutil_check(__wt_snprintf(savedir, sizeof(savedir), "%s.SAVE", home)); /* @@ -788,10 +804,30 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) "Copy database home directory using direct I/O to run recovery,\n" "along with a saved 'pre-recovery' copy.\n"); copy_directory(home, checkdir, directio); + /* Copy the original home directory explicitly without direct I/O. */ + copy_directory(home, dbgdir, false); copy_directory(checkdir, savedir, false); printf("Open database, run recovery and verify content\n"); - testutil_check(wiredtiger_open(checkdir, NULL, ENV_CONFIG_REC, &conn)); + ret = wiredtiger_open(checkdir, NULL, ENV_CONFIG_REC, &conn); + /* If this fails, abort the child process before we die so we can see what it was doing. */ + if (ret != 0) { + if (pid != 0) { + /* + * The child is stopped, it won't process an abort until it is continued. First signal + * the abort, then signal continue so that the child process will process the abort and + * dump core. + */ + printf("Send abort to child process ID %d\n", (int)pid); + if (kill(pid, SIGABRT) != 0) + testutil_die(errno, "kill"); + if (kill(pid, SIGCONT) != 0) + testutil_die(errno, "kill"); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); + } + testutil_check(ret); + } testutil_check(conn->open_session(conn, NULL, NULL, &session)); testutil_check(session->open_cursor(session, uri_main, NULL, NULL, &cursor)); testutil_check(session->open_cursor(session, uri_rev, NULL, NULL, &rev)); @@ -1025,6 +1061,7 @@ main(int argc, char *argv[]) working_dir = "WT_TEST.random-directio"; method = "none"; pid = 0; + schema_frequency = SCHEMA_FREQUENCY_DEFAULT; memset(args, 0, sizeof(args)); if (!has_direct_io()) { @@ -1038,7 +1075,7 @@ main(int argc, char *argv[]) __wt_snprintf_len_set(p, sizeof(args) - (size_t)(p - args), &size, " %s", argv[i])); p += size; } - while ((ch = __wt_getopt(progname, argc, argv, "d:h:i:m:n:pS:T:t:v")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "d:f:h:i:m:n:pS:T:t:v")) != EOF) switch (ch) { case 'd': datasize = (uint32_t)atoi(__wt_optarg); @@ -1047,6 +1084,9 @@ main(int argc, char *argv[]) return (EXIT_FAILURE); } break; + case 'f': + schema_frequency = (uint64_t)atoi(__wt_optarg); + break; case 'h': working_dir = __wt_optarg; break; @@ -1131,9 +1171,7 @@ main(int argc, char *argv[]) } if (!LF_ISSET(SCHEMA_INTEGRATED) && LF_ISSET(SCHEMA_CREATE_CHECK | SCHEMA_DATA_CHECK | SCHEMA_DROP_CHECK)) { - fprintf(stderr, - "Schema '*check' options cannot be used " - "without 'integrated'\n"); + fprintf(stderr, "Schema '*check' options cannot be used without 'integrated'\n"); usage(); } printf("CONFIG:%s\n", args); @@ -1191,7 +1229,7 @@ main(int argc, char *argv[]) testutil_die(errno, "kill"); printf("Check DB\n"); fflush(stdout); - if (!check_db(nth, datasize, true, flags)) + if (!check_db(nth, datasize, pid, true, flags)) return (EXIT_FAILURE); if (kill(pid, SIGCONT) != 0) testutil_die(errno, "kill"); @@ -1206,7 +1244,7 @@ main(int argc, char *argv[]) if (waitpid(pid, &status, 0) == -1) testutil_die(errno, "waitpid"); } - if (verify_only && !check_db(nth, datasize, false, flags)) { + if (verify_only && !check_db(nth, datasize, 0, false, flags)) { printf("FAIL\n"); return (EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh index 815b8a21c9f..7d5b96d9b57 100755 --- a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh @@ -23,7 +23,7 @@ for threads in $TEST_THREADS; do for method in $TEST_METHODS; do RUN_TEST="$RUN_TEST_CMD -t 5 -m $method" $RUN_TEST -T $threads || exit 1 - $RUN_TEST -T $threads -S create,drop,verbose || exit 1 + $RUN_TEST -f 20 -T $threads -S create,drop,verbose || exit 1 # Here are successively tougher schema tests that do not yet # reliably pass. 'verbose' can be added to any. diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index 68c5fc7f9e9..9fc3ceb6cde 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -94,6 +94,12 @@ static volatile THREAD_TS th_ts[MAX_TH]; ",transaction_sync=(enabled,method=none)" #define ENV_CONFIG_REC "log=(archive=false,recover=on)" +/* + * A minimum width of 10, along with zero filling, means that all the keys sort according to their + * integer value, making each thread's key space distinct. + */ +#define KEY_FORMAT ("%010" PRIu64) + typedef struct { uint64_t absent_key; /* Last absent key */ uint64_t exist_key; /* First existing key after miss */ @@ -490,7 +496,7 @@ thread_ts_run(void *arg) } } else ts_wait: - __wt_sleep(0, 1000); + __wt_sleep(0, 1000); } /* NOTREACHED */ } @@ -539,10 +545,8 @@ thread_ckpt_run(void *arg) if (WT_TIMEDIFF_SEC(now, start) >= 1) printf("CKPT: !stable_set time %" PRIu64 "\n", WT_TIMEDIFF_SEC(now, start)); if (WT_TIMEDIFF_SEC(now, start) > MAX_STARTUP) { - fprintf(stderr, - "After %d seconds stable still not " - "set. Aborting.\n", - MAX_STARTUP); + fprintf( + stderr, "After %d seconds stable still not set. Aborting.\n", MAX_STARTUP); /* * For the checkpoint thread the info contains the number of threads. */ @@ -690,7 +694,7 @@ thread_run(void *arg) } if (use_ts) stable_ts = __wt_atomic_addv64(&global_ts, 1); - testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, i)); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, i)); testutil_check(session->begin_transaction(session, NULL)); if (use_prep) @@ -871,9 +875,7 @@ print_missing(REPORT *r, const char *fname, const char *msg) { if (r->exist_key != INVALID_KEY) printf("%s: %s error %" PRIu64 " absent records %" PRIu64 "-%" PRIu64 ". Then keys %" PRIu64 - "-%" PRIu64 - " exist." - " Key range %" PRIu64 "-%" PRIu64 "\n", + "-%" PRIu64 " exist. Key range %" PRIu64 "-%" PRIu64 "\n", fname, msg, (r->exist_key - r->first_miss) - 1, r->first_miss, r->exist_key - 1, r->exist_key, r->last_key, r->first_key, r->last_key); } @@ -990,9 +992,7 @@ main(int argc, char *argv[]) nth = MIN_TH; } - printf( - "Parent: compatibility: %s, " - "in-mem log sync: %s, timestamp in use: %s\n", + printf("Parent: compatibility: %s, in-mem log sync: %s, timestamp in use: %s\n", compat ? "true" : "false", inmem ? "true" : "false", use_ts ? "true" : "false"); printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, @@ -1047,9 +1047,7 @@ main(int argc, char *argv[]) * particularly in automated testing. */ testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " - "cp -p * ../%s.SAVE", - home, home, home)); + "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && cp -p * ../%s.SAVE", home, home, home)); if ((status = system(buf)) < 0) testutil_die(status, "system: %s", buf); printf("Open database, run recovery and verify content\n"); @@ -1119,7 +1117,7 @@ main(int argc, char *argv[]) key, last_key); break; } - testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, key)); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, key)); cur_coll->set_key(cur_coll, kname); cur_local->set_key(cur_local, kname); cur_oplog->set_key(cur_oplog, kname); @@ -1134,9 +1132,8 @@ main(int argc, char *argv[]) * larger than the saved one. */ if (!inmem && stable_fp != 0 && stable_fp <= stable_val) { - printf( - "%s: COLLECTION no record with " - "key %" PRIu64 " record ts %" PRIu64 " <= stable ts %" PRIu64 "\n", + printf("%s: COLLECTION no record with key %" PRIu64 " record ts %" PRIu64 + " <= stable ts %" PRIu64 "\n", fname, key, stable_fp, stable_val); absent_coll++; } @@ -1155,9 +1152,8 @@ main(int argc, char *argv[]) * If we found a record, the stable timestamp written to our file better be no * larger than the checkpoint one. */ - printf( - "%s: COLLECTION record with " - "key %" PRIu64 " record ts %" PRIu64 " > stable ts %" PRIu64 "\n", + printf("%s: COLLECTION record with key %" PRIu64 " record ts %" PRIu64 + " > stable ts %" PRIu64 "\n", fname, key, stable_fp, stable_val); fatal = true; } diff --git a/src/third_party/wiredtiger/test/csuite/scope/main.c b/src/third_party/wiredtiger/test/csuite/scope/main.c index 145472fdc01..ff7057c4104 100644 --- a/src/third_party/wiredtiger/test/csuite/scope/main.c +++ b/src/third_party/wiredtiger/test/csuite/scope/main.c @@ -38,8 +38,9 @@ handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const ch (void)(handler); /* Skip the error messages we're expecting to see. */ - if (ignore_errors > 0 && (strstr(message, "requires key be set") != NULL || - strstr(message, "requires value be set") != NULL)) { + if (ignore_errors > 0 && + (strstr(message, "requires key be set") != NULL || + strstr(message, "requires value be set") != NULL)) { --ignore_errors; return (0); } @@ -87,41 +88,52 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) RESERVE, UPDATE } func; - } * op, ops[] = {/* - * The ops order is specific: insert has to happen first so - * other operations are possible, and remove has to be last. - */ - { - "insert", INSERT_GET_KEY, - }, - { - "insert", INSERT_GET_VALUE, - }, - { - "search", SEARCH, - }, - { - "search", SEARCH_NEAR, - }, - { - "reserve", RESERVE, - }, - { - "insert", MODIFY, - }, - { - "update", UPDATE, - }, - { - "remove", REMOVE_GET_KEY, - }, - { - "remove", REMOVE_GET_VALUE, - }, - { - "remove", REMOVE_POS, - }, - {NULL, INSERT_GET_KEY}}; + } * op, + ops[] = {/* + * The ops order is specific: insert has to happen first so + * other operations are possible, and remove has to be last. + */ + { + "insert", + INSERT_GET_KEY, + }, + { + "insert", + INSERT_GET_VALUE, + }, + { + "search", + SEARCH, + }, + { + "search", + SEARCH_NEAR, + }, + { + "reserve", + RESERVE, + }, + { + "insert", + MODIFY, + }, + { + "update", + UPDATE, + }, + { + "remove", + REMOVE_GET_KEY, + }, + { + "remove", + REMOVE_GET_VALUE, + }, + { + "remove", + REMOVE_POS, + }, + {NULL, INSERT_GET_KEY}}; WT_CURSOR *cursor; #define MODIFY_ENTRIES 2 WT_MODIFY entries[MODIFY_ENTRIES]; diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index a9ebfcd085b..048547a4739 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -78,21 +78,38 @@ static const char *const uri_shadow = "shadow"; static const char *const ckpt_file = "checkpoint_done"; -static bool compat, inmem, use_ts; +static bool compat, inmem, stress, use_ts; static volatile uint64_t global_ts = 1; -#define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" +/* + * The configuration sets the eviction update and dirty targets at 20% so that on average, each + * thread can have a couple of dirty pages before eviction threads kick in. See below where these + * symbols are used for cache sizing - we'll have about 10 pages allocated per thread. On the other + * side, the eviction update and dirty triggers are 90%, so application threads aren't involved in + * eviction until we're close to running out of cache. + */ +#define ENV_CONFIG_ADD_COMPAT ",compatibility=(release=\"2.9\")" +#define ENV_CONFIG_ADD_EVICT_DIRTY ",eviction_dirty_target=20,eviction_dirty_trigger=90" +#define ENV_CONFIG_ADD_STRESS ",timing_stress_for_test=[prepare_checkpoint_delay]" + #define ENV_CONFIG_DEF \ - "cache_size=20M,create," \ + "cache_size=%" PRIu32 \ + "M,create," \ "debug_mode=(table_logging=true,checkpoint_retention=5)," \ - "eviction_updates_trigger=95,eviction_updates_target=80," \ + "eviction_updates_target=20,eviction_updates_trigger=90," \ "log=(archive=true,file_max=10M,enabled),session_max=%d," \ - "statistics=(fast),statistics_log=(wait=1,json=true)," + "statistics=(fast),statistics_log=(wait=1,json=true)" #define ENV_CONFIG_TXNSYNC \ ENV_CONFIG_DEF \ - "transaction_sync=(enabled,method=none)" + ",transaction_sync=(enabled,method=none)" #define ENV_CONFIG_REC "log=(archive=false,recover=on)" +/* + * A minimum width of 10, along with zero filling, means that all the keys sort according to their + * integer value, making each thread's key space distinct. + */ +#define KEY_FORMAT ("%010" PRIu64) + typedef struct { uint64_t absent_key; /* Last absent key */ uint64_t exist_key; /* First existing key after miss */ @@ -232,6 +249,7 @@ thread_run(void *arg) { FILE *fp; WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_shadow; + WT_DECL_RET; WT_ITEM data; WT_RAND_STATE rnd; WT_SESSION *prepared_session, *session; @@ -308,7 +326,7 @@ thread_run(void *arg) printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->info, td->start); active_ts = 0; for (i = td->start;; ++i) { - testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, i)); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, i)); testutil_check(session->begin_transaction(session, NULL)); if (use_prep) @@ -344,7 +362,9 @@ thread_run(void *arg) data.size = __wt_random(&rnd) % MAX_VAL; data.data = cbuf; cur_coll->set_value(cur_coll, &data); - testutil_check(cur_coll->insert(cur_coll)); + if ((ret = cur_coll->insert(cur_coll)) == WT_ROLLBACK) + goto rollback; + testutil_check(ret); cur_shadow->set_value(cur_shadow, &data); if (use_ts) { /* @@ -356,11 +376,13 @@ thread_run(void *arg) __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, active_ts)); testutil_check(session->timestamp_transaction(session, tscfg)); } - testutil_check(cur_shadow->insert(cur_shadow)); + if ((ret = cur_shadow->insert(cur_shadow)) == WT_ROLLBACK) + goto rollback; data.size = __wt_random(&rnd) % MAX_VAL; data.data = obuf; cur_oplog->set_value(cur_oplog, &data); - testutil_check(cur_oplog->insert(cur_oplog)); + if ((ret = cur_oplog->insert(cur_oplog)) == WT_ROLLBACK) + goto rollback; if (use_prep) { /* * Run with prepare every once in a while. And also yield after prepare sometimes too. @@ -382,7 +404,9 @@ thread_run(void *arg) } testutil_check(session->commit_transaction(session, NULL)); /* - * Insert into the local table outside the timestamp txn. + * Insert into the local table outside the timestamp txn. This must occur after the + * timestamp transaction, not before, because of the possibility of rollback in the + * transaction. The local table must stay in sync with the other tables. */ data.size = __wt_random(&rnd) % MAX_VAL; data.data = lbuf; @@ -394,6 +418,13 @@ thread_run(void *arg) */ if (fprintf(fp, "%" PRIu64 " %" PRIu64 "\n", active_ts, i) < 0) testutil_die(EIO, "fprintf"); + + if (0) { +rollback: + testutil_check(session->rollback_transaction(session, NULL)); + if (use_prep) + testutil_check(prepared_session->rollback_transaction(prepared_session, NULL)); + } } /* NOTREACHED */ } @@ -410,20 +441,42 @@ run_workload(uint32_t nth) WT_SESSION *session; THREAD_DATA *td; wt_thread_t *thr; - uint32_t ckpt_id, i, ts_id; + uint32_t cache_mb, ckpt_id, i, ts_id; char envconf[512], uri[128]; thr = dcalloc(nth + 2, sizeof(*thr)); td = dcalloc(nth + 2, sizeof(THREAD_DATA)); + + /* + * Size the cache appropriately for the number of threads. Each thread adds keys sequentially to + * its own portion of the key space, so each thread will be dirtying one page at a time. By + * default, a leaf page grows to 32K in size before it splits and the thread begins to fill + * another page. We'll budget for 10 full size leaf pages per thread in the cache plus a little + * extra in the total for overhead. + */ + cache_mb = ((32 * WT_KILOBYTE * 10) * nth) / WT_MEGABYTE + 20; + if (chdir(home) != 0) testutil_die(errno, "Child chdir: %s", home); if (inmem) - testutil_check(__wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_DEF, SESSION_MAX)); + testutil_check( + __wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_DEF, cache_mb, SESSION_MAX)); else - testutil_check(__wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_TXNSYNC, SESSION_MAX)); + testutil_check( + __wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_TXNSYNC, cache_mb, SESSION_MAX)); if (compat) - strcat(envconf, ENV_CONFIG_COMPAT); + strcat(envconf, ENV_CONFIG_ADD_COMPAT); + if (stress) + strcat(envconf, ENV_CONFIG_ADD_STRESS); + /* + * The eviction dirty target and trigger configurations are not compatible with certain other + * configurations. + */ + if (!compat && !inmem) + strcat(envconf, ENV_CONFIG_ADD_EVICT_DIRTY); + + printf("wiredtiger_open configuration: %s\n", envconf); testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* @@ -503,9 +556,7 @@ print_missing(REPORT *r, const char *fname, const char *msg) { if (r->exist_key != INVALID_KEY) printf("%s: %s error %" PRIu64 " absent records %" PRIu64 "-%" PRIu64 ". Then keys %" PRIu64 - "-%" PRIu64 - " exist." - " Key range %" PRIu64 "-%" PRIu64 "\n", + "-%" PRIu64 " exist. Key range %" PRIu64 "-%" PRIu64 "\n", fname, msg, (r->exist_key - r->first_miss) - 1, r->first_miss, r->exist_key - 1, r->exist_key, r->last_key, r->first_key, r->last_key); } @@ -549,7 +600,7 @@ main(int argc, char *argv[]) (void)testutil_set_progname(argv); - compat = inmem = false; + compat = inmem = stress = false; use_ts = true; nth = MIN_TH; rand_th = rand_time = true; @@ -557,7 +608,7 @@ main(int argc, char *argv[]) verify_only = false; working_dir = "WT_TEST.timestamp-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmT:t:vz")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmsT:t:vz")) != EOF) switch (ch) { case 'C': compat = true; @@ -571,14 +622,15 @@ main(int argc, char *argv[]) case 'm': inmem = true; break; + case 's': + stress = true; + break; case 'T': rand_th = false; nth = (uint32_t)atoi(__wt_optarg); if (nth > MAX_TH) { - fprintf(stderr, - "Number of threads is larger than the" - " maximum %" PRId32 "\n", - MAX_TH); + fprintf( + stderr, "Number of threads is larger than the maximum %" PRId32 "\n", MAX_TH); return (EXIT_FAILURE); } break; @@ -626,12 +678,14 @@ main(int argc, char *argv[]) } printf( - "Parent: compatibility: %s, " - "in-mem log sync: %s, timestamp in use: %s\n", - compat ? "true" : "false", inmem ? "true" : "false", use_ts ? "true" : "false"); + "Parent: compatibility: %s, in-mem log sync: %s, add timing stress: %s, timestamp in " + "use: %s\n", + compat ? "true" : "false", inmem ? "true" : "false", stress ? "true" : "false", + use_ts ? "true" : "false"); printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); - printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, - compat ? " -C" : "", inmem ? " -m" : "", !use_ts ? " -z" : "", working_dir, nth, timeout); + printf("CONFIG: %s%s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, + compat ? " -C" : "", inmem ? " -m" : "", stress ? " -s" : "", !use_ts ? " -z" : "", + working_dir, nth, timeout); /* * Fork a child to insert as many items. We will then randomly kill the child, run recovery * and make sure all items we wrote exist after recovery runs. @@ -682,9 +736,7 @@ main(int argc, char *argv[]) * particularly in automated testing. */ testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " - "cp -p * ../%s.SAVE", - home, home, home)); + "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && cp -p * ../%s.SAVE", home, home, home)); if ((status = system(buf)) < 0) testutil_die(status, "system: %s", buf); printf("Open database, run recovery and verify content\n"); @@ -759,7 +811,7 @@ main(int argc, char *argv[]) key, last_key); break; } - testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, key)); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, key)); cur_coll->set_key(cur_coll, kname); cur_local->set_key(cur_local, kname); cur_oplog->set_key(cur_oplog, kname); @@ -779,9 +831,8 @@ main(int argc, char *argv[]) * larger than the saved one. */ if (!inmem && stable_fp != 0 && stable_fp <= stable_val) { - printf( - "%s: COLLECTION no record with " - "key %" PRIu64 " record ts %" PRIu64 " <= stable ts %" PRIu64 "\n", + printf("%s: COLLECTION no record with key %" PRIu64 " record ts %" PRIu64 + " <= stable ts %" PRIu64 "\n", fname, key, stable_fp, stable_val); absent_coll++; } @@ -807,9 +858,8 @@ main(int argc, char *argv[]) * If we found a record, the stable timestamp written to our file better be no * larger than the checkpoint one. */ - printf( - "%s: COLLECTION record with " - "key %" PRIu64 " record ts %" PRIu64 " > stable ts %" PRIu64 "\n", + printf("%s: COLLECTION record with key %" PRIu64 " record ts %" PRIu64 + " > stable ts %" PRIu64 "\n", fname, key, stable_fp, stable_val); fatal = true; } else if ((ret = cur_shadow->search(cur_shadow)) != 0) diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh index 324e981dd70..0cac6a56ea4 100755 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh @@ -2,16 +2,25 @@ set -e -# Smoke-test timestamp-abort as part of running "make check". +# Smoke-test timestamp-abort as part of running "make check". Use the -s option +# to add a stress timing in checkpoint prepare. + +default_test_args="-t 10 -T 5" +while getopts ":s" opt; do + case $opt in + s) default_test_args="$default_test_args -s" ;; + esac +done + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix # and running in test/csuite. top_builddir=${top_builddir:-../../build_posix} top_srcdir=${top_srcdir:-../..} - $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -t 10 -T 5 -#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -t 10 -T 5 -L - $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m -t 10 -T 5 -#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m -t 10 -T 5 -L - $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C -t 10 -T 5 - $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C -m -t 10 -T 5 +$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort $default_test_args +#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort $default_test_args -L +$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m $default_test_args +#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m $default_test_args -L +$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C $default_test_args +$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C -m $default_test_args diff --git a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c index a518f6bed0e..f4b428fda2e 100644 --- a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c @@ -114,10 +114,8 @@ main(int argc, char *argv[]) testutil_make_work_dir(opts->home); testutil_check(wiredtiger_open(opts->home, NULL, - "create,cache_size=1G,checkpoint=(wait=30)," - "eviction_trigger=80,eviction_target=64,eviction_dirty_target=65," - "log=(enabled,file_max=10M)," - "transaction_sync=(enabled=true,method=none)", + "create,cache_size=1G,checkpoint=(wait=30),eviction_trigger=80,eviction_target=64,eviction_" + "dirty_target=65,log=(enabled,file_max=10M),transaction_sync=(enabled=true,method=none)", &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c index 0131c020994..e227a6fd5d7 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c @@ -107,16 +107,12 @@ main(int argc, char *argv[]) testutil_make_work_dir(opts->home); testutil_check(__wt_snprintf(buf, sizeof(buf), - "create," - "cache_size=%s," - "eviction=(threads_max=5)," - "statistics=(fast)", + "create,cache_size=%s,eviction=(threads_max=5),statistics=(fast)", opts->table_type == TABLE_FIX ? "500MB" : "2GB")); testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); testutil_check(__wt_snprintf(buf, sizeof(buf), - "key_format=r,value_format=%s," - "allocation_size=4K,leaf_page_max=64K", + "key_format=r,value_format=%s,allocation_size=4K,leaf_page_max=64K", opts->table_type == TABLE_FIX ? "8t" : "S")); testutil_check(session->create(session, opts->uri, buf)); testutil_check(session->close(session, NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c index 95bc274b0f8..e8a1637730e 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c @@ -151,9 +151,8 @@ test_join(TEST_OPTS *opts, SHARED_OPTS *sharedopts, bool bloom, bool sometimes_r * Note: id is repeated as id2. This makes it easier to identify the primary key in dumps of the * index files. */ - testutil_check(session->create(session, opts->uri, - "key_format=i,value_format=iiSii," - "columns=(id,post,bal,extra,flag,id2)")); + testutil_check(session->create( + session, opts->uri, "key_format=i,value_format=iiSii,columns=(id,post,bal,extra,flag,id2)")); testutil_check(session->create(session, sharedopts->posturi, "columns=(post)")); testutil_check(session->create(session, sharedopts->baluri, "columns=(bal)")); @@ -203,9 +202,8 @@ test_join(TEST_OPTS *opts, SHARED_OPTS *sharedopts, bool bloom, bool sometimes_r } for (i = 0; i < N_INSERT_THREAD; ++i) fprintf(stderr, - " insert thread %d did " - "%d inserts, %d removes, %d notfound, %d rollbacks\n", - i, insert_args[i].inserts, insert_args[i].removes, insert_args[i].notfounds, + " insert thread %d did %d inserts, %d removes, %d notfound, %d rollbacks\n", i, + insert_args[i].inserts, insert_args[i].removes, insert_args[i].notfounds, insert_args[i].rollbacks); testutil_check(session->drop(session, sharedopts->posturi, NULL)); @@ -363,9 +361,7 @@ thread_join(void *arg) */ testutil_check(joincur->get_key(joincur, &key)); testutil_check(joincur->get_value(joincur, &post, &bal, &extra, &flag, &key2)); - fprintf(stderr, - "FAIL: iteration %d: " - "key=%d/%d, postal_code=%d, balance=%d, flag=%d\n", + fprintf(stderr, "FAIL: iteration %d: key=%d/%d, postal_code=%d, balance=%d, flag=%d\n", threadargs->joins, key, key2, post, bal, flag); /* Save the results. */ testutil_check(opts->conn->close(opts->conn, NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c index e49d0cbe0aa..f8b5098b1a3 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c @@ -62,15 +62,10 @@ main(int argc, char *argv[]) testutil_make_work_dir(opts->home); testutil_check(wiredtiger_open(opts->home, NULL, - "create," - "cache_size=2G," - "eviction=(threads_max=5)," - "statistics=(fast)", - &opts->conn)); + "create,cache_size=2G,eviction=(threads_max=5),statistics=(fast)", &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, opts->uri, - "key_format=Q,value_format=Q," - "leaf_page_max=32k,")); + testutil_check( + session->create(session, opts->uri, "key_format=Q,value_format=Q,leaf_page_max=32k,")); /* Create the single record. */ testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &c)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c b/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c index 4f211493c57..c101c824fc0 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c @@ -74,10 +74,8 @@ main(int argc, char *argv[]) testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=200M", &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, opts->uri, - "key_format=r," - "value_format=5sHQ," - "columns=(id,country,year,population)")); + testutil_check(session->create( + session, opts->uri, "key_format=r,value_format=5sHQ,columns=(id,country,year,population)")); /* Create an index with a simple key. */ testutil_check(session->create(session, countryuri, "columns=(country)")); diff --git a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c index bee38e812ab..1cb98c6b717 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c @@ -74,9 +74,8 @@ main(int argc, char *argv[]) * Note: repeated primary key 'id' as 'id2'. This makes it easier to dump an index and know * which record we're looking at. */ - testutil_check(session->create(session, opts->uri, - "key_format=i,value_format=iiii," - "columns=(id,post,balance,flag,id2)")); + testutil_check(session->create( + session, opts->uri, "key_format=i,value_format=iiii,columns=(id,post,balance,flag,id2)")); tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); @@ -136,10 +135,8 @@ main(int argc, char *argv[]) */ testutil_check(joincur->get_key(joincur, &key)); testutil_check(joincur->get_value(joincur, &post, &balance, &flag, &key2)); - fprintf(stderr, - "FAIL: " - "key=%d/%d, postal_code=%d, balance=%d, flag=%d\n", - key, key2, post, balance, flag); + fprintf(stderr, "FAIL: key=%d/%d, postal_code=%d, balance=%d, flag=%d\n", key, key2, post, + balance, flag); count++; } testutil_assert(ret == WT_NOTFOUND); diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index 9005c474801..66be61ad271 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -116,9 +116,8 @@ main(int argc, char *argv[]) * Note: id is repeated as id2. This makes it easier to identify the primary key in dumps of the * index files. */ - testutil_check(session->create(session, opts->uri, - "key_format=i,value_format=iiSii," - "columns=(id,post,bal,extra,flag,id2)")); + testutil_check(session->create( + session, opts->uri, "key_format=i,value_format=iiSii,columns=(id,post,bal,extra,flag,id2)")); tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); @@ -188,10 +187,8 @@ main(int argc, char *argv[]) */ if (nfail != 0) fprintf(stderr, - "ERROR: %d failures when a single commit" - " took more than %d seconds.\n" - "This may indicate a real problem or a" - " particularly slow machine.\n", + "ERROR: %d failures when a single commit took more than %d seconds.\n" + "This may indicate a real problem or a particularly slow machine.\n", nfail, GAP_ERROR); testutil_assert(nfail == 0); testutil_progress(opts, "cleanup starting"); diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index 1285109e706..92bd2c3cd4c 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -342,9 +342,7 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) bool got_failure, got_success; if (opts->verbose) - printf( - "Determining best range of operations until failure, " - "with close_test %s.\n", + printf("Determining best range of operations until failure, with close_test %s.\n", (close_test ? "enabled" : "disabled")); run_check_subtest(opts, debugger, 1, close_test, &cutoff); @@ -386,9 +384,7 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) if (!got_failure || !got_success) { fprintf(stderr, "Warning: did not find a reliable test range.\n" - "midpoint=%" PRIu64 - ", close_test=%d, got_failure=%d, " - "got_success=%d\n", + "midpoint=%" PRIu64 ", close_test=%d, got_failure=%d, got_success=%d\n", mid, (int)close_test, (int)got_failure, (int)got_success); return (EAGAIN); } @@ -511,17 +507,14 @@ subtest_main(int argc, char *argv[], bool close_test) if ((p = getenv("top_builddir")) == NULL) p = "../../build_posix"; testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_size=250M,log=(enabled)," - "transaction_sync=(enabled,method=none)," - "extensions=(%s/%s=" - "(early_load,config={environment=true,verbose=true}))", + "create,cache_size=250M,log=(enabled),transaction_sync=(enabled,method=none),extensions=(%s/" + "%s=(early_load,config={environment=true,verbose=true}))", p, WT_FAIL_FS_LIB)); testutil_check(wiredtiger_open(opts->home, &event_handler, config, &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, "table:subtest", - "key_format=i,value_format=iiiS," - "columns=(id,v0,v1,v2,big)")); + testutil_check(session->create( + session, "table:subtest", "key_format=i,value_format=iiiS,columns=(id,v0,v1,v2,big)")); testutil_check(session->create(session, "table:subtest2", "key_format=i,value_format=i")); diff --git a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c index 9c433ed127e..baca92846bc 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c @@ -188,10 +188,8 @@ monitor(void *args) if (ctr != last_ops[i]) last_ops[i] = ctr; else { - printf( - "Thread %d had a task running" - " for more than %d seconds\n", - i, MAX_EXECUTION_TIME / 2); + printf("Thread %d had a task running for more than %d seconds\n", i, + MAX_EXECUTION_TIME / 2); abort(); } } diff --git a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c index a63cfb2724a..c831180c130 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c @@ -54,8 +54,9 @@ handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const ch (void)(handler); /* Skip the error messages we're expecting to see. */ - if (ignore_errors > 0 && (strstr(message, "requires key be set") != NULL || - strstr(message, "requires value be set") != NULL)) { + if (ignore_errors > 0 && + (strstr(message, "requires key be set") != NULL || + strstr(message, "requires value be set") != NULL)) { --ignore_errors; return (0); } @@ -83,15 +84,11 @@ main(int argc, char *argv[]) testutil_make_work_dir(opts->home); testutil_check(wiredtiger_open(opts->home, &event_handler, - "create," - "cache_size=1G," - "statistics_log=(json,wait=1)", - &opts->conn)); + "create,cache_size=1G,statistics_log=(json,wait=1)", &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); testutil_check(session->create(session, uri, - "key_format=Q,value_format=u," - "leaf_item_max=64M,leaf_page_max=32k,memory_page_max=1M")); + "key_format=Q,value_format=u,leaf_item_max=64M,leaf_page_max=32k,memory_page_max=1M")); testutil_check(session->open_cursor(session, uri, NULL, NULL, &c)); @@ -133,7 +130,7 @@ main(int argc, char *argv[]) modify_entry.offset = offset; modify_entry.size = modify_entry.data.size; /* FIXME-WT-6113: extend timeout to pass the test */ - (void)alarm(7); + (void)alarm(15); testutil_check(c->modify(c, &modify_entry, 1)); (void)alarm(0); testutil_check(session2->commit_transaction(session2, NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c index 4e1393b0449..ef51a9dd19c 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c @@ -312,9 +312,7 @@ copy_database(const char *sfx) char buf[1024]; testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ./%s.%s; mkdir ./%s.%s; " - "cp -p %s/* ./%s.%s", - home, sfx, home, sfx, home, home, sfx)); + "rm -rf ./%s.%s; mkdir ./%s.%s; cp -p %s/* ./%s.%s", home, sfx, home, sfx, home, home, sfx)); printf("copy: %s\n", buf); if ((ret = system(buf)) < 0) testutil_die(ret, "system: %s", buf); diff --git a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c index b91d3c2222e..d3283d6007b 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c @@ -57,9 +57,8 @@ uri_init(void) /* Initialize the file contents. */ for (i = 0; i < uris; ++i) { - testutil_check(__wt_snprintf(buf, sizeof(buf), - "key_format=S,value_format=S," - "allocation_size=4K,leaf_page_max=32KB,")); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "key_format=S,value_format=S,allocation_size=4K,leaf_page_max=32KB,")); testutil_check(session->create(session, uri_list[i], buf)); testutil_check(session->open_cursor(session, uri_list[i], NULL, NULL, &cursor)); for (key = 1; key < MAXKEY; ++key) { @@ -305,9 +304,18 @@ run(int argc, char *argv[]) u_int uris; bool cache_cursors; } runs[] = { - {1, 1, false}, {1, 1, true}, {8, 1, false}, {8, 1, true}, {16, 1, false}, {16, 1, true}, - {16, WT_ELEMENTS(uri_list), false}, {16, WT_ELEMENTS(uri_list), true}, {64, 100, false}, - {64, 100, true}, {64, WT_ELEMENTS(uri_list), false}, {64, WT_ELEMENTS(uri_list), true}, + {1, 1, false}, + {1, 1, true}, + {8, 1, false}, + {8, 1, true}, + {16, 1, false}, + {16, 1, true}, + {16, WT_ELEMENTS(uri_list), false}, + {16, WT_ELEMENTS(uri_list), true}, + {64, 100, false}, + {64, 100, true}, + {64, WT_ELEMENTS(uri_list), false}, + {64, WT_ELEMENTS(uri_list), true}, }; WT_RAND_STATE rnd; u_int i, n; diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 68ac9001277..6e97452af14 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -224,22 +224,29 @@ functions: set -o verbose for i in $(seq ${times|1}); do - # Run the various combinations of args. Let time and threads be random. + # Run the various combinations of args. Let time and threads be random. Add a + # timing stress to test_timestamp_abort every other run. + if [ $(( $i % 2 )) -eq 0 ]; then + test_timestamp_abort_args=-s + else + test_timestamp_abort_args= + fi + # Run current version with write-no-sync txns. ${test_env_vars|} ./test_random_abort 2>&1 - ${test_env_vars|} ./test_timestamp_abort 2>&1 + ${test_env_vars|} ./test_timestamp_abort $test_timestamp_abort_args 2>&1 # Current version with memory-based txns (MongoDB usage). ${test_env_vars|} ./test_random_abort -m 2>&1 - ${test_env_vars|} ./test_timestamp_abort -m 2>&1 + ${test_env_vars|} ./test_timestamp_abort -m $test_timestamp_abort_args 2>&1 # V1 log compatibility mode with write-no-sync txns. ${test_env_vars|} ./test_random_abort -C 2>&1 - ${test_env_vars|} ./test_timestamp_abort -C 2>&1 + ${test_env_vars|} ./test_timestamp_abort -C $test_timestamp_abort_args 2>&1 # V1 log compatibility mode with memory-based txns. ${test_env_vars|} ./test_random_abort -C -m 2>&1 - ${test_env_vars|} ./test_timestamp_abort -C -m 2>&1 + ${test_env_vars|} ./test_timestamp_abort -C -m $test_timestamp_abort_args 2>&1 ${test_env_vars|} ./test_truncated_log ${truncated_log_args|} 2>&1 @@ -549,6 +556,7 @@ tasks: posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - func: "make check all" vars: + smp_command: -j 1 test_env_vars: MSAN_OPTIONS=abort_on_error=1:disable_coredump=0 MSAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer TESTUTIL_SLOW_MACHINE=1 - name: make-check-asan-test @@ -564,6 +572,7 @@ tasks: posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - func: "make check all" vars: + smp_command: -j 1 test_env_vars: ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer TESTUTIL_SLOW_MACHINE=1 - name: make-check-linux-no-ftruncate-test @@ -894,6 +903,21 @@ tasks: ${test_env_vars|} $(pwd)/../test/csuite/timestamp_abort/smoke.sh 2>&1 + - name: csuite-timestamp-abort-stress-test + tags: ["pull_request"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/../test/csuite/timestamp_abort/smoke.sh -s 2>&1 + - name: csuite-scope-test tags: ["pull_request"] depends_on: @@ -1631,7 +1655,18 @@ tasks: ulimit -c unlimited largescale/run-million-collection-test.sh . - - name: compatibility-test-for-releases-full + - name: compatibility-test-for-newer-releases + commands: + - func: "get project" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + test/evergreen/compatibility_test_for_releases.sh -n + + - name: compatibility-test-for-older-releases commands: - func: "get project" - command: shell.exec @@ -1640,9 +1675,9 @@ tasks: script: | set -o errexit set -o verbose - test/evergreen/compatibility_test_for_releases.sh -l + test/evergreen/compatibility_test_for_releases.sh -o - - name: compatibility-test-for-releases + - name: compatibility-test-for-wt-standalone-releases commands: - func: "get project" - command: shell.exec @@ -1651,7 +1686,7 @@ tasks: script: | set -o errexit set -o verbose - test/evergreen/compatibility_test_for_releases.sh + test/evergreen/compatibility_test_for_releases.sh -w - name: generate-datafile-little-endian depends_on: @@ -1879,15 +1914,17 @@ tasks: vars: # Don't use diagnostic - this test looks for timing problems that are more likely to occur without it posix_configure_flags: --enable-strict - - func: "checkpoint test" - vars: - checkpoint_args: -t m -n 1000000 -k 5000000 -C cache_size=100MB + # Temporarily disable mix and column file type + # FIXME after WT-6608 + #- func: "checkpoint test" + # vars: + # checkpoint_args: -t m -n 1000000 -k 5000000 -C cache_size=100MB + #- func: "checkpoint test" + # vars: + # checkpoint_args: -t c -n 1000000 -k 5000000 -C cache_size=100MB - func: "checkpoint test" vars: checkpoint_args: -t r -n 1000000 -k 5000000 -C cache_size=100MB - - func: "checkpoint test" - vars: - checkpoint_args: -t c -n 1000000 -k 5000000 -C cache_size=100MB - name: coverage-report commands: @@ -2195,7 +2232,7 @@ tasks: - func: "checkpoint stress test" vars: times: 1 # No of times to run the loop - no_of_procs: 8 # No of processes to run in the background + no_of_procs: 10 # No of processes to run in the background - name: split-stress-test tags: ["stress-test-1", "stress-test-zseries-1"] @@ -2481,20 +2518,21 @@ buildvariants: tasks: - name: million-collection-test -- name: compatibility-tests-full - display_name: Compatibility tests full +- name: compatibility-tests-less-frequent + display_name: Compatibility tests (less frequent) batchtime: 10080 # 7 days run_on: - ubuntu1804-test tasks: - - name: compatibility-test-for-releases-full + - name: compatibility-test-for-older-releases + - name: compatibility-test-for-wt-standalone-releases - name: compatibility-tests display_name: Compatibility tests run_on: - ubuntu1804-test tasks: - - name: compatibility-test-for-releases + - name: compatibility-test-for-newer-releases - name: windows-64 display_name: "! Windows 64-bit" diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh index 551bbd78764..bf5f9fed505 100755 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash ############################################################################################## # Check branches to ensure forward/backward compatibility, including some upgrade/downgrade testing. -# Pass in -l to trigger the longer version of this test which tests more variants. ############################################################################################## set -e @@ -151,53 +150,90 @@ upgrade_downgrade() done } +# Only one of below flags will be set by the 1st argument of the script. +older=false +newer=false +wt_standalone=false + +# Branches in below 2 arrays should be put in newer-to-older order. +# +# An overlap (last element of the 1st array & first element of the 2nd array) +# is expected to avoid missing the edge testing coverage. +# +# The 2 arrays should be adjusted over time when newer branches are created, +# or older branches are EOL. +newer_release_branches=(develop mongodb-4.6 mongodb-4.4 mongodb-4.2) +older_release_branches=(mongodb-4.2 mongodb-4.0 mongodb-3.6) + +declare -A scopes +scopes[newer]="newer stable release branches" +scopes[older]="older stable release branches" +scopes[wt_standalone]="WiredTiger standalone releases" + ############################################################# -# usage strinf +# usage string ############################################################# usage() { - echo "Usage: \tcompatibility_test_for_releases [-l]" - echo "\t-l\trun additional variants of wiredtiger" + echo -e "Usage: \tcompatibility_test_for_releases [-n|-o|-w]" + echo -e "\t-n\trun compatibility tests for ${scopes[newer]}" + echo -e "\t-o\trun compatibility tests for ${scopes[older]}" + echo -e "\t-w\trun compatibility tests for ${scopes[wt_standalone]}" exit 1 } -long=false - -if [ $# -eq 1 ]; then - if [ $1 = "-l" ]; then - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo "Performing long compatibility test run" - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - long=true - else - (usage) - fi -else - if [ $# -gt 1 ]; then - (usage) - fi +if [ $# -ne 1 ]; then + usage fi +# Script argument processing +case $1 in +"-n") + newer=true + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "Performing compatibility tests for ${scopes[newer]}" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" +;; +"-o") + older=true + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "Performing compatibility tests for ${scopes[older]}" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" +;; +"-w") + wt_standalone=true + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "Performing compatibility tests for ${scopes[wt_standalone]}" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" +;; +*) + usage +;; +esac + # Create a directory in which to do the work. top="test-compatibility-run" rm -rf "$top" && mkdir "$top" cd "$top" # Build the branches. -if [ "$long" = true ]; then - (build_branch mongodb-3.4) - (build_branch mongodb-3.6) - (build_branch mongodb-4.0) +if [ "$newer" = true ]; then + for b in ${newer_release_branches[@]}; do + (build_branch $b) + done +fi + +if [ "$older" = true ]; then + for b in ${older_release_branches[@]}; do + (build_branch $b) + done fi -(build_branch mongodb-4.2) -(build_branch mongodb-4.4) -(build_branch mongodb-4.6) -(build_branch develop) # Get the names of the last two WiredTiger releases, wt1 is the most recent release, wt2 is the # release before that. Minor trickiness, we depend on the "develop" directory already existing # so we have a source in which to do git commands. -if [ "$long" = true ]; then +if [ "${wt_standalone}" = true ]; then + (build_branch develop) cd develop; wt1=$(get_prev_version 1); cd .. (build_branch "$wt1") cd develop; wt2=$(get_prev_version 2); cd .. @@ -205,42 +241,67 @@ if [ "$long" = true ]; then fi # Run format in each branch for supported access methods. -if [ "$long" = true ]; then - (run_format mongodb-3.4 "fix row var") - (run_format mongodb-3.6 "fix row var") - (run_format mongodb-4.0 "fix row var") +if [ "$newer" = true ]; then + for b in ${newer_release_branches[@]}; do + (run_format $b "row") + done fi -(run_format mongodb-4.2 "fix row var") -(run_format mongodb-4.4 "row") -(run_format mongodb-4.6 "row") -(run_format develop "row") -if [ "$long" = true ]; then + +if [ "$older" = true ]; then + for b in ${older_release_branches[@]}; do + (run_format $b "fix row var") + done +fi + +if [ "${wt_standalone}" = true ]; then (run_format "$wt1" "fix row var") (run_format "$wt2" "fix row var") fi # Verify backward compatibility for supported access methods. -if [ "$long" = true ]; then - (verify_branches mongodb-3.6 mongodb-3.4 "fix row var") - (verify_branches mongodb-4.0 mongodb-3.6 "fix row var") - (verify_branches mongodb-4.2 mongodb-4.0 "fix row var") +# +# The branch array includes a list of branches in newer-to-older order. +# For backport compatibility, the binary of the newer branch should +# be used to verify the data files generated by the older branch. +# e.g. (verify_branches mongodb-4.4 mongodb-4.2 "row") +if [ "$newer" = true ]; then + for i in ${!newer_release_branches[@]}; do + [[ $((i+1)) < ${#newer_release_branches[@]} ]] && \ + (verify_branches ${newer_release_branches[$i]} ${newer_release_branches[$((i+1))]} "row") + done fi -(verify_branches mongodb-4.4 mongodb-4.2 "fix row var") -(verify_branches mongodb-4.6 mongodb-4.4 "row") -(verify_branches develop mongodb-4.6 "row") -if [ "$long" = true ]; then - (verify_branches "$wt1" "$wt2" "row") + +if [ "$older" = true ]; then + for i in ${!older_release_branches[@]}; do + [[ $((i+1)) < ${#older_release_branches[@]} ]] && \ + (verify_branches ${older_release_branches[$i]} ${older_release_branches[$((i+1))]} "fix row var") + done +fi + +if [ "${wt_standalone}" = true ]; then (verify_branches develop "$wt1" "row") + (verify_branches "$wt1" "$wt2" "row") fi # Verify forward compatibility for supported access methods. -(verify_branches mongodb-4.2 mongodb-4.4 "row") -(verify_branches mongodb-4.4 mongodb-4.6 "row") -(verify_branches mongodb-4.6 develop "row") +# +# The branch array includes a list of branches in newer-to-older order. +# For forward compatibility, the binary of the older branch should +# be used to verify the data files generated by the newer branch. +# e.g. (verify_branches mongodb-4.2 mongodb-4.4 "row") +if [ "$newer" = true ]; then + for i in ${!newer_release_branches[@]}; do + [[ $((i+1)) < ${#newer_release_branches[@]} ]] && \ + (verify_branches ${newer_release_branches[$((i+1))]} ${newer_release_branches[$i]} "row") + done +fi # Upgrade/downgrade testing for supported access methods. -(upgrade_downgrade mongodb-4.2 mongodb-4.4 "row") -(upgrade_downgrade mongodb-4.4 mongodb-4.6"row") -(upgrade_downgrade mongodb-4.6 develop "row") +if [ "$newer" = true ]; then + for i in ${!newer_release_branches[@]}; do + [[ $((i+1)) < ${#newer_release_branches[@]} ]] && \ + (upgrade_downgrade ${newer_release_branches[$((i+1))]} ${newer_release_branches[$i]} "row") + done +fi exit 0 diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c index d73fe635abe..db366bfb22c 100644 --- a/src/third_party/wiredtiger/test/fops/t.c +++ b/src/third_party/wiredtiger/test/fops/t.c @@ -57,11 +57,12 @@ main(int argc, char *argv[]) const char *uri; const char *desc; const char *config; - } * cp, configs[] = {{"file:wt", NULL, NULL}, {"table:wt", NULL, NULL}, + } * cp, + configs[] = {{"file:wt", NULL, NULL}, {"table:wt", NULL, NULL}, /* Configure for a modest cache size. */ #define LSM_CONFIG "lsm=(chunk_size=1m,merge_max=2),leaf_page_max=4k" - {"lsm:wt", NULL, LSM_CONFIG}, {"table:wt", " [lsm]", "type=lsm," LSM_CONFIG}, - {NULL, NULL, NULL}}; + {"lsm:wt", NULL, LSM_CONFIG}, {"table:wt", " [lsm]", "type=lsm," LSM_CONFIG}, + {NULL, NULL, NULL}}; u_int nthreads; int ch, cnt, runs; char *config_open, *working_dir; @@ -150,9 +151,8 @@ wt_startup(char *config_open) testutil_make_work_dir(home); testutil_check(__wt_snprintf(config_buf, sizeof(config_buf), - "create,error_prefix=\"%s\",cache_size=5MB%s%s," - "operation_tracking=(enabled=false)", - progname, config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); + "create,error_prefix=\"%s\",cache_size=5MB%s%s,operation_tracking=(enabled=false)", progname, + config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); testutil_check(wiredtiger_open(home, &event_handler, config_buf, &conn)); } @@ -233,9 +233,7 @@ static int usage(void) { fprintf(stderr, - "usage: %s " - "[-C wiredtiger-config] [-l log] [-n ops] [-r runs] [-t threads] " - "[-x] \n", + "usage: %s [-C wiredtiger-config] [-l log] [-n ops] [-r runs] [-t threads] [-x] \n", progname); fprintf(stderr, "%s", "\t-C specify wiredtiger_open configuration arguments\n" diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index bfe964cf871..8a358343aeb 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -41,10 +41,34 @@ check_copy(void) size_t len; char *path; - len = strlen(g.home) + strlen("BACKUP") + 2; + len = WT_MAX(strlen(BACKUP_INFO_FILE), strlen(BACKUP_INFO_FILE_TMP)); + len = WT_MAX(len, strlen("BACKUP")); + len += strlen(g.home) + 2; path = dmalloc(len); + /* + * Remove any backup info files that exist. We're about the run recovery in the backup directory + * so we cannot use the backup directory after that restart. We must remove the files before the + * restart to avoid a window where they could exist and the backup directory has had recovery + * run. + */ + testutil_check(__wt_snprintf(path, len, "%s/%s", g.home, BACKUP_INFO_FILE_TMP)); + ret = unlink(path); + /* It is fine if the file does not exist. */ + if (ret == 0 || errno == ENOENT) + ret = 0; + else + testutil_die(errno, "unlink %s", path); + + testutil_check(__wt_snprintf(path, len, "%s/%s", g.home, BACKUP_INFO_FILE)); + ret = unlink(path); + /* It is fine if the file does not exist. */ + if (ret == 0 || errno == ENOENT) + ret = 0; + else + testutil_die(errno, "unlink %s", path); + + /* Now setup and open the path for real. */ testutil_check(__wt_snprintf(path, len, "%s/BACKUP", g.home)); - wts_open(path, &conn, &session, true); /* @@ -266,10 +290,18 @@ copy_blocks(WT_SESSION *session, WT_CURSOR *bkup_c, const char *name) tmp = drealloc(tmp, size); tmp_sz = size; } - error_sys_check(lseek(rfd, (wt_off_t)offset, SEEK_SET)); + /* + * Don't use the system checker for lseek. The system check macro uses an int which is + * often 4 bytes and checks for any negative value. The offset returned from lseek is + * 8 bytes and we can have a false positive error check. + */ + if (lseek(rfd, (wt_off_t)offset, SEEK_SET) == -1) + testutil_die(errno, "backup-read: lseek"); error_sys_check(rdsize = read(rfd, tmp, size)); - error_sys_check(lseek(wfd1, (wt_off_t)offset, SEEK_SET)); - error_sys_check(lseek(wfd2, (wt_off_t)offset, SEEK_SET)); + if (lseek(wfd1, (wt_off_t)offset, SEEK_SET) == -1) + testutil_die(errno, "backup-write1: lseek"); + if (lseek(wfd2, (wt_off_t)offset, SEEK_SET) == -1) + testutil_die(errno, "backup-write2: lseek"); /* Use the read size since we may have read less than the granularity. */ error_sys_check(write(wfd1, tmp, (size_t)rdsize)); error_sys_check(write(wfd2, tmp, (size_t)rdsize)); @@ -340,6 +372,119 @@ copy_file(WT_SESSION *session, const char *name) */ #define HOME_BACKUP_INIT_CMD "rm -rf %s/BACKUP %s/BACKUP.copy && mkdir %s/BACKUP %s/BACKUP.copy" +#define RESTORE_SKIP 1 +#define RESTORE_SUCCESS 0 +/* + * restore_backup_info -- + * If it exists, restore the backup information. Return 0 on success. + */ +static int +restore_backup_info(WT_SESSION *session, ACTIVE_FILES *active) +{ + FILE *fp; + WT_CURSOR *cursor; + WT_DECL_RET; + size_t len; + uint64_t id; + uint32_t i; + char buf[512], *path; + + testutil_assert(g.c_backup_incr_flag == INCREMENTAL_BLOCK); + len = strlen(g.home) + strlen(BACKUP_INFO_FILE) + 2; + path = dmalloc(len); + testutil_check(__wt_snprintf(path, len, "%s/%s", g.home, BACKUP_INFO_FILE)); + errno = 0; + ret = RESTORE_SUCCESS; + if ((fp = fopen(path, "r")) == NULL && errno != ENOENT) + testutil_die(errno, "restore_backup_info fopen: %s", path); + free(path); + if (errno == ENOENT) + return (RESTORE_SKIP); + ret = fscanf(fp, "%" SCNu64 "\n", &id); + if (ret != 1) + testutil_die(EINVAL, "restore_backup_info ID"); + /* + * Try to open the backup cursor. We may get ENOENT if the source ID we wrote to the program + * file was not yet checkpointed. Sometimes it will, sometimes it won't. If we don't find it + * then return non-zero so that we skip incremental restart testing. + * + * NOTE: This call to open a backup cursor to check the source id uses a made up 'this_id' that + * tries to generate one that cannot possibly be in use. This call can/should be changed if the + * API ever allows us to open a cursor with a source id that does not require a this id. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), + "incremental=(enabled,src_id=%" PRIu64 ",this_id=%" PRIu64 ")", id, id / 2)); + while ((ret = session->open_cursor(session, "backup:", NULL, buf, &cursor)) == EBUSY) + __wt_yield(); + if (ret != 0) { + if (ret == ENOENT) { + ret = RESTORE_SKIP; + goto out; + } + testutil_die(ret, "session.open_cursor: backup"); + } + testutil_check(cursor->close(cursor)); + + active_files_init(active); + ret = fscanf(fp, "%" SCNu32 "\n", &active->count); + /* We could save just an ID if the file count was 0, so return if we find that case. */ + if (ret != 1) { + ret = RESTORE_SKIP; + goto out; + } + + /* Set global id after error paths. */ + g.backup_id = id + 1; + active->names = drealloc(active->names, sizeof(char *) * active->count); + for (i = 0; i < active->count; ++i) { + memset(buf, 0, sizeof(buf)); + ret = fscanf(fp, "%511s\n", buf); + if (ret != 1) { + ret = RESTORE_SKIP; + goto out; + } + active->names[i] = strdup(buf); + } + ret = RESTORE_SUCCESS; +out: + fclose_and_clear(&fp); + return (ret); +} + +/* + * save_backup_info -- + * Save backup information to a text file format can use to restore on a reopen. + */ +static void +save_backup_info(ACTIVE_FILES *active, uint64_t id) +{ + FILE *fp; + size_t len; + uint32_t i; + char *from_path, *to_path; + + if (g.c_backup_incr_flag != INCREMENTAL_BLOCK) + return; + len = strlen(g.home) + strlen(BACKUP_INFO_FILE_TMP) + 2; + from_path = dmalloc(len); + testutil_check(__wt_snprintf(from_path, len, "%s/%s", g.home, BACKUP_INFO_FILE_TMP)); + if ((fp = fopen(from_path, "w")) == NULL) + testutil_die(errno, "save_backup_info fopen: %s", from_path); + fprintf(fp, "%" PRIu64 "\n", id); + if (active->count > 0) { + fprintf(fp, "%" PRIu32 "\n", active->count); + for (i = 0; i < active->count; ++i) + fprintf(fp, "%s\n", active->names[i]); + } + fclose_and_clear(&fp); + len = strlen(g.home) + strlen(BACKUP_INFO_FILE) + 2; + to_path = dmalloc(len); + testutil_check(__wt_snprintf(to_path, len, "%s/%s", g.home, BACKUP_INFO_FILE)); + error_sys_check(rename(from_path, to_path)); + free(from_path); + free(to_path); +} + /* * backup -- * Periodically do a backup and verify it. @@ -354,7 +499,7 @@ backup(void *arg) WT_SESSION *session; size_t len; u_int incremental, period; - uint64_t src_id; + uint64_t src_id, this_id; const char *config, *key; char cfg[512], *cmd; bool full, incr_full; @@ -362,22 +507,34 @@ backup(void *arg) (void)(arg); conn = g.wts_conn; - - /* Guarantee backup ID uniqueness, we might be reopening an existing database. */ - __wt_seconds(NULL, &g.backup_id); - /* Open a session. */ testutil_check(conn->open_session(conn, NULL, NULL, &session)); + __wt_seconds(NULL, &g.backup_id); + active_files_init(&active[0]); + active_files_init(&active[1]); + active_now = active_prev = NULL; + incr_full = true; + incremental = 0; + /* + * If we're reopening an existing database and doing incremental backup we reset the initialized + * variables based on whatever they were at the end of the previous run. We want to make sure + * that we can take an incremental backup and use the older id as a source identifier. We force + * that only if the restore function was successful in restoring the backup information. + */ + if (g.reopen && g.c_backup_incr_flag == INCREMENTAL_BLOCK && + restore_backup_info(session, &active[0]) == RESTORE_SUCCESS) { + incr_full = false; + full = false; + incremental = 1; + active_prev = &active[0]; + } + /* * Perform a full backup at somewhere under 10 seconds (that way there's at least one), then at * larger intervals, optionally do incremental backups between full backups. */ - incr_full = true; - incremental = 0; - active_files_init(&active[0]); - active_files_init(&active[1]); - active_now = active_prev = NULL; + this_id = 0; for (period = mmrand(NULL, 1, 10);; period = mmrand(NULL, 20, 45)) { /* Sleep for short periods so we don't make the run wait. */ while (period > 0 && !g.workers_finished) { @@ -407,8 +564,8 @@ backup(void *arg) active_now = &active[g.backup_id % 2]; active_prev = NULL; testutil_check(__wt_snprintf(cfg, sizeof(cfg), - "incremental=(enabled,granularity=%" PRIu32 "K,this_id=ID%" PRIu64 ")", - g.c_backup_incr_granularity, g.backup_id++)); + "incremental=(enabled,granularity=%" PRIu32 "K,this_id=%" PRIu64 ")", + g.c_backup_incr_granularity, g.backup_id)); full = true; incr_full = false; } else { @@ -418,12 +575,13 @@ backup(void *arg) active_now = &active[0]; src_id = g.backup_id - 1; testutil_check(__wt_snprintf(cfg, sizeof(cfg), - "incremental=(enabled,src_id=ID%" PRIu64 ",this_id=ID%" PRIu64 ")", src_id, - g.backup_id++)); + "incremental=(enabled,src_id=%" PRIu64 ",this_id=%" PRIu64 ")", src_id, + g.backup_id)); /* Restart a full incremental every once in a while. */ full = false; incr_full = mmrand(NULL, 1, 8) == 1; } + this_id = g.backup_id++; config = cfg; /* Free up the old active file list we're going to overwrite. */ active_files_free(active_now); @@ -486,6 +644,8 @@ backup(void *arg) lock_writeunlock(session, &g.backup_lock); active_files_sort(active_now); active_files_remove_missing(active_prev, active_now); + /* Save the backup information to a file so we can restart on a reopen. */ + save_backup_info(active_now, this_id); active_prev = active_now; /* @@ -494,8 +654,13 @@ backup(void *arg) * intermediate states, once we perform recovery on the backup database, we can't do any * more incremental backups). */ - if (full) - incremental = g.c_logging_archive ? 1 : mmrand(NULL, 1, 8); + if (full) { + incremental = 1; + if (g.c_backup_incr_flag == INCREMENTAL_LOG) + incremental = g.c_logging_archive ? 1 : mmrand(NULL, 1, 8); + else if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) + incremental = mmrand(NULL, 1, 8); + } if (--incremental == 0) { check_copy(); /* We ran recovery in the backup directory, so next time it must be a full backup. */ diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 374bc470f5b..a6eb4c5f898 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -380,12 +380,6 @@ config_backward_compatible(void) if (!backward_compatible) return; - if (g.c_backup_incr_flag != INCREMENTAL_OFF) { - if (config_is_perm("backup.incremental")) - testutil_die(EINVAL, "incremental backup not supported in backward compatibility mode"); - config_single("backup.incremental=off", false); - } - if (g.c_mmap_all) { if (config_is_perm("disk.mmap_all")) testutil_die(EINVAL, "disk.mmap_all not supported in backward compatibility mode"); @@ -814,9 +808,11 @@ config_pct(void) uint32_t *vp; /* Value store */ u_int order; /* Order of assignment */ } list[] = { - {"ops.pct.delete", &g.c_delete_pct, 0}, {"ops.pct.insert", &g.c_insert_pct, 0}, + {"ops.pct.delete", &g.c_delete_pct, 0}, + {"ops.pct.insert", &g.c_insert_pct, 0}, #define CONFIG_MODIFY_ENTRY 2 - {"ops.pct.modify", &g.c_modify_pct, 0}, {"ops.pct.read", &g.c_read_pct, 0}, + {"ops.pct.modify", &g.c_modify_pct, 0}, + {"ops.pct.read", &g.c_read_pct, 0}, {"ops.pct.write", &g.c_write_pct, 0}, }; u_int i, max_order, max_slot, n, pct; diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index b8f2922021e..f1aa17e85fd 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -300,6 +300,10 @@ static CONFIG c[] = { {"stress.checkpoint", "stress checkpoints", C_BOOL, 2, 0, 0, &g.c_timing_stress_checkpoint, NULL}, /* 2% */ + {"stress.checkpoint_prepare", "stress checkpoint prepare", C_BOOL, 2, 0, 0, + &g.c_timing_stress_checkpoint_prepare, NULL}, + + /* 2% */ {"stress.hs_checkpoint_delay", "stress history store checkpoint delay", C_BOOL, 2, 0, 0, &g.c_timing_stress_hs_checkpoint_delay, NULL}, diff --git a/src/third_party/wiredtiger/test/format/config_compat.c b/src/third_party/wiredtiger/test/format/config_compat.c index 0a5fe4424f7..93616108769 100644 --- a/src/third_party/wiredtiger/test/format/config_compat.c +++ b/src/third_party/wiredtiger/test/format/config_compat.c @@ -29,43 +29,185 @@ #include "format.h" static const char *list[] = { - "abort=", "format.abort", "alter=", "ops.alter", "assert_commit_timestamp=", - "assert.commit_timestamp", "assert_read_timestamp=", "assert.read_timestamp", "auto_throttle=", - "lsm.auto_throttle", "backup_incremental=", "backup.incremental", "backups=", "backup", "bitcnt=", - "btree.bitcnt", "bloom=", "lsm.bloom", "bloom_bit_count=", "lsm.bloom_bit_count", - "bloom_hash_count=", "lsm.bloom_hash_count", "bloom_oldest=", "lsm.bloom_oldest", "cache=", - "cache", "cache_minimum=", "cache.minimum", "checkpoint_log_size=", "checkpoint.log_size", - "checkpoint_wait=", "checkpoint.wait", "checkpoints=", "checkpoint", "checksum=", "disk.checksum", - "chunk_size=", "lsm.chunk_size", "compaction=", "ops.compaction", "compression=", - "btree.compression", "data_extend=", "disk.data_extend", "data_source=", "runs.source", - "delete_pct=", "ops.pct.delete", "dictionary=", "btree.dictionary", "direct_io=", - "disk.direct_io", "encryption=", "disk.encryption", "evict_max=", "cache.evict_max", "file_type=", - "runs.type", "firstfit=", "disk.firstfit", "huffman_key=", "btree.huffman_key", "huffman_value=", - "btree.huffman_value", "in_memory=", "runs.in_memory", "independent_thread_rng=", - "format.independent_thread_rng", "insert_pct=", "ops.pct.insert", "internal_key_truncation=", - "btree.internal_key_truncation", "internal_page_max=", "btree.internal_page_max", "isolation=", - "transaction.isolation", "key_gap=", "btree.key_gap", "key_max=", "btree.key_max", "key_min=", - "btree.key_min", "leaf_page_max=", "btree.leaf_page_max", "leak_memory=", - "wiredtiger.leak_memory", "logging_archive=", "logging.archive", "logging_compression=", - "logging.compression", "logging_file_max=", "logging.file_max", "logging_prealloc=", - "logging.prealloc", "lsm_worker_threads=", "lsm.worker_threads", "major_timeout=", - "format.major_timeout", "memory_page_max=", "btree.memory_page_max", "merge_max=", - "lsm.merge_max", "mmap=", "disk.mmap", "mmap_all=", "disk.mmap_all", "modify_pct=", - "ops.pct.modify", "ops=", "runs.ops", "prefix_compression=", "btree.prefix_compression", - "prefix_compression_min=", "btree.prefix_compression_min", "prepare=", "ops.prepare", - "random_cursor=", "ops.random_cursor", "read_pct=", "ops.pct.read", "rebalance=", "ops.rebalance", - "repeat_data_pct=", "btree.repeat_data_pct", "reverse=", "btree.reverse", "rows=", "runs.rows", - "salvage=", "ops.salvage", "split_pct=", "btree.split_pct", "statistics=", "statistics", - "statistics_server=", "statistics.server", "threads=", "runs.threads", "timer=", "runs.timer", - "timing_stress_aggressive_sweep=", "stress.aggressive_sweep", "timing_stress_checkpoint=", - "stress.checkpoint", "timing_stress_hs_sweep=", "stress.hs_sweep", "timing_stress_split_1=", - "stress.split_1", "timing_stress_split_2=", "stress.split_2", "timing_stress_split_3=", - "stress.split_3", "timing_stress_split_4=", "stress.split_4", "timing_stress_split_5=", - "stress.split_5", "timing_stress_split_6=", "stress.split_6", "timing_stress_split_7=", - "stress.split_7", "timing_stress_split_8=", "stress.split_8", "transaction-frequency=", - "transaction.frequency", "transaction_timestamps=", "transaction.timestamps", "truncate=", - "ops.truncate", "value_max=", "btree.value_max", "value_min=", "btree.value_min", "verify=", - "ops.verify", "wiredtiger_config=", "wiredtiger.config", "write_pct=", "ops.pct.write", NULL, + "abort=", + "format.abort", + "alter=", + "ops.alter", + "assert_commit_timestamp=", + "assert.commit_timestamp", + "assert_read_timestamp=", + "assert.read_timestamp", + "auto_throttle=", + "lsm.auto_throttle", + "backup_incremental=", + "backup.incremental", + "backups=", + "backup", + "bitcnt=", + "btree.bitcnt", + "bloom=", + "lsm.bloom", + "bloom_bit_count=", + "lsm.bloom_bit_count", + "bloom_hash_count=", + "lsm.bloom_hash_count", + "bloom_oldest=", + "lsm.bloom_oldest", + "cache=", + "cache", + "cache_minimum=", + "cache.minimum", + "checkpoint_log_size=", + "checkpoint.log_size", + "checkpoint_wait=", + "checkpoint.wait", + "checkpoints=", + "checkpoint", + "checksum=", + "disk.checksum", + "chunk_size=", + "lsm.chunk_size", + "compaction=", + "ops.compaction", + "compression=", + "btree.compression", + "data_extend=", + "disk.data_extend", + "data_source=", + "runs.source", + "delete_pct=", + "ops.pct.delete", + "dictionary=", + "btree.dictionary", + "direct_io=", + "disk.direct_io", + "encryption=", + "disk.encryption", + "evict_max=", + "cache.evict_max", + "file_type=", + "runs.type", + "firstfit=", + "disk.firstfit", + "huffman_key=", + "btree.huffman_key", + "huffman_value=", + "btree.huffman_value", + "in_memory=", + "runs.in_memory", + "independent_thread_rng=", + "format.independent_thread_rng", + "insert_pct=", + "ops.pct.insert", + "internal_key_truncation=", + "btree.internal_key_truncation", + "internal_page_max=", + "btree.internal_page_max", + "isolation=", + "transaction.isolation", + "key_gap=", + "btree.key_gap", + "key_max=", + "btree.key_max", + "key_min=", + "btree.key_min", + "leaf_page_max=", + "btree.leaf_page_max", + "leak_memory=", + "wiredtiger.leak_memory", + "logging_archive=", + "logging.archive", + "logging_compression=", + "logging.compression", + "logging_file_max=", + "logging.file_max", + "logging_prealloc=", + "logging.prealloc", + "lsm_worker_threads=", + "lsm.worker_threads", + "major_timeout=", + "format.major_timeout", + "memory_page_max=", + "btree.memory_page_max", + "merge_max=", + "lsm.merge_max", + "mmap=", + "disk.mmap", + "mmap_all=", + "disk.mmap_all", + "modify_pct=", + "ops.pct.modify", + "ops=", + "runs.ops", + "prefix_compression=", + "btree.prefix_compression", + "prefix_compression_min=", + "btree.prefix_compression_min", + "prepare=", + "ops.prepare", + "random_cursor=", + "ops.random_cursor", + "read_pct=", + "ops.pct.read", + "rebalance=", + "ops.rebalance", + "repeat_data_pct=", + "btree.repeat_data_pct", + "reverse=", + "btree.reverse", + "rows=", + "runs.rows", + "salvage=", + "ops.salvage", + "split_pct=", + "btree.split_pct", + "statistics=", + "statistics", + "statistics_server=", + "statistics.server", + "threads=", + "runs.threads", + "timer=", + "runs.timer", + "timing_stress_aggressive_sweep=", + "stress.aggressive_sweep", + "timing_stress_checkpoint=", + "stress.checkpoint", + "timing_stress_hs_sweep=", + "stress.hs_sweep", + "timing_stress_split_1=", + "stress.split_1", + "timing_stress_split_2=", + "stress.split_2", + "timing_stress_split_3=", + "stress.split_3", + "timing_stress_split_4=", + "stress.split_4", + "timing_stress_split_5=", + "stress.split_5", + "timing_stress_split_6=", + "stress.split_6", + "timing_stress_split_7=", + "stress.split_7", + "timing_stress_split_8=", + "stress.split_8", + "transaction-frequency=", + "transaction.frequency", + "transaction_timestamps=", + "transaction.timestamps", + "truncate=", + "ops.truncate", + "value_max=", + "btree.value_max", + "value_min=", + "btree.value_min", + "verify=", + "ops.verify", + "wiredtiger_config=", + "wiredtiger.config", + "write_pct=", + "ops.pct.write", + NULL, NULL, }; diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 4f7db16a3c0..a184f5eb9e3 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -51,7 +51,9 @@ #undef MEGABYTE #define MEGABYTE(v) ((v)*WT_MEGABYTE) -#define WT_NAME "wt" /* Object name */ +#define BACKUP_INFO_FILE "BACKUP_INFO" /* Format's backup information for restart */ +#define BACKUP_INFO_FILE_TMP "BACKUP_INFO.TMP" /* Format's backup information for restart */ +#define WT_NAME "wt" /* Object name */ #define DATASOURCE(v) (strcmp(v, g.c_data_source) == 0 ? 1 : 0) @@ -215,6 +217,7 @@ typedef struct { uint32_t c_timing_stress_checkpoint; uint32_t c_timing_stress_hs_checkpoint_delay; uint32_t c_timing_stress_hs_sweep; + uint32_t c_timing_stress_checkpoint_prepare; uint32_t c_timing_stress_split_1; uint32_t c_timing_stress_split_2; uint32_t c_timing_stress_split_3; diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 069340df59b..f28c9125a30 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -116,19 +116,21 @@ track(const char *tag, uint64_t cnt, TINFO *tinfo) track_ts_dots(old_dot_cnt), track_ts_diff(old_ts, cur_ts), track_ts_dots(cur_dot_cnt))); } - testutil_check(__wt_snprintf_len_set(msg, sizeof(msg), &len, "%4" PRIu32 ": %s: " - "S %" PRIu64 "%s, " - "I %" PRIu64 "%s, " - "U %" PRIu64 "%s, " - "R %" PRIu64 "%s%s", - g.run_cnt, tag, tinfo->search > M(9) ? tinfo->search / M(1) : tinfo->search, - tinfo->search > M(9) ? "M" : "", - tinfo->insert > M(9) ? tinfo->insert / M(1) : tinfo->insert, - tinfo->insert > M(9) ? "M" : "", - tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, - tinfo->update > M(9) ? "M" : "", - tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, - tinfo->remove > M(9) ? "M" : "", ts_msg)); + testutil_check( + __wt_snprintf_len_set(msg, sizeof(msg), &len, + "%4" PRIu32 ": %s: " + "S %" PRIu64 "%s, " + "I %" PRIu64 "%s, " + "U %" PRIu64 "%s, " + "R %" PRIu64 "%s%s", + g.run_cnt, tag, tinfo->search > M(9) ? tinfo->search / M(1) : tinfo->search, + tinfo->search > M(9) ? "M" : "", + tinfo->insert > M(9) ? tinfo->insert / M(1) : tinfo->insert, + tinfo->insert > M(9) ? "M" : "", + tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, + tinfo->update > M(9) ? "M" : "", + tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, + tinfo->remove > M(9) ? "M" : "", ts_msg)); } if (last_len > len) { memset(msg + len, ' ', (size_t)(last_len - len)); diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index cc27ec32213..3d605d45617 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -185,8 +185,7 @@ create_database(const char *home, WT_CONNECTION **connp) /* Logging configuration. */ if (g.c_logging) CONFIG_APPEND(p, - ",log=(enabled=true,archive=%d," - "prealloc=%d,file_max=%" PRIu32 ",compressor=\"%s\")", + ",log=(enabled=true,archive=%d,prealloc=%d,file_max=%" PRIu32 ",compressor=\"%s\")", g.c_logging_archive ? 1 : 0, g.c_logging_prealloc ? 1 : 0, KILOBYTE(g.c_logging_file_max), compressor(g.c_logging_compression_flag)); @@ -216,13 +215,10 @@ create_database(const char *home, WT_CONNECTION **connp) */ if (g.c_statistics_server) { if (mmrand(NULL, 0, 5) == 1 && memcmp(g.uri, "file:", strlen("file:")) == 0) - CONFIG_APPEND(p, - ",statistics=(fast),statistics_log=" - "(json,on_close,wait=5,sources=(\"file:\"))"); + CONFIG_APPEND( + p, ",statistics=(fast),statistics_log=(json,on_close,wait=5,sources=(\"file:\"))"); else - CONFIG_APPEND(p, - ",statistics=(fast),statistics_log=" - "(json,on_close,wait=5)"); + CONFIG_APPEND(p, ",statistics=(fast),statistics_log=(json,on_close,wait=5)"); } else CONFIG_APPEND(p, ",statistics=(%s)", g.c_statistics ? "fast" : "none"); @@ -232,6 +228,8 @@ create_database(const char *home, WT_CONNECTION **connp) CONFIG_APPEND(p, ",aggressive_sweep"); if (g.c_timing_stress_checkpoint) CONFIG_APPEND(p, ",checkpoint_slow"); + if (g.c_timing_stress_checkpoint_prepare) + CONFIG_APPEND(p, ",prepare_checkpoint_delay"); if (g.c_timing_stress_hs_checkpoint_delay) CONFIG_APPEND(p, ",history_store_checkpoint_delay"); if (g.c_timing_stress_hs_sweep) @@ -255,9 +253,7 @@ create_database(const char *home, WT_CONNECTION **connp) CONFIG_APPEND(p, "]"); /* Extensions. */ - CONFIG_APPEND(p, - ",extensions=[" - "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", + CONFIG_APPEND(p, ",extensions=[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", g.c_reverse ? REVERSE_PATH : "", access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "", access(ROTN_PATH, R_OK) == 0 ? ROTN_PATH : "", access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "", @@ -296,10 +292,8 @@ create_object(WT_CONNECTION *conn) max = sizeof(config); CONFIG_APPEND(p, - "key_format=%s" - ",allocation_size=512" - ",%s" - ",internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32 ",memory_page_max=%" PRIu32, + "key_format=%s,allocation_size=512,%s,internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32 + ",memory_page_max=%" PRIu32, (g.type == ROW) ? "u" : "r", g.c_firstfit ? "block_allocation=first" : "", g.intl_page_max, g.leaf_page_max, MEGABYTE(g.c_memory_page_max)); diff --git a/src/third_party/wiredtiger/test/huge/huge.c b/src/third_party/wiredtiger/test/huge/huge.c index d9b51ecc8a8..25101046265 100644 --- a/src/third_party/wiredtiger/test/huge/huge.c +++ b/src/third_party/wiredtiger/test/huge/huge.c @@ -86,8 +86,9 @@ run(CONFIG *cp, int bigkey, size_t bytes) printf(SIZET_FMT "%s%s: %s %s big %s\n", bytes < MEGABYTE ? bytes : (bytes < GIGABYTE ? bytes / MEGABYTE : bytes / GIGABYTE), - bytes < MEGABYTE ? "" : (bytes < GIGABYTE ? (bytes % MEGABYTE == 0 ? "" : "+") : - (bytes % GIGABYTE == 0 ? "" : "+")), + bytes < MEGABYTE ? "" : + (bytes < GIGABYTE ? (bytes % MEGABYTE == 0 ? "" : "+") : + (bytes % GIGABYTE == 0 ? "" : "+")), bytes < MEGABYTE ? "B" : (bytes < GIGABYTE ? "MB" : "GB"), cp->uri, cp->config, bigkey ? "key" : "value"); diff --git a/src/third_party/wiredtiger/test/packing/intpack-test3.c b/src/third_party/wiredtiger/test/packing/intpack-test3.c index 43cb8834997..0d63af4b204 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test3.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test3.c @@ -63,10 +63,12 @@ test_value(int64_t val) } /* Ensure that decoding used the correct amount of buffer */ if (cp != p) { - fprintf(stderr, "Unpack consumed wrong size for %" PRId64 ", expected %" WT_SIZET_FMT - ", got %" WT_SIZET_FMT "\n", - sinput, used_len, cp > p ? used_len + (size_t)(cp - p) : /* More than buf used */ - used_len - (size_t)(p - cp)); /* Less than buf used */ + fprintf(stderr, + "Unpack consumed wrong size for %" PRId64 ", expected %" WT_SIZET_FMT + ", got %" WT_SIZET_FMT "\n", + sinput, used_len, + cp > p ? used_len + (size_t)(cp - p) : /* More than buf used */ + used_len - (size_t)(p - cp)); /* Less than buf used */ abort(); } @@ -86,8 +88,9 @@ test_value(int64_t val) } /* Ensure that decoding used the correct amount of buffer */ if (cp != p) { - fprintf(stderr, "Unpack consumed wrong size for %" PRId64 ", expected %" WT_SIZET_FMT - ", got %" WT_SIZET_FMT "\n", + fprintf(stderr, + "Unpack consumed wrong size for %" PRId64 ", expected %" WT_SIZET_FMT + ", got %" WT_SIZET_FMT "\n", sinput, used_len, cp > p ? used_len + (size_t)(cp - p) : used_len - (size_t)(p - cp)); abort(); } diff --git a/src/third_party/wiredtiger/test/readonly/readonly.c b/src/third_party/wiredtiger/test/readonly/readonly.c index c7e3279db06..3741109a625 100644 --- a/src/third_party/wiredtiger/test/readonly/readonly.c +++ b/src/third_party/wiredtiger/test/readonly/readonly.c @@ -253,9 +253,8 @@ main(int argc, char *argv[]) testutil_die(status, "system: %s", cmd); testutil_check(__wt_snprintf(cmd, sizeof(cmd), - "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " - "chmod 0555 %s; chmod -R 0444 %s/*", - home, home_rd2, home_rd2, home_rd2, home_rd2)); + "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; chmod 0555 %s; chmod -R 0444 %s/*", home, home_rd2, + home_rd2, home_rd2, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index b378fbf5845..af61d8e11ec 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -510,26 +510,20 @@ build(int ikey, int ivalue, int cnt) switch (page_type) { case WT_PAGE_COL_FIX: testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=r,value_format=7t," - "allocation_size=%d," - "internal_page_max=%d,internal_item_max=%d," - "leaf_page_max=%d,leaf_item_max=%d", + "key_format=r,value_format=7t,allocation_size=%d,internal_page_max=%d,internal_item_max=%" + "d,leaf_page_max=%d,leaf_item_max=%d", PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; case WT_PAGE_COL_VAR: testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=r," - "allocation_size=%d," - "internal_page_max=%d,internal_item_max=%d," - "leaf_page_max=%d,leaf_item_max=%d", + "key_format=r,allocation_size=%d,internal_page_max=%d,internal_item_max=%d,leaf_page_max=" + "%d,leaf_item_max=%d", PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; case WT_PAGE_ROW_LEAF: testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=u," - "allocation_size=%d," - "internal_page_max=%d,internal_item_max=%d," - "leaf_page_max=%d,leaf_item_max=%d", + "key_format=u,allocation_size=%d,internal_page_max=%d,internal_item_max=%d,leaf_page_max=" + "%d,leaf_item_max=%d", PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; default: diff --git a/src/third_party/wiredtiger/test/suite/test_backup02.py b/src/third_party/wiredtiger/test/suite/test_backup02.py index 6bb9f23ca1a..c8886cc6e7b 100755 --- a/src/third_party/wiredtiger/test/suite/test_backup02.py +++ b/src/third_party/wiredtiger/test/suite/test_backup02.py @@ -58,35 +58,42 @@ class test_backup02(wttest.WiredTigerTestCase): # ckpt = checkpoint_thread(self.conn, done) # ckpt.start() bkp = backup_thread(self.conn, 'backup.dir', done) - bkp.start() - work_queue = queue.Queue() - my_data = 'a' * self.dsize - for i in range(self.nops): - work_queue.put_nowait(('gi', i, my_data)) - opthreads = [] - for i in range(self.nthreads): - t = op_thread(self.conn, uris, self.fmt, work_queue, done) - opthreads.append(t) - t.start() + try: + bkp.start() - # Add 200 update entries into the queue every .1 seconds. - more_time = self.time - while more_time > 0: - time.sleep(0.1) - my_data = str(more_time) + 'a' * (self.dsize - len(str(more_time))) - more_time = more_time - 0.1 + my_data = 'a' * self.dsize for i in range(self.nops): - work_queue.put_nowait(('gu', i, my_data)) + work_queue.put_nowait(('gi', i, my_data)) + + for i in range(self.nthreads): + t = op_thread(self.conn, uris, self.fmt, work_queue, done) + opthreads.append(t) + t.start() - work_queue.join() - done.set() -# # Wait for checkpoint thread to notice status change. -# ckpt.join() - for t in opthreads: - t.join() - bkp.join() + # Add 200 update entries into the queue every .1 seconds. + more_time = self.time + while more_time > 0: + time.sleep(0.1) + my_data = str(more_time) + 'a' * (self.dsize - len(str(more_time))) + more_time = more_time - 0.1 + for i in range(self.nops): + work_queue.put_nowait(('gu', i, my_data)) + except: + # Deplete the work queue if there's an error. + while not work_queue.empty(): + work_queue.get() + work_queue.task_done() + raise + finally: + work_queue.join() + done.set() + # Wait for checkpoint thread to notice status change. + # ckpt.join() + for t in opthreads: + t.join() + bkp.join() if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug024.py b/src/third_party/wiredtiger/test/suite/test_bug024.py new file mode 100644 index 00000000000..bc3d56d7501 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_bug024.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from helper import copy_wiredtiger_home +import wiredtiger, wttest +from wtdataset import SimpleDataSet +import os, shutil + +# test_bug024.py +# WT-6526: test that we can successfully open a readonly connection after it was stopped while +# the temporary turtle file existed. We simulate that by copying the turtle file to its temporary name +# and then opening the connection readonly. +class test_bug024(wttest.WiredTigerTestCase): + conn_config = ('cache_size=50MB') + + # Create a table. + uri = "table:test_bug024" + + def test_bug024(self): + nrows = 10 + ds = SimpleDataSet(self, self.uri, nrows, key_format="S", value_format='u') + ds.populate() + + self.conn.close() + # Copying the file manually to recreate the issue described in WT-6526. + shutil.copy('WiredTiger.turtle', 'WiredTiger.turtle.set') + + # Open wiredtiger in new directory and in readonly mode. + conn = self.wiredtiger_open(self.home, "readonly") + conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py index bda058b10a9..e40d72b5615 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py @@ -52,28 +52,35 @@ class test_checkpoint02(wttest.WiredTigerTestCase): self.session.create(self.uri, "key_format=" + self.fmt + ",value_format=S") ckpt = checkpoint_thread(self.conn, done) - ckpt.start() - - uris = list() - uris.append(self.uri) work_queue = queue.Queue() - my_data = 'a' * self.dsize - for i in xrange(self.nops): - if i % 191 == 0 and i != 0: - work_queue.put_nowait(('b', i, my_data)) - work_queue.put_nowait(('i', i, my_data)) - opthreads = [] - for i in range(self.nthreads): - t = op_thread(self.conn, uris, self.fmt, work_queue, done) - opthreads.append(t) - t.start() + try: + ckpt.start() + + uris = list() + uris.append(self.uri) + my_data = 'a' * self.dsize + for i in xrange(self.nops): + if i % 191 == 0 and i != 0: + work_queue.put_nowait(('b', i, my_data)) + work_queue.put_nowait(('i', i, my_data)) - work_queue.join() - done.set() - for t in opthreads: - t.join() - ckpt.join() + for i in range(self.nthreads): + t = op_thread(self.conn, uris, self.fmt, work_queue, done) + opthreads.append(t) + t.start() + except: + # Deplete the work queue if there's an error. + while not work_queue.empty(): + work_queue.get() + work_queue.task_done() + raise + finally: + work_queue.join() + done.set() + for t in opthreads: + t.join() + ckpt.join() # Create a cursor - ensure all items have been put. cursor = self.session.open_cursor(self.uri, None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index e466aa81a95..0b68b695e38 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -149,7 +149,7 @@ class test_compact02(wttest.WiredTigerTestCase): # Compact can collide with eviction, if that happens we retry. Wait for # a long time, the check for EBUSY means we're not retrying on any real # errors. - for i in range(1, 80): + for i in range(1, 100): if not self.raisesBusy( lambda: self.session.compact(self.uri, None)): break diff --git a/src/third_party/wiredtiger/test/suite/test_config09.py b/src/third_party/wiredtiger/test/suite/test_config09.py new file mode 100644 index 00000000000..d55c0a8c87d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_config09.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_config09.py +# Test the configuration that enables/disables dirty table flushing. +# + +import wiredtiger, wttest +from wiredtiger import stat + +class test_config09(wttest.WiredTigerTestCase): + ntables = 50 + nentries = 5 + uri = 'table:config09.' + conn_config = 'hash=(buckets=256,dhandle_buckets=1024),statistics=(fast)' + + # Create, populate and checkpoint the initial tables. + def create_tables(self): + for i in range(self.ntables): + uri = self.uri + str(i) + self.session.create(uri, 'key_format=i,value_format=i') + c = self.session.open_cursor(uri) + for j in range(self.nentries): + c[j] = j + c.close() + self.session.checkpoint() + + # Update half the tables. + def update_tables(self): + for i in range(self.ntables//2): + uri = self.uri + str(i) + c = self.session.open_cursor(uri) + for j in range(self.nentries): + c[j] = j + 100 + c.close() + self.session.checkpoint() + + # Verify statistics. + def get_stat(self, stat): + stat_cursor = self.session.open_cursor('statistics:', None, None) + val = stat_cursor[stat][2] + stat_cursor.close() + return val + + def test_config09_invalid(self): + self.conn.close() + + # Verify the message when using non-power-of-two values. + msg = '/power of 2/' + config = 'hash=(buckets=255,dhandle_buckets=1024)' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open('.', config), msg) + config = 'hash=(buckets=256,dhandle_buckets=1000)' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open('.', config), msg) + config = 'hash=(dhandle_buckets=1000)' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open('.', config), msg) + + def test_config09(self): + self.create_tables() + + val = self.get_stat(stat.conn.buckets) + self.assertEqual(val, 256) + val = self.get_stat(stat.conn.buckets_dh) + self.assertEqual(val, 1024) + + self.update_tables() + val = self.get_stat(stat.conn.txn_checkpoint_handle_applied) + # We cannot assert it is equal to half because there could be other + # internal tables in the count. Assert it is less than 75% and at least + # half. + self.assertGreaterEqual(val, self.ntables // 2) + self.assertLess(val, self.ntables // 4 * 3) + val = self.get_stat(stat.conn.txn_checkpoint_handle_skipped) + self.assertNotEqual(val, 0) + + self.conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor13.py b/src/third_party/wiredtiger/test/suite/test_cursor13.py index c90db17e299..371f9a36ea1 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor13.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor13.py @@ -42,12 +42,40 @@ class test_cursor13_base(wttest.WiredTigerTestCase): stat_cursor_reopen = 0 # Returns a list: [cursor_cached, cursor_reopened] + # + # We want the statistics for operations triggered from our program. The challenge is that + # eviction threads may cache history store cursors in the background. We address this by + # subtracting out operations from the history store file. This is tricky because we can't + # atomically check the connections stats and the history store stats. So we look at the + # history store stats before and after the connection stats and only accept a result where + # the history store stats haven't changed. def caching_stats(self): - stat_cursor = self.session.open_cursor('statistics:', None, None) - cache = stat_cursor[stat.conn.cursor_cache][2] - reopen = stat_cursor[stat.conn.cursor_reopen][2] - stat_cursor.close() - return [cache, reopen] + hs_stats_uri = 'statistics:file:WiredTigerHS.wt' + max_tries = 100 + for i in range(max_tries): + hs_stats_before = self.session.open_cursor(hs_stats_uri, None, None) + conn_stats = self.session.open_cursor('statistics:', None, None) + hs_stats_after = self.session.open_cursor(hs_stats_uri, None, None) + + totals = [ conn_stats [stat.conn.cursor_cache][2], + conn_stats [stat.conn.cursor_reopen][2] ] + hs_before = [ hs_stats_before[stat.dsrc.cursor_cache][2], + hs_stats_before[stat.dsrc.cursor_reopen][2] ] + hs_after = [ hs_stats_after[stat.dsrc.cursor_cache][2], + hs_stats_after[stat.dsrc.cursor_reopen][2] ] + + hs_stats_before.close() + hs_stats_after.close() + conn_stats.close() + + if hs_before[0] == hs_after[0] and hs_before[1] == hs_after[1]: + break + + # Fail if we haven't been able to get stable hs stats after too many attempts. + # Seems impossible, but better to check than to have an accidental infinite loop. + self.assertNotEqual(i, max_tries - 1) + + return [totals[0] - hs_after[0], totals[1] - hs_after[1]] # Returns a list: [cursor_sweep, cursor_sweep_buckets, # cursor_sweep_examined, cursor_sweep_closed] @@ -65,18 +93,16 @@ class test_cursor13_base(wttest.WiredTigerTestCase): if expect_change: self.assertGreater(stats[0], self.stat_cursor_cache) self.stat_cursor_cache = stats[0] - # Stats may change due to background operations in history store cursor - #else: - # self.assertEqual(stats[0], self.stat_cursor_cache) + else: + self.assertEqual(stats[0], self.stat_cursor_cache) def assert_cursor_reopened(self, expect_change): stats = self.caching_stats() if expect_change: self.assertGreater(stats[1], self.stat_cursor_reopen) self.stat_cursor_reopen = stats[1] - # Stats may change due to background operations in history store cursor - #else: - # self.assertEqual(stats[1], self.stat_cursor_reopen) + else: + self.assertEqual(stats[1], self.stat_cursor_reopen) def cursor_stats_init(self): stats = self.caching_stats() @@ -450,10 +476,8 @@ class test_cursor13_big(test_cursor13_big_base): # ', closes = ' + str(self.closecount)) #self.tty('stats after = ' + str(end_stats)) - # Stats won't be exact because they may include operations triggered by other - # threads (e.g., eviction) opening and closing history store cursors. - self.assertGreaterEqual(end_stats[0] - begin_stats[0], self.closecount) - self.assertGreaterEqual(end_stats[1] - begin_stats[1], self.opencount) + self.assertEquals(end_stats[0] - begin_stats[0], self.closecount) + self.assertEquals(end_stats[1] - begin_stats[1], self.opencount) class test_cursor13_sweep(test_cursor13_big_base): # Set dhandle sweep configuration so that dhandles should be closed within @@ -512,7 +536,7 @@ class test_cursor13_sweep(test_cursor13_big_base): # ', closes = ' + str(self.closecount)) #self.tty('stats after = ' + str(end_stats)) #self.tty('sweep stats after = ' + str(end_sweep_stats)) - self.assertGreaterEqual(end_stats[0] - begin_stats[0], self.closecount) + self.assertEquals(end_stats[0] - begin_stats[0], self.closecount) swept = end_sweep_stats[3] - begin_sweep_stats[3] # Although this is subject to tuning parameters, we know that diff --git a/src/third_party/wiredtiger/test/suite/test_cursor16.py b/src/third_party/wiredtiger/test/suite/test_cursor16.py index b5fbd7996eb..facc4eafc51 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor16.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor16.py @@ -39,7 +39,7 @@ class test_cursor16(wttest.WiredTigerTestCase): uri_count = 100 session_count = 100 - conn_config = 'cache_cursors=true,statistics=(fast)' + conn_config = 'cache_cursors=true,statistics=(fast),in_memory=true' # Returns the number of cursors cached def cached_stats(self): @@ -64,8 +64,7 @@ class test_cursor16(wttest.WiredTigerTestCase): for j in range(0, 10): cursor[str(j)] = str(j) - # Skewed by internal history store operations - #self.assertEqual(0, self.cached_stats()) + self.assertEqual(0, self.cached_stats()) sessions = [] for i in range(0, self.session_count): @@ -90,8 +89,7 @@ class test_cursor16(wttest.WiredTigerTestCase): session.close() #self.tty('end cursors cached=' + str(self.cached_stats())) - # Skewed by internal history store operations - #self.assertEqual(0, self.cached_stats()) + self.assertEqual(0, self.cached_stats()) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt01.py b/src/third_party/wiredtiger/test/suite/test_encrypt01.py index 92032a765fe..09031c67a28 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt01.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt01.py @@ -61,13 +61,20 @@ class test_encrypt01(wttest.WiredTigerTestCase): ('none-snappy', dict(log_compress=None, block_compress='snappy')), ('snappy-lz4', dict(log_compress='snappy', block_compress='lz4')), ] - scenarios = make_scenarios(types, encrypt, compress) + loadExt = [ + ('earlyLoadTrue', dict(earlyLoad=True)), + ('earlyLoadFalse', dict(earlyLoad=False)), + ] + + scenarios = make_scenarios(types, encrypt, compress, loadExt) nrecords = 5000 bigvalue = "abcdefghij" * 1001 # len(bigvalue) = 10010 def conn_extensions(self, extlist): extlist.skip_if_missing = True + if self.earlyLoad == True: + extlist.early_load_ext = True extlist.extension('encryptors', self.sys_encrypt) extlist.extension('encryptors', self.file_encrypt) extlist.extension('compressors', self.block_compress) diff --git a/src/third_party/wiredtiger/test/suite/test_hs06.py b/src/third_party/wiredtiger/test/suite/test_hs06.py index b00a196c809..967f1519807 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs06.py +++ b/src/third_party/wiredtiger/test/suite/test_hs06.py @@ -95,7 +95,12 @@ class test_hs06(wttest.WiredTigerTestCase): self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) self.session.checkpoint() - # Check the checkpoint wrote the expected values. Todo: Fix checkpoint cursors WT-5492. + # Check the checkpoint wrote the expected values. + # + # FIXME-WT-5927: Checkpoint cursors are known to have issues in durable history so we've + # removing the use of checkpoint handles in this test. As part of WT-5927, we should either + # re-enable the testing of checkpoint cursors or remove this comment. + # # cursor2 = self.session.open_cursor(uri, None, 'checkpoint=WiredTigerCheckpoint') cursor2 = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) diff --git a/src/third_party/wiredtiger/test/suite/test_hs15.py b/src/third_party/wiredtiger/test/suite/test_hs15.py index 8124f1d5f1e..ffebad7afa9 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs15.py +++ b/src/third_party/wiredtiger/test/suite/test_hs15.py @@ -33,7 +33,7 @@ def timestamp_str(t): # test_hs15.py # Ensure eviction doesn't clear the history store again after checkpoint has done so because of the same update without timestamp. -class test_hs14(wttest.WiredTigerTestCase): +class test_hs15(wttest.WiredTigerTestCase): conn_config = 'cache_size=5MB' session_config = 'isolation=snapshot' diff --git a/src/third_party/wiredtiger/test/suite/test_prepare08.py b/src/third_party/wiredtiger/test/suite/test_prepare08.py index 87e9997b110..9b2ac655ede 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare08.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare08.py @@ -34,12 +34,12 @@ from wtdataset import SimpleDataSet def timestamp_str(t): return '%x' % t -# test_prepare07.py -# Test to ensure prepared tombstones are properly aborted even when they are written +# test_prepare08.py +# Test to ensure prepared tombstones are properly aborted/committed even when they are written # to the data store. class test_prepare08(wttest.WiredTigerTestCase): # Force a small cache. - conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' + conn_config = 'cache_size=5MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) @@ -47,7 +47,16 @@ class test_prepare08(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor.set_key(ds.key(i)) cursor.set_value(value) - self.assertEquals(cursor.insert(), 0) + self.assertEquals(cursor.update(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + cursor.close() + + def removes(self, ds, uri, nrows, ts): + cursor = self.session.open_cursor(uri) + self.session.begin_transaction('isolation=snapshot') + for i in range(1, nrows): + cursor.set_key(ds.key(i)) + self.assertEquals(cursor.remove(), 0) self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) cursor.close() @@ -56,50 +65,220 @@ class test_prepare08(wttest.WiredTigerTestCase): self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) - self.assertEquals(cursor.search(), 0) - self.assertEquals(cursor.get_value(),value) + if value == None: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + else: + self.assertEquals(cursor.search(), 0) + self.assertEquals(cursor.get_value(),value) self.session.commit_transaction() cursor.close() - def test_prepare(self): + def test_prepare_delete_rollback(self): + nrows = 2000 + # Create a small table. - uri = "table:test" - nrows = 1000 - ds = SimpleDataSet(self, uri, 0, key_format="S", value_format='u') - ds.populate() + uri_1 = "table:test_prepare08_1" + ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1.populate() + + uri_2 = "table:test_prepare08_2" + ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2.populate() value_a = b"aaaaa" * 100 value_b = b"bbbbb" * 100 + value_c = b"ccccc" * 100 + value_d = b"ddddd" * 100 + value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) # Initially load huge data - self.updates(ds, uri, nrows, value_a, 20) + self.updates(ds_1, uri_1, nrows, value_a, 20) + self.updates(ds_2, uri_2, nrows, value_a, 20) + # Add some more updates - self.updates(ds, uri, nrows, value_b, 30) + self.updates(ds_1, uri_1, nrows, value_b, 30) + self.updates(ds_2, uri_2, nrows, value_b, 30) + + # Verify the updates + self.check(ds_1, uri_1, nrows, value_a, 20) + self.check(ds_1, uri_1, nrows, value_b, 30) + + self.check(ds_2, uri_2, nrows, value_a, 20) + self.check(ds_2, uri_2, nrows, value_b, 30) # Checkpoint self.session.checkpoint() - # Remove the updates from a prepare session and and keep it open. + # Remove the updates from a prepare session and keep it open. session_p = self.conn.open_session() - cursor_p = session_p.open_cursor(uri) + cursor_p = session_p.open_cursor(uri_1) session_p.begin_transaction('isolation=snapshot') for i in range(1, nrows): - cursor_p.set_key(ds.key(i)) + cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) - self.check(ds, uri, nrows, value_a, 20) - self.check(ds, uri, nrows, value_b, 50) + # Adding more updates to other table should trigger eviction on uri_1 + self.updates(ds_2, uri_2, nrows, value_c, 40) + self.updates(ds_2, uri_2, nrows, value_d, 50) + self.updates(ds_2, uri_2, nrows, value_e, 60) + + self.check(ds_1, uri_1, nrows, value_a, 20) + self.check(ds_1, uri_1, nrows, value_b, 50) #rollback the prepared session session_p.rollback_transaction() - self.check(ds, uri, nrows, value_a, 20) - self.check(ds, uri, nrows, value_b, 50) + self.check(ds_1, uri_1, nrows, value_a, 20) + self.check(ds_1, uri_1, nrows, value_b, 50) + + # close sessions. + cursor_p.close() + session_p.close() + self.session.close() + + def test_prepare_update_delete_commit(self): + nrows = 2000 + + # Create a small table. + uri_1 = "table:test_prepare10_1" + ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1.populate() + + # Create another small table. + uri_2 = "table:test_prepare10_2" + ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2.populate() + + value_a = b"aaaaa" * 100 + value_b = b"bbbbb" * 100 + value_c = b"ccccc" * 100 + value_d = b"ddddd" * 100 + value_e = b"eeeee" * 100 + + # Commit some updates along with a prepared update, which is not resolved. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + + # Initially load huge data + self.updates(ds_1, uri_1, nrows, value_a, 20) + self.updates(ds_2, uri_2, nrows, value_a, 20) + + # Add some more updates + self.updates(ds_1, uri_1, nrows, value_b, 30) + self.updates(ds_2, uri_2, nrows, value_b, 30) + + # Verify the updates + self.check(ds_1, uri_1, nrows, value_a, 20) + self.check(ds_1, uri_1, nrows, value_b, 30) + + self.check(ds_2, uri_2, nrows, value_a, 20) + self.check(ds_2, uri_2, nrows, value_b, 30) + + # Checkpoint + self.session.checkpoint() + + # Remove the updates from a prepare session and keep it open. + session_p = self.conn.open_session() + cursor_p = session_p.open_cursor(uri_1) + session_p.begin_transaction('isolation=snapshot') + for i in range(1, nrows): + cursor_p.set_key(ds_1.key(i)) + cursor_p.set_value(value_c) + self.assertEquals(cursor_p.update(), 0) + cursor_p.set_key(ds_1.key(i)) + self.assertEquals(cursor_p.remove(), 0) + session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + + # Adding more updates to other table should trigger eviction on uri_1 + self.updates(ds_2, uri_2, nrows, value_c, 40) + self.updates(ds_2, uri_2, nrows, value_d, 50) + self.updates(ds_2, uri_2, nrows, value_e, 60) + + self.check(ds_1, uri_1, nrows, value_a, 20) + self.check(ds_1, uri_1, nrows, value_b, 30) + self.check(ds_1, uri_1, nrows, value_b, 50) + + # Commit the prepared session + session_p.commit_transaction('commit_timestamp=' + timestamp_str(50) + ',durable_timestamp=' + timestamp_str(60)) + + self.check(ds_1, uri_1, nrows, value_a, 20) + self.check(ds_1, uri_1, nrows, value_b, 30) + self.check(ds_1, uri_1, 0, None, 50) + + # close sessions. + cursor_p.close() + session_p.close() + self.session.close() + + def test_prepare_update_delete_commit_with_no_base_update(self): + nrows = 2000 + + # Create a small table. + uri_1 = "table:test_prepare10_1" + ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1.populate() + + # Create another small table. + uri_2 = "table:test_prepare10_2" + ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2.populate() + + value_a = b"aaaaa" * 100 + value_b = b"bbbbb" * 100 + value_c = b"ccccc" * 100 + value_d = b"ddddd" * 100 + value_e = b"eeeee" * 100 + + # Commit some updates along with a prepared update, which is not resolved. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + + # Initially load huge data + self.updates(ds_1, uri_1, nrows, value_a, 20) + self.updates(ds_2, uri_2, nrows, value_a, 20) + + # Remove updates from one table and add some more updates to another table + self.removes(ds_1, uri_1, nrows, 30) + self.updates(ds_2, uri_2, nrows, value_b, 30) + + # Checkpoint + self.session.checkpoint() + + # Remove the updates from a prepare session and and keep it open. + session_p = self.conn.open_session() + cursor_p = session_p.open_cursor(uri_1) + session_p.begin_transaction('isolation=snapshot') + for i in range(1, nrows): + cursor_p.set_key(ds_1.key(i)) + cursor_p.set_value(value_c) + self.assertEquals(cursor_p.update(), 0) + cursor_p.set_key(ds_1.key(i)) + cursor_p.set_value(value_d) + self.assertEquals(cursor_p.update(), 0) + cursor_p.set_key(ds_1.key(i)) + self.assertEquals(cursor_p.remove(), 0) + session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + + # Adding more updates to other table should trigger eviction on uri_1 + self.updates(ds_2, uri_2, nrows, value_c, 40) + self.updates(ds_2, uri_2, nrows, value_d, 50) + self.updates(ds_2, uri_2, nrows, value_e, 60) + + self.check(ds_1, uri_1, nrows, value_a, 20) + self.check(ds_1, uri_1, 0, None, 30) + self.check(ds_1, uri_1, 0, None, 50) + + # Commit the prepared session + session_p.commit_transaction('commit_timestamp=' + timestamp_str(50) + ',durable_timestamp=' + timestamp_str(60)) + + self.check(ds_1, uri_1, nrows, value_a, 20) + self.check(ds_1, uri_1, 0, None, 30) + self.check(ds_1, uri_1, 0, None, 50) # close sessions. cursor_p.close() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare10.py b/src/third_party/wiredtiger/test/suite/test_prepare10.py index 8a6548d9ee5..42408fac3b0 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare10.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare10.py @@ -133,7 +133,7 @@ class test_prepare10(wttest.WiredTigerTestCase): cursor3.reset() session3.begin_transaction() - # Remove the updates from a prepare session and and keep it open. + # Insert the updates from a prepare session and keep it open. session_p = self.conn.open_session() cursor_p = session_p.open_cursor(uri) session_p.begin_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py index e79c55203bf..4a322c61998 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -29,14 +29,36 @@ import fnmatch, os, shutil, threading, time from helper import copy_wiredtiger_home from test_rollback_to_stable01 import test_rollback_to_stable_base -from wiredtiger import stat +from wiredtiger import stat, wiredtiger_strerror, WiredTigerError, WT_ROLLBACK from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wtthread import checkpoint_thread, op_thread +from time import sleep def timestamp_str(t): return '%x' % t +def retry_rollback(self, name, code): + retry_limit = 100 + retries = 0 + completed = False + saved_exception = None + while not completed and retries < retry_limit: + if retries != 0: + self.pr("Retrying operation for " + name) + sleep(0.1) + try: + code() + completed = True + except WiredTigerError as e: + rollback_str = wiredtiger_strerror(WT_ROLLBACK) + if rollback_str not in str(e): + raise(e) + retries += 1 + saved_exception = e + if not completed and saved_exception: + raise(saved_exception) + # test_rollback_to_stable10.py # Test the rollback to stable operation performs sweeping history store. class test_rollback_to_stable10(test_rollback_to_stable_base): @@ -50,7 +72,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): scenarios = make_scenarios(prepare_values) def conn_config(self): - config = 'cache_size=5MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true),timing_stress_for_test=[history_store_checkpoint_delay]' + config = 'cache_size=6MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true),timing_stress_for_test=[history_store_checkpoint_delay]' return config def simulate_crash_restart(self, olddir, newdir): @@ -80,6 +102,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): nrows = 1000 # Create a table without logging. + self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" ds_1 = SimpleDataSet( self, uri_1, 0, key_format="i", value_format="S", config='log=(enabled=false)') @@ -103,6 +126,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): value_f = "fffff" * 100 # Perform several updates. + self.pr("large updates") self.large_updates(uri_1, value_d, ds_1, nrows, 20) self.large_updates(uri_1, value_c, ds_1, nrows, 30) self.large_updates(uri_1, value_b, ds_1, nrows, 40) @@ -133,19 +157,173 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Create a checkpoint thread done = threading.Event() ckpt = checkpoint_thread(self.conn, done) - ckpt.start() - - # Perform several updates in parallel with checkpoint. - self.large_updates(uri_1, value_e, ds_1, nrows, 70) - self.large_updates(uri_2, value_e, ds_2, nrows, 70) - self.large_updates(uri_1, value_f, ds_1, nrows, 80) - self.large_updates(uri_2, value_f, ds_2, nrows, 80) + try: + self.pr("start checkpoint") + ckpt.start() - done.set() - ckpt.join() + # Perform several updates in parallel with checkpoint. + # Rollbacks may occur when checkpoint is running, so retry as needed. + self.pr("updates") + retry_rollback(self, 'update ds1, e', + lambda: self.large_updates(uri_1, value_e, ds_1, nrows, 70)) + retry_rollback(self, 'update ds2, e', + lambda: self.large_updates(uri_2, value_e, ds_2, nrows, 70)) + retry_rollback(self, 'update ds1, f', + lambda: self.large_updates(uri_1, value_f, ds_1, nrows, 80)) + retry_rollback(self, 'update ds2, f', + lambda: self.large_updates(uri_2, value_f, ds_2, nrows, 80)) + finally: + done.set() + ckpt.join() # Simulate a server crash and restart. + self.pr("restart") self.simulate_crash_restart(".", "RESTART") + self.pr("restart complete") + + # Check that the correct data is seen at and after the stable timestamp. + self.check(value_a, uri_1, nrows, 50) + self.check(value_a, uri_1, nrows, 80) + self.check(value_b, uri_1, nrows, 40) + self.check(value_c, uri_1, nrows, 30) + self.check(value_d, uri_1, nrows, 20) + + # Check that the correct data is seen at and after the stable timestamp. + self.check(value_c, uri_2, nrows, 30) + self.check(value_a, uri_2, nrows, 50) + self.check(value_a, uri_2, nrows, 80) + self.check(value_b, uri_2, nrows, 40) + self.check(value_d, uri_2, nrows, 20) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + calls = stat_cursor[stat.conn.txn_rts][2] + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2] + keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] + keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] + pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + stat_cursor.close() + + self.assertEqual(calls, 0) + self.assertEqual(keys_removed, 0) + self.assertEqual(keys_restored, 0) + self.assertGreaterEqual(upd_aborted, 0) + self.assertGreater(pages_visited, 0) + self.assertGreaterEqual(hs_removed, 0) + self.assertGreater(hs_sweep, 0) + + def test_rollback_to_stable_prepare(self): + nrows = 1000 + + # Create a table without logging. + self.pr("create/populate tables") + uri_1 = "table:rollback_to_stable10_1" + ds_1 = SimpleDataSet( + self, uri_1, 0, key_format="i", value_format="S", config='log=(enabled=false)') + ds_1.populate() + + # Create another table without logging. + uri_2 = "table:rollback_to_stable10_2" + ds_2 = SimpleDataSet( + self, uri_2, 0, key_format="i", value_format="S", config='log=(enabled=false)') + ds_2.populate() + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + + ',stable_timestamp=' + timestamp_str(10)) + + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + value_e = "eeeee" * 100 + value_f = "fffff" * 100 + + # Perform several updates. + self.pr("large updates") + self.large_updates(uri_1, value_d, ds_1, nrows, 20) + self.large_updates(uri_1, value_c, ds_1, nrows, 30) + self.large_updates(uri_1, value_b, ds_1, nrows, 40) + self.large_updates(uri_1, value_a, ds_1, nrows, 50) + + self.large_updates(uri_2, value_d, ds_2, nrows, 20) + self.large_updates(uri_2, value_c, ds_2, nrows, 30) + self.large_updates(uri_2, value_b, ds_2, nrows, 40) + self.large_updates(uri_2, value_a, ds_2, nrows, 50) + + # Verify data is visible and correct. + self.check(value_d, uri_1, nrows, 20) + self.check(value_c, uri_1, nrows, 30) + self.session.breakpoint() + self.check(value_b, uri_1, nrows, 40) + self.check(value_a, uri_1, nrows, 50) + + self.check(value_d, uri_2, nrows, 20) + self.check(value_c, uri_2, nrows, 30) + self.session.breakpoint() + self.check(value_b, uri_2, nrows, 40) + self.check(value_a, uri_2, nrows, 50) + + # Pin stable to timestamp 60 if prepare otherwise 50. + if self.prepare: + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + else: + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + + # Here's the update operation we'll perform, encapsulated so we can easily retry + # it if we get a rollback. Rollbacks may occur when checkpoint is running. + def simple_update(cursor, key, value): + cursor.set_key(key) + cursor.set_value(value) + self.assertEquals(cursor.update(), 0) + + # Create a checkpoint thread + done = threading.Event() + ckpt = checkpoint_thread(self.conn, done) + try: + self.pr("start checkpoint") + ckpt.start() + + # Perform several updates in parallel with checkpoint. + session_p1 = self.conn.open_session() + cursor_p1 = session_p1.open_cursor(uri_1) + session_p1.begin_transaction('isolation=snapshot') + self.pr("updates 1") + for i in range(1, nrows): + retry_rollback(self, 'update ds1', + lambda: simple_update(cursor_p1, ds_1.key(i), value_e)) + self.pr("prepare 1") + session_p1.prepare_transaction('prepare_timestamp=' + timestamp_str(69)) + + # Perform several updates in parallel with checkpoint. + session_p2 = self.conn.open_session() + cursor_p2 = session_p2.open_cursor(uri_2) + session_p2.begin_transaction('isolation=snapshot') + self.pr("updates 2") + for i in range(1, nrows): + retry_rollback(self, 'update ds2', + lambda: simple_update(cursor_p2, ds_2.key(i), value_e)) + self.pr("prepare 2") + session_p2.prepare_transaction('prepare_timestamp=' + timestamp_str(69)) + finally: + done.set() + ckpt.join() + + # Simulate a crash by copying to a new directory(RESTART). + copy_wiredtiger_home(".", "RESTART") + + # Commit the prepared transaction. + session_p1.commit_transaction('commit_timestamp=' + timestamp_str(70) + ',durable_timestamp=' + timestamp_str(71)) + session_p2.commit_transaction('commit_timestamp=' + timestamp_str(70) + ',durable_timestamp=' + timestamp_str(71)) + session_p1.close() + session_p2.close() + + # Open the new directory. + self.pr("restart") + self.conn = self.setUpConnectionOpen("RESTART") + self.session = self.setUpSessionOpen(self.conn) + self.pr("restart complete") # Check that the correct data is seen at and after the stable timestamp. self.check(value_a, uri_1, nrows, 50) diff --git a/src/third_party/wiredtiger/test/suite/test_txn21.py b/src/third_party/wiredtiger/test/suite/test_txn21.py index 5abf907ac84..06db7ce76aa 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn21.py +++ b/src/third_party/wiredtiger/test/suite/test_txn21.py @@ -43,7 +43,17 @@ class test_txn21(wttest.WiredTigerTestCase): # Transaction-level configuration. def test_operation_timeout_txn(self): + # Test during begin. self.session.begin_transaction('operation_timeout_ms=2000') + self.session.rollback_transaction() + + # Test during rollback. + self.session.begin_transaction() + self.session.rollback_transaction('operation_timeout_ms=2000') + + # Test during commit. + self.session.begin_transaction() + self.session.commit_transaction('operation_timeout_ms=2000') if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn22.py b/src/third_party/wiredtiger/test/suite/test_txn22.py new file mode 100755 index 00000000000..31d569951a7 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_txn22.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_txn22.py +# Transactions: test salvage with removed + +import fnmatch, os, shutil, time +from wtscenario import make_scenarios +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +def copy_for_crash_restart(olddir, newdir): + ''' Simulate a crash from olddir and restart in newdir. ''' + # with the connection still open, copy files to new directory + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + # Skip lock file on Windows since it is locked + if os.path.isfile(fullname) and \ + "WiredTiger.lock" not in fullname and \ + "Tmplog" not in fullname and \ + "Preplog" not in fullname: + shutil.copy(fullname, newdir) + +class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): + base_config = 'cache_size=1GB' + conn_config = base_config + + # File to be corrupted + filename_scenarios = [ + ('WiredTiger', dict(filename='WiredTiger')), + ('WiredTiger.basecfg', dict(filename='WiredTiger.basecfg')), + ('WiredTiger.turtle', dict(filename='WiredTiger.turtle')), + ('WiredTiger.wt', dict(filename='WiredTiger.wt')), + ('WiredTigerHS.wt', dict(filename='WiredTigerHS.wt')), + ('test_txn22.wt', dict(filename='test_txn22.wt')), + ] + + # In many cases, wiredtiger_open without any salvage options will + # just work. We list those cases here. + openable = [ + "removal:WiredTiger.basecfg", + "removal:WiredTiger.turtle", + ] + + # The cases for which salvage will not work + not_salvageable = [ + "removal:WiredTiger.turtle", + "removal:WiredTiger.wt", + ] + + scenarios = make_scenarios(filename_scenarios) + uri = 'table:test_txn22' + create_params = 'key_format=i,value_format=S' + nrecords = 1000 # records per table. + + def valuegen(self, i): + return str(i) + 'A' * 1024 + + # Insert a list of keys + def inserts(self, keylist): + c = self.session.open_cursor(self.uri) + for i in keylist: + c[i] = self.valuegen(i) + c.close() + + def checks(self): + c = self.session.open_cursor(self.uri) + gotlist = [] + for key, value in c: + gotlist.append(key) + self.assertEqual(self.valuegen(key), value) + c.close() + + def corrupt_meta(self, homedir): + # Mark this test has having corrupted files + self.databaseCorrupted() + filename = os.path.join(homedir, self.filename) + os.remove(filename) + + def is_openable(self): + key = 'removal:' + self.filename + return key in self.openable + + def is_salvageable(self): + key = 'removal:' + self.filename + return key not in self.not_salvageable + + def test_corrupt_meta(self): + newdir = "RESTART" + newdir2 = "RESTART2" + expect = list(range(0, self.nrecords)) + salvage_config = self.base_config + ',salvage=true' + + self.session.create(self.uri, self.create_params) + self.inserts(expect) + + # Simulate a crash by copying the contents of the directory + # before closing. After we corrupt the copy, make another + # copy of the corrupted directory. + # + # The first corrupted copy will be used to run: + # wiredtiger_open without salvage flag, followed by: + # wiredtiger_open with salvage flag. + # The second directory will be used to run: + # wiredtiger_open with salvage flag first. + + copy_for_crash_restart(self.home, newdir) + self.close_conn() + self.corrupt_meta(newdir) + copy_for_crash_restart(newdir, newdir2) + + for salvagedir in [ newdir, newdir2 ]: + # Removing the 'WiredTiger.turtle' file has weird behavior: + # Immediately doing wiredtiger_open (without salvage) succeeds. + # Following that, wiredtiger_open w/ salvage also succeeds. + # + # But, immediately after the corruption, if we run + # wiredtiger_open with salvage, it will fail. + # This anomoly should be fixed or explained. + if self.filename == 'WiredTiger.turtle': + continue + + if self.is_salvageable(): + if self.filename == 'WiredTigerHS.wt': + # Without salvage, they result in an error during the wiredtiger_open. + # But the nature of the messages produced during the error is variable + # by which case it is, and even variable from system to system. + with self.expectedStdoutPattern('.'): + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.reopen_conn(salvagedir, self.base_config), + '/.*/') + + self.reopen_conn(salvagedir, salvage_config) + if self.filename == 'test_txn22': + self.checks() + else: + # Certain cases are not currently salvageable, they result in + # an error during the wiredtiger_open. But the nature of the + # messages produced during the error is variable by which case + # it is, and even variable from system to system. + with self.expectedStdoutPattern('.'): + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.reopen_conn(salvagedir, salvage_config), + '/.*/') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index f88f26b5da7..a0a86731f1c 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -263,8 +263,11 @@ class WiredTigerTestCase(unittest.TestCase): result = '' extfiles = {} skipIfMissing = False + earlyLoading = '' if hasattr(exts, 'skip_if_missing'): skipIfMissing = exts.skip_if_missing + if hasattr(exts, 'early_load_ext') and exts.early_load_ext == True: + earlyLoading = '=(early_load=true)' for ext in exts: extconf = '' if '=' in ext: @@ -302,7 +305,7 @@ class WiredTigerTestCase(unittest.TestCase): else: extfiles[ext] = complete if len(extfiles) != 0: - result = ',extensions=[' + ','.join(list(extfiles.values())) + ']' + result = ',extensions=[' + ','.join(list(extfiles.values())) + earlyLoading + ']' return result # Can be overridden, but first consider setting self.conn_config diff --git a/src/third_party/wiredtiger/test/thread/file.c b/src/third_party/wiredtiger/test/thread/file.c index b536889b582..8260b612bd8 100644 --- a/src/third_party/wiredtiger/test/thread/file.c +++ b/src/third_party/wiredtiger/test/thread/file.c @@ -38,11 +38,8 @@ file_create(const char *name) testutil_check(conn->open_session(conn, NULL, NULL, &session)); testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=%s," - "internal_page_max=%d," - "leaf_page_max=%d," - "%s", - ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024, ftype == FIX ? ",value_format=3t" : "")); + "key_format=%s,internal_page_max=%d,leaf_page_max=%d,%s", ftype == ROW ? "u" : "r", 16 * 1024, + 128 * 1024, ftype == FIX ? ",value_format=3t" : "")); if ((ret = session->create(session, name, config)) != 0) if (ret != EEXIST) diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index 299df0a5118..513835c2c2e 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -246,14 +246,10 @@ testutil_sleep_wait(uint32_t seconds, pid_t pid) while (seconds > 0) { if ((got = waitpid(pid, &status, WNOHANG | WUNTRACED)) == pid) { if (WIFEXITED(status)) - testutil_die(EINVAL, "Child process %" PRIu64 - " exited early" - " with status %d", + testutil_die(EINVAL, "Child process %" PRIu64 " exited early with status %d", (uint64_t)pid, WEXITSTATUS(status)); if (WIFSIGNALED(status)) - testutil_die(EINVAL, "Child process %" PRIu64 - " terminated " - " with signal %d", + testutil_die(EINVAL, "Child process %" PRIu64 " terminated with signal %d", (uint64_t)pid, WTERMSIG(status)); } else if (got == -1) testutil_die(errno, "waitpid"); |