diff options
author | Luke Chen <luke.chen@mongodb.com> | 2019-03-25 16:50:07 +1100 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2019-03-25 16:50:07 +1100 |
commit | f4bea6b0f6b247f89929cc0a0558f0bf141e5a72 (patch) | |
tree | 4e6a3828e6315c57a3bdeb51cb7261741d6fb146 /src/third_party | |
parent | e35e8076dbddc863205cf24517e1b16dc9104d07 (diff) | |
download | mongo-f4bea6b0f6b247f89929cc0a0558f0bf141e5a72.tar.gz |
Import wiredtiger: 67066a329d365cb1de6e198184016d4a9d280679 from branch mongodb-4.2
ref: bb36cf83cd..67066a329d
for: 4.1.10
WT-4324 Ensure checkpoints rewrite pages with data in the future
WT-4609 Extend timestamp usage documentation
WT-4621 Support rounding up prepare timestamp to oldest
WT-4629 Research timestamping and add to test coverage for timestamping
WT-4630 Add Clang Format script
WT-4645 Fill gaps in test coverage regarding transactions
WT-4646 Coverity doesn't like testutil_assert() on fopen calls
WT-4648 Coverity false positive on fopen call
WT-4651 Remove 'incase' from s_string.ok and its uses
WT-4654 Split task for Evergreen Windows build variant to reduce makespan
WT-4655 Avoid C and C++ compiler incompatibilities
Diffstat (limited to 'src/third_party')
26 files changed, 959 insertions, 91 deletions
diff --git a/src/third_party/wiredtiger/.clang-format b/src/third_party/wiredtiger/.clang-format new file mode 100644 index 00000000000..1c4722d4870 --- /dev/null +++ b/src/third_party/wiredtiger/.clang-format @@ -0,0 +1,125 @@ +--- +Language: Cpp +# BasedOnStyle: LLVM +AccessModifierOffset: 0 +AlignAfterOpenBracket: DontAlign +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: false +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: AllDefinitions +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: false +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 100 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH + - TAILQ_FOREACH + - WT_CKPT_FOREACH + - WT_EXT_FOREACH + - WT_EXT_FOREACH_OFF + - WT_FIX_FOREACH + - WT_CELL_FOREACH_BEGIN + - WT_CELL_FOREACH_VRFY + - WT_COL_FOREACH + - WT_INTL_FOREACH_BEGIN + - WT_ROW_FOREACH + - WT_SKIP_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '(Test)?$' +IndentCaseLabels: false +IndentPPDirectives: None +IndentWidth: 8 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '^WT_PACKED_STRUCT_BEGIN$' +MacroBlockEnd: '^WT_PACKED_STRUCT_END$' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Right +RawStringFormats: + - Delimiter: pb + Language: TextProto + BasedOnStyle: google +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 8 +UseTab: ForIndentation +... diff --git a/src/third_party/wiredtiger/build_posix/configure.ac.in b/src/third_party/wiredtiger/build_posix/configure.ac.in index d0377ce32e0..ea3d51d009c 100644 --- a/src/third_party/wiredtiger/build_posix/configure.ac.in +++ b/src/third_party/wiredtiger/build_posix/configure.ac.in @@ -25,14 +25,22 @@ AM_PROG_AS(as gas) define([AC_LIBTOOL_LANG_CXX_CONFIG], [:])dnl define([AC_LIBTOOL_LANG_F77_CONFIG], [:])dnl +# We only turn on C++ if the compiler is named "cc", otherwise, there's no +# reason to believe "c++" can build compatible objects. +# # Check whether the C++ compiler works by linking a trivial program. -AC_CACHE_CHECK([whether the C++ compiler works], - [wt_cv_prog_cxx_works], - [AC_LANG_PUSH([C++]) - AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], - [wt_cv_prog_cxx_works=yes], - [wt_cv_prog_cxx_works=no]) - AC_LANG_POP([C++])]) +if test $CC = "cc"; then + AC_CACHE_CHECK([whether the C++ compiler works], + [wt_cv_prog_cxx_works], + [AC_LANG_PUSH([C++]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], + [wt_cv_prog_cxx_works=yes], + [wt_cv_prog_cxx_works=no]) + AC_LANG_POP([C++])]) +else + AC_MSG_WARN([C++ compiler ignored unless compiler is named "cc"]) + wt_cv_prog_cxx_works=no +fi AM_CONDITIONAL([HAVE_CXX], [test "$wt_cv_prog_cxx_works" = "yes"]) LT_PREREQ(2.2.6) diff --git a/src/third_party/wiredtiger/dist/api_config.py b/src/third_party/wiredtiger/dist/api_config.py index 0471bde51fd..78f4f4fa91c 100644 --- a/src/third_party/wiredtiger/dist/api_config.py +++ b/src/third_party/wiredtiger/dist/api_config.py @@ -152,11 +152,13 @@ tfile.write('''/* DO NOT EDIT: automatically built by dist/api_config.py. */ ''') # Make a TextWrapper that wraps at commas. -w = textwrap.TextWrapper(width=64, break_on_hyphens=False) +w = textwrap.TextWrapper(width=64, break_on_hyphens=False, + break_long_words=False) w.wordsep_re = w.wordsep_simple_re = re.compile(r'(,)') # TextWrapper that wraps at whitespace. -ws = textwrap.TextWrapper(width=64, break_on_hyphens=False) +ws = textwrap.TextWrapper(width=64, break_on_hyphens=False, + break_long_words=False) def checkstr(c): '''Generate the function reference and JSON string used by __wt_config_check diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index d585557d968..190044dca0a 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1286,13 +1286,31 @@ methods = { if read timestamp is earlier than oldest timestamp, read timestamp will be rounded to oldest timestamp''', type='boolean'), + Config('roundup_timestamps', '', r''' + round up timestamps of the transaction. This setting alters the + visibility expected in a transaction. See @ref + transaction_timestamps''', + type='category', subconfig= [ + Config('prepared', 'false', r''' + applicable only for prepared transactions. Indicates if the prepare + timestamp and the commit timestamp of this transaction can be + rounded up. If the prepare timestamp is less than the oldest + timestamp, the prepare timestamp will be rounded to the oldest + timestamp. If the commit timestamp is less than the prepare + timestamp, the commit timestamp will be rounded up to the prepare + timestamp''', type='boolean'), + Config('read', 'false', r''' + if the read timestamp is less than the oldest timestamp, the + read timestamp will be rounded up to the oldest timestamp''', + type='boolean'), + ]), Config('snapshot', '', r''' use a named, in-memory snapshot, see @ref transaction_named_snapshots'''), Config('sync', '', r''' whether to sync log records when the transaction commits, inherited from ::wiredtiger_open \c transaction_sync''', - type='boolean'), + type='boolean') ]), 'WT_SESSION.commit_transaction' : Method([ diff --git a/src/third_party/wiredtiger/dist/s_clang-format b/src/third_party/wiredtiger/dist/s_clang-format new file mode 100644 index 00000000000..a16a8d4af17 --- /dev/null +++ b/src/third_party/wiredtiger/dist/s_clang-format @@ -0,0 +1,33 @@ +#! /bin/sh + +# Installation of the clang development package isn't standard, list a +# couple of the places we're using. +export PATH=$PATH:/usr/local/clang60/bin:/usr/local/llvm-devel/bin + +# Find the top-level WiredTiger directory and move to there. +cd `git rev-parse --show-toplevel` || exit 1 + +# Ensure that we have the correct version of clang-format. +clang_format_version="6.0.0" +clang-format --version | grep "version $clang_format_version" >/dev/null 2>&1 +if test $? -ne 0; then + echo "$0: found incorrect version of clang-format ($clang_format_version required)" + exit 1 +fi + +case $# in +0) + # Get all source files that aren't in s_clang-format.list. + search=`find src -name '*.[chi]'` + for f in `cat dist/s_clang-format.list`; do + search=`echo "$search" | sed "\#$f#d"` + done;; +1) + search="$1";; +*) + echo "usage: $0 [file]" + exit 1;; +esac + +# Format each file inplace. +clang-format -i --fallback-style=none $search diff --git a/src/third_party/wiredtiger/dist/s_clang-format.list b/src/third_party/wiredtiger/dist/s_clang-format.list new file mode 100644 index 00000000000..1b9c2d75eac --- /dev/null +++ b/src/third_party/wiredtiger/dist/s_clang-format.list @@ -0,0 +1,5 @@ +src/include/bitstring.i +src/include/queue.h +src/os_posix/os_getopt.c +src/support/hash_city.c +src/support/hash_fnv.c diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 3868f96c431..c2e70ba2646 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -840,7 +840,6 @@ iiu ikey im impl -incase incr incrementals incrementing diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 933bc76bbff..c9b3aed9f2d 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "bb36cf83cd952262af49a589d15940a8c865174c", + "commit": "67066a329d365cb1de6e198184016d4a9d280679", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index 473b55274a9..3fb9803e7ad 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -251,7 +251,7 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* * In case of append, the recno (key) for the value is assigned * now. Set the recno in the transaction operation to be used - * incase this transaction is prepared to retrieve the update + * in case this transaction is prepared to retrieve the update * corresponding to this operation. */ __wt_txn_op_set_recno(session, cbt->recno); diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index 05d0dfb4041..cf51f1c2e67 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -208,7 +208,7 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, if (logged && modify_type != WT_UPDATE_RESERVE) { WT_ERR(__wt_txn_log_op(session, cbt)); /* - * Set the key in the transaction operation to be used incase + * Set the key in the transaction operation to be used in case * this transaction is prepared to retrieve the update * corresponding to this operation. */ diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 1b2f6ce5ed6..303e4b39a38 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -271,6 +271,13 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK + confchk_WT_SESSION_begin_transaction_roundup_timestamps_subconfigs[] = { + { "prepared", "boolean", NULL, NULL, NULL, 0 }, + { "read", "boolean", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_begin_transaction[] = { { "ignore_prepare", "boolean", NULL, NULL, NULL, 0 }, { "isolation", "string", @@ -281,6 +288,9 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_begin_transaction[] = { { "priority", "int", NULL, "min=-100,max=100", NULL, 0 }, { "read_timestamp", "string", NULL, NULL, NULL, 0 }, { "round_to_oldest", "boolean", NULL, NULL, NULL, 0 }, + { "roundup_timestamps", "category", + NULL, NULL, + confchk_WT_SESSION_begin_transaction_roundup_timestamps_subconfigs, 2 }, { "snapshot", "string", NULL, NULL, NULL, 0 }, { "sync", "boolean", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } @@ -1360,8 +1370,9 @@ static const WT_CONFIG_ENTRY config_entries[] = { }, { "WT_SESSION.begin_transaction", "ignore_prepare=false,isolation=,name=,priority=0,read_timestamp=" - ",round_to_oldest=false,snapshot=,sync=", - confchk_WT_SESSION_begin_transaction, 8 + ",round_to_oldest=false,roundup_timestamps=(prepared=false," + "read=false),snapshot=,sync=", + confchk_WT_SESSION_begin_transaction, 9 }, { "WT_SESSION.checkpoint", "drop=,force=false,name=,target=,use_timestamp=true", diff --git a/src/third_party/wiredtiger/src/docs/transactions.dox b/src/third_party/wiredtiger/src/docs/transactions.dox index 25845ce0209..451be5953cb 100644 --- a/src/third_party/wiredtiger/src/docs/transactions.dox +++ b/src/third_party/wiredtiger/src/docs/transactions.dox @@ -209,6 +209,58 @@ durable timestamp, instead of commit timestamp for persisting the data updates. If the durable timestamp is not specified, then the commit timestamp will be considered as the durable timestamp. +There are a number of constraints around assigning timestamps for running +transactions - the table below summarizes those constraints: + +| API | Prepared | Constraint | Enforced | Description | +|---------|----------|------------|----------|-------------| +| Prepare | During | prepare_timestamp >= stable_timestamp | Y | None | +| Commit | No | commit_timestamp > stable_timestamp | Y | None | +| Commit | Yes | commit_timestamp >= prepare_timestamp | Y | The commit timestamp may be older than the oldest timestamp at the time of commit. | +| Commit | Yes | durable_timestamp > stable_timestamp | Y | None | +| Commit | Yes | durable_timestamp != 0 \|\| commit_timestamp > stable_timestamp | N | If no durable timestamp is given when committing a prepared transaction, the commit timestamp must be greater than the stable timestamp. | + +@subsection timestamp_roundup Automatic rounding of timestamps +Applications setting timestamps for a transaction have to comply with the +constraints based on the global timestamp state. In order to be compliant with +the constraints applications need to query the global timestamp state and +check their timestamps for compliance and adjust timestamps if required. +To simplify the burden on applications related to rounding up timestamps +WiredTiger supports automatic rounding of timestamps in some scenarios. + +Applications can configure +<code>roundup_timestamps=(prepared=true,read=true)</code> with +WT_SESSION::begin_transaction. + +The configuration <code>roundup_timestamps=(prepared=true)</code> will be valid +only for prepared transactions. It indicates that the prepare timestamp could be +rounded up to the oldest timestamp, if the prepare timestamp is less than the +oldest timestamp. This setting also indicates that the commit timestamp of the +transaction could be rounded up to the prepare timestamp, if the commit +timestamp is less than the prepare timestamp. Based on the timestamps values +and constraints, enabling this configuration could result in only one of +timestamps being rounded up. For example, for the timestamp values +<code>prepare_timestamp=100, commit_timestamp=300, oldest_timestamp=200</code> +with configuration <code>roundup_timestamps=(prepared=true)</code> only the +prepare timestamp will be rounded up to the oldest timestamp and the commit +timestamp will not be adjusted and the result will be +<code>prepare_timestamp=200, commit_timestamp=300, oldest_timestamp=200</code>. +For cases where both the prepare timestamp and the commit timestamp needs to be +rounded up, first the prepare timestamp will be rounded to the oldest timestamp +and then the commit timestamp will be rounded up to the new prepare timestamp. +For example, for the timestamp values <code>prepare_timestamp=100, + commit_timestamp=150, oldest_timestamp=200</code> with configuration + <code>roundup_timestamps=(prepared=true)</code>, the prepare timestamp is + rounded up to the oldest timestamp, as part of the + WT_SESSION::prepare_transaction, as <code>prepare_timestamp=200</code> and + subsequently as part of WT_SESSION::commit_transaction, the commit + timestamp is rounded up to the new prepare timestamp as + <code>commit_timestamp=200</code>. + +Configuring <code>roundup_timestamps=(read=true)</code> causes the read +timestamp to be rounded up to the oldest timestamp, if the read timestamp is +greater than the oldest timestamp no change will be made. + @subsection timestamp_connection Managing global timestamp state Applications that use timestamps need to manage some global state in order @@ -234,6 +286,16 @@ does not automatically reset the maximum commit timestamp it is tracking. The application should explicitly do so by setting a commit timestamp in WT_CONNECTION::set_timestamp. +| Timestamp | Description | Constraint | +|-----------|-------------|------------| +| all_committed | The oldest timestamp at which all previous write transactions have committed. | | +| last_checkpoint | The point at which the last checkpoint ran. If no checkpoint has run it's value will be 0. | last_checkpoint <= stable timestamp | +| oldest | Point in time readers cannot be created using a timestamp older than the oldest timestamp, as explained above modification history is discarded prior to the oldest timestamp. This timestamp can be set via the API. | 0 <= oldest <= stable | +| oldest_reader | The timestamp of the oldest currently active read transaction, if there is no current read transaction then querying for the oldest_reader with WT_CONNECTION::query_timestamp will return WT_NOTFOUND. | | +| pinned | Minimum of the oldest_reader and oldest timestamp. | | +| recovery | The stable timestamp used, if any, in the most recent checkpoint prior to the last shutdown. | | +| stable | Any active transaction with a commit timestamp less than or equal to the current stable timestamp will not be able to modify data, except in the instance of prepared transactions. This timestamp can be set via the API. | stable >= oldest | + @subsection timestamp_extensions Timestamp support in the extension API The extension API, used by modules that extend WiredTiger via diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 777e4d2db3a..75275cc6097 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -862,7 +862,7 @@ extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force extern int __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_commit_timestamp_validate(WT_SESSION_IMPL *session, const char *name, wt_timestamp_t ts, WT_CONFIG_ITEM *cval, bool durable_ts) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_parse_prepare_timestamp(WT_SESSION_IMPL *session, const char *cfg[], wt_timestamp_t *timestamp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_parse_prepare_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_parse_read_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session); extern void __wt_txn_clear_commit_timestamp(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 1ac34a4defd..166950410d5 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -341,7 +341,9 @@ struct __wt_txn { #define WT_TXN_TS_DURABLE_ALWAYS 0x040000u #define WT_TXN_TS_DURABLE_KEYS 0x080000u #define WT_TXN_TS_DURABLE_NEVER 0x100000u -#define WT_TXN_UPDATE 0x200000u +#define WT_TXN_TS_ROUND_PREPARED 0x200000u +#define WT_TXN_TS_ROUND_READ 0x400000u +#define WT_TXN_UPDATE 0x800000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index a358e74d925..d3c28f485ef 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -72,7 +72,7 @@ __wt_txn_op_set_recno(WT_SESSION_IMPL *session, uint64_t recno) * again. Even though only prepared updates can be evicted, at this * stage we don't know whether this transaction will be prepared or * not, hence we are copying the key for all operations, so that we can - * use this key to fetch the update incase this transaction is + * use this key to fetch the update in case this transaction is * prepared. */ op->u.op_col.recno = recno; @@ -108,7 +108,7 @@ __wt_txn_op_set_key(WT_SESSION_IMPL *session, const WT_ITEM *key) * again. Even though only prepared updates can be evicted, at this * stage we don't know whether this transaction will be prepared or * not, hence we are copying the key for all operations, so that we can - * use this key to fetch the update incase this transaction is + * use this key to fetch the update in case this transaction is * prepared. */ return (__wt_buf_set(session, &op->u.op_row.key, key->data, key->size)); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 028a7663975..c7e0352d644 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1794,6 +1794,23 @@ struct __wt_session { * @config{round_to_oldest, if read timestamp is earlier than oldest * timestamp\, read timestamp will be rounded to oldest timestamp., a * boolean flag; default \c false.} + * @config{roundup_timestamps = (, round up timestamps of the + * transaction. This setting alters the visibility expected in a + * transaction. See @ref transaction_timestamps., a set of related + * configuration options defined below.} + * @config{ prepared, applicable only for + * prepared transactions. Indicates if the prepare timestamp and the + * commit timestamp of this transaction can be rounded up. If the + * prepare timestamp is less than the oldest timestamp\, the prepare + * timestamp will be rounded to the oldest timestamp. If the commit + * timestamp is less than the prepare timestamp\, the commit timestamp + * will be rounded up to the prepare timestamp., a boolean flag; default + * \c false.} + * @config{ read, if the read + * timestamp is less than the oldest timestamp\, the read timestamp will + * be rounded up to the oldest timestamp., a boolean flag; default \c + * false.} + * @config{ ),,} * @config{snapshot, use a named\, in-memory snapshot\, see @ref * transaction_named_snapshots., a string; default empty.} * @config{sync, whether to sync log records when the transaction diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 1791317eb1b..4c36a615b0d 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -683,6 +683,16 @@ __rec_write_page_status(WT_SESSION_IMPL *session, WT_RECONCILE *r) WT_ASSERT(session, !F_ISSET(r, WT_REC_EVICT) || F_ISSET(r, WT_REC_LOOKASIDE | WT_REC_UPDATE_RESTORE)); + + /* + * We have written the page, but something prevents it from + * being evicted. If we wrote the newest versions of updates, + * the on-disk page may contain records that are newer than + * what checkpoint would write. Make sure that checkpoint + * visits the page and (if necessary) fixes things up. + */ + if (r->las_skew_newest) + mod->first_dirty_txn = WT_TXN_FIRST; } else { /* * Track the page's maximum transaction ID (used to decide if diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 1482e02c7ea..364474dd362 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -531,6 +531,25 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) if (cval.val) F_SET(txn, WT_TXN_IGNORE_PREPARE); + /* + * Check if the prepare timestamp and the commit timestamp of a + * prepared transaction need to be rounded up. + */ + WT_RET(__wt_config_gets_def( + session, cfg, "roundup_timestamps.prepared", 0, &cval)); + if (cval.val) + F_SET(txn, WT_TXN_TS_ROUND_PREPARED); + else + F_CLR(txn, WT_TXN_TS_ROUND_PREPARED); + + /* Check if read timestamp needs to be rounded up. */ + WT_RET(__wt_config_gets_def( + session, cfg, "roundup_timestamps.read", 0, &cval)); + if (cval.val) + F_SET(txn, WT_TXN_TS_ROUND_READ); + else + F_CLR(txn, WT_TXN_TS_ROUND_READ); + WT_RET(__wt_txn_parse_read_timestamp(session, cfg)); return (0); @@ -791,6 +810,13 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) readonly = txn->mod_count == 0; prepare = F_ISSET(txn, WT_TXN_PREPARE); + /* + * Clear the prepared round up flag if the transaction is not prepared. + * There is no rounding up to do in that case. + */ + if (!prepare) + F_CLR(txn, WT_TXN_TS_ROUND_PREPARED); + /* Look for a commit timestamp. */ WT_ERR( __wt_config_gets_def(session, cfg, "commit_timestamp", 0, &cval)); @@ -1047,7 +1073,6 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN *txn; WT_TXN_OP *op; WT_UPDATE *upd; - wt_timestamp_t ts; u_int i; txn = &session->txn; @@ -1060,8 +1085,7 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_txn_context_check(session, true)); /* Parse and validate the prepare timestamp. */ - WT_RET(__wt_txn_parse_prepare_timestamp(session, cfg, &ts)); - txn->prepare_timestamp = ts; + WT_RET(__wt_txn_parse_prepare_timestamp(session, cfg)); /* * We are about to release the snapshot: copy values into any @@ -1106,7 +1130,7 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) } /* Set prepare timestamp. */ - upd->start_ts = ts; + upd->start_ts = txn->prepare_timestamp; WT_PUBLISH(upd->prepare_state, WT_PREPARE_INPROGRESS); op->u.op_upd = NULL; diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 0b9f790c1e6..594be12e6b1 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -640,11 +640,20 @@ __wt_txn_commit_timestamp_validate(WT_SESSION_IMPL *session, const char *name, * timestamp. */ if (F_ISSET(txn, WT_TXN_PREPARE) && ts < txn->prepare_timestamp) { - __wt_timestamp_to_string(txn->prepare_timestamp, ts_string[0]); - WT_RET_MSG(session, EINVAL, - "%s timestamp %.*s older than the prepare timestamp %s " - "for this transaction", - name, (int)cval->len, cval->str, ts_string[0]); + /* + * If roundup timestamps is configured, the commit timestamp + * will be rounded up to the prepare timestamp. + */ + if (F_ISSET(txn, WT_TXN_TS_ROUND_PREPARED)) + ts = txn->prepare_timestamp; + else { + __wt_timestamp_to_string( + txn->prepare_timestamp, ts_string[0]); + WT_RET_MSG(session, EINVAL, + "%s timestamp %.*s older than the prepare " + "timestamp %s for this transaction", + name, (int)cval->len, cval->str, ts_string[0]); + } } if (F_ISSET(txn, WT_TXN_HAS_TS_DURABLE) && @@ -674,8 +683,8 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) bool prepare, prepare_allowed; txn = &session->txn; + prepare_allowed = false; prepare = F_ISSET(txn, WT_TXN_PREPARE); - prepare_allowed = prepare == false; /* Look for a commit timestamp. */ ret = __wt_config_gets_def(session, cfg, "commit_timestamp", 0, &cval); @@ -699,7 +708,7 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) prepare_allowed = true; } - /* Look for a durable timestamp incase of prepared transaction. */ + /* If we have a prepared transaction, look for a durable timestamp. */ if (prepare) { ret = __wt_config_gets_def( session, cfg, "durable_timestamp", 0, &cval); @@ -740,74 +749,95 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) */ int __wt_txn_parse_prepare_timestamp( - WT_SESSION_IMPL *session, const char *cfg[], wt_timestamp_t *timestamp) + WT_SESSION_IMPL *session, const char *cfg[]) { WT_CONFIG_ITEM cval; - WT_TXN *prev; + WT_TXN *prev, *txn; WT_TXN_GLOBAL *txn_global; - wt_timestamp_t oldest_ts; - char ts_string[WT_TS_INT_STRING_SIZE]; + wt_timestamp_t oldest_ts, timestamp; + char ts_string[2][WT_TS_INT_STRING_SIZE]; + txn = &session->txn; txn_global = &S2C(session)->txn_global; WT_RET(__wt_config_gets_def(session, cfg, "prepare_timestamp", 0, &cval)); - if (cval.len > 0) { - if (F_ISSET(&session->txn, WT_TXN_HAS_TS_COMMIT)) - WT_RET_MSG(session, EINVAL, - "commit timestamp should not have been set before " - "prepare transaction"); + if (cval.len == 0) + WT_RET_MSG(session, EINVAL, "prepare timestamp is required"); + + if (F_ISSET(&session->txn, WT_TXN_HAS_TS_COMMIT)) + WT_RET_MSG(session, EINVAL, "commit timestamp " + "should not have been set before prepare transaction"); - WT_RET(__wt_txn_parse_timestamp( - session, "prepare", timestamp, &cval)); + WT_RET(__wt_txn_parse_timestamp(session, "prepare", ×tamp, &cval)); + __wt_readlock(session, &txn_global->read_timestamp_rwlock); + oldest_ts = txn_global->oldest_timestamp; + /* + * Prepare timestamp must be greater than the latest active read + * timestamp, if any. + */ + prev = TAILQ_LAST( + &txn_global->read_timestamph, __wt_txn_rts_qh); + while (prev != NULL) { + /* Skip self and non-active transactions. */ + if (prev->clear_read_q || prev == txn) { + prev = TAILQ_PREV( + prev, __wt_txn_rts_qh, read_timestampq); + continue; + } + + if (prev->read_timestamp >= timestamp) { + __wt_readunlock(session, + &txn_global->read_timestamp_rwlock); + __wt_timestamp_to_string( + prev->read_timestamp, ts_string[0]); + WT_RET_MSG(session, EINVAL, + "prepare timestamp %.*s must be greater than the " + "latest active read timestamp %s ", + (int)cval.len, cval.str, ts_string[0]); + } + break; + } + /* Unlock here to have less code branches. */ + __wt_readunlock(session, &txn_global->read_timestamp_rwlock); + /* + * Check whether the prepare timestamp is less than the oldest + * timestamp. + */ + if (timestamp < oldest_ts) { /* - * Prepare timestamp must be later/greater than latest active - * read timestamp. + * Check whether the prepare timestamp needs to be rounded up to + * the oldest timestamp. */ - __wt_readlock(session, &txn_global->read_timestamp_rwlock); - prev = TAILQ_LAST(&txn_global->read_timestamph, - __wt_txn_rts_qh); - while (prev != NULL) { + if (F_ISSET(txn, WT_TXN_TS_ROUND_PREPARED)) { /* - * Skip any transactions that are not active. + * Check that there are no active readers. That would + * be a violation of preconditions for rounding + * timestamps of prepared transactions. */ - if (prev->clear_read_q) { - prev = TAILQ_PREV( - prev, __wt_txn_rts_qh, read_timestampq); - continue; - } - if (prev->read_timestamp >= *timestamp) { - __wt_readunlock(session, - &txn_global->read_timestamp_rwlock); + WT_ASSERT(session, prev == NULL); + + if (WT_VERBOSE_ISSET(session, WT_VERB_TIMESTAMP)) { __wt_timestamp_to_string( - prev->read_timestamp, ts_string); - WT_RET_MSG(session, EINVAL, - "prepare timestamp %.*s not later than " - "an active read timestamp %s ", - (int)cval.len, cval.str, ts_string); + timestamp, ts_string[0]); + __wt_timestamp_to_string( + oldest_ts, ts_string[1]); + __wt_verbose(session, WT_VERB_TIMESTAMP, + "prepare timestamp %s rounded to oldest " + "timestamp %s", ts_string[0], ts_string[1]); } - break; + timestamp = oldest_ts; + } else { + __wt_timestamp_to_string( + oldest_ts, ts_string[0]); + WT_RET_MSG(session, EINVAL, + "prepare timestamp %.*s is older than the oldest " + "timestamp %s ", (int)cval.len, cval.str, + ts_string[0]); } - __wt_readunlock(session, &txn_global->read_timestamp_rwlock); - - /* - * If there are no active readers, prepare timestamp must not - * be older than oldest timestamp. - */ - if (prev == NULL) { - oldest_ts = txn_global->oldest_timestamp; - - if (*timestamp < oldest_ts) { - __wt_timestamp_to_string(oldest_ts, ts_string); - WT_RET_MSG(session, EINVAL, - "prepare timestamp %.*s is older than the " - "oldest timestamp %s ", (int)cval.len, - cval.str, ts_string); - } - } - } else - WT_RET_MSG(session, EINVAL, "prepare timestamp is required"); + } + txn->prepare_timestamp = timestamp; return (0); } @@ -854,6 +884,11 @@ __wt_txn_parse_read_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) cfg, "round_to_oldest", 0, &cval)); round_to_oldest = cval.val; /* + * The read timestamp could be rounded to the oldest timestamp. + */ + if (F_ISSET(txn, WT_TXN_TS_ROUND_READ)) + round_to_oldest = true; + /* * This code is not using the timestamp validate function to * avoid a race between checking and setting transaction * timestamp. diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 9189a993732..b370e8507c3 100644 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -1069,7 +1069,14 @@ buildvariants: tasks: - name: compile - name: compile-windows-alt - - name: unit-test + - name: unit-test-bucket00 + - name: unit-test-bucket01 + - name: unit-test-bucket02 + - name: unit-test-bucket03 + - name: unit-test-bucket04 + - name: unit-test-bucket05 + - name: unit-test-bucket06 + - name: unit-test-bucket07 #- name: format - Enable when we have a solution for hangs and crashses - name: fops diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index d3429788bfc..06386e5e86e 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -160,7 +160,7 @@ run(int r) testutil_make_work_dir(HOME); - testutil_assert((res_fp = fopen(RSLT, "w")) != NULL); + testutil_checksys((res_fp = fopen(RSLT, "w")) == NULL); /* * Each run builds the LOAD file, and then appends the first page of @@ -571,7 +571,7 @@ copy(u_int gen, u_int recno) uint32_t cksum32, gen32; char buf[PSIZE]; - testutil_assert((ifp = fopen(LOAD, "r")) != NULL); + testutil_checksys((ifp = fopen(LOAD, "r")) == NULL); /* * If the salvage file doesn't exist, then we're creating it: @@ -579,9 +579,9 @@ copy(u_int gen, u_int recno) * Otherwise, we are appending to an existing file. */ if (file_exists(SLVG)) - testutil_assert((ofp = fopen(SLVG, "a")) != NULL); + testutil_checksys((ofp = fopen(SLVG, "a")) == NULL); else { - testutil_assert((ofp = fopen(SLVG, "w")) != NULL); + testutil_checksys((ofp = fopen(SLVG, "w")) == NULL); testutil_assert(fread(buf, 1, PSIZE, ifp) == PSIZE); testutil_assert(fwrite(buf, 1, PSIZE, ofp) == PSIZE); } @@ -656,7 +656,7 @@ process(void) testutil_check(conn->close(conn, 0)); /* Dump. */ - testutil_assert((fp = fopen(DUMP, "w")) != NULL); + testutil_checksys((fp = fopen(DUMP, "w")) == NULL); testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); testutil_check(session->open_cursor( diff --git a/src/third_party/wiredtiger/test/suite/test_prepare05.py b/src/third_party/wiredtiger/test/suite/test_prepare05.py index e158e81b79e..c236c1f0783 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare05.py @@ -80,7 +80,7 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( 'prepare_timestamp=' + timestamp_str(4)), - "/not later than an active read timestamp/") + "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() # Check setting the prepare timestamp as later than active read diff --git a/src/third_party/wiredtiger/test/suite/test_prepare06.py b/src/third_party/wiredtiger/test/suite/test_prepare06.py new file mode 100644 index 00000000000..a4dc312cd7b --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare06.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_prepare06.py +# Prepare: Rounding up prepared transactions Timestamps. +# + +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +def timestamp_str(t): + return '%x' % t + +class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_prepare06' + uri = 'table:' + tablename + + def test_timestamp_api(self): + self.session.create(self.uri, 'key_format=i,value_format=i') + c = self.session.open_cursor(self.uri) + + # It is illegal to set the prepare timestamp older than the oldest + # timestamp. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20)) + self.session.begin_transaction() + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.prepare_transaction( + 'prepare_timestamp=' + timestamp_str(10)), + "/older than the oldest timestamp/") + self.session.rollback_transaction() + + # Check setting a prepared transaction timestamps earlier than the + # oldest timestamp is valid with roundup_timestamps settings. + self.session.begin_transaction('roundup_timestamps=(prepared=true)') + self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + + # Check the cases with an active reader. + # Start a new reader to have an active read timestamp. + s_reader = self.conn.open_session() + s_reader.begin_transaction('read_timestamp=' + timestamp_str(40)) + + # It is illegal to set the prepare timestamp as earlier than an active + # read timestamp even with roundup_timestamps settings. + self.session.begin_transaction() + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.prepare_transaction( + 'prepare_timestamp=' + timestamp_str(10)), + "/must be greater than the latest active read timestamp/") + self.session.rollback_transaction() + + # It is illegal to set the prepare timestamp the same as an active read + # timestamp even with roundup_timestamps settings. + self.session.begin_transaction() + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.prepare_transaction( + 'prepare_timestamp=' + timestamp_str(40)), + "/must be greater than the latest active read timestamp/") + self.session.rollback_transaction() + + # It is illegal to set a commit timestamp older than the prepare + # timestamp of a transaction. + self.session.begin_transaction() + c[1] = 1 + self.session.prepare_transaction( + 'prepare_timestamp=' + timestamp_str(45)) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.commit_transaction( + 'commit_timestamp=' + timestamp_str(30)), + "/older than the prepare timestamp/") + + # It is legal to set a commit timestamp older than prepare timestamp of + # a transaction with roundup_timestamps settings. + self.session.begin_transaction('roundup_timestamps=(prepared=true)') + c[1] = 1 + self.session.prepare_transaction( + 'prepare_timestamp=' + timestamp_str(45)) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + + s_reader.commit_transaction() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp14.py b/src/third_party/wiredtiger/test/suite/test_timestamp14.py new file mode 100644 index 00000000000..c73cf0ee45d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp14.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_timestamp14.py +# Global timestamps: oldest reader, all committed, pinned +# + +import random +from suite_subprocess import suite_subprocess +import wiredtiger, wttest +from wtscenario import make_scenarios + +def timestamp_str(t): + return '%x' % t + +class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_timestamp14' + uri = 'table:' + tablename + + def test_all_committed(self): + all_committed_uri = self.uri + '_all_committed' + session1 = self.setUpSessionOpen(self.conn) + session2 = self.setUpSessionOpen(self.conn) + session1.create(all_committed_uri, 'key_format=i,value_format=i') + session2.create(all_committed_uri, 'key_format=i,value_format=i') + + # Scenario 0: No commit timestamp has ever been specified therefore + # There is no all_committed timestamp and we will get an error + # Querying for it. + session1.begin_transaction() + cur1 = session1.open_cursor(all_committed_uri) + cur1[1]=1 + session1.commit_transaction() + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.conn.query_timestamp('get=all_committed')) + + # Scenario 1: A single transaction with a commit timestamp, will + # result in the all_committed timestamp being set. + session1.begin_transaction() + cur1[1]=1 + session1.commit_transaction('commit_timestamp=1') + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "1") + + # Scenario 2: A transaction begins and specifies that it intends + # to commit at timestamp 2, a second transaction begins and commits + # at timestamp 3. + session1.begin_transaction() + session1.timestamp_transaction('commit_timestamp=2') + + session2.begin_transaction() + cur2 = session2.open_cursor(all_committed_uri) + cur2[2] = 2 + session2.commit_transaction('commit_timestamp=3') + + # As the original transaction is still running the all_committed + # timestamp is being held at 1. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "1") + cur1[1] = 2 + session1.commit_transaction() + + # Now that the original transaction has finished the all_committed + # timestamp has moved to 3, skipping 2 as there is a commit with + # a greater timestamp already existing. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "3") + + # Senario 3: Commit with a commit timestamp of 5 and then begin a + # transaction intending to commit at 4, the all_committed timestamp + # should move back to 3. Until the transaction at 4 completes. + session1.begin_transaction() + cur1[1] = 3 + session1.commit_transaction('commit_timestamp=5') + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "5") + + session1.begin_transaction() + # All committed will now move back to 3 as it is the point at which + # all transactions up to that point have committed. + session1.timestamp_transaction('commit_timestamp=4') + + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "3") + + session1.commit_transaction() + + # Now that the transaction at timestamp 4 has completed the + # all committed timestamp is back at 5. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "5") + + # Scenario 4: Holding a transaction open without a commit timestamp + # Will not affect the all_committed timestamp. + session1.begin_transaction() + session2.begin_transaction() + cur2[2] = 2 + session2.commit_transaction('commit_timestamp=6') + + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "6") + cur1[1] = 2 + session1.commit_transaction() + + def test_oldest_reader(self): + oldest_reader_uri = self.uri + '_oldest_reader_pinned' + session1 = self.setUpSessionOpen(self.conn) + session2 = self.setUpSessionOpen(self.conn) + session1.create(oldest_reader_uri, 'key_format=i,value_format=i') + session2.create(oldest_reader_uri, 'key_format=i,value_format=i') + + # Nothing is reading so there is no oldest reader. + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.conn.query_timestamp('get=oldest_reader')) + + # Write some data for reading. + session1.begin_transaction() + cur1 = session1.open_cursor(oldest_reader_uri) + cur1[1]=1 + session1.commit_transaction('commit_timestamp=5') + + # No active sessions so no oldest reader. + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.conn.query_timestamp('get=oldest_reader')) + + # Create an active read session. + session1.begin_transaction('read_timestamp=5') + # Oldest reader should now exist and be equal to our read timestamp. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=oldest_reader'), '5') + + # Start transaction without read timestamp specified + # Should not affect the current oldest reader. + session2.begin_transaction() + cur2 = session2.open_cursor(oldest_reader_uri) + cur2[2] = 2 + + self.assertTimestampsEqual( + self.conn.query_timestamp('get=oldest_reader'), '5') + + session2.commit_transaction('commit_timestamp=7') + + # Open read transaction with newer read timestamp, oldest + # Reader should therefore be unchanged. + session2.begin_transaction('read_timestamp=7') + + self.assertTimestampsEqual( + self.conn.query_timestamp('get=oldest_reader'), '5') + + # End current oldest reader transaction, it will have now moved + # up to our transaction created before. + session1.commit_transaction() + + self.assertTimestampsEqual( + self.conn.query_timestamp('get=oldest_reader'), '7') + + session2.commit_transaction() + + # Now that all read transactions have completed we will be back + # to having no oldest reader. + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.conn.query_timestamp('get=oldest_reader')) + + def test_pinned_oldest(self): + pinned_oldest_uri = self.uri + 'pinned_oldest' + session1 = self.setUpSessionOpen(self.conn) + session1.create(pinned_oldest_uri, 'key_format=i,value_format=i') + # Confirm no oldest timestamp exists. + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.conn.query_timestamp('get=oldest')) + + # Confirm no pinned timestamp exists. + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.conn.query_timestamp('get=pinned')) + + # Write some data for reading. + session1.begin_transaction() + cur1 = session1.open_cursor(pinned_oldest_uri) + cur1[1]=1 + session1.commit_transaction('commit_timestamp=5') + + # Confirm no oldest timestamp exists. + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.conn.query_timestamp('get=oldest')) + + # Confirm no pinned timestamp exists. + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.conn.query_timestamp('get=pinned')) + + self.conn.set_timestamp('oldest_timestamp=5') + + # Pinned timestamp should now match oldest timestamp + self.assertTimestampsEqual(self.conn.query_timestamp('get=pinned'), '5') + + # Write some more data for reading. + session1.begin_transaction() + cur1[2]=2 + session1.commit_transaction('commit_timestamp=8') + + # Create an active read session. + session1.begin_transaction('read_timestamp=5') + + # Move oldest timestamp past active read session. + self.conn.set_timestamp('oldest_timestamp=8') + + # Pinned timestamp should now reflect oldest reader. + self.assertTimestampsEqual(self.conn.query_timestamp('get=pinned'), '5') + + # End active read session. + session1.commit_transaction() + + # Pinned timestamp should now match oldest timestamp. + self.assertTimestampsEqual(self.conn.query_timestamp('get=pinned'), '8') + + def test_all(self): + all_uri = self.uri + 'pinned_oldest' + session1 = self.setUpSessionOpen(self.conn) + session2 = self.setUpSessionOpen(self.conn) + session1.create(all_uri, 'key_format=i,value_format=i') + session2.create(all_uri, 'key_format=i,value_format=i') + cur1 = session1.open_cursor(all_uri) + cur2 = session2.open_cursor(all_uri) + # Set up oldest timestamp. + self.conn.set_timestamp('oldest_timestamp=1') + + # Write some data for reading. + + session1.begin_transaction() + cur1[1]=1 + session1.commit_transaction('commit_timestamp=2') + session1.begin_transaction() + cur1[1]=2 + session1.commit_transaction('commit_timestamp=4') + # Confirm all_committed is now 4. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "4") + + # Create a read session. + session1.begin_transaction('read_timestamp=2') + # Confirm oldest reader is 2 and the the value we read is 1. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=oldest_reader'), "2") + + self.assertEqual(cur1[1], 1) + # Commit some data at timestamp 7. + session2.begin_transaction() + cur2[2] = 2 + session2.commit_transaction('commit_timestamp=7') + # All_committed should now be 7. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "7") + + # Move oldest to 5. + self.conn.set_timestamp('oldest_timestamp=5') + + # Confirm pinned timestamp is pointing at oldest_reader. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=pinned'), + self.conn.query_timestamp('get=oldest_reader')) + + # Begin a write transaction pointing at timestamp 6, + # this is below our current all_committed so it should move back + # to the oldest timestamp. + session2.begin_transaction() + session2.timestamp_transaction('commit_timestamp=6') + cur2[2] = 3 + + # Confirm all_committed is now equal to oldest. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), + self.conn.query_timestamp('get=oldest')) + + session2.commit_transaction() + self.assertTimestampsEqual( + self.conn.query_timestamp('get=all_committed'), "7") + # End our read transaction. + session1.commit_transaction() + + # Pinned will now match oldest. + self.assertTimestampsEqual( + self.conn.query_timestamp('get=pinned'), + self.conn.query_timestamp('get=oldest')) + +if __name__ == '__main__': + wttest.run()
\ No newline at end of file diff --git a/src/third_party/wiredtiger/test/suite/test_txn20.py b/src/third_party/wiredtiger/test/suite/test_txn20.py new file mode 100644 index 00000000000..381435ff472 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_txn20.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_txn20.py +# Transactions: more granular testing of isolation levels +# + +import wttest +from wtscenario import make_scenarios + +class test_txn20(wttest.WiredTigerTestCase): + + uri = 'table:test_txn' + iso_types = [ + ('isolation_read_uncommitted', dict(isolation='read-uncommitted')), + ('isolation_read_committed', dict(isolation='read-committed')), + ('isolation_snapshot', dict(isolation='snapshot')) + ] + scenarios = make_scenarios(iso_types) + key = 'key' + old_value = 'value: old' + new_value = 'value: new' + + def test_isolation_level(self): + self.session.create(self.uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(self.uri, None) + cursor[self.key] = self.old_value + + # Make an update and don't commit it just yet. We should see the update + # from the 'read-uncommitted' isolation level. + self.session.begin_transaction() + cursor[self.key] = self.new_value + + s = self.conn.open_session() + cursor = s.open_cursor(self.uri, None) + s.begin_transaction('isolation=' + self.isolation) + + if self.isolation == 'read-uncommitted': + # Unlike the 'read-committed' and 'snapshot' isolation levels, we're + # not protected from dirty reads so we'll see the update above even + # though its respective transaction has not been committed. + self.assertEqual(cursor[self.key], self.new_value) + else: + self.assertEqual(cursor[self.key], self.old_value) + + # Commit the update now. We should see the update from the + # 'read-committed' and 'read-uncommitted' isolation levels. + self.session.commit_transaction() + + if self.isolation == 'snapshot': + # We should never see the updates above since it wasn't committed at + # the time of the snapshot. + self.assertEqual(cursor[self.key], self.old_value) + else: + # Unlike the 'snapshot' isolation level, 'read-committed' is not + # protected from non-repeatable reads so we'll see an update + # that wasn't visible earlier in our previous read. As before, + # 'read-uncommitted' will still see the new value. + self.assertEqual(cursor[self.key], self.new_value) + + # Cleanup. + self.session.close() + s.rollback_transaction() + s.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index 6ecf8513abd..bebfb5f200a 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -166,8 +166,8 @@ testutil_progress(TEST_OPTS *opts, const char *message) uint64_t now; if (opts->progress_fp == NULL) - testutil_assert((opts->progress_fp = - fopen(opts->progress_file_name, "w")) != NULL); + testutil_checksys((opts->progress_fp = + fopen(opts->progress_file_name, "w")) == NULL); fp = opts->progress_fp; __wt_seconds(NULL, &now); |