diff options
author | Luke Chen <luke.chen@mongodb.com> | 2019-04-26 16:30:33 +1000 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2019-04-26 16:30:33 +1000 |
commit | 36ed7dbb3c1cf8f6bc5e55278f9960aa5c2c6c6c (patch) | |
tree | 547ba5881561a8f5a736953c7f327810ae9a12cd | |
parent | fe5ef2303f53773cabd01e0ce7e7cc83965fe211 (diff) | |
download | mongo-36ed7dbb3c1cf8f6bc5e55278f9960aa5c2c6c6c.tar.gz |
Import wiredtiger: 617a81369c810c0b99493a09d3fce862f4d0cdec from branch mongodb-4.2
ref: 9416282c42..617a81369c
for: 4.1.11
WT-4450 Add testing around timestamping updates in the future
WT-4575 Use wt_err instead of wt_errx for corruption messages
WT-4647 Avoid splitting words when wrapping lines in wiredtiger.in
WT-4717 Remove usages of "round_to_oldest" configuration
WT-4720 Coverity complaint #111697: handle tiny sizes in block manager reads
WT-4721 Add verbose output for failures in the wt2853_perf test
WT-4731 Add new retries to the checkpoint integrity test
21 files changed, 374 insertions, 123 deletions
diff --git a/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py b/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py index 3486f40d403..3c039dfebd9 100644 --- a/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py +++ b/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py @@ -82,8 +82,8 @@ conn_config="create,cache_size=2GB,session_max=1000,eviction=(threads_min=4,thre table_config="allocation_size=4k,memory_page_max=10MB,prefix_compression=false,split_pct=90,leaf_page_max=32k,internal_page_max=16k,type=file" compression_opts = { "none" : "block_compressor=none", - "lz4" : "block_compressor=lz4" - "snappy" : "block_compressor=snappy" + "lz4" : "block_compressor=lz4", + "snappy" : "block_compressor=snappy", "zlib" : "block_compressor=zlib", "zlib_onepage" : "block_compressor=zlib,memory_page_image_max=32k", "zlib_tenpage" : "block_compressor=zlib,memory_page_image_max=320k", diff --git a/src/third_party/wiredtiger/dist/api_config.py b/src/third_party/wiredtiger/dist/api_config.py index 2b13097f108..0618599b459 100755 --- a/src/third_party/wiredtiger/dist/api_config.py +++ b/src/third_party/wiredtiger/dist/api_config.py @@ -127,8 +127,12 @@ for line in open(f, 'r'): w = textwrap.TextWrapper(width=80-len(prefix.expandtabs()), break_on_hyphens=False, + break_long_words=False, replace_whitespace=False, fix_sentence_endings=True) + # Separate at spaces, and after a set of non-breaking space indicators. + w.wordsep_re = w.wordsep_simple_re = \ + re.compile(r'(\s+|(?<= )[\w_,.;:]*)') for c in api_data.methods[config_name].config: if 'undoc' in c.flags: continue diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 96009a78193..5b097964fbf 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "9416282c42d40328dfb7ff0f28831f639f98d3cb", + "commit": "617a81369c810c0b99493a09d3fce862f4d0cdec", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 9614e1c2810..0c2ef31254f 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -253,6 +253,16 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, buf->size = size; /* + * Ensure we don't read information that isn't there. It shouldn't ever + * happen, but it's a cheap test. + */ + if (size < block->allocsize) + WT_RET_MSG(session, EINVAL, + "%s: impossibly small block size of %" PRIu32 "B, less than " + "allocation size of %" PRIu32, + block->name, size, block->allocsize); + + /* * We incrementally read through the structure before doing a checksum, * do little- to big-endian handling early on, and then select from the * original or swapped structure as needed. diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index aa26a627a94..655c1649fb8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -639,12 +639,12 @@ __wt_btree_tree_open( * Try to provide a helpful failure message. */ if (ret != 0 && WT_IS_METADATA(session->dhandle)) { - __wt_errx(session, + __wt_err(session, ret, "WiredTiger has failed to open its metadata"); - __wt_errx(session, "This may be due to the database" + __wt_err(session, ret, "This may be due to the database" " files being encrypted, being from an older" " version or due to corruption on disk"); - __wt_errx(session, "You should confirm that you have" + __wt_err(session, ret, "You should confirm that you have" " opened the database with the correct options including" " all encryption and compression options"); } diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 890fbb26a74..51ef34ae9f2 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1349,9 +1349,9 @@ struct __wt_session { * @config{ bloom_bit_count, * the number of bits used per item for LSM bloom filters., an integer * between 2 and 1000; default \c 16.} - * @config{ bloom_config, config string used when - * creating Bloom filter files\, passed to WT_SESSION::create., a - * string; default empty.} + * @config{ + * bloom_config, config string used when creating Bloom filter files\, + * passed to WT_SESSION::create., a string; default empty.} * @config{ bloom_hash_count, the number of hash * values per item used for LSM bloom filters., an integer between 2 and * 100; default \c 8.} @@ -1379,10 +1379,10 @@ struct __wt_session { * @config{ prefix, * custom data source prefix instead of \c "file"., a string; default * empty.} - * @config{ start - * _generation, merge generation at which the custom data source is used - * (zero indicates no custom data source)., an integer between 0 and 10; - * default \c 0.} + * @config{ + * start_generation, merge generation at which the custom data source is + * used (zero indicates no custom data source)., an integer between 0 + * and 10; default \c 0.} * @config{ suffix, * custom data source suffix instead of \c ".lsm"., a string; default * empty.} @@ -1390,10 +1390,10 @@ struct __wt_session { * @config{ merge_max, the * maximum number of chunks to include in a merge operation., an integer * between 2 and 100; default \c 15.} - * @config{ merge_min, the minimum number of - * chunks to include in a merge operation. If set to 0 or 1 half the - * value of merge_max is used., an integer no more than 100; default \c - * 0.} + * @config{ + * merge_min, the minimum number of chunks to include in a merge + * operation. If set to 0 or 1 half the value of merge_max is used., an + * integer no more than 100; default \c 0.} * @config{ ),,} * @config{memory_page_image_max, the maximum in-memory page image * represented by a single storage block. Depending on compression @@ -2017,12 +2017,14 @@ struct __wt_session { * related configuration options defined below.} * @config{ all, drop all named snapshots., a * boolean flag; default \c false.} - * @config{ before, drop all snapshots up to but - * not including the specified name., a string; default empty.} - * @config{ names, drop specific named - * snapshots., a list of strings; default empty.} - * @config{ to, drop all snapshots up to and - * including the specified name., a string; default empty.} + * @config{ + * before, drop all snapshots up to but not including the specified + * name., a string; default empty.} + * @config{ + * names, drop specific named snapshots., a list of strings; default + * empty.} + * @config{ to, drop all snapshots up to + * and including the specified name., a string; default empty.} * @config{ * ),,} * @config{include_updates, make updates from the current transaction @@ -2333,8 +2335,9 @@ struct __wt_connection { * integer between 0 and 100; default \c 0.} * @config{ prealloc, pre-allocate log files., a * boolean flag; default \c true.} - * @config{ zero_fill, manually write zeroes into - * log files., a boolean flag; default \c false.} + * @config{ + * zero_fill, manually write zeroes into log files., a boolean flag; + * default \c false.} * @config{ ),,} * @config{lsm_manager = (, configure database wide options for LSM tree * management. The LSM manager is started automatically the first time @@ -2802,11 +2805,11 @@ struct __wt_connection { * @config{ ops_max, * maximum number of expected simultaneous asynchronous operations., an integer * between 1 and 4096; default \c 1024.} - * @config{ threads, the number of worker threads to - * service asynchronous requests. Each worker thread uses a session from the - * configured session_max., an integer between 1 and 20; default \c 2.} - * @config{ - * ),,} + * @config{ + * threads, the number of worker threads to service asynchronous requests. Each + * worker thread uses a session from the configured session_max., an integer + * between 1 and 20; default \c 2.} + * @config{ ),,} * @config{buffer_alignment, in-memory alignment (in bytes) for buffers used for * I/O. The default value of -1 indicates a platform-specific alignment value * should be used (4KB on Linux systems when direct I/O is configured\, zero @@ -2856,10 +2859,10 @@ struct __wt_connection { * below.} * @config{ release, compatibility release * version string., a string; default empty.} - * @config{ require_max, required maximum compatibility - * version of existing data files. Must be greater than or equal to any release - * version set in the \c release setting. Has no effect if creating the - * database., a string; default empty.} + * @config{ + * require_max, required maximum compatibility version of existing data files. + * Must be greater than or equal to any release version set in the \c release + * setting. Has no effect if creating the database., a string; default empty.} * @config{ require_min, required minimum compatibility * version of existing data files. Must be less than or equal to any release * version set in the \c release setting. Has no effect if creating the @@ -2908,17 +2911,16 @@ struct __wt_connection { * empty.} * @config{eviction = (, eviction configuration options., a set of related * configuration options defined below.} - * @config{ threads_max, maximum number of threads + * @config{ + * threads_max, maximum number of threads WiredTiger will start to help evict + * pages from cache. The number of threads started will vary depending on the + * current eviction load. Each eviction worker thread uses a session from the + * configured session_max., an integer between 1 and 20; default \c 8.} + * @config{ threads_min, minimum number of threads * WiredTiger will start to help evict pages from cache. The number of threads - * started will vary depending on the current eviction load. Each eviction - * worker thread uses a session from the configured session_max., an integer - * between 1 and 20; default \c 8.} - * @config{ threads_min, - * minimum number of threads WiredTiger will start to help evict pages from - * cache. The number of threads currently running will vary depending on the - * current eviction load., an integer between 1 and 20; default \c 1.} - * @config{ - * ),,} + * currently running will vary depending on the current eviction load., an + * integer between 1 and 20; default \c 1.} + * @config{ ),,} * @config{eviction_checkpoint_target, perform eviction at the beginning of * checkpoints to bring the dirty content in cache to this level. It is a * percentage of the cache size if the value is within the range of 0 to 100 or @@ -2967,13 +2969,13 @@ struct __wt_connection { * @config{ close_handle_minimum, number of handles open * before the file manager will look for handles to close., an integer greater * than or equal to 0; default \c 250.} - * @config{ close_idle_time, amount of time in seconds a - * file handle needs to be idle before attempting to close it. A setting of 0 - * means that idle handles are not closed., an integer between 0 and 100000; - * default \c 30.} - * @config{ close_scan_interval, interval - * in seconds at which to check for files that are inactive and close them., an - * integer between 1 and 100000; default \c 10.} + * @config{ + * close_idle_time, amount of time in seconds a file handle needs to be idle + * before attempting to close it. A setting of 0 means that idle handles are + * not closed., an integer between 0 and 100000; default \c 30.} + * @config{ close_scan_interval, interval in seconds at + * which to check for files that are inactive and close them., an integer + * between 1 and 100000; default \c 10.} * @config{ ),,} * @config{in_memory, keep data in-memory only. See @ref in_memory for more * information., a boolean flag; default \c false.} @@ -3025,10 +3027,10 @@ struct __wt_connection { * session_max., a set of related configuration options defined below.} * @config{ merge, merge LSM chunks where possible., a * boolean flag; default \c true.} - * @config{ worker_thread_max, Configure a set of threads - * to manage merging LSM trees in the database. Each worker thread uses a - * session handle from the configured session_max., an integer between 3 and 20; - * default \c 4.} + * @config{ + * worker_thread_max, Configure a set of threads to manage merging LSM trees in + * the database. Each worker thread uses a session handle from the configured + * session_max., an integer between 3 and 20; default \c 4.} * @config{ ),,} * @config{mmap, Use memory mapping to access files when possible., a boolean * flag; default \c true.} diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 1dc6c60a137..90d269c3b3e 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -2602,12 +2602,12 @@ err: WT_STAT_CONN_INCR(session, log_scans); * a helpful failure message. */ if (ret != 0 && firstrecord && LF_ISSET(WT_LOGSCAN_RECOVER)) { - __wt_errx(session, + __wt_err(session, ret, "WiredTiger is unable to read the recovery log."); - __wt_errx(session, "This may be due to the log" + __wt_err(session, ret, "This may be due to the log" " files being encrypted, being from an older" " version or due to corruption on disk"); - __wt_errx(session, "You should confirm that you have" + __wt_err(session, ret, "You should confirm that you have" " opened the database with the correct options including" " all encryption and compression options"); } diff --git a/src/third_party/wiredtiger/src/meta/meta_track.c b/src/third_party/wiredtiger/src/meta/meta_track.c index f411b9675eb..f763c49ef2a 100644 --- a/src/third_party/wiredtiger/src/meta/meta_track.c +++ b/src/third_party/wiredtiger/src/meta/meta_track.c @@ -348,7 +348,8 @@ err: /* WT_ASSERT(session, unroll || saved_ret != 0 || session->txn.mod_count == 0); #ifdef WT_ENABLE_SCHEMA_TXN - __wt_errx(session, "TRACK: Abort internal schema txn"); + __wt_err(session, saved_ret, + "TRACK: Abort internal schema txn"); WT_TRET(__wt_txn_rollback(session, NULL)); #endif } diff --git a/src/third_party/wiredtiger/src/os_common/os_fhandle.c b/src/third_party/wiredtiger/src/os_common/os_fhandle.c index ca2fe730444..387aa0d4aa3 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fhandle.c +++ b/src/third_party/wiredtiger/src/os_common/os_fhandle.c @@ -301,7 +301,6 @@ __handle_close(WT_SESSION_IMPL *session, WT_FH *fh, bool locked) if (fh->ref != 0) { __wt_errx(session, "Closing a file handle with open references: %s", fh->name); - WT_TRET(EBUSY); } /* Remove from the list. */ diff --git a/src/third_party/wiredtiger/src/os_win/os_dir.c b/src/third_party/wiredtiger/src/os_win/os_dir.c index d0734dea0c6..08fae209b33 100644 --- a/src/third_party/wiredtiger/src/os_win/os_dir.c +++ b/src/third_party/wiredtiger/src/os_win/os_dir.c @@ -53,10 +53,11 @@ __directory_list_worker(WT_FILE_SYSTEM *file_system, findhandle = FindFirstFileW(pathbuf_wide->data, &finddata); if (findhandle == INVALID_HANDLE_VALUE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: directory-list: FindFirstFile: %s", pathbuf->data, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); + WT_ERR(ret); } for (count = 0;;) { @@ -89,10 +90,11 @@ skip: if (FindNextFileW(findhandle, &finddata) != 0) windows_error = __wt_getlasterror(); if (windows_error == ERROR_NO_MORE_FILES) break; - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: directory-list: FindNextFileW: %s", pathbuf->data, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); + WT_ERR(ret); } *dirlistp = entries; @@ -101,12 +103,12 @@ skip: if (FindNextFileW(findhandle, &finddata) != 0) err: if (findhandle != INVALID_HANDLE_VALUE) if (FindClose(findhandle) == 0) { windows_error = __wt_getlasterror(); - __wt_errx(session, + if (ret == 0) + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: directory-list: FindClose: %s", pathbuf->data, __wt_formatmessage(session, windows_error)); - if (ret == 0) - ret = __wt_map_windows_error(windows_error); } __wt_free(session, dir_copy); diff --git a/src/third_party/wiredtiger/src/os_win/os_dlopen.c b/src/third_party/wiredtiger/src/os_win/os_dlopen.c index b5a54b9843a..bed886f7a57 100644 --- a/src/third_party/wiredtiger/src/os_win/os_dlopen.c +++ b/src/third_party/wiredtiger/src/os_win/os_dlopen.c @@ -28,10 +28,11 @@ __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) if (GetModuleHandleExW( 0, NULL, (HMODULE *)&dlh->handle) == FALSE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "GetModuleHandleExW: %s: %s", path, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); + WT_ERR(ret); } } else { // TODO: load dll here @@ -54,6 +55,7 @@ int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret) { + WT_DECL_RET; DWORD windows_error; void *sym; @@ -62,11 +64,12 @@ __wt_dlsym(WT_SESSION_IMPL *session, sym = GetProcAddress(dlh->handle, name); if (sym == NULL && fail) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "GetProcAddress: %s in %s: %s", name, dlh->name, __wt_formatmessage(session, windows_error)); - WT_RET(__wt_map_windows_error(windows_error)); + WT_RET(ret); } *(void **)sym_ret = sym; @@ -85,9 +88,9 @@ __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) if (FreeLibrary(dlh->handle) == FALSE) { windows_error = __wt_getlasterror(); - __wt_errx(session, "FreeLibrary: %s: %s", - dlh->name, __wt_formatmessage(session, windows_error)); ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "FreeLibrary: %s: %s", + dlh->name, __wt_formatmessage(session, windows_error)); } __wt_free(session, dlh->name); diff --git a/src/third_party/wiredtiger/src/os_win/os_getenv.c b/src/third_party/wiredtiger/src/os_win/os_getenv.c index 74fecae1ba9..8cd53f9b2d0 100644 --- a/src/third_party/wiredtiger/src/os_win/os_getenv.c +++ b/src/third_party/wiredtiger/src/os_win/os_getenv.c @@ -15,6 +15,7 @@ int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) { + WT_DECL_RET; DWORD size, windows_error; *envp = NULL; @@ -29,8 +30,9 @@ __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) return (0); windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "GetEnvironmentVariableA: %s: %s", variable, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } diff --git a/src/third_party/wiredtiger/src/os_win/os_map.c b/src/third_party/wiredtiger/src/os_win/os_map.c index 209ef6718c2..30964791615 100644 --- a/src/third_party/wiredtiger/src/os_win/os_map.c +++ b/src/third_party/wiredtiger/src/os_win/os_map.c @@ -16,6 +16,7 @@ int __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) { + WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; wt_off_t file_size; @@ -42,25 +43,27 @@ __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, win_fh->filehandle, NULL, PAGE_READONLY, 0, 0, NULL); if (mapped_cookie == NULL) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: memory-map: CreateFileMappingW: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } if ((map = MapViewOfFile(mapped_cookie, FILE_MAP_READ, 0, 0, len)) == NULL) { /* Retrieve the error before cleaning up. */ windows_error = __wt_getlasterror(); + ret = __wt_map_windows_error(windows_error); (void)CloseHandle(mapped_cookie); - __wt_errx(session, + __wt_err(session, ret, "%s: memory-map: MapViewOfFile: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } *(void **)mapped_cookiep = mapped_cookie; @@ -91,20 +94,20 @@ __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, if (UnmapViewOfFile(mapped_region) == 0) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: memory-unmap: UnmapViewOfFile: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - ret = __wt_map_windows_error(windows_error); } if (CloseHandle(*(void **)mapped_cookie) == 0) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: memory-unmap: CloseHandle: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - ret = __wt_map_windows_error(windows_error); } return (ret); diff --git a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c index 2a971572293..a91c409e1b0 100644 --- a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c @@ -120,7 +120,9 @@ skipping: *signalled = false; if (sleepret != 0) return; - __wt_errx(session, "SleepConditionVariableCS: %s: %s", + __wt_err(session, + __wt_map_windows_error(windows_error), + "SleepConditionVariableCS: %s: %s", cond->name, __wt_formatmessage(session, windows_error)); WT_PANIC_MSG(session, __wt_map_windows_error(windows_error), "SleepConditionVariableCS: %s", cond->name); diff --git a/src/third_party/wiredtiger/src/os_win/os_thread.c b/src/third_party/wiredtiger/src/os_win/os_thread.c index fde03cbd13e..6524f8f23b3 100644 --- a/src/third_party/wiredtiger/src/os_win/os_thread.c +++ b/src/third_party/wiredtiger/src/os_win/os_thread.c @@ -40,6 +40,7 @@ __wt_thread_create(WT_SESSION_IMPL *session, int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t *tid) { + WT_DECL_RET; DWORD windows_error; /* Only attempt to join if thread was created successfully */ @@ -58,7 +59,8 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t *tid) WaitForSingleObject(tid->id, INFINITE)) != WAIT_OBJECT_0) { if (windows_error == WAIT_FAILED) windows_error = __wt_getlasterror(); - __wt_errx(session, "thread join: WaitForSingleObject: %s", + __wt_err(session, __wt_map_windows_error(windows_error), + "thread join: WaitForSingleObject: %s", __wt_formatmessage(session, windows_error)); /* If we fail to wait, we will leak handles, do not continue. */ @@ -67,9 +69,10 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t *tid) if (CloseHandle(tid->id) == 0) { windows_error = __wt_getlasterror(); - __wt_errx(session, "thread join: CloseHandle: %s", + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "thread join: CloseHandle: %s", __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } return (0); diff --git a/src/third_party/wiredtiger/src/os_win/os_utf8.c b/src/third_party/wiredtiger/src/os_win/os_utf8.c index c136a28dab2..f27203be353 100644 --- a/src/third_party/wiredtiger/src/os_win/os_utf8.c +++ b/src/third_party/wiredtiger/src/os_win/os_utf8.c @@ -16,6 +16,7 @@ int __wt_to_utf16_string( WT_SESSION_IMPL *session, const char *utf8, WT_ITEM **outbuf) { + WT_DECL_RET; DWORD windows_error; int bufferSize; @@ -23,9 +24,10 @@ __wt_to_utf16_string( windows_error = __wt_getlasterror(); if (bufferSize == 0 && windows_error != ERROR_INSUFFICIENT_BUFFER) { - __wt_errx(session, "MultiByteToWideChar: %s", + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "MultiByteToWideChar: %s", __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } WT_RET(__wt_scr_alloc(session, bufferSize * sizeof(wchar_t), outbuf)); @@ -35,9 +37,10 @@ __wt_to_utf16_string( if (bufferSize == 0) { windows_error = __wt_getlasterror(); __wt_scr_free(session, outbuf); - __wt_errx(session, "MultiByteToWideChar: %s", + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "MultiByteToWideChar: %s", __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } (*outbuf)->size = bufferSize; @@ -52,6 +55,7 @@ int __wt_to_utf8_string( WT_SESSION_IMPL *session, const wchar_t *wide, WT_ITEM **outbuf) { + WT_DECL_RET; DWORD windows_error; int bufferSize; @@ -60,9 +64,10 @@ __wt_to_utf8_string( windows_error = __wt_getlasterror(); if (bufferSize == 0 && windows_error != ERROR_INSUFFICIENT_BUFFER) { - __wt_errx(session, "WideCharToMultiByte: %s", + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "WideCharToMultiByte: %s", __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } WT_RET(__wt_scr_alloc(session, bufferSize, outbuf)); @@ -72,9 +77,10 @@ __wt_to_utf8_string( if (bufferSize == 0) { windows_error = __wt_getlasterror(); __wt_scr_free(session, outbuf); - __wt_errx(session, "WideCharToMultiByte: %s", + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "WideCharToMultiByte: %s", __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } (*outbuf)->size = bufferSize; diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index c45afbf5730..9e545063e2b 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -532,11 +532,6 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) if (cval.val) F_SET(txn, WT_TXN_IGNORE_PREPARE); - /* Check if read timestamp to be rounded up to the oldest timestamp. */ - WT_RET(__wt_config_gets_def(session, cfg, "round_to_oldest", 0, &cval)); - if (cval.val) - F_SET(txn, WT_TXN_TS_ROUND_READ); - /* * Check if the prepare timestamp and the commit timestamp of a * prepared transaction need to be rounded up. diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 50d24778ffb..438dad5359d 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -950,16 +950,10 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) { WT_CONFIG_ITEM cval; WT_DECL_RET; - WT_TXN *txn = &session->txn; wt_timestamp_t ts; WT_TRET(__wt_txn_context_check(session, true)); - /* Look for round_to_oldest configuration. */ - ret = __wt_config_gets_def(session, cfg, "round_to_oldest", 0, &cval); - if (cval.val) - F_SET(txn, WT_TXN_TS_ROUND_READ); - /* Look for a commit timestamp. */ ret = __wt_config_gets_def(session, cfg, "commit_timestamp", 0, &cval); WT_RET_NOTFOUND_OK(ret); diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index 1f14056eca1..be6811b38af 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -193,6 +193,16 @@ main(int argc, char *argv[]) nfail += get_args[i].nfail; } + /* + * Note that slow machines can be skipped for this test. + * See the bypass code earlier. + */ + if (nfail != 0) + fprintf(stderr, + "ERROR: %d failures when a single commit" + " took more than %d seconds.\n" + "This may indicate a real problem or a" + " particularly slow machine.\n", nfail, MAX_GAP); testutil_assert(nfail == 0); testutil_progress(opts, "cleanup starting"); testutil_cleanup(opts); diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index 41fe6cf953c..bec69105679 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -79,7 +79,8 @@ #define MAX_OP_RANGE 1000 #define STDERR_FILE "stderr.txt" #define STDOUT_FILE "stdout.txt" -#define TESTS_PER_OP_VALUE 3 +#define TESTS_PER_CALIBRATION 2 +#define TESTS_WITH_RECALIBRATION 5 #define VERBOSE_PRINT 10000 static int check_results(TEST_OPTS *, uint64_t *); @@ -93,7 +94,8 @@ static void generate_value(uint32_t, uint64_t, char *, int *, int *, int *, char **); static void run_check_subtest(TEST_OPTS *, const char *, uint64_t, bool, uint64_t *); -static void run_check_subtest_range(TEST_OPTS *, const char *, bool); +static int run_check_subtest_range(TEST_OPTS *, const char *, bool); +static void run_check_subtest_range_retry(TEST_OPTS *, const char *, bool); static int run_process(TEST_OPTS *, const char *, char *[], int *); static void subtest_main(int, char *[], bool); static void subtest_populate(TEST_OPTS *, bool); @@ -353,17 +355,16 @@ run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops, /* * run_check_subtest_range -- + * Run successive tests via binary search that determines the approximate + * crossover point between when data is recoverable or not. Once that is + * determined, run the subtest in a range near that crossover point. * - * Run successive tests via binary search that determines the approximate - * crossover point between when data is recoverable or not. Once that is - * determined, run the subtest in a range near that crossover point. - * - * The theory is that running at the crossover point will tend to trigger - * "interesting" failures at the borderline when the checkpoint is about to, - * or has, succeeded. If any of those failures creates a WT home directory - * that cannot be recovered, the top level test will fail. - */ -static void + * The theory is that running at the crossover point will tend to trigger + * "interesting" failures at the borderline when the checkpoint is about + * to, or has, succeeded. If any of those failures creates a WiredTiger + * home directory that cannot be recovered, the top level test will fail. + */ +static int run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) { uint64_t cutoff, high, low, mid, nops, nresults; @@ -402,7 +403,7 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) got_failure = false; got_success = false; for (i = 0; - i < TESTS_PER_OP_VALUE && (!got_failure || !got_success); i++) + i < TESTS_PER_CALIBRATION && (!got_failure || !got_success); i++) for (nops = mid - 10; nops < mid + 10; nops++) { run_check_subtest(opts, debugger, nops, close_test, &nresults); @@ -414,9 +415,51 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) /* * Check that it really ran with a crossover point. + * If not, perhaps we calibrated the range incorrectly. + * Tell caller to try again. */ - testutil_assert(got_failure); - testutil_assert(got_success); + if (!got_failure || !got_success) { + fprintf(stderr, "Warning: did not find a reliable test range.\n" + "midpoint=%" PRIu64 ", close_test=%d, got_failure=%d, " + "got_success=%d\n", mid, (int)close_test, (int)got_failure, + (int)got_success); + return (EAGAIN); + } + return (0); +} + +/* + * run_check_subtest_range_retry -- + * Repeatedly run the subtest range test, retrying some number of times + * as long as EBUSY is returned, a warning that the test did not + * adequately cover "both sides" of the test threshold. Such warning + * returns should be rare and are not hard failures, no WiredTiger bug + * is demonstrated. Rerunning the subtest range test will determine + * a new calibration for the range. + */ +static void +run_check_subtest_range_retry(TEST_OPTS *opts, const char *debugger, + bool close_test) +{ + WT_DECL_RET; + int tries; + + for (tries = 0; tries < TESTS_WITH_RECALIBRATION; tries++) { + if (tries != 0) { + fprintf(stderr, "Retrying after sleep...\n"); + sleep(5); + } + if ((ret = run_check_subtest_range( + opts, debugger, close_test)) == 0) + break; + testutil_assert(ret == EAGAIN); + } + if (tries == TESTS_WITH_RECALIBRATION) + /* + * If we couldn't successfully perform the test, + * we want to know about it. + */ + testutil_die(ret, "too many retries"); } /* @@ -679,8 +722,8 @@ main(int argc, char *argv[]) " (change with -n N)\n", opts->nrecords); } if (opts->nops == 0) { - run_check_subtest_range(opts, debugger, false); - run_check_subtest_range(opts, debugger, true); + run_check_subtest_range_retry(opts, debugger, false); + run_check_subtest_range_retry(opts, debugger, true); } else run_check_subtest(opts, debugger, opts->nops, opts->nrecords, &nresults); diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp17.py b/src/third_party/wiredtiger/test/suite/test_timestamp17.py new file mode 100644 index 00000000000..cbda72af39c --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp17.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_timestamp17.py +# Test unintended timestamp usage on an update and ensure behaviour +# matches expectations. Additionally move the timestamp to ensure +# that values read are still consistent after those timestamps are +# moved. +# + +import random +from suite_subprocess import suite_subprocess +import wiredtiger, wttest +from wtscenario import make_scenarios + +def timestamp_str(t): + return '%x' % t + +class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_timestamp17' + uri = 'table:' + tablename + + def test_inconsistent_timestamping(self): + self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.begin_transaction() + cur1 = self.session.open_cursor(self.uri) + cur1[1] = 1 + self.session.commit_transaction('commit_timestamp=25') + + self.session.begin_transaction() + cur1[1] = 2 + self.session.commit_transaction('commit_timestamp=50') + + self.session.begin_transaction() + cur1[1] = 3 + self.session.commit_transaction('commit_timestamp=200') + + self.session.begin_transaction() + cur1.set_key(1) + cur1.remove() + self.session.commit_transaction('commit_timestamp=100') + + # Read before any updates and ensure we cannot find the key or value. + self.session.begin_transaction('read_timestamp=20') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + self.session.commit_transaction() + + # Read at 25 and we should see 1. + self.session.begin_transaction('read_timestamp=25') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, 0) + value1 = cur1.get_value() + self.session.commit_transaction() + self.assertEqual(1, value1) + + # Read at 50 and we should see 2. + self.session.begin_transaction('read_timestamp=50') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, 0) + value1 = cur1.get_value() + self.session.commit_transaction() + self.assertEqual(2, value1) + + # Read at 100 and we should not find anything. + self.session.begin_transaction('read_timestamp=100') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + self.session.commit_transaction() + + # Read at 200 and we should still not find anything. + self.session.begin_transaction('read_timestamp=200') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + self.session.commit_transaction() + + # Read at 300 for further validation. + self.session.begin_transaction('read_timestamp=300') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + self.session.commit_transaction() + + # Move oldest timestamp forward and + # confirm we see the correct numbers. + self.conn.set_timestamp('oldest_timestamp=49') + + # Read at 49 and we should see 1. + self.session.begin_transaction('read_timestamp=49') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, 0) + value1 = cur1.get_value() + self.session.commit_transaction() + self.assertEqual(1, value1) + + self.conn.set_timestamp('oldest_timestamp=99') + + # Read at 99 and we should see 2. + self.session.begin_transaction('read_timestamp=99') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, 0) + value1 = cur1.get_value() + self.session.commit_transaction() + self.assertEqual(2, value1) + + # Move oldest to the point at which we deleted. + self.conn.set_timestamp('oldest_timestamp=100') + + # Read at 100 and we should not find anything. + self.session.begin_transaction('read_timestamp=100') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + self.session.commit_transaction() + + # Read at 200 and we should not find anything. + self.session.begin_transaction('read_timestamp=200') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + self.session.commit_transaction() + + # Move oldest timestamp to 200 to ensure history + # works as expected and we do not see the value 3. + self.conn.set_timestamp('oldest_timestamp=200') + + self.session.begin_transaction('read_timestamp=200') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + self.session.commit_transaction() + + self.session.begin_transaction('read_timestamp=250') + cur1.set_key(1) + search_success = cur1.search() + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + self.session.commit_transaction() + +if __name__ == '__main__': + wttest.run() |