diff options
author | Luke Chen <luke.chen@mongodb.com> | 2018-08-14 16:16:26 +1000 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2018-08-14 16:16:26 +1000 |
commit | 957f4b0daeb542269b62982e2eba2f73e699fe42 (patch) | |
tree | d7ae2551288259234de77b0060f540531cd31551 /src/third_party/wiredtiger | |
parent | 5b523ccf4a5df3021dc025a05853879fda05cd0a (diff) | |
download | mongo-957f4b0daeb542269b62982e2eba2f73e699fe42.tar.gz |
Import wiredtiger: aeeee2053a380ebd2ba90f606a56176737882aad from branch mongodb-4.0
ref: 2bb717625d..aeeee2053a
for: 4.0.2
WT-3856 Create a test that runs recovery to different points of time with schema operations
WT-4026 Add implementation for existing file extension configuration API
WT-4193 test/format snapshot-isolation search mismatch
WT-4194 Improve fairness of eviction with multiple tables
WT-4207 Coverity #1394567: null pointer dereference
WT-4210 schema abort child process failing prematurely
WT-4215 Allow recovery of backup without salvage
WT-4229 Lint
WT-4234 Remove documentation mention of legacy tool statlog.py
WT-4235 Fix workgen tracking of table state across workloads
WT-4242 New log file extension Python test failure
Diffstat (limited to 'src/third_party/wiredtiger')
35 files changed, 521 insertions, 118 deletions
diff --git a/src/third_party/wiredtiger/.gitignore b/src/third_party/wiredtiger/.gitignore index 4a9e098a17f..b57a37aad83 100644 --- a/src/third_party/wiredtiger/.gitignore +++ b/src/third_party/wiredtiger/.gitignore @@ -104,6 +104,7 @@ _wiredtiger.pyd **/test/checkpoint/t **/test/csuite/test_random_abort **/test/csuite/test_rwlock +**/test/csuite/test_schema_abort **/test/csuite/test_scope **/test/csuite/test_timestamp_abort **/test/csuite/test_truncated_log diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx index 74393b13681..9ae63682f9c 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx +++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx @@ -254,12 +254,11 @@ ContextInternal::~ContextInternal() { } int ContextInternal::create_all() { - if (_runtime_alloced != _tint_last) { + if (_runtime_alloced < _tint_last) { // The array references are 1-based, we'll waste one entry. TableRuntime *new_table_runtime = new TableRuntime[_tint_last + 1]; - memcpy(new_table_runtime, _table_runtime, sizeof(uint64_t) * _runtime_alloced); - memset(&new_table_runtime[_runtime_alloced], 0, - sizeof(uint64_t) * (_tint_last - _runtime_alloced + 1)); + for (int i = 0; i < _runtime_alloced; i++) + new_table_runtime[i + 1] = _table_runtime[i + 1]; delete _table_runtime; _table_runtime = new_table_runtime; _runtime_alloced = _tint_last; diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 66825754a6a..36f78816b84 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -708,7 +708,8 @@ wiredtiger_open_log_configuration = [ information'''), Config('file_max', '100MB', r''' the maximum size of log files''', - min='100KB', max='2GB'), + min='100KB', # !!! Must match WT_LOG_FILE_MIN + max='2GB'), # !!! Must match WT_LOG_FILE_MAX Config('path', '"."', r''' the name of a directory into which log files are written. The directory must already exist. If the value is not an absolute path, @@ -862,7 +863,11 @@ wiredtiger_open_common =\ file extension configuration. If set, extend files of the set type in allocations of the set size, instead of a block at a time as each new block is written. For example, - <code>file_extend=(data=16MB)</code>''', + <code>file_extend=(data=16MB)</code>. If set to 0, disable the file + extension for the set type. For log files, the allowed range is + between 100KB and 2GB; values larger than the configured maximum log + size and the default config would extend log files in allocations of + the maximum log file size.''', type='list', choices=['data', 'log']), Config('hazard_max', '1000', r''' maximum number of simultaneous hazard pointers per session diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index e7f5dc2f681..333541535df 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "2bb717625d1e81e6a0abfb910ead00afdde7fe2a", + "commit": "aeeee2053a380ebd2ba90f606a56176737882aad", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.0" diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index b6172966eae..75d868eed08 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -1414,10 +1414,6 @@ __wt_btcur_modify(WT_CURSOR_BTREE *cbt, WT_MODIFY *entries, int nentries) /* Save the cursor state. */ __cursor_state_save(cursor, &state); - if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) - WT_ERR_MSG(session, ENOTSUP, - "not supported in read-uncommitted transactions"); - /* * Get the current value and apply the modification to it, for a few * reasons: first, we set the updated value so the application can @@ -1428,7 +1424,23 @@ __wt_btcur_modify(WT_CURSOR_BTREE *cbt, WT_MODIFY *entries, int nentries) * trouble if we attempt to modify a value that doesn't exist. For the * fifth reason, verify we're not in a read-uncommitted transaction, * that implies a value that might disappear out from under us. + * + * Also, an application might read a value outside of a transaction and + * then call modify. For that to work, the read must be part of the + * transaction that performs the update for correctness, otherwise we + * could race with another thread and end up modifying the wrong value. + * A clever application could get this right (imagine threads that only + * updated non-overlapping, fixed-length byte strings), but it's unsafe + * because it will work most of the time and the failure is unlikely to + * be detected. Require explicit transactions for modify operations. */ + if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) + WT_ERR_MSG(session, ENOTSUP, + "not supported in read-uncommitted transactions"); + if (F_ISSET(&session->txn, WT_TXN_AUTOCOMMIT)) + WT_ERR_MSG(session, ENOTSUP, + "not supported in implicit transactions"); + if (!F_ISSET(cursor, WT_CURSTD_KEY_INT) || !F_ISSET(cursor, WT_CURSTD_VALUE_INT)) WT_ERR(__wt_btcur_search(cbt)); diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 58853bff6ac..7f3391984b2 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -553,7 +553,7 @@ err: /* int __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) { @@ -683,7 +683,7 @@ read: /* */ #ifdef HAVE_DIAGNOSTIC WT_RET( - __wt_hazard_set(session, ref, &busy, file, line)); + __wt_hazard_set(session, ref, &busy, func, line)); #else WT_RET(__wt_hazard_set(session, ref, &busy)); #endif diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index 6434142824e..3dde08b43c1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -443,8 +443,8 @@ restart: /* goto done; } - WT_ASSERT(session, ret == WT_NOTFOUND); - WT_ERR_NOTFOUND_OK(ret); + WT_ASSERT(session, ret == WT_NOTFOUND); + WT_ERR_NOTFOUND_OK(ret); __wt_spin_backoff(&yield_count, &sleep_usecs); } diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 47ab622a7f4..3d06bac02c5 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2626,6 +2626,12 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, #endif WT_ERR(__wt_config_gets(session, cfg, "file_extend", &cval)); + /* + * If the log extend length is not set use the default of the configured + * maximum log file size. That size is not known until it is initialized + * as part of the log server initialization. + */ + conn->log_extend_len = WT_CONFIG_UNSET; for (ft = file_types; ft->name != NULL; ft++) { ret = __wt_config_subgets(session, &cval, ft->name, &sval); if (ret == 0) { @@ -2634,7 +2640,21 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, conn->data_extend_len = sval.val; break; case WT_DIRECT_IO_LOG: - conn->log_extend_len = sval.val; + /* + * When using "file_extend=(log=)", the val + * returned is 1. Unset the log extend length + * in that case to use the default. + */ + if (sval.val == 1) + conn->log_extend_len = WT_CONFIG_UNSET; + else if (sval.val == 0 || + (sval.val >= WT_LOG_FILE_MIN && + sval.val <= WT_LOG_FILE_MAX)) + conn->log_extend_len = sval.val; + else + WT_ERR_MSG(session, EINVAL, + "invalid log extend length: %" + PRId64, sval.val); break; } } else diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index d42789c809e..9f4af1aba88 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -277,6 +277,15 @@ __logmgr_config( if (!reconfig) { WT_RET(__wt_config_gets(session, cfg, "log.file_max", &cval)); conn->log_file_max = (wt_off_t)cval.val; + /* + * With the default log file extend configuration or if the log + * file extension size is larger than the configured maximum log + * file size, set the log file extension size to the configured + * maximum log file size. + */ + if (conn->log_extend_len == WT_CONFIG_UNSET || + conn->log_extend_len > conn->log_file_max) + conn->log_extend_len = conn->log_file_max; WT_STAT_CONN_SET(session, log_max_filesize, conn->log_file_max); } @@ -890,7 +899,6 @@ __log_wrlsn_server(void *arg) __wt_log_wrlsn(session, NULL); if (0) { err: WT_PANIC_MSG(session, ret, "log wrlsn server error"); - } return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/src/docs/statistics.dox b/src/third_party/wiredtiger/src/docs/statistics.dox index 26c4b66fa40..19b7b17257b 100644 --- a/src/third_party/wiredtiger/src/docs/statistics.dox +++ b/src/third_party/wiredtiger/src/docs/statistics.dox @@ -156,9 +156,4 @@ currently open in the database, nor will any statistics requiring the traversal of a tree (as if the \c statistics_fast configuration string were set). -A Python script that parses the default logging output and uses the -<a href="http://www.gnuplot.info/">gnuplot</a>, utility to generate -Portable Network Graphics (PNG) format graphs is included in the -WiredTiger distribution in the file \c tools/statlog.py. - */ diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index 7e89d23230f..0d8e5e1b428 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,4 +1,20 @@ /*! @page upgrading Upgrading WiredTiger applications + +</dl><hr> +@section version_311 Upgrading to Version 3.1.1 +<dl> + +<dt>WT_CURSOR::modify transaction requirements</dt> +<dd> +In previous releases of WiredTiger, it was possible to use implicit +transactions in combination with WT_CURSOR::modify operations. This +requires applications be extraordinarily careful to avoid multiple +threads which are changing the same values racing with each other. In +the 3.1.1 release, WT_CURSOR::modify operations must be performed in an +explicit transaction, and will fail if that's not the case. +</dd> + +</dl><hr> @section version_310 Upgrading to Version 3.1.0 <dl> diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 13b5dfc4c8d..b072c47a9ba 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -318,7 +318,7 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_ERR(__evict_lru_pages(session, false)); if (0) { -err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); +err: WT_PANIC_RET(session, ret, "cache eviction thread error"); } return (ret); } @@ -357,7 +357,7 @@ __wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_VERB_EVICTSERVER, "%s", "cache eviction thread exiting"); if (0) { -err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); +err: WT_PANIC_RET(session, ret, "cache eviction thread error"); } return (ret); } @@ -810,10 +810,8 @@ __evict_clear_walk(WT_SESSION_IMPL *session) cache = S2C(session)->cache; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_PASS)); - if (session->dhandle == cache->walk_tree) { + if (session->dhandle == cache->walk_tree) cache->walk_tree = NULL; - cache->walk_target = 0; - } if ((ref = btree->evict_ref) == NULL) return (0); @@ -1414,10 +1412,8 @@ retry: while (slot < max_entries) { */ if ((dhandle = cache->walk_tree) != NULL) cache->walk_tree = NULL; - else { + else dhandle = TAILQ_FIRST(&conn->dhqh); - cache->walk_target = 0; - } } else { if (incr) { WT_ASSERT(session, dhandle->session_inuse > 0); @@ -1427,7 +1423,6 @@ retry: while (slot < max_entries) { cache->walk_tree = NULL; } dhandle = TAILQ_NEXT(dhandle, q); - cache->walk_target = 0; } /* If we reach the end of the list, we're done. */ @@ -1601,8 +1596,7 @@ __evict_push_candidate(WT_SESSION_IMPL *session, * Calculate how many pages to queue for a given tree. */ static uint32_t -__evict_walk_target( - WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_entries) +__evict_walk_target(WT_SESSION_IMPL *session, u_int max_entries) { WT_CACHE *cache; uint64_t btree_inuse, bytes_per_slot, cache_inuse; @@ -1611,7 +1605,7 @@ __evict_walk_target( cache = S2C(session)->cache; target_pages_clean = target_pages_dirty = 0; - total_slots = max_entries - queue->evict_entries; + total_slots = max_entries; /* * The number of times we should fill the queue by the end of @@ -1717,14 +1711,13 @@ __evict_walk_tree(WT_SESSION_IMPL *session, */ start = queue->evict_queue + *slotp; remaining_slots = max_entries - *slotp; - if (cache->walk_target != 0) { - WT_ASSERT(session, cache->walk_progress <= cache->walk_target); - target_pages = cache->walk_target - cache->walk_progress; - } else { - target_pages = cache->walk_target = - __evict_walk_target(session, queue, max_entries); - cache->walk_progress = 0; + if (btree->evict_walk_progress >= btree->evict_walk_target) { + btree->evict_walk_target = + __evict_walk_target(session, max_entries); + btree->evict_walk_progress = 0; } + target_pages = WT_MIN(btree->evict_walk_target / QUEUE_FILLS_PER_PASS, + btree->evict_walk_target - btree->evict_walk_progress); if (target_pages > remaining_slots) target_pages = remaining_slots; @@ -2017,7 +2010,7 @@ fast: /* If the page can't be evicted, give up. */ continue; ++evict; ++pages_queued; - ++cache->walk_progress; + ++btree->evict_walk_progress; __wt_verbose(session, WT_VERB_EVICTSERVER, "select: %p, size %" WT_SIZET_FMT, diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 2a5be782b06..93e1d8e44ea 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -192,6 +192,8 @@ struct __wt_btree { */ WT_REF *evict_ref; /* Eviction thread's location */ uint64_t evict_priority; /* Relative priority of cached pages */ + uint32_t evict_walk_progress;/* Eviction walk progress */ + uint32_t evict_walk_target; /* Eviction walk target */ u_int evict_walk_period; /* Skip this many LRU walks */ u_int evict_walk_saved; /* Saved walk skips for checkpoints */ u_int evict_walk_skips; /* Number of walks skipped */ diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index ed3480a40d0..86159e971db 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1668,7 +1668,7 @@ static inline int __wt_page_swap_func( WT_SESSION_IMPL *session, WT_REF *held, WT_REF *want, uint32_t flags #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) { @@ -1691,7 +1691,7 @@ __wt_page_swap_func( /* Get the wanted page. */ ret = __wt_page_in_func(session, want, flags #ifdef HAVE_DIAGNOSTIC - , file, line + , func, line #endif ); diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 7d07e6dfd98..8afedb30832 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -149,7 +149,6 @@ struct __wt_cache { WT_SPINLOCK evict_pass_lock; /* Eviction pass lock */ WT_SESSION_IMPL *walk_session; /* Eviction pass session */ WT_DATA_HANDLE *walk_tree; /* LRU walk current tree */ - uint32_t walk_progress, walk_target;/* Progress in current tree */ WT_SPINLOCK evict_queue_lock; /* Eviction current queue lock */ WT_EVICT_QUEUE evict_queues[WT_EVICT_QUEUE_MAX]; diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 06847117b7d..3ae4fe2aaea 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -50,6 +50,7 @@ struct __wt_config_parser_impl { "", 0, 0, WT_CONFIG_ITEM_NUM \ } +#define WT_CONFIG_UNSET -1 /* * DO NOT EDIT: automatically built by dist/api_config.py. * configuration section: BEGIN diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index 6f28dfae2c8..1d2d21617a6 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -98,7 +98,7 @@ /* Called on unexpected code path: locate the failure. */ #define __wt_illegal_value(session, v) \ - __wt_illegal_value_func(session, (uintmax_t)v, __func__, __LINE__) + __wt_illegal_value_func(session, (uintmax_t)(v), __func__, __LINE__) /* Return and branch-to-err-label cases for switch statements. */ #define WT_ILLEGAL_VALUE(session, v) \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 34cee52df99..bebc3ad7bc1 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -163,7 +163,7 @@ extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBU extern int __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ); extern int __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -675,8 +675,8 @@ extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_ extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep); extern void __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler); -extern void __wt_err_func(WT_SESSION_IMPL *session, int error, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_errx_func(WT_SESSION_IMPL *session, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_err_func(WT_SESSION_IMPL *session, int error, const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_errx_func(WT_SESSION_IMPL *session, const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_set_return_func(WT_SESSION_IMPL *session, const char*func, int line, int err) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_err_printf(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)); @@ -708,7 +708,7 @@ extern uint64_t __wt_hash_fnv64(const void *string, size_t len); extern int __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ); extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -757,7 +757,7 @@ extern const char *__wt_buf_set_size(WT_SESSION_IMPL *session, uint64_t size, bo extern int __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index 2613808e74e..d3e36d37da9 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -87,6 +87,12 @@ union __wt_lsn { #define WT_LOGC_KEY_FORMAT WT_UNCHECKED_STRING(III) #define WT_LOGC_VALUE_FORMAT WT_UNCHECKED_STRING(qIIIuu) +/* + * Size range for the log files. + */ +#define WT_LOG_FILE_MAX ((int64_t)2 * WT_GIGABYTE) +#define WT_LOG_FILE_MIN (100 * WT_KILOBYTE) + #define WT_LOG_SKIP_HEADER(data) \ ((const uint8_t *)(data) + offsetof(WT_LOG_RECORD, record)) #define WT_LOG_REC_SIZE(size) \ diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 1180c641222..d7540c333e7 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -352,7 +352,7 @@ union __wt_rand_state { * buffer. If not concatenating, clear the size so we don't use \ * any existing contents. \ */ \ - if (!concatenate) \ + if (!(concatenate)) \ (buf)->size = 0; \ for (;;) { \ WT_ASSERT(session, (buf)->memsize >= (buf)->size); \ diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index cbf572f9a23..10ff7bd48dc 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -25,7 +25,7 @@ struct __wt_data_handle_cache { struct __wt_hazard { WT_REF *ref; /* Page reference */ #ifdef HAVE_DIAGNOSTIC - const char *file; /* File/line where hazard acquired */ + const char *func; /* Function/line hazard acquired */ int line; #endif }; @@ -120,7 +120,7 @@ struct __wt_session_impl { * to applications, create a parallel structure instead. */ struct __wt_scratch_track { - const char *file; /* Allocating file, line */ + const char *func; /* Allocating function, line */ int line; } *scratch_track; #endif diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index c80aaef53fb..99adacb0a1b 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -495,10 +495,22 @@ struct __wt_cursor { * \c S), or raw byte arrays accessed using a WT_ITEM structure (value * format type \c u). * - * Calling the WT_CURSOR::modify method outside of snapshot isolation - * can lead to unexpected results. While \c read-committed isolation - * is supported with the WT_CURSOR::modify method, \c read-uncommitted - * isolation is not. + * The WT_CURSOR::modify method can only be called from within an + * explicit transaction configured at a higher isolation level than + * \c read-uncommitted. Using \c read-committed isolation is allowed, + * but requires caution: reading a value, re-positioning the cursor + * and then modifying the value based on the initial read could lead + * to unexpected results. Using \c snapshot isolation is recommended. + * + * The WT_CURSOR::modify method stores a change record in cache and + * writes a change record to the log instead of the usual complete + * values. Note that WT_CURSOR::modify is generally slower than the + * WT_CURSOR::update method, and can result in slower reads because + * the complete value must be assembled during retrieval. The + * WT_CURSOR::modify method is intended for applications modifying + * large records where there is cache or I/O pressure, that is, + * applications that will benefit when data updates require less cache + * and they write less logging information. * * @snippet ex_all.c Modify an existing record * @@ -510,12 +522,6 @@ struct __wt_cursor { * (as it partially depends on the underlying file configuration), but * is always a small number of bytes less than 4GB. * - * The WT_CURSOR::modify method stores a change record in cache and - * writes a change record to the log, instead of the usual complete - * value. This can reduce cache and logging requirements, but may result - * in slower reads because the complete value must be assembled during - * retrieval. - * * @param cursor the cursor handle * @param entries an array of modification data structures * @param nentries the number of modification data structures @@ -2870,8 +2876,12 @@ struct __wt_connection { * @config{file_extend, file extension configuration. If set\, extend files of * the set type in allocations of the set size\, instead of a block at a time as * each new block is written. For example\, - * <code>file_extend=(data=16MB)</code>., a list\, with values chosen from the - * following options: \c "data"\, \c "log"; default empty.} + * <code>file_extend=(data=16MB)</code>. If set to 0\, disable the file + * extension for the set type. For log files\, the allowed range is between + * 100KB and 2GB; values larger than the configured maximum log size and the + * default config would extend log files in allocations of the maximum log file + * size., a list\, with values chosen from the following options: \c "data"\, \c + * "log"; default empty.} * @config{file_manager = (, control how file handles are managed., a set of * related configuration options defined below.} * @config{ close_handle_minimum, number of handles open diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index d8cff7afa10..c1d7b06ad6d 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -208,7 +208,7 @@ __log_fs_write(WT_SESSION_IMPL *session, WT_RET(__wt_log_force_sync(session, &slot->slot_release_lsn)); } if ((ret = __wt_write(session, slot->slot_fh, offset, len, buf)) != 0) - WT_PANIC_MSG(session, ret, + WT_PANIC_RET(session, ret, "%s: fatal log failure", slot->slot_fh->name); return (ret); } @@ -674,11 +674,15 @@ __log_prealloc(WT_SESSION_IMPL *session, WT_FH *fh) return (__log_zero(session, fh, WT_LOG_END_HEADER, conn->log_file_max)); + /* If configured to not extend the file, we're done. */ + if (conn->log_extend_len == 0) + return (0); + /* * We have exclusive access to the log file and there are no other * writes happening concurrently, so there are no locking issues. */ - ret = __wt_fextend(session, fh, conn->log_file_max); + ret = __wt_fextend(session, fh, conn->log_extend_len); return (ret == EBUSY || ret == ENOTSUP ? 0 : ret); } @@ -1862,7 +1866,7 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size, WT_LOG *log; WT_LOG_RECORD *logrec; wt_off_t off, remainder; - size_t buf_left, bufsz, rdlen; + size_t allocsize, buf_left, bufsz, rdlen; char *buf, *p, *zerobuf; bool corrupt; @@ -1898,6 +1902,7 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size, remainder -= (wt_off_t)rdlen, off += (wt_off_t)rdlen) { rdlen = WT_MIN(bufsz, (size_t)remainder); WT_ERR(__wt_read(session, fh, off, rdlen, buf)); + allocsize = (log == NULL ? WT_LOG_ALIGN : log->allocsize); if (memcmp(buf, zerobuf, rdlen) != 0) { /* * Find where the next log record starts after the @@ -1905,7 +1910,7 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size, */ for (p = buf, buf_left = rdlen; buf_left > 0; buf_left -= rdlen, p += rdlen) { - rdlen = WT_MIN(log->allocsize, buf_left); + rdlen = WT_MIN(allocsize, buf_left); if (memcmp(p, zerobuf, rdlen) != 0) break; } @@ -1915,14 +1920,21 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size, * present in the buffer, we either have a valid header * or corruption. Verify the header of this record to * determine whether it is just a hole or corruption. + * + * We don't bother making this check for backup copies, + * as records may have their beginning zeroed, hence + * the part after a hole may in fact be the middle of + * the record. */ - logrec = (WT_LOG_RECORD *)p; - if (buf_left >= sizeof(WT_LOG_RECORD)) { - off += p - buf; - WT_ERR(__log_record_verify(session, fh, - (uint32_t)off, logrec, &corrupt)); - if (corrupt) - *error_offset = off; + if (!F_ISSET(conn, WT_CONN_WAS_BACKUP)) { + logrec = (WT_LOG_RECORD *)p; + if (buf_left >= sizeof(WT_LOG_RECORD)) { + off += p - buf; + WT_ERR(__log_record_verify(session, fh, + (uint32_t)off, logrec, &corrupt)); + if (corrupt) + *error_offset = off; + } } *hole = true; break; @@ -1945,12 +1957,6 @@ err: __wt_free(session, buf); * padded with zeroes before writing). The only way we have any certainty * is if the last byte is non-zero, when that happens, we know that * the write cannot be partial. - * - * When we have a checksum mismatch, it is important to know that whether - * it may be 1) the result of a partial write or 2) the result of - * corruption. The former can happen in normal operations, and we - * will silently truncate the log when it occurs. The latter will - * result in an error during recovery, and requires salvage to fix. */ static bool __log_check_partial_write(WT_SESSION_IMPL *session, WT_ITEM *buf, @@ -2480,7 +2486,30 @@ advance: */ if (log != NULL) log->trunc_lsn = rd_lsn; - /* Make a check to see if it may be a partial write. */ + /* + * If the user asked for a specific LSN and it is not + * a valid LSN, return WT_NOTFOUND. + */ + if (LF_ISSET(WT_LOGSCAN_ONE)) + ret = WT_NOTFOUND; + + /* + * When we have a checksum mismatch, we determine + * whether it may be the result of: + * 1) some expected corruption that can occur during + * backups + * 2) a partial write that can naturally occur when + * an application crashes + * 3) some other corruption + * + * As the first and second happen commonly in normal + * operations, we will silently truncate the log when + * it occurs. The third will result in an error during + * recovery, and requires salvage to fix. + */ + if (F_ISSET(conn, WT_CONN_WAS_BACKUP)) + break; + if (!__log_check_partial_write(session, buf, reclen)) { /* * It's not a partial write, and we have a bad @@ -2505,12 +2534,6 @@ advance: log_fh->name, "", rd_lsn.l.offset)); } } - /* - * If the user asked for a specific LSN and it is not - * a valid LSN, return WT_NOTFOUND. - */ - if (LF_ISSET(WT_LOGSCAN_ONE)) - ret = WT_NOTFOUND; break; } __wt_log_record_byteswap(logrec); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 03feafe3c8c..3dd7222630f 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -407,6 +407,7 @@ __lsm_tree_find(WT_SESSION_IMPL *session, { WT_LSM_TREE *lsm_tree; + *treep = NULL; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); /* See if the tree is already open. */ diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index a38b2ed0f02..145009be4df 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -595,8 +595,6 @@ static int __rec_write_check_complete( WT_SESSION_IMPL *session, WT_RECONCILE *r, int tret, bool *lookaside_retryp) { - WT_UNUSED(session); - /* * Tests in this function are lookaside tests and tests to decide if * rewriting a page in memory is worth doing. In-memory configurations diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 1c6cd8e50ae..33dc1a0a0d4 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -174,7 +174,7 @@ __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler) */ static int __eventv(WT_SESSION_IMPL *session, bool msg_event, int error, - const char *func_name, int line_number, const char *fmt, va_list ap) + const char *func, int line, const char *fmt, va_list ap) WT_GCC_FUNC_ATTRIBUTE((cold)) { struct timespec ts; @@ -231,8 +231,8 @@ __eventv(WT_SESSION_IMPL *session, bool msg_event, int error, WT_ERROR_APPEND(p, remain, ", %s", prefix); WT_ERROR_APPEND(p, remain, ": "); - if (func_name != NULL) - WT_ERROR_APPEND(p, remain, "%s, %d: ", func_name, line_number); + if (func != NULL) + WT_ERROR_APPEND(p, remain, "%s, %d: ", func, line); WT_ERROR_APPEND_AP(p, remain, fmt, ap); @@ -314,7 +314,7 @@ err: if (fprintf(stderr, */ void __wt_err_func(WT_SESSION_IMPL *session, - int error, const char *func_name, int line_number, const char *fmt, ...) + int error, const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) WT_GCC_FUNC_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) @@ -326,8 +326,7 @@ __wt_err_func(WT_SESSION_IMPL *session, * an error value to return. */ va_start(ap, fmt); - WT_IGNORE_RET(__eventv(session, - false, error, func_name, line_number, fmt, ap)); + WT_IGNORE_RET(__eventv(session, false, error, func, line, fmt, ap)); va_end(ap); } @@ -337,7 +336,7 @@ __wt_err_func(WT_SESSION_IMPL *session, */ void __wt_errx_func(WT_SESSION_IMPL *session, - const char *func_name, int line_number, const char *fmt, ...) + const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) @@ -349,8 +348,7 @@ __wt_errx_func(WT_SESSION_IMPL *session, * an error value to return. */ va_start(ap, fmt); - WT_IGNORE_RET(__eventv(session, - false, 0, func_name, line_number, fmt, ap)); + WT_IGNORE_RET(__eventv(session, false, 0, func, line, fmt, ap)); va_end(ap); } diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index 4116638e31c..eb65c00741c 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -67,7 +67,7 @@ hazard_grow(WT_SESSION_IMPL *session) int __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) { @@ -146,7 +146,7 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp */ hp->ref = ref; #ifdef HAVE_DIAGNOSTIC - hp->file = file; + hp->func = func; hp->line = line; #endif /* Publish the hazard pointer before reading page's state. */ @@ -419,7 +419,7 @@ __wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor) } __wt_errx(session, "hazard pointer reference to discarded object: (%p: %s, line %d)", - (void *)hp->ref, hp->file, hp->line); + (void *)hp->ref, hp->func, hp->line); return (false); } @@ -438,6 +438,6 @@ __hazard_dump(WT_SESSION_IMPL *session) __wt_errx(session, "session %p: hazard pointer %p: %s, line %d", (void *)session, - (void *)hp->ref, hp->file, hp->line); + (void *)hp->ref, hp->func, hp->line); } #endif diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index 21c83146a31..a0f7de3179f 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -242,7 +242,7 @@ __wt_buf_set_size( int __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) @@ -330,15 +330,14 @@ __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp F_SET(*best, WT_ITEM_INUSE); #ifdef HAVE_DIAGNOSTIC - session->scratch_track[best - session->scratch].file = file; + session->scratch_track[best - session->scratch].func = func; session->scratch_track[best - session->scratch].line = line; #endif *scratchp = *best; return (0); -err: WT_RET_MSG(session, ret, - "session unable to allocate a scratch buffer"); +err: WT_RET_MSG(session, ret, "session unable to allocate a scratch buffer"); } /* @@ -361,7 +360,7 @@ __wt_scr_discard(WT_SESSION_IMPL *session) "scratch buffer allocated and never discarded" ": %s: %d", session-> - scratch_track[bufp - session->scratch].file, + scratch_track[bufp - session->scratch].func, session-> scratch_track[bufp - session->scratch].line ); diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index 735faa417c3..ff0b74edcad 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -537,7 +537,6 @@ thread_ckpt_run(void *arg) testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); first_ckpt = true; ts = 0; - __wt_epoch(NULL, &start); for (i = 0; ;++i) { sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL; sleep(sleep_time); @@ -555,9 +554,12 @@ thread_ckpt_run(void *arg) /* * Create the checkpoint file so that the parent process knows * at least one checkpoint has finished and can start its - * timer. + * timer. Start the timer for stable after the first checkpoint + * completes because a slow I/O lag during the checkpoint can + * cause a false positive for a timeout. */ if (first_ckpt) { + __wt_epoch(NULL, &start); testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL); first_ckpt = false; testutil_checksys(fclose(fp) != 0); diff --git a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c index 1efe22c1816..2dd0f86db20 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c @@ -101,8 +101,7 @@ main(int argc, char *argv[]) "key_format=Q,value_format=u," "leaf_item_max=64M,leaf_page_max=32k,memory_page_max=1M")); - testutil_check( - session->open_cursor(session, uri, NULL, NULL, &c)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &c)); /* Value is initialized with 'v' and has not significance to it. */ large_doc = dmalloc(DATASIZE); @@ -137,6 +136,8 @@ main(int argc, char *argv[]) while (++j < MODIFY_COUNT) { for (i = 0; i < NUM_DOCS; i++) { /* Position the cursor. */ + testutil_check( + session2->begin_transaction(session2, NULL)); c->set_key(c, i); modify_entry.data.data = "abcdefghijklmnopqrstuvwxyz"; @@ -146,6 +147,8 @@ main(int argc, char *argv[]) (void)alarm(1); testutil_check(c->modify(c, &modify_entry, 1)); (void)alarm(0); + testutil_check( + session2->commit_transaction(session2, NULL)); } /* * Modify operations are done similar to append sequence. diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 705594a2156..f92f438a4f1 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -353,8 +353,15 @@ snap_check(WT_CURSOR *cursor, testutil_assert(start->keyno != 0); } - /* Check for subsequent changes to this record. */ + /* + * Check for subsequent changes to this record. If we find a + * read, don't treat it was a subsequent change, that way we + * verify the results of the change as well as the results of + * the read. + */ for (p = start + 1; p < stop; ++p) { + if (p->op == READ) + continue; if (p->keyno == start->keyno) break; @@ -872,10 +879,11 @@ ops(void *arg) break; case MODIFY: /* - * Change modify into update if in a read-uncommitted - * transaction, modify isn't supported in that case. + * Change modify into update if not in a transaction + * or in a read-uncommitted transaction, modify isn't + * supported in those cases. */ - if (iso_config == ISOLATION_READ_UNCOMMITTED) + if (!intxn || iso_config == ISOLATION_READ_UNCOMMITTED) goto update_instead_of_chosen_op; ++tinfo->update; diff --git a/src/third_party/wiredtiger/test/suite/test_config07.py b/src/third_party/wiredtiger/test/suite/test_config07.py new file mode 100644 index 00000000000..edf37577d4b --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_config07.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch, os, time +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_config07.py +# Test that log files extend as configured and as documented. +class test_config07(wttest.WiredTigerTestCase): + uri = "table:test" + entries = 100000 + K = 1024 + log_size = K * K + + extend_len = [ + ('default', dict(log_extend_len='()', expected_log_size = log_size)), + ('empty', dict(log_extend_len='(log=)', expected_log_size = log_size)), + ('disable', dict(log_extend_len='(log=0)', expected_log_size = 128)), + ('100K', dict(log_extend_len='(log=100K)', expected_log_size = 100 * K)), + ('too_small', dict(log_extend_len='(log=20K)', expected_log_size = None)), + ('too_large', dict(log_extend_len='(log=20G)', expected_log_size = None)), + ('small_in_allowed range', dict(log_extend_len='(log=200K)', + expected_log_size = 200 * K)), + ('large_in_allowed_range', dict(log_extend_len='(log=900K)', + expected_log_size = 900 * K)), + ('larger_than_log_file_size', dict(log_extend_len='(log=20M)', + expected_log_size = log_size)), + ('with_data_file_extend_conf', dict(log_extend_len='(log=100K,data=16M)', + expected_log_size = 100 * K)), + ] + + scenarios = make_scenarios(extend_len) + + def populate(self): + cur = self.session.open_cursor(self.uri, None, None) + for i in range(0, self.entries): + cur[i] = i + cur.close() + + def checkLogFileSize(self, size): + # Wait for a log file to be preallocated. Avoid timing problems, but + # assert that a file is created within 1 minute. + for i in range(1,60): + logs = fnmatch.filter(os.listdir('.'), "*Prep*") + if logs: + f = logs[-1] + file_size = os.stat(f).st_size + self.assertEqual(size, file_size) + break + time.sleep(1) + self.assertTrue(logs) + + def test_log_extend(self): + self.conn.close() + msg = '/invalid log extend length/' + + config = 'log=(enabled,file_max=1M),file_extend=' + self.log_extend_len + configarg = 'create,statistics=(fast)' + ',' + config + + # Expect an error when an invalid log extend size is provided. + if self.expected_log_size is None: + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open('.', configarg), msg) + return + + self.conn = self.wiredtiger_open('.', configarg) + self.session = self.conn.open_session(None) + + # Create a table, insert data in it to trigger log file writes. + self.session.create(self.uri, 'key_format=i,value_format=i') + self.populate() + self.session.checkpoint() + + self.checkLogFileSize(self.expected_log_size) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor12.py b/src/third_party/wiredtiger/test/suite/test_cursor12.py index 50204274b94..96bf48ef83c 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor12.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor12.py @@ -194,12 +194,14 @@ class test_cursor12(wttest.WiredTigerTestCase): self.assertEquals(c.update(), 0) c.reset() + self.session.begin_transaction() c.set_key(ds.key(row)) mods = [] for j in i['mods']: mod = wiredtiger.Modify(j[0], j[1], j[2]) mods.append(mod) self.assertEquals(c.modify(mods), 0) + self.session.commit_transaction() c.reset() c.set_key(ds.key(row)) @@ -288,6 +290,7 @@ class test_cursor12(wttest.WiredTigerTestCase): ds.populate() c = self.session.open_cursor(self.uri, None) + self.session.begin_transaction() c.set_key(ds.key(10)) orig = 'abcdefghijklmnopqrstuvwxyz' c.set_value(orig) @@ -299,6 +302,7 @@ class test_cursor12(wttest.WiredTigerTestCase): mod = wiredtiger.Modify(new, 10, 5) mods.append(mod) self.assertEquals(c.modify(mods), 0) + self.session.commit_transaction() c.set_key(ds.key(10)) self.assertEquals(c.search(), 0) @@ -314,12 +318,14 @@ class test_cursor12(wttest.WiredTigerTestCase): c.set_key(ds.key(10)) self.assertEquals(c.remove(), 0) + self.session.begin_transaction() mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(10)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) + self.session.commit_transaction() # Check that modify returns not-found when an insert is not yet committed # and after it's aborted. @@ -347,6 +353,7 @@ class test_cursor12(wttest.WiredTigerTestCase): # Test that another transaction cannot modify our uncommitted record. xs = self.conn.open_session() xc = xs.open_cursor(self.uri, None) + xs.begin_transaction() xc.set_key(ds.key(30)) xc.set_value(ds.value(30)) mods = [] @@ -354,16 +361,19 @@ class test_cursor12(wttest.WiredTigerTestCase): mods.append(mod) xc.set_key(ds.key(30)) self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND) + xs.rollback_transaction() # Rollback our transaction. self.session.rollback_transaction() # Test that we can't modify our aborted insert. + self.session.begin_transaction() mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(30)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_las01.py b/src/third_party/wiredtiger/test/suite/test_las01.py index 8b6132f8418..f47df3be9ac 100644 --- a/src/third_party/wiredtiger/test/suite/test_las01.py +++ b/src/third_party/wiredtiger/test/suite/test_las01.py @@ -139,8 +139,10 @@ class test_las01(wttest.WiredTigerTestCase): session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') # Apply two modify operations - replacing the first two items with 'A' + self.session.begin_transaction() self.large_modifies(self.session, uri, 0, ds, nrows) self.large_modifies(self.session, uri, 1, ds, nrows) + self.session.commit_transaction() # Check to see the value after recovery self.durable_check(bigvalue3, uri, ds, nrows) session2.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_schema08.py b/src/third_party/wiredtiger/test/suite/test_schema08.py new file mode 100644 index 00000000000..e7b44219ba3 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_schema08.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch, os, shutil, sys +from suite_subprocess import suite_subprocess +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_schema08.py +# Test schema operations on recovery. +# Test all schema operations alter, create, drop, rename. +# After doing the operation, create a backup copy of the directory, +# walk the log recording each LSN, truncate the backup copy of the +# log walking backward from the LSNs and then run recovery. +class test_schema08(wttest.WiredTigerTestCase, suite_subprocess): + # We want to copy, truncate and run recovery so keep the log + # file small and don't pre-allocate any. We expect a small log. + conn_config = 'log=(enabled,archive=false,file_max=100k,prealloc=false)' + types = [ + ('file', dict(uri='file:', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), + ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), + ('table-index', dict(uri='table:', use_cg=False, use_index=True)), + ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ] + ops = [ + ('none', dict(schema_ops='none')), + ('alter', dict(schema_ops='alter')), + ('drop', dict(schema_ops='drop')), + ('rename', dict(schema_ops='rename')), + ] + ckpt = [ + ('no_ckpt', dict(ckpt=False)), + ('with_ckpt', dict(ckpt=True)), + ] + scenarios = make_scenarios(types, ops, ckpt) + count = 0 + lsns = [] + backup_pfx = "BACKUP." + + def do_alter(self, uri, suburi): + alter_param = 'cache_resident=true' + self.session.alter(uri, alter_param) + if suburi != None: + self.session.alter(suburi, alter_param) + + def do_ops(self, uri, suburi): + if (self.schema_ops == 'none'): + return + if (self.schema_ops == 'alter'): + self.do_alter(uri, suburi) + elif (self.schema_ops == 'drop'): + self.session.drop(uri, None) + elif (self.schema_ops == 'rename'): + newuri = self.uri + "new-table" + self.session.rename(uri, newuri, None) + + # Count actual log records in the log. Log cursors walk the individual + # operations of a transaction as well as the entire record. Skip counting + # any individual commit operations and only count entire records. + def find_logrecs(self): + self.count = 0 + self.session.log_flush('sync=on') + c = self.session.open_cursor('log:', None, None) + self.lsns.append(0) + while c.next() == 0: + # lsn.file, lsn.offset, opcount + keys = c.get_key() + # We don't expect to need more than one log file. We only store + # the offsets in a list so assert lsn.file is 1. + self.assertTrue(keys[0] == 1) + + # Only count whole records, which is when opcount is zero. + # If opcount is not zero it is an operation of a commit. + # Skip LSN 128, that is a system record and its existence + # is assumed within the system. + if keys[2] == 0 and keys[1] != 128: + self.count += 1 + self.lsns.append(keys[1]) + c.close() + self.pr("Find " + str(self.count) + " logrecs LSNS: ") + self.pr(str(self.lsns)) + + def make_backups(self): + # With the connection still open, copy files to the new directory. + # Make an initial copy as well as a copy for each LSN we save. + # Truncate the log to the appropriate offset as we make each copy. + olddir = "." + log1 = 'WiredTigerLog.0000000001' + for lsn in self.lsns: + newdir = self.backup_pfx + str(lsn) + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + # Skip lock file on Windows since it is locked + if os.path.isfile(fullname) and \ + "WiredTiger.lock" not in fullname and \ + "Tmplog" not in fullname and \ + "Preplog" not in fullname: + shutil.copy(fullname, newdir) + # Truncate the file to the LSN offset. + # NOTE: This removes the record at that offset + # resulting in recovery running to just before + # that record. + if lsn != 0: + logf = os.path.join(newdir + '/' + log1) + f = open(logf, "r+") + f.truncate(lsn) + f.close() + # print "New size " + logf + ": " + str(os.path.getsize(logf)) + + def run_recovery(self, uri, suburi): + # With the connection still open, copy files to the new directory. + # Make an initial copy as well as a copy for each LSN we save. + # Truncate the log to the appropriate offset as we make each copy. + olddir = "." + for lsn in self.lsns: + newdir = self.backup_pfx + str(lsn) + outfile = newdir + '.txt' + self.runWt(['-R', '-h', newdir, 'list', '-v'], outfilename=outfile) + + # Test that creating and dropping tables does not write individual + # log records. + def test_schema08_create(self): + self.count = 0 + self.lsns = [] + uri = self.uri + 'table0' + create_params = 'key_format=i,value_format=S,' + + cgparam = '' + suburi = None + if self.use_cg or self.use_index: + cgparam = 'columns=(k,v),' + if self.use_cg: + cgparam += 'colgroups=(g0),' + + # Create main table. + self.session.create(uri, create_params + cgparam) + + # Checkpoint after the main table creation if wanted. + if self.ckpt: + self.session.checkpoint() + + # Add in column group or index tables. + if self.use_cg: + # Create. + cgparam = 'columns=(v),' + suburi = 'colgroup:table0:g0' + self.session.create(suburi, cgparam) + + if self.use_index: + # Create. + suburi = 'index:table0:i0' + self.session.create(suburi, cgparam) + + self.do_ops(uri, suburi) + self.find_logrecs() + # print "Found " + str(self.count) + " log records" + self.make_backups() + self.run_recovery(uri, suburi) + +if __name__ == '__main__': + wttest.run() |