diff options
author | Luke Chen <luke.chen@mongodb.com> | 2018-08-13 15:46:37 +1000 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2018-08-13 15:46:37 +1000 |
commit | ec6f2a54d8b901e95e556d53647944be8ce441d5 (patch) | |
tree | a75c521f1918cf0343acf2ced1669567e2a7b854 /src/third_party/wiredtiger/src | |
parent | 1c52af393ebc9040fbc300293835f68f827304ce (diff) | |
download | mongo-ec6f2a54d8b901e95e556d53647944be8ce441d5.tar.gz |
Import wiredtiger: c91b80412603f283532e267893f9238dd4a5ec0f from branch mongodb-4.2
ref: 65531924e3..c91b804126
for: 4.1.2
WT-3856 Create a test that runs recovery to different points of time with schema operations
WT-4026 Add implementation for existing file extension configuration API
WT-4193 test/format snapshot-isolation search mismatch
WT-4194 Improve fairness of eviction with multiple tables
WT-4207 Coverity #1394567: null pointer dereference
WT-4210 schema abort child process failing prematurely
WT-4215 Allow recovery of backup without salvage
WT-4229 Lint
WT-4234 Remove documentation mention of legacy tool statlog.py
WT-4235 Fix workgen tracking of table state across workloads
WT-4242 New log file extension Python test failure
Diffstat (limited to 'src/third_party/wiredtiger/src')
24 files changed, 184 insertions, 103 deletions
diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index b6172966eae..75d868eed08 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -1414,10 +1414,6 @@ __wt_btcur_modify(WT_CURSOR_BTREE *cbt, WT_MODIFY *entries, int nentries) /* Save the cursor state. */ __cursor_state_save(cursor, &state); - if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) - WT_ERR_MSG(session, ENOTSUP, - "not supported in read-uncommitted transactions"); - /* * Get the current value and apply the modification to it, for a few * reasons: first, we set the updated value so the application can @@ -1428,7 +1424,23 @@ __wt_btcur_modify(WT_CURSOR_BTREE *cbt, WT_MODIFY *entries, int nentries) * trouble if we attempt to modify a value that doesn't exist. For the * fifth reason, verify we're not in a read-uncommitted transaction, * that implies a value that might disappear out from under us. + * + * Also, an application might read a value outside of a transaction and + * then call modify. For that to work, the read must be part of the + * transaction that performs the update for correctness, otherwise we + * could race with another thread and end up modifying the wrong value. + * A clever application could get this right (imagine threads that only + * updated non-overlapping, fixed-length byte strings), but it's unsafe + * because it will work most of the time and the failure is unlikely to + * be detected. Require explicit transactions for modify operations. */ + if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) + WT_ERR_MSG(session, ENOTSUP, + "not supported in read-uncommitted transactions"); + if (F_ISSET(&session->txn, WT_TXN_AUTOCOMMIT)) + WT_ERR_MSG(session, ENOTSUP, + "not supported in implicit transactions"); + if (!F_ISSET(cursor, WT_CURSTD_KEY_INT) || !F_ISSET(cursor, WT_CURSTD_VALUE_INT)) WT_ERR(__wt_btcur_search(cbt)); diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 58853bff6ac..7f3391984b2 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -553,7 +553,7 @@ err: /* int __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) { @@ -683,7 +683,7 @@ read: /* */ #ifdef HAVE_DIAGNOSTIC WT_RET( - __wt_hazard_set(session, ref, &busy, file, line)); + __wt_hazard_set(session, ref, &busy, func, line)); #else WT_RET(__wt_hazard_set(session, ref, &busy)); #endif diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index 6434142824e..3dde08b43c1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -443,8 +443,8 @@ restart: /* goto done; } - WT_ASSERT(session, ret == WT_NOTFOUND); - WT_ERR_NOTFOUND_OK(ret); + WT_ASSERT(session, ret == WT_NOTFOUND); + WT_ERR_NOTFOUND_OK(ret); __wt_spin_backoff(&yield_count, &sleep_usecs); } diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 47ab622a7f4..3d06bac02c5 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2626,6 +2626,12 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, #endif WT_ERR(__wt_config_gets(session, cfg, "file_extend", &cval)); + /* + * If the log extend length is not set use the default of the configured + * maximum log file size. That size is not known until it is initialized + * as part of the log server initialization. + */ + conn->log_extend_len = WT_CONFIG_UNSET; for (ft = file_types; ft->name != NULL; ft++) { ret = __wt_config_subgets(session, &cval, ft->name, &sval); if (ret == 0) { @@ -2634,7 +2640,21 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, conn->data_extend_len = sval.val; break; case WT_DIRECT_IO_LOG: - conn->log_extend_len = sval.val; + /* + * When using "file_extend=(log=)", the val + * returned is 1. Unset the log extend length + * in that case to use the default. + */ + if (sval.val == 1) + conn->log_extend_len = WT_CONFIG_UNSET; + else if (sval.val == 0 || + (sval.val >= WT_LOG_FILE_MIN && + sval.val <= WT_LOG_FILE_MAX)) + conn->log_extend_len = sval.val; + else + WT_ERR_MSG(session, EINVAL, + "invalid log extend length: %" + PRId64, sval.val); break; } } else diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index d42789c809e..9f4af1aba88 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -277,6 +277,15 @@ __logmgr_config( if (!reconfig) { WT_RET(__wt_config_gets(session, cfg, "log.file_max", &cval)); conn->log_file_max = (wt_off_t)cval.val; + /* + * With the default log file extend configuration or if the log + * file extension size is larger than the configured maximum log + * file size, set the log file extension size to the configured + * maximum log file size. + */ + if (conn->log_extend_len == WT_CONFIG_UNSET || + conn->log_extend_len > conn->log_file_max) + conn->log_extend_len = conn->log_file_max; WT_STAT_CONN_SET(session, log_max_filesize, conn->log_file_max); } @@ -890,7 +899,6 @@ __log_wrlsn_server(void *arg) __wt_log_wrlsn(session, NULL); if (0) { err: WT_PANIC_MSG(session, ret, "log wrlsn server error"); - } return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/src/docs/statistics.dox b/src/third_party/wiredtiger/src/docs/statistics.dox index 26c4b66fa40..19b7b17257b 100644 --- a/src/third_party/wiredtiger/src/docs/statistics.dox +++ b/src/third_party/wiredtiger/src/docs/statistics.dox @@ -156,9 +156,4 @@ currently open in the database, nor will any statistics requiring the traversal of a tree (as if the \c statistics_fast configuration string were set). -A Python script that parses the default logging output and uses the -<a href="http://www.gnuplot.info/">gnuplot</a>, utility to generate -Portable Network Graphics (PNG) format graphs is included in the -WiredTiger distribution in the file \c tools/statlog.py. - */ diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index 7e89d23230f..0d8e5e1b428 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,4 +1,20 @@ /*! @page upgrading Upgrading WiredTiger applications + +</dl><hr> +@section version_311 Upgrading to Version 3.1.1 +<dl> + +<dt>WT_CURSOR::modify transaction requirements</dt> +<dd> +In previous releases of WiredTiger, it was possible to use implicit +transactions in combination with WT_CURSOR::modify operations. This +requires applications be extraordinarily careful to avoid multiple +threads which are changing the same values racing with each other. In +the 3.1.1 release, WT_CURSOR::modify operations must be performed in an +explicit transaction, and will fail if that's not the case. +</dd> + +</dl><hr> @section version_310 Upgrading to Version 3.1.0 <dl> diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 13b5dfc4c8d..b072c47a9ba 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -318,7 +318,7 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_ERR(__evict_lru_pages(session, false)); if (0) { -err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); +err: WT_PANIC_RET(session, ret, "cache eviction thread error"); } return (ret); } @@ -357,7 +357,7 @@ __wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_VERB_EVICTSERVER, "%s", "cache eviction thread exiting"); if (0) { -err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); +err: WT_PANIC_RET(session, ret, "cache eviction thread error"); } return (ret); } @@ -810,10 +810,8 @@ __evict_clear_walk(WT_SESSION_IMPL *session) cache = S2C(session)->cache; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_PASS)); - if (session->dhandle == cache->walk_tree) { + if (session->dhandle == cache->walk_tree) cache->walk_tree = NULL; - cache->walk_target = 0; - } if ((ref = btree->evict_ref) == NULL) return (0); @@ -1414,10 +1412,8 @@ retry: while (slot < max_entries) { */ if ((dhandle = cache->walk_tree) != NULL) cache->walk_tree = NULL; - else { + else dhandle = TAILQ_FIRST(&conn->dhqh); - cache->walk_target = 0; - } } else { if (incr) { WT_ASSERT(session, dhandle->session_inuse > 0); @@ -1427,7 +1423,6 @@ retry: while (slot < max_entries) { cache->walk_tree = NULL; } dhandle = TAILQ_NEXT(dhandle, q); - cache->walk_target = 0; } /* If we reach the end of the list, we're done. */ @@ -1601,8 +1596,7 @@ __evict_push_candidate(WT_SESSION_IMPL *session, * Calculate how many pages to queue for a given tree. */ static uint32_t -__evict_walk_target( - WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_entries) +__evict_walk_target(WT_SESSION_IMPL *session, u_int max_entries) { WT_CACHE *cache; uint64_t btree_inuse, bytes_per_slot, cache_inuse; @@ -1611,7 +1605,7 @@ __evict_walk_target( cache = S2C(session)->cache; target_pages_clean = target_pages_dirty = 0; - total_slots = max_entries - queue->evict_entries; + total_slots = max_entries; /* * The number of times we should fill the queue by the end of @@ -1717,14 +1711,13 @@ __evict_walk_tree(WT_SESSION_IMPL *session, */ start = queue->evict_queue + *slotp; remaining_slots = max_entries - *slotp; - if (cache->walk_target != 0) { - WT_ASSERT(session, cache->walk_progress <= cache->walk_target); - target_pages = cache->walk_target - cache->walk_progress; - } else { - target_pages = cache->walk_target = - __evict_walk_target(session, queue, max_entries); - cache->walk_progress = 0; + if (btree->evict_walk_progress >= btree->evict_walk_target) { + btree->evict_walk_target = + __evict_walk_target(session, max_entries); + btree->evict_walk_progress = 0; } + target_pages = WT_MIN(btree->evict_walk_target / QUEUE_FILLS_PER_PASS, + btree->evict_walk_target - btree->evict_walk_progress); if (target_pages > remaining_slots) target_pages = remaining_slots; @@ -2017,7 +2010,7 @@ fast: /* If the page can't be evicted, give up. */ continue; ++evict; ++pages_queued; - ++cache->walk_progress; + ++btree->evict_walk_progress; __wt_verbose(session, WT_VERB_EVICTSERVER, "select: %p, size %" WT_SIZET_FMT, diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 2a5be782b06..93e1d8e44ea 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -192,6 +192,8 @@ struct __wt_btree { */ WT_REF *evict_ref; /* Eviction thread's location */ uint64_t evict_priority; /* Relative priority of cached pages */ + uint32_t evict_walk_progress;/* Eviction walk progress */ + uint32_t evict_walk_target; /* Eviction walk target */ u_int evict_walk_period; /* Skip this many LRU walks */ u_int evict_walk_saved; /* Saved walk skips for checkpoints */ u_int evict_walk_skips; /* Number of walks skipped */ diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index ed3480a40d0..86159e971db 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1668,7 +1668,7 @@ static inline int __wt_page_swap_func( WT_SESSION_IMPL *session, WT_REF *held, WT_REF *want, uint32_t flags #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) { @@ -1691,7 +1691,7 @@ __wt_page_swap_func( /* Get the wanted page. */ ret = __wt_page_in_func(session, want, flags #ifdef HAVE_DIAGNOSTIC - , file, line + , func, line #endif ); diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 7d07e6dfd98..8afedb30832 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -149,7 +149,6 @@ struct __wt_cache { WT_SPINLOCK evict_pass_lock; /* Eviction pass lock */ WT_SESSION_IMPL *walk_session; /* Eviction pass session */ WT_DATA_HANDLE *walk_tree; /* LRU walk current tree */ - uint32_t walk_progress, walk_target;/* Progress in current tree */ WT_SPINLOCK evict_queue_lock; /* Eviction current queue lock */ WT_EVICT_QUEUE evict_queues[WT_EVICT_QUEUE_MAX]; diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 06847117b7d..3ae4fe2aaea 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -50,6 +50,7 @@ struct __wt_config_parser_impl { "", 0, 0, WT_CONFIG_ITEM_NUM \ } +#define WT_CONFIG_UNSET -1 /* * DO NOT EDIT: automatically built by dist/api_config.py. * configuration section: BEGIN diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index 6f28dfae2c8..1d2d21617a6 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -98,7 +98,7 @@ /* Called on unexpected code path: locate the failure. */ #define __wt_illegal_value(session, v) \ - __wt_illegal_value_func(session, (uintmax_t)v, __func__, __LINE__) + __wt_illegal_value_func(session, (uintmax_t)(v), __func__, __LINE__) /* Return and branch-to-err-label cases for switch statements. */ #define WT_ILLEGAL_VALUE(session, v) \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 34cee52df99..bebc3ad7bc1 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -163,7 +163,7 @@ extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBU extern int __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ); extern int __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -675,8 +675,8 @@ extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_ extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep); extern void __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler); -extern void __wt_err_func(WT_SESSION_IMPL *session, int error, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_errx_func(WT_SESSION_IMPL *session, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_err_func(WT_SESSION_IMPL *session, int error, const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_errx_func(WT_SESSION_IMPL *session, const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_set_return_func(WT_SESSION_IMPL *session, const char*func, int line, int err) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_err_printf(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)); @@ -708,7 +708,7 @@ extern uint64_t __wt_hash_fnv64(const void *string, size_t len); extern int __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ); extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -757,7 +757,7 @@ extern const char *__wt_buf_set_size(WT_SESSION_IMPL *session, uint64_t size, bo extern int __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index 2613808e74e..d3e36d37da9 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -87,6 +87,12 @@ union __wt_lsn { #define WT_LOGC_KEY_FORMAT WT_UNCHECKED_STRING(III) #define WT_LOGC_VALUE_FORMAT WT_UNCHECKED_STRING(qIIIuu) +/* + * Size range for the log files. + */ +#define WT_LOG_FILE_MAX ((int64_t)2 * WT_GIGABYTE) +#define WT_LOG_FILE_MIN (100 * WT_KILOBYTE) + #define WT_LOG_SKIP_HEADER(data) \ ((const uint8_t *)(data) + offsetof(WT_LOG_RECORD, record)) #define WT_LOG_REC_SIZE(size) \ diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 1180c641222..d7540c333e7 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -352,7 +352,7 @@ union __wt_rand_state { * buffer. If not concatenating, clear the size so we don't use \ * any existing contents. \ */ \ - if (!concatenate) \ + if (!(concatenate)) \ (buf)->size = 0; \ for (;;) { \ WT_ASSERT(session, (buf)->memsize >= (buf)->size); \ diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index cbf572f9a23..10ff7bd48dc 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -25,7 +25,7 @@ struct __wt_data_handle_cache { struct __wt_hazard { WT_REF *ref; /* Page reference */ #ifdef HAVE_DIAGNOSTIC - const char *file; /* File/line where hazard acquired */ + const char *func; /* Function/line hazard acquired */ int line; #endif }; @@ -120,7 +120,7 @@ struct __wt_session_impl { * to applications, create a parallel structure instead. */ struct __wt_scratch_track { - const char *file; /* Allocating file, line */ + const char *func; /* Allocating function, line */ int line; } *scratch_track; #endif diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index c80aaef53fb..99adacb0a1b 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -495,10 +495,22 @@ struct __wt_cursor { * \c S), or raw byte arrays accessed using a WT_ITEM structure (value * format type \c u). * - * Calling the WT_CURSOR::modify method outside of snapshot isolation - * can lead to unexpected results. While \c read-committed isolation - * is supported with the WT_CURSOR::modify method, \c read-uncommitted - * isolation is not. + * The WT_CURSOR::modify method can only be called from within an + * explicit transaction configured at a higher isolation level than + * \c read-uncommitted. Using \c read-committed isolation is allowed, + * but requires caution: reading a value, re-positioning the cursor + * and then modifying the value based on the initial read could lead + * to unexpected results. Using \c snapshot isolation is recommended. + * + * The WT_CURSOR::modify method stores a change record in cache and + * writes a change record to the log instead of the usual complete + * values. Note that WT_CURSOR::modify is generally slower than the + * WT_CURSOR::update method, and can result in slower reads because + * the complete value must be assembled during retrieval. The + * WT_CURSOR::modify method is intended for applications modifying + * large records where there is cache or I/O pressure, that is, + * applications that will benefit when data updates require less cache + * and they write less logging information. * * @snippet ex_all.c Modify an existing record * @@ -510,12 +522,6 @@ struct __wt_cursor { * (as it partially depends on the underlying file configuration), but * is always a small number of bytes less than 4GB. * - * The WT_CURSOR::modify method stores a change record in cache and - * writes a change record to the log, instead of the usual complete - * value. This can reduce cache and logging requirements, but may result - * in slower reads because the complete value must be assembled during - * retrieval. - * * @param cursor the cursor handle * @param entries an array of modification data structures * @param nentries the number of modification data structures @@ -2870,8 +2876,12 @@ struct __wt_connection { * @config{file_extend, file extension configuration. If set\, extend files of * the set type in allocations of the set size\, instead of a block at a time as * each new block is written. For example\, - * <code>file_extend=(data=16MB)</code>., a list\, with values chosen from the - * following options: \c "data"\, \c "log"; default empty.} + * <code>file_extend=(data=16MB)</code>. If set to 0\, disable the file + * extension for the set type. For log files\, the allowed range is between + * 100KB and 2GB; values larger than the configured maximum log size and the + * default config would extend log files in allocations of the maximum log file + * size., a list\, with values chosen from the following options: \c "data"\, \c + * "log"; default empty.} * @config{file_manager = (, control how file handles are managed., a set of * related configuration options defined below.} * @config{ close_handle_minimum, number of handles open diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index d8cff7afa10..c1d7b06ad6d 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -208,7 +208,7 @@ __log_fs_write(WT_SESSION_IMPL *session, WT_RET(__wt_log_force_sync(session, &slot->slot_release_lsn)); } if ((ret = __wt_write(session, slot->slot_fh, offset, len, buf)) != 0) - WT_PANIC_MSG(session, ret, + WT_PANIC_RET(session, ret, "%s: fatal log failure", slot->slot_fh->name); return (ret); } @@ -674,11 +674,15 @@ __log_prealloc(WT_SESSION_IMPL *session, WT_FH *fh) return (__log_zero(session, fh, WT_LOG_END_HEADER, conn->log_file_max)); + /* If configured to not extend the file, we're done. */ + if (conn->log_extend_len == 0) + return (0); + /* * We have exclusive access to the log file and there are no other * writes happening concurrently, so there are no locking issues. */ - ret = __wt_fextend(session, fh, conn->log_file_max); + ret = __wt_fextend(session, fh, conn->log_extend_len); return (ret == EBUSY || ret == ENOTSUP ? 0 : ret); } @@ -1862,7 +1866,7 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size, WT_LOG *log; WT_LOG_RECORD *logrec; wt_off_t off, remainder; - size_t buf_left, bufsz, rdlen; + size_t allocsize, buf_left, bufsz, rdlen; char *buf, *p, *zerobuf; bool corrupt; @@ -1898,6 +1902,7 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size, remainder -= (wt_off_t)rdlen, off += (wt_off_t)rdlen) { rdlen = WT_MIN(bufsz, (size_t)remainder); WT_ERR(__wt_read(session, fh, off, rdlen, buf)); + allocsize = (log == NULL ? WT_LOG_ALIGN : log->allocsize); if (memcmp(buf, zerobuf, rdlen) != 0) { /* * Find where the next log record starts after the @@ -1905,7 +1910,7 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size, */ for (p = buf, buf_left = rdlen; buf_left > 0; buf_left -= rdlen, p += rdlen) { - rdlen = WT_MIN(log->allocsize, buf_left); + rdlen = WT_MIN(allocsize, buf_left); if (memcmp(p, zerobuf, rdlen) != 0) break; } @@ -1915,14 +1920,21 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size, * present in the buffer, we either have a valid header * or corruption. Verify the header of this record to * determine whether it is just a hole or corruption. + * + * We don't bother making this check for backup copies, + * as records may have their beginning zeroed, hence + * the part after a hole may in fact be the middle of + * the record. */ - logrec = (WT_LOG_RECORD *)p; - if (buf_left >= sizeof(WT_LOG_RECORD)) { - off += p - buf; - WT_ERR(__log_record_verify(session, fh, - (uint32_t)off, logrec, &corrupt)); - if (corrupt) - *error_offset = off; + if (!F_ISSET(conn, WT_CONN_WAS_BACKUP)) { + logrec = (WT_LOG_RECORD *)p; + if (buf_left >= sizeof(WT_LOG_RECORD)) { + off += p - buf; + WT_ERR(__log_record_verify(session, fh, + (uint32_t)off, logrec, &corrupt)); + if (corrupt) + *error_offset = off; + } } *hole = true; break; @@ -1945,12 +1957,6 @@ err: __wt_free(session, buf); * padded with zeroes before writing). The only way we have any certainty * is if the last byte is non-zero, when that happens, we know that * the write cannot be partial. - * - * When we have a checksum mismatch, it is important to know that whether - * it may be 1) the result of a partial write or 2) the result of - * corruption. The former can happen in normal operations, and we - * will silently truncate the log when it occurs. The latter will - * result in an error during recovery, and requires salvage to fix. */ static bool __log_check_partial_write(WT_SESSION_IMPL *session, WT_ITEM *buf, @@ -2480,7 +2486,30 @@ advance: */ if (log != NULL) log->trunc_lsn = rd_lsn; - /* Make a check to see if it may be a partial write. */ + /* + * If the user asked for a specific LSN and it is not + * a valid LSN, return WT_NOTFOUND. + */ + if (LF_ISSET(WT_LOGSCAN_ONE)) + ret = WT_NOTFOUND; + + /* + * When we have a checksum mismatch, we determine + * whether it may be the result of: + * 1) some expected corruption that can occur during + * backups + * 2) a partial write that can naturally occur when + * an application crashes + * 3) some other corruption + * + * As the first and second happen commonly in normal + * operations, we will silently truncate the log when + * it occurs. The third will result in an error during + * recovery, and requires salvage to fix. + */ + if (F_ISSET(conn, WT_CONN_WAS_BACKUP)) + break; + if (!__log_check_partial_write(session, buf, reclen)) { /* * It's not a partial write, and we have a bad @@ -2505,12 +2534,6 @@ advance: log_fh->name, "", rd_lsn.l.offset)); } } - /* - * If the user asked for a specific LSN and it is not - * a valid LSN, return WT_NOTFOUND. - */ - if (LF_ISSET(WT_LOGSCAN_ONE)) - ret = WT_NOTFOUND; break; } __wt_log_record_byteswap(logrec); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 03feafe3c8c..3dd7222630f 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -407,6 +407,7 @@ __lsm_tree_find(WT_SESSION_IMPL *session, { WT_LSM_TREE *lsm_tree; + *treep = NULL; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); /* See if the tree is already open. */ diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index a38b2ed0f02..145009be4df 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -595,8 +595,6 @@ static int __rec_write_check_complete( WT_SESSION_IMPL *session, WT_RECONCILE *r, int tret, bool *lookaside_retryp) { - WT_UNUSED(session); - /* * Tests in this function are lookaside tests and tests to decide if * rewriting a page in memory is worth doing. In-memory configurations diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 1c6cd8e50ae..33dc1a0a0d4 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -174,7 +174,7 @@ __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler) */ static int __eventv(WT_SESSION_IMPL *session, bool msg_event, int error, - const char *func_name, int line_number, const char *fmt, va_list ap) + const char *func, int line, const char *fmt, va_list ap) WT_GCC_FUNC_ATTRIBUTE((cold)) { struct timespec ts; @@ -231,8 +231,8 @@ __eventv(WT_SESSION_IMPL *session, bool msg_event, int error, WT_ERROR_APPEND(p, remain, ", %s", prefix); WT_ERROR_APPEND(p, remain, ": "); - if (func_name != NULL) - WT_ERROR_APPEND(p, remain, "%s, %d: ", func_name, line_number); + if (func != NULL) + WT_ERROR_APPEND(p, remain, "%s, %d: ", func, line); WT_ERROR_APPEND_AP(p, remain, fmt, ap); @@ -314,7 +314,7 @@ err: if (fprintf(stderr, */ void __wt_err_func(WT_SESSION_IMPL *session, - int error, const char *func_name, int line_number, const char *fmt, ...) + int error, const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) WT_GCC_FUNC_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) @@ -326,8 +326,7 @@ __wt_err_func(WT_SESSION_IMPL *session, * an error value to return. */ va_start(ap, fmt); - WT_IGNORE_RET(__eventv(session, - false, error, func_name, line_number, fmt, ap)); + WT_IGNORE_RET(__eventv(session, false, error, func, line, fmt, ap)); va_end(ap); } @@ -337,7 +336,7 @@ __wt_err_func(WT_SESSION_IMPL *session, */ void __wt_errx_func(WT_SESSION_IMPL *session, - const char *func_name, int line_number, const char *fmt, ...) + const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) @@ -349,8 +348,7 @@ __wt_errx_func(WT_SESSION_IMPL *session, * an error value to return. */ va_start(ap, fmt); - WT_IGNORE_RET(__eventv(session, - false, 0, func_name, line_number, fmt, ap)); + WT_IGNORE_RET(__eventv(session, false, 0, func, line, fmt, ap)); va_end(ap); } diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index 4116638e31c..eb65c00741c 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -67,7 +67,7 @@ hazard_grow(WT_SESSION_IMPL *session) int __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) { @@ -146,7 +146,7 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp */ hp->ref = ref; #ifdef HAVE_DIAGNOSTIC - hp->file = file; + hp->func = func; hp->line = line; #endif /* Publish the hazard pointer before reading page's state. */ @@ -419,7 +419,7 @@ __wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor) } __wt_errx(session, "hazard pointer reference to discarded object: (%p: %s, line %d)", - (void *)hp->ref, hp->file, hp->line); + (void *)hp->ref, hp->func, hp->line); return (false); } @@ -438,6 +438,6 @@ __hazard_dump(WT_SESSION_IMPL *session) __wt_errx(session, "session %p: hazard pointer %p: %s, line %d", (void *)session, - (void *)hp->ref, hp->file, hp->line); + (void *)hp->ref, hp->func, hp->line); } #endif diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index 21c83146a31..a0f7de3179f 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -242,7 +242,7 @@ __wt_buf_set_size( int __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp #ifdef HAVE_DIAGNOSTIC - , const char *file, int line + , const char *func, int line #endif ) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) @@ -330,15 +330,14 @@ __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp F_SET(*best, WT_ITEM_INUSE); #ifdef HAVE_DIAGNOSTIC - session->scratch_track[best - session->scratch].file = file; + session->scratch_track[best - session->scratch].func = func; session->scratch_track[best - session->scratch].line = line; #endif *scratchp = *best; return (0); -err: WT_RET_MSG(session, ret, - "session unable to allocate a scratch buffer"); +err: WT_RET_MSG(session, ret, "session unable to allocate a scratch buffer"); } /* @@ -361,7 +360,7 @@ __wt_scr_discard(WT_SESSION_IMPL *session) "scratch buffer allocated and never discarded" ": %s: %d", session-> - scratch_track[bufp - session->scratch].file, + scratch_track[bufp - session->scratch].func, session-> scratch_track[bufp - session->scratch].line ); |