diff options
author | Michael Cahill <michael.cahill@mongodb.com> | 2015-11-11 16:35:13 +1100 |
---|---|---|
committer | Michael Cahill <michael.cahill@mongodb.com> | 2015-11-11 16:35:13 +1100 |
commit | 76ef930d453d4de3ad24245eeb0043c94a0cf258 (patch) | |
tree | 58a2f77b0506ba040bb37e10a6c0704df0c0608a /src/third_party | |
parent | 80dfa9ff4da8935ff294483e14ad5a6cdcd05985 (diff) | |
download | mongo-76ef930d453d4de3ad24245eeb0043c94a0cf258.tar.gz |
Import wiredtiger-wiredtiger-mongodb-3.2.0-rc2-44-gd9ec1ff.tar.gz from wiredtiger branch mongodb-3.2
Diffstat (limited to 'src/third_party')
22 files changed, 215 insertions, 101 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 70ed6e0220b..6a2b0497d15 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -167,6 +167,7 @@ if useTcmalloc: if conf.CheckCHeader('gperftools/tcmalloc.h'): wtlibs.append("libtcmalloc_minimal") conf.env.Append(CPPDEFINES=['HAVE_LIBTCMALLOC']) + conf.env.Append(CPPDEFINES=['HAVE_POSIX_MEMALIGN']) else: print 'tcmalloc.h must be installed!' Exit(1) @@ -406,8 +407,11 @@ Default(t) t = env.Program("t_huge", "test/huge/huge.c", LIBS=[wtlib] + wtlibs) -#env.Alias("check", env.SmokeTest(t)) -Default(t) + +#t = env.Program("t_recovery", +# "test/recovery/recovery.c", +# LIBS=[wtlib] + wtlibs) +#Default(t) t = env.Program("t_fops", ["test/fops/file.c", diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index 0fa9e030659..e2f128a48df 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -31,5 +31,6 @@ test/fops test/format test/huge test/packing +test/recovery test/salvage test/thread diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 99e08282e49..9afff74ca71 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -594,15 +594,16 @@ wiredtiger_open_common = connection_runtime_config + [ checkpoints''', type='boolean'), Config('direct_io', '', r''' - Use \c O_DIRECT to access files. Options are given as a list, - such as <code>"direct_io=[data]"</code>. Configuring - \c direct_io requires care, see @ref - tuning_system_buffer_cache_direct_io for important warnings. - Including \c "data" will cause WiredTiger data files to use - \c O_DIRECT, including \c "log" will cause WiredTiger log files - to use \c O_DIRECT, and including \c "checkpoint" will cause - WiredTiger data files opened at a checkpoint (i.e: read only) to - use \c O_DIRECT''', + Use \c O_DIRECT on POSIX systems, and \c FILE_FLAG_NO_BUFFERING on + Windows to access files. Options are given as a list, such as + <code>"direct_io=[data]"</code>. Configuring \c direct_io requires + care, see @ref tuning_system_buffer_cache_direct_io for important + warnings. Including \c "data" will cause WiredTiger data files to use + direct I/O, including \c "log" will cause WiredTiger log files to use + direct I/O, and including \c "checkpoint" will cause WiredTiger data + files opened at a checkpoint (i.e: read only) to use direct I/O. + \c direct_io should be combined with \c write_through to get the + equivalent of \c O_DIRECT on Windows.''', type='list', choices=['checkpoint', 'data', 'log']), Config('encryption', '', r''' configure an encryptor for system wide metadata and logs. @@ -674,6 +675,17 @@ wiredtiger_open_common = connection_runtime_config + [ @ref tune_durability for more information''', choices=['dsync', 'fsync', 'none']), ]), + Config('write_through', '', r''' + Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to files. Ignored + on non-Windows systems. Options are given as a list, such as + <code>"write_through=[data]"</code>. Configuring \c write_through + requires care, see @ref tuning_system_buffer_cache_direct_io for + important warnings. Including \c "data" will cause WiredTiger data + files to write through cache, including \c "log" will cause WiredTiger + log files to write through cache. \c write_through should be combined + with \c direct_io to get the equivalent of POSIX \c O_DIRECT on + Windows.''', + type='list', choices=['data', 'log']), ] wiredtiger_open = wiredtiger_open_common + [ diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index da677c17389..0e2bad0910c 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -8,12 +8,6 @@ flags = { ################################################### # Internal routine flag declarations ################################################### - 'cache_flush' : [ - 'SYNC_CHECKPOINT', - 'SYNC_CLOSE', - 'SYNC_DISCARD', - 'SYNC_WRITE_LEAVES', - ], 'file_types' : [ 'FILE_TYPE_CHECKPOINT', 'FILE_TYPE_DATA', diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 26c0a905b82..c14f4c961e6 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -584,6 +584,7 @@ fopen fp fprintf free'd +fscanf fstat fsync fsyncs diff --git a/src/third_party/wiredtiger/lang/java/wiredtiger.i b/src/third_party/wiredtiger/lang/java/wiredtiger.i index ae370ec89f5..1326b9ebb90 100644 --- a/src/third_party/wiredtiger/lang/java/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/java/wiredtiger.i @@ -395,15 +395,20 @@ javaCloseHandler(WT_EVENT_HANDLER *handler, WT_SESSION *session, WT_CURSOR *cursor) { int ret; + JAVA_CALLBACK *jcb; WT_UNUSED(handler); - if (cursor != NULL) - ret = cursorCloseHandler(NULL, session, (JAVA_CALLBACK *) - cursor->lang_private); - else - ret = closeHandler(NULL, session, (JAVA_CALLBACK *) - ((WT_SESSION_IMPL *)session)->lang_private); + ret = 0; + if (cursor != NULL) { + if ((jcb = (JAVA_CALLBACK *)cursor->lang_private) != NULL) { + ret = cursorCloseHandler(NULL, session, jcb); + cursor->lang_private = NULL; + } + } else if ((jcb = ((WT_SESSION_IMPL *)session)->lang_private) != NULL) { + ret = closeHandler(NULL, session, jcb); + ((WT_SESSION_IMPL *)session)->lang_private = NULL; + } return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c index 67e70d0cdb9..32418a9c063 100644 --- a/src/third_party/wiredtiger/src/btree/bt_discard.c +++ b/src/third_party/wiredtiger/src/btree/bt_discard.c @@ -51,12 +51,14 @@ __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) *pagep = NULL; /* - * We should never discard a dirty page, the file's current eviction - * point or a page queued for LRU eviction. + * We should never discard ... */ - WT_ASSERT(session, !__wt_page_is_modified(page)); - WT_ASSERT(session, !F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)); - WT_ASSERT(session, !__wt_fair_islocked(session, &page->page_lock)); + WT_ASSERT( /* ... a dirty page */ + session, !__wt_page_is_modified(page)); + WT_ASSERT( /* ... a page queued for LRU eviction */ + session, !F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)); + WT_ASSERT( /* ... a locked page */ + session, !__wt_fair_islocked(session, &page->page_lock)); #ifdef HAVE_DIAGNOSTIC { diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index c0c739d68ad..9e45bf10a5c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -796,7 +796,7 @@ __split_multi_inmem_final(WT_PAGE *orig, WT_MULTI *multi) uint32_t i, slot; /* - * We've successfully created new in-memory pages. For error-handling + * We successfully created new in-memory pages. For error-handling * reasons, we've left the update chains referenced by both the original * and new pages. We're ready to discard the original page, terminate * the original page's reference to any update list we moved. @@ -818,6 +818,25 @@ __split_multi_inmem_final(WT_PAGE *orig, WT_MULTI *multi) } /* + * __split_multi_inmem_fail -- + * Discard allocated pages after failure. + */ +static void +__split_multi_inmem_fail(WT_SESSION_IMPL *session, WT_REF *ref) +{ + /* + * We failed creating new in-memory pages. For error-handling reasons, + * we've left the update chains referenced by both the original and + * new pages. Discard the new pages, setting a flag so the discard code + * doesn't discard the updates on the page. + */ + if (ref->page != NULL) { + F_SET_ATOMIC(ref->page, WT_PAGE_UPDATE_IGNORE); + __wt_free_ref(session, ref->page, ref, true); + } +} + +/* * __wt_multi_to_ref -- * Move a multi-block list into an array of WT_REF structures. */ @@ -1555,6 +1574,7 @@ __wt_split_reverse(WT_SESSION_IMPL *session, WT_REF *ref) int __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref) { + WT_DECL_RET; WT_PAGE *page; WT_PAGE_MODIFY *mod; WT_REF new; @@ -1573,7 +1593,7 @@ __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref) * Build the new page. */ memset(&new, 0, sizeof(new)); - WT_RET(__split_multi_inmem(session, page, &new, &mod->mod_multi[0])); + WT_ERR(__split_multi_inmem(session, page, &new, &mod->mod_multi[0])); /* * The rewrite succeeded, we can no longer fail. @@ -1597,6 +1617,9 @@ __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref) WT_PUBLISH(ref->state, WT_REF_MEM); return (0); + +err: __split_multi_inmem_fail(session, &new); + return (ret); } /* @@ -1655,24 +1678,8 @@ __split_multi(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) __wt_page_out(session, &page); if (0) { -err: /* - * A note on error handling: when handling unresolved changes, - * we create new in-memory pages with those unresolved changes. - * The problem is the new pages are given references to the - * original page's update lists, and once all of the pages are - * created, there's a second pass to remove the updates from the - * original page. If an error occurs, we can't simply free the - * newly created pages, that would discard the original page's - * updates. Set a flag so the discard function doesn't discard - * the updates on the page. - */ - for (i = 0; i < new_entries; ++i) - if (ref_new[i]->page != NULL) { - F_SET_ATOMIC( - ref_new[i]->page, WT_PAGE_UPDATE_IGNORE); - __wt_free_ref(session, - ref_new[i]->page, ref_new[i], true); - } +err: for (i = 0; i < new_entries; ++i) + __split_multi_inmem_fail(session, ref_new[i]); } __wt_free(session, ref_new); diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 110f9f8fb0c..7395cce11e1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -13,7 +13,7 @@ * Flush pages for a specific file. */ static int -__sync_file(WT_SESSION_IMPL *session, int syncop) +__sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) { struct timespec end, start; WT_BTREE *btree; @@ -128,14 +128,21 @@ __sync_file(WT_SESSION_IMPL *session, int syncop) if (walk == NULL) break; - page = walk->page; - mod = page->modify; - /* Skip clean pages. */ - if (!__wt_page_is_modified(page)) + if (!__wt_page_is_modified(walk->page)) continue; /* + * Take a local reference to the page modify structure + * now that we know the page is dirty. It needs to be + * done in this order otherwise the page modify + * structure could have been created between taking the + * reference and checking modified. + */ + page = walk->page; + mod = page->modify; + + /* * Write dirty pages, unless we can be sure they only * became dirty after the checkpoint started. * @@ -169,6 +176,9 @@ __sync_file(WT_SESSION_IMPL *session, int syncop) WT_ERR(__wt_reconcile(session, walk, NULL, 0)); } break; + case WT_SYNC_CLOSE: + case WT_SYNC_DISCARD: + WT_ILLEGAL_VALUE_ERR(session); } if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { @@ -250,7 +260,7 @@ err: /* On error, clear any left-over tree walk. */ * Cache operations. */ int -__wt_cache_op(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, int op) +__wt_cache_op(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_CACHE_OP op) { WT_DECL_RET; WT_BTREE *btree; @@ -269,6 +279,9 @@ __wt_cache_op(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, int op) WT_ASSERT(session, btree->ckpt == NULL); btree->ckpt = ckptbase; break; + case WT_SYNC_DISCARD: + case WT_SYNC_WRITE_LEAVES: + break; } switch (op) { @@ -280,7 +293,6 @@ __wt_cache_op(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, int op) case WT_SYNC_DISCARD: WT_ERR(__wt_evict_file(session, op)); break; - WT_ILLEGAL_VALUE_ERR(session); } err: switch (op) { @@ -288,6 +300,9 @@ err: switch (op) { case WT_SYNC_CLOSE: btree->ckpt = NULL; break; + case WT_SYNC_DISCARD: + case WT_SYNC_WRITE_LEAVES: + break; } return (ret); diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index 87929d8a457..7b21f1e40bb 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -487,7 +487,13 @@ leaf_match: cbt->compare = 0; return (0); -err: if (leaf != NULL) +err: /* + * Release the current page if the search started at the root. If the + * search didn't start at the root we should never have gone looking + * beyond the start page. + */ + WT_ASSERT(session, leaf == NULL || leaf == current); + if (leaf == NULL) WT_TRET(__wt_page_release(session, current, 0)); return (ret); } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index c8aca15d103..311ddd56b7a 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -552,6 +552,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { "\"split\",\"temporary\",\"transaction\",\"verify\",\"version\"," "\"write\"]", NULL, 0 }, + { "write_through", "list", + NULL, "choices=[\"data\",\"log\"]", + NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -629,6 +632,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { "\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, + { "write_through", "list", + NULL, "choices=[\"data\",\"log\"]", + NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -701,6 +707,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { "\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, + { "write_through", "list", + NULL, "choices=[\"data\",\"log\"]", + NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -772,6 +781,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { "\"split\",\"temporary\",\"transaction\",\"verify\",\"version\"," "\"write\"]", NULL, 0 }, + { "write_through", "list", + NULL, "choices=[\"data\",\"log\"]", + NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -984,8 +996,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),use_environment_priv=0" - ",verbose=", - confchk_wiredtiger_open, 35 + ",verbose=,write_through=", + confchk_wiredtiger_open, 36 }, { "wiredtiger_open_all", "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," @@ -1005,8 +1017,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),use_environment_priv=0" - ",verbose=,version=(major=0,minor=0)", - confchk_wiredtiger_open_all, 36 + ",verbose=,version=(major=0,minor=0),write_through=", + confchk_wiredtiger_open_all, 37 }, { "wiredtiger_open_basecfg", "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," @@ -1025,8 +1037,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),verbose=," - "version=(major=0,minor=0)", - confchk_wiredtiger_open_basecfg, 31 + "version=(major=0,minor=0),write_through=", + confchk_wiredtiger_open_basecfg, 32 }, { "wiredtiger_open_usercfg", "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," @@ -1044,8 +1056,9 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",name=,quota=0,reserve=0,size=500MB),statistics=none," "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),verbose=", - confchk_wiredtiger_open_usercfg, 30 + "transaction_sync=(enabled=0,method=fsync),verbose=," + "write_through=", + confchk_wiredtiger_open_usercfg, 31 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index d86b02287f0..c65b74e4e4e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1947,6 +1947,16 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, goto err; } + WT_ERR(__wt_config_gets(session, cfg, "write_through", &cval)); + for (ft = file_types; ft->name != NULL; ft++) { + ret = __wt_config_subgets(session, &cval, ft->name, &sval); + if (ret == 0) { + if (sval.val) + FLD_SET(conn->write_through, ft->flag); + } else if (ret != WT_NOTFOUND) + goto err; + } + /* * If buffer alignment is not configured, use zero unless direct I/O is * also configured, in which case use the build-time default. diff --git a/src/third_party/wiredtiger/src/evict/evict_file.c b/src/third_party/wiredtiger/src/evict/evict_file.c index 448de57d88e..043fbf6bbeb 100644 --- a/src/third_party/wiredtiger/src/evict/evict_file.c +++ b/src/third_party/wiredtiger/src/evict/evict_file.c @@ -13,7 +13,7 @@ * Discard pages for a specific file. */ int -__wt_evict_file(WT_SESSION_IMPL *session, int syncop) +__wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) { WT_DECL_RET; WT_PAGE *page; diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index caf8996e68b..d8a3829863f 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -39,6 +39,14 @@ struct __wt_evict_worker { uint32_t flags; }; +/* Cache operations. */ +typedef enum __wt_cache_op { + WT_SYNC_CHECKPOINT, + WT_SYNC_CLOSE, + WT_SYNC_DISCARD, + WT_SYNC_WRITE_LEAVES +} WT_CACHE_OP; + /* * WiredTiger cache structure. */ diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index a585e08ef1a..35a83d7c50f 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -412,7 +412,9 @@ struct __wt_connection_impl { wt_off_t data_extend_len; /* file_extend data length */ wt_off_t log_extend_len; /* file_extend log length */ - uint32_t direct_io; /* O_DIRECT file type flags */ + /* O_DIRECT/FILE_FLAG_NO_BUFFERING file type flags */ + uint32_t direct_io; + uint32_t write_through; /* FILE_FLAG_WRITE_THROUGH type flags */ bool mmap; /* mmap configuration */ uint32_t verbose; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 75064c56334..032b94b7040 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -159,7 +159,7 @@ extern int __wt_split_reverse(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing); extern int __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst); -extern int __wt_cache_op(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, int op); +extern int __wt_cache_op(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_CACHE_OP op); extern int __wt_upgrade(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_verify_dsk_image(WT_SESSION_IMPL *session, const char *tag, const WT_PAGE_HEADER *dsk, size_t size, bool empty_page_ok); @@ -317,7 +317,7 @@ extern void __wt_curtable_set_key(WT_CURSOR *cursor, ...); extern void __wt_curtable_set_value(WT_CURSOR *cursor, ...); extern int __wt_table_range_truncate(WT_CURSOR_TABLE *start, WT_CURSOR_TABLE *stop); extern int __wt_curtable_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); -extern int __wt_evict_file(WT_SESSION_IMPL *session, int syncop); +extern int __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop); extern void __wt_evict_list_clear_page(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_evict_server_wake(WT_SESSION_IMPL *session); extern int __wt_evict_create(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h index 99b6f1c483f..95fe18b9ecb 100644 --- a/src/third_party/wiredtiger/src/include/flags.h +++ b/src/third_party/wiredtiger/src/include/flags.h @@ -64,10 +64,6 @@ #define WT_SESSION_NO_SCHEMA_LOCK 0x00008000 #define WT_SESSION_QUIET_CORRUPT_FILE 0x00010000 #define WT_SESSION_SERVER_ASYNC 0x00020000 -#define WT_SYNC_CHECKPOINT 0x00000001 -#define WT_SYNC_CLOSE 0x00000002 -#define WT_SYNC_DISCARD 0x00000004 -#define WT_SYNC_WRITE_LEAVES 0x00000008 #define WT_TXN_LOG_CKPT_CLEANUP 0x00000001 #define WT_TXN_LOG_CKPT_PREPARE 0x00000002 #define WT_TXN_LOG_CKPT_START 0x00000004 diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 037399625ea..044611d655e 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2130,15 +2130,17 @@ struct __wt_connection { * true.} * @config{create, create the database if it does not exist., a boolean flag; * default \c false.} - * @config{direct_io, Use \c O_DIRECT to access files. Options are given as a + * @config{direct_io, Use \c O_DIRECT on POSIX systems\, and \c + * FILE_FLAG_NO_BUFFERING on Windows to access files. Options are given as a * list\, such as <code>"direct_io=[data]"</code>. Configuring \c direct_io * requires care\, see @ref tuning_system_buffer_cache_direct_io for important - * warnings. Including \c "data" will cause WiredTiger data files to use \c - * O_DIRECT\, including \c "log" will cause WiredTiger log files to use \c - * O_DIRECT\, and including \c "checkpoint" will cause WiredTiger data files - * opened at a checkpoint (i.e: read only) to use \c O_DIRECT., a list\, with - * values chosen from the following options: \c "checkpoint"\, \c "data"\, \c - * "log"; default empty.} + * warnings. Including \c "data" will cause WiredTiger data files to use direct + * I/O\, including \c "log" will cause WiredTiger log files to use direct I/O\, + * and including \c "checkpoint" will cause WiredTiger data files opened at a + * checkpoint (i.e: read only) to use direct I/O. \c direct_io should be + * combined with \c write_through to get the equivalent of \c O_DIRECT on + * Windows., a list\, with values chosen from the following options: \c + * "checkpoint"\, \c "data"\, \c "log"; default empty.} * @config{encryption = (, configure an encryptor for system wide metadata and * logs. If a system wide encryptor is set\, it is also used for encrypting * data files and tables\, unless encryption configuration is explicitly set for @@ -2339,6 +2341,15 @@ struct __wt_connection { * "overflow"\, \c "read"\, \c "reconcile"\, \c "recovery"\, \c "salvage"\, \c * "shared_cache"\, \c "split"\, \c "temporary"\, \c "transaction"\, \c * "verify"\, \c "version"\, \c "write"; default empty.} + * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to + * files. Ignored on non-Windows systems. Options are given as a list\, such + * as <code>"write_through=[data]"</code>. Configuring \c write_through requires + * care\, see @ref tuning_system_buffer_cache_direct_io for important warnings. + * Including \c "data" will cause WiredTiger data files to write through cache\, + * including \c "log" will cause WiredTiger log files to write through cache. + * \c write_through should be combined with \c direct_io to get the equivalent + * of POSIX \c O_DIRECT on Windows., a list\, with values chosen from the + * following options: \c "data"\, \c "log"; default empty.} * @configend * Additionally, if files named \c WiredTiger.config or \c WiredTiger.basecfg * appear in the WiredTiger home directory, they are read for configuration diff --git a/src/third_party/wiredtiger/src/os_win/os_open.c b/src/third_party/wiredtiger/src/os_win/os_open.c index c7b30408e63..c3106763452 100644 --- a/src/third_party/wiredtiger/src/os_win/os_open.c +++ b/src/third_party/wiredtiger/src/os_win/os_open.c @@ -78,11 +78,20 @@ __wt_open(WT_SESSION_IMPL *session, } else dwCreationDisposition = OPEN_EXISTING; + /* + * direct_io means no OS file caching. This requires aligned buffer + * allocations like O_DIRECT. + */ if (dio_type && FLD_ISSET(conn->direct_io, dio_type)) { - f |= FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH; + f |= FILE_FLAG_NO_BUFFERING; direct_io = true; } + /* FILE_FLAG_WRITE_THROUGH does not require aligned buffers */ + if (dio_type && FLD_ISSET(conn->write_through, dio_type)) { + f |= FILE_FLAG_WRITE_THROUGH; + } + if (dio_type == WT_FILE_TYPE_LOG && FLD_ISSET(conn->txn_logsync, WT_LOG_DSYNC)) { f |= FILE_FLAG_WRITE_THROUGH; diff --git a/src/third_party/wiredtiger/src/schema/schema_stat.c b/src/third_party/wiredtiger/src/schema/schema_stat.c index 88f92b71599..d73d66cd399 100644 --- a/src/third_party/wiredtiger/src/schema/schema_stat.c +++ b/src/third_party/wiredtiger/src/schema/schema_stat.c @@ -96,15 +96,17 @@ __curstat_size_only(WT_SESSION_IMPL *session, * are concurrent schema level operations (for example drop). That is * fine - failing here results in falling back to the slow path of * opening the handle. + * !!! Deliberately discard the return code from a failed call - the + * error is flagged by not setting fast to true. */ - WT_ERR(__wt_filesize_name(session, namebuf.data, true, &filesize)); + if (__wt_filesize_name(session, namebuf.data, true, &filesize) == 0) { + /* Setup and populate the statistics structure */ + __wt_stat_dsrc_init_single(&cst->u.dsrc_stats); + cst->u.dsrc_stats.block_size = filesize; + __wt_curstat_dsrc_final(cst); - /* Setup and populate the statistics structure */ - __wt_stat_dsrc_init_single(&cst->u.dsrc_stats); - cst->u.dsrc_stats.block_size = filesize; - __wt_curstat_dsrc_final(cst); - - *was_fast = true; + *was_fast = true; + } err: __wt_free(session, tableconf); __wt_buf_free(session, &namebuf); diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 2045329b8ff..ed0e016dcb2 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -147,7 +147,8 @@ __session_close(WT_SESSION *wt_session, const char *config) * Notify the user that we are closing the cursor handle * via the registered close callback. */ - if (session->event_handler->handle_close != NULL) + if (session->event_handler->handle_close != NULL && + !WT_STREQ(cursor->uri, WT_LAS_URI)) WT_TRET(session->event_handler->handle_close( session->event_handler, wt_session, cursor)); WT_TRET(cursor->close(cursor)); diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 346e9c0ab38..ec2f0921ef2 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -132,6 +132,25 @@ __wt_session_lock_dhandle( want_exclusive = LF_ISSET(WT_DHANDLE_EXCLUSIVE); /* + * If this session already has exclusive access to the handle, there is + * no point trying to lock it again. + * + * This should only happen if a checkpoint handle is locked multiple + * times during a checkpoint operation, or the handle is already open + * without any special flags. In particular, it must fail if + * attempting to checkpoint a handle opened for a bulk load, even in + * the same session. + */ + if (dhandle->excl_session == session) { + if (!LF_ISSET(WT_DHANDLE_LOCK_ONLY) && + (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || + F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS))) + return (EBUSY); + ++dhandle->excl_ref; + return (0); + } + + /* * Check that the handle is open. We've already incremented * the reference count, so once the handle is open it won't be * closed by another thread. @@ -207,6 +226,11 @@ __wt_session_lock_dhandle( /* We have an exclusive lock, we're done. */ F_SET(dhandle, WT_DHANDLE_EXCLUSIVE); + WT_ASSERT(session, + dhandle->excl_session == NULL && + dhandle->excl_ref == 0); + dhandle->excl_session = session; + dhandle->excl_ref = 1; WT_ASSERT(session, !F_ISSET(dhandle, WT_DHANDLE_DEAD)); return (0); } @@ -454,19 +478,10 @@ __wt_session_get_btree(WT_SESSION_IMPL *session, WT_RET(__session_get_dhandle(session, uri, checkpoint)); dhandle = session->dhandle; - /* - * If this session already owns the handle, increase - * the owner ref count. - */ - if (dhandle->excl_session == session) - dhandle->excl_ref++; - else { - /* Try to lock the handle. */ - WT_RET(__wt_session_lock_dhandle( - session, flags, &is_dead)); - if (is_dead) - continue; - } + /* Try to lock the handle. */ + WT_RET(__wt_session_lock_dhandle(session, flags, &is_dead)); + if (is_dead) + continue; /* If the handle is open in the mode we want, we're done. */ if (LF_ISSET(WT_DHANDLE_LOCK_ONLY) || |