diff options
158 files changed, 3691 insertions, 2176 deletions
diff --git a/src/third_party/wiredtiger/NEWS b/src/third_party/wiredtiger/NEWS index 9af9a41a5c6..806c153a7ec 100644 --- a/src/third_party/wiredtiger/NEWS +++ b/src/third_party/wiredtiger/NEWS @@ -1,3 +1,85 @@ +WiredTiger release 2.5.2, 2015-03-23 +------------------------------------ + +The WiredTiger 2.5.2 release contains important bug fixes. + +API changes: + +* Allow memory_page_max to be at most a quater of the cache size not half. + This avoids operations getting stalled due to the cache being filled with + one or two pages. + +Bug fixes and other important changes: + +* When skipping a dirty page during a checkpoint, make sure the tree is marked + dirty. + refs SUPPORT-1248, SERVER-17319, SERVER-17506, #1404, #1643, #1721, #1735 + +* Fix a bug in range truncate where we could remove the wrong records. + refs SERVER-17345 + +* Fix a bug in LSM management where we could let the cache get full - leading + to a operations being blocked. + refs #1720 + +* Fix several bugs in the checkpoint implementation that could lead to a tree + being marked clean when it had updates in memory. If shutdown occurred at + a specific time those updates would be discarded without being written. + refs SUPPORT-1248 + +* Fix some bugs in logging - where system crashes could leave empty files that + would stop recovery working on re-start. + refs #1717, #1719, SERVER-17451 + +* Fix a bug in recovery. Force recovery instead of returning an error if the + LSN given doesn't exist. + refs #1700, #1704 + +* Move writing into log worker thread to avoid latency in application threads. + refs #1683 + +* Fix a bug in the reconfigure API related to adhering to shared cache quotas. + refs #1712, #1713 + +* Fix a bug in WiredTiger statistics where we weren't recording overflow + record statistics. + refs #1520, #1703, #1711 + +* Several enhancements to eviction of large pages including: + - Don't do forced eviction of a page if it is the current walk point. + - Don't update the read generation on page in if it's set to oldest. + - Clear the walk positions before the eviction server sleeps. + - Reverse the direction of the LRU walk regularly. + - Add all pages that would block to the eviction queue. + - If evicting dirty pages use the worker threads not the server. + refs #1706 + +* Use raw mode when dumping indices. + refs #1709 + +* Fix a bug where we could race opening files while a WT_CONNECTION::close is + in progress. + refs SERVER-17319 + +* Fix a bug in LSM where snapshot transaction updates could have the wrong + visibility check applied. Leading to invalid updates. + refs #1641, #1701, #1702 + +* Fix a bug in checkpoint where it could get an EBUSY return unnecessarily. + refs #1404, #1589, #1705 + +* Fix a bug when writing a page from memory to disk (reconciling). We could + overwrite the end of a temporary buffer in some cases. + refs #1697, #1699 + +* Sometimes we would choose a sub-optimal layout for on disk pages when + writing them out from memory. + refs #1699 + +* Improve the performance of in-memory lookups by making the content of the + page structure more cache friendly. + + WiredTiger release 2.5.1, 2015-03-07 ------------------------------------ diff --git a/src/third_party/wiredtiger/README b/src/third_party/wiredtiger/README index d89756c9f42..f304bade13e 100644 --- a/src/third_party/wiredtiger/README +++ b/src/third_party/wiredtiger/README @@ -1,6 +1,6 @@ -WiredTiger 2.5.2: (March 9, 2015) +WiredTiger 2.5.3: (March 26, 2015) -This is version 2.5.2 of WiredTiger. +This is version 2.5.3 of WiredTiger. WiredTiger release packages and documentation can be found at: @@ -9,7 +9,7 @@ WiredTiger release packages and documentation can be found at: Information on configuring, building and installing WiredTiger can be found at: - http://source.wiredtiger.com/2.5.2/install.html + http://source.wiredtiger.com/2.5.3/install.html WiredTiger licensing information can be found at: diff --git a/src/third_party/wiredtiger/RELEASE_INFO b/src/third_party/wiredtiger/RELEASE_INFO index ac5ff8ac028..e19e825e64a 100644 --- a/src/third_party/wiredtiger/RELEASE_INFO +++ b/src/third_party/wiredtiger/RELEASE_INFO @@ -1,6 +1,6 @@ WIREDTIGER_VERSION_MAJOR=2 WIREDTIGER_VERSION_MINOR=5 -WIREDTIGER_VERSION_PATCH=2 +WIREDTIGER_VERSION_PATCH=3 WIREDTIGER_VERSION="$WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH" WIREDTIGER_RELEASE_DATE=`date "+%B %e, %Y"` diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 1c109c973c0..8e9fae28ac6 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -25,6 +25,9 @@ AddOption("--enable-attach", dest="attach", action="store_true", default=False, AddOption("--enable-diagnostic", dest="diagnostic", action="store_true", default=False, help="Configure WiredTiger to perform various run-time diagnostic tests. DO NOT configure this option in production environments.") +AddOption("--enable-lz4", dest="lz4", type="string", nargs=1, action="store", + help="Use LZ4 compression") + AddOption("--enable-python", dest="lang-python", type="string", nargs=1, action="store", help="Build Python extension, specify location of swig.exe binary") @@ -49,46 +52,66 @@ swig_binary = GetOption("lang-python") # Initialize environment # +var = Variables() + +var.Add('MSVC_USE_SCRIPT', 'Path to vcvars.bat to override SCons default VS tool search'); + +var.Add('CPPPATH', 'C Preprocessor include path', [ + "#/src/include/", + "#/build_win", + "#/test/windows", + "#/.", +]) + +var.Add('CFLAGS', 'C Compiler Flags', [ + "/Z7", # Generate debugging symbols + "/wd4090", # Ignore warning about mismatched const qualifiers + "/wd4996", # Ignore deprecated functions + "/W3", # Warning level 3 + #"/we4244", # Possible loss of data + "/we4013", # Error on undefined functions + #"/we4047", # Indirection differences in types + #"/we4024", # Differences in parameter types + #"/we4100", # Unreferenced local parameter + "/TC", # Compile as C code + #"/Od", # Disable optimization + "/Ob1", # inline expansion + "/O2", # optimize for speed + "/GF", # enable string pooling + "/EHsc", # extern "C" does not throw + #"/RTC1", # enable stack checks + "/GS", # enable security checks + "/Gy", # separate functions for linker + "/Zc:wchar_t", + "/Gd", + "/MD" if GetOption("dynamic-crt") else "/MT", +]) + +var.Add('LINKFLAGS', 'Linker Flags', [ + "/DEBUG", # Generate debug symbols + "/INCREMENTAL:NO", # Disable incremental linking + "/OPT:REF", # Remove dead code + "/DYNAMICBASE", + "/NXCOMPAT", +]) + +var.Add('TOOLS', 'SCons tools', [ + "default", + "swig", + "textfile" +]) + +var.Add('SWIG', 'SWIG binary location', swig_binary) + env = Environment( - CPPPATH = ["#/src/include/", - "#/build_win", - "#/test/windows", - "#/.", - ], - CFLAGS = [ - "/Z7", # Generate debugging symbols - "/wd4090", # Ignore warning about mismatched const qualifiers - "/wd4996", - "/W3", # Warning level 3 - "/we4013", # Error on undefined functions - "/TC", # Compile as C code - #"/Od", # Disable optimization - "/Ob1", # inline expansion - "/O2", # optimize for speed - "/GF", # enable string pooling - "/EHsc", # extern "C" does not throw - #"/RTC1", # enable stack checks - "/GS", # enable secrutiy checks - "/Gy", # separate functions for linker - "/Zc:wchar_t", - "/Gd", - "/MD" if GetOption("dynamic-crt") else "/MT", - ], - LINKFLAGS = [ - "/DEBUG", # Generate debug symbols - "/INCREMENTAL:NO", # Disable incremental linking - "/OPT:REF", # Remove dead code - "/DYNAMICBASE", - "/NXCOMPAT", - ], - tools=["default", "swig", "textfile"], - SWIG=swig_binary + variables = var ) env['STATIC_AND_SHARED_OBJECTS_ARE_THE_SAME'] = 1 useZlib = GetOption("zlib") useSnappy = GetOption("snappy") +useLz4 = GetOption("lz4") useBdb = GetOption("bdb") wtlibs = [] @@ -117,6 +140,16 @@ if useSnappy: print 'snappy-c.h must be installed!' Exit(1) +if useLz4: + conf.env.Append(CPPPATH=[useLz4 + "/include"]) + conf.env.Append(LIBPATH=[useLz4 + "/lib"]) + if conf.CheckCHeader('lz4.h'): + conf.env.Append(CPPDEFINES=['HAVE_BUILTIN_EXTENSION_LZ4']) + wtlibs.append("lz4") + else: + print 'lz4.h must be installed!' + Exit(1) + if useBdb: conf.env.Append(CPPPATH=[useBdb+ "/include"]) conf.env.Append(LIBPATH=[useBdb+ "/lib"]) @@ -204,6 +237,9 @@ if useZlib: if useSnappy: wtsources.append("ext/compressors/snappy/snappy_compress.c") +if useLz4: + wtsources.append("ext/compressors/lz4/lz4_compress.c") + wt_objs = [env.Object(a) for a in wtsources] # Static Library - libwiredtiger.lib diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index 4c072b85c70..e27b4861bed 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -149,6 +149,9 @@ config_compress(CONFIG *cfg) } else if (strcmp(s, "bzip") == 0) { cfg->compress_ext = BZIP_EXT; cfg->compress_table = BZIP_BLK; + } else if (strcmp(s, "lz4") == 0) { + cfg->compress_ext = LZ4_EXT; + cfg->compress_table = LZ4_BLK; } else if (strcmp(s, "snappy") == 0) { cfg->compress_ext = SNAPPY_EXT; cfg->compress_table = SNAPPY_BLK; diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/parallel-pop-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/parallel-pop-stress.wtperf new file mode 100644 index 00000000000..71da5044c9e --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/parallel-pop-stress.wtperf @@ -0,0 +1,9 @@ +# wtperf options file: Run populate multi-threaded with a btree. This can +# uncover conditions where pages are too busy for eviction to work as +# expected +conn_config="cache_size=1GB" +table_config="type=file" +icount=10000000 +report_interval=5 +populate_threads=20 +run_time=0 diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index 976847e9a21..201623c7859 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -72,6 +72,9 @@ typedef struct __config_thread CONFIG_THREAD; #define BZIP_BLK BLKCMP_PFX "bzip2" #define BZIP_EXT \ EXT_PFX EXTPATH "bzip2/.libs/libwiredtiger_bzip2.so" EXT_SFX +#define LZ4_BLK BLKCMP_PFX "lz4" +#define LZ4_EXT \ + EXT_PFX EXTPATH "lz4/.libs/libwiredtiger_lz4.so" EXT_SFX #define SNAPPY_BLK BLKCMP_PFX "snappy" #define SNAPPY_EXT \ EXT_PFX EXTPATH "snappy/.libs/libwiredtiger_snappy.so" EXT_SFX diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i index 58d56e512ce..705b95a7450 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i @@ -97,7 +97,7 @@ DEF_OPT_AS_CONFIG_STRING(conn_config, "create", DEF_OPT_AS_BOOL(compact, 0, "post-populate compact for LSM merging activity") DEF_OPT_AS_STRING(compression, "none", "compression extension. Allowed configuration values are: " - "'none', 'bzip', 'snappy', 'zlib'") + "'none', 'bzip', 'lz4', 'snappy', 'zlib'") DEF_OPT_AS_BOOL(create, 1, "do population phase; false to use existing database") DEF_OPT_AS_UINT32(database_count, 1, diff --git a/src/third_party/wiredtiger/build_posix/Make.base b/src/third_party/wiredtiger/build_posix/Make.base index 51a8e77cebe..3037d70528a 100644 --- a/src/third_party/wiredtiger/build_posix/Make.base +++ b/src/third_party/wiredtiger/build_posix/Make.base @@ -67,6 +67,9 @@ $(srcdir)/docs/man/man1/wt.1: $(srcdir)/docs/index.html $(srcdir)/docs/man/man3/wiredtiger.3: $(srcdir)/docs/index.html libwiredtiger_la_LIBADD = +if HAVE_BUILTIN_EXTENSION_LZ4 +libwiredtiger_la_LIBADD += ext/compressors/lz4/libwiredtiger_lz4.la +endif if HAVE_BUILTIN_EXTENSION_SNAPPY libwiredtiger_la_LIBADD += ext/compressors/snappy/libwiredtiger_snappy.la endif diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index 82feee58aa1..8f8614a7016 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -7,6 +7,7 @@ # If a condition is included, the subdir is made conditional via AM_CONDITIONAL ext/collators/reverse ext/compressors/bzip2 BZIP2 +ext/compressors/lz4 LZ4 ext/compressors/nop ext/compressors/snappy SNAPPY ext/compressors/zlib ZLIB diff --git a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 index 2682c8ea82c..a5ad50416ed 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 @@ -13,6 +13,8 @@ no) wt_cv_enable_attach=no;; esac AC_MSG_RESULT($wt_cv_enable_attach) +AH_TEMPLATE(HAVE_BUILTIN_EXTENSION_LZ4, + [LZ4 support automatically loaded.]) AH_TEMPLATE(HAVE_BUILTIN_EXTENSION_SNAPPY, [Snappy support automatically loaded.]) AH_TEMPLATE(HAVE_BUILTIN_EXTENSION_ZLIB, @@ -20,7 +22,7 @@ AH_TEMPLATE(HAVE_BUILTIN_EXTENSION_ZLIB, AC_MSG_CHECKING(if --with-builtins option specified) AC_ARG_WITH(builtins, [AS_HELP_STRING([--with-builtins], - [builtin extension names (snappy, zlib).])], + [builtin extension names (lz4, snappy, zlib).])], [with_builtins=$withval], [with_builtins=]) @@ -28,6 +30,8 @@ AC_ARG_WITH(builtins, builtin_list=`echo "$with_builtins"|tr -s , ' '` for builtin_i in $builtin_list; do case "$builtin_i" in + lz4) AC_DEFINE(HAVE_BUILTIN_EXTENSION_LZ4) + wt_cv_with_builtin_extension_lz4=yes;; snappy) AC_DEFINE(HAVE_BUILTIN_EXTENSION_SNAPPY) wt_cv_with_builtin_extension_snappy=yes;; zlib) AC_DEFINE(HAVE_BUILTIN_EXTENSION_ZLIB) @@ -35,6 +39,8 @@ for builtin_i in $builtin_list; do *) AC_MSG_ERROR([Unknown builtin extension "$builtin_i"]);; esac done +AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_LZ4], + [test "$wt_cv_with_builtin_extension_lz4" = "yes"]) AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_SNAPPY], [test "$wt_cv_with_builtin_extension_snappy" = "yes"]) AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_ZLIB], @@ -168,6 +174,34 @@ if test "$wt_cv_enable_snappy" = "yes"; then fi AM_CONDITIONAL([SNAPPY], [test "$wt_cv_enable_snappy" = "yes"]) +AC_MSG_CHECKING(if --enable-lz4 option specified) +AC_ARG_ENABLE(lz4, + [AS_HELP_STRING([--enable-lz4], + [Build the lz4 compressor extension.])], r=$enableval, r=no) +case "$r" in +no) if test "$wt_cv_with_builtin_extension_lz4" = "yes"; then + wt_cv_enable_lz4=yes + else + wt_cv_enable_lz4=no + fi + ;; +*) if test "$wt_cv_with_builtin_extension_lz4" = "yes"; then + AC_MSG_ERROR( + [Only one of --enable-lz4 --with-builtins=lz4 allowed]) + fi + wt_cv_enable_lz4=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_lz4) +if test "$wt_cv_enable_lz4" = "yes"; then + AC_LANG_PUSH([C++]) + AC_CHECK_HEADER(lz4.h,, + [AC_MSG_ERROR([--enable-lz4 requires lz4.h])]) + AC_LANG_POP([C++]) + AC_CHECK_LIB(lz4, LZ4_compress,, + [AC_MSG_ERROR([--enable-lz4 requires lz4 library])]) +fi +AM_CONDITIONAL([LZ4], [test "$wt_cv_enable_lz4" = "yes"]) + AH_TEMPLATE(SPINLOCK_TYPE, [Spinlock type from mutex.h.]) AC_MSG_CHECKING(if --with-spinlock option specified) AC_ARG_WITH(spinlock, diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 index cbd389ea40d..b19418fc29d 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 @@ -2,8 +2,8 @@ dnl build by dist/s_version VERSION_MAJOR=2 VERSION_MINOR=5 -VERSION_PATCH=2 -VERSION_STRING='"WiredTiger 2.5.2: (March 9, 2015)"' +VERSION_PATCH=3 +VERSION_STRING='"WiredTiger 2.5.3: (March 26, 2015)"' AC_SUBST(VERSION_MAJOR) AC_SUBST(VERSION_MINOR) diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version.m4 index 340f77e5474..94a84e72955 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/version.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/version.m4 @@ -1,2 +1,2 @@ dnl WiredTiger product version for AC_INIT. Maintained by dist/s_version -2.5.2 +2.5.3 diff --git a/src/third_party/wiredtiger/build_win/filelist.win b/src/third_party/wiredtiger/build_win/filelist.win index 78ddcb14a59..e297ca16b06 100644 --- a/src/third_party/wiredtiger/build_win/filelist.win +++ b/src/third_party/wiredtiger/build_win/filelist.win @@ -103,6 +103,7 @@ src/os_posix/os_alloc.c src/os_posix/os_getline.c src/os_posix/os_getopt.c src/os_posix/os_mtx_rw.c +src/os_posix/os_stdio.c src/os_posix/os_strtouq.c src/os_win/os_dir.c src/os_win/os_dlopen.c diff --git a/src/third_party/wiredtiger/build_win/wiredtiger_config.h b/src/third_party/wiredtiger/build_win/wiredtiger_config.h index 93317b6d81d..d52b3b3ae33 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger_config.h +++ b/src/third_party/wiredtiger/build_win/wiredtiger_config.h @@ -10,6 +10,9 @@ /* Build the LevelDB API with Basho LevelDB support. */ /* #undef HAVE_BASHOLEVELDB */ +/* LZ4 support automatically loaded. */ +/* #undef HAVE_BUILTIN_EXTENSION_LZ4 */ + /* Snappy support automatically loaded. */ /* #undef HAVE_BUILTIN_EXTENSION_SNAPPY */ @@ -55,6 +58,9 @@ /* Define to 1 if you have the `dl' library (-ldl). */ /* #undef HAVE_LIBDL */ +/* Define to 1 if you have the `lz4' library (-llz4). */ +/* #undef HAVE_LIBLZ4 */ + /* Define to 1 if you have the `pthread' library (-lpthread). */ /* #undef HAVE_LIBPTHREAD */ diff --git a/src/third_party/wiredtiger/dist/api_config.py b/src/third_party/wiredtiger/dist/api_config.py index 83a57ad4e0d..5608933defb 100644 --- a/src/third_party/wiredtiger/dist/api_config.py +++ b/src/third_party/wiredtiger/dist/api_config.py @@ -165,9 +165,12 @@ w = textwrap.TextWrapper(width=64, break_on_hyphens=False) w.wordsep_re = w.wordsep_simple_re = re.compile(r'(,)') def checkstr(c): - '''Generate the JSON string used by __wt_config_check to validate the - config string''' + '''Generate the function reference and JSON string used by __wt_config_check + to validate the config string''' checks = c.flags + cfunc = str(checks.get('func', '')) + if not cfunc: + cfunc = 'NULL'; cmin = str(checks.get('min', '')) cmax = str(checks.get('max', '')) choices = checks.get('choices', []) @@ -180,9 +183,9 @@ def checkstr(c): result.append('choices=' + '[' + ','.join('\\"' + s + '\\"' for s in choices) + ']') if result: - return '"' + ','.join(result) + '"' + return cfunc + ', "' + ','.join(result) + '"' else: - return 'NULL' + return cfunc + ', NULL' def get_default(c): t = gettype(c) @@ -206,7 +209,7 @@ def add_subconfig(c): tfile.write(''' static const WT_CONFIG_CHECK confchk_%(name)s_subconfigs[] = { \t%(check)s -\t{ NULL, NULL, NULL, NULL } +\t{ NULL, NULL, NULL, NULL, NULL } }; ''' % { 'name' : c.name, @@ -230,7 +233,7 @@ for name in sorted(api_data.methods.keys()): tfile.write(''' static const WT_CONFIG_CHECK confchk_%(name)s[] = { \t%(check)s -\t{ NULL, NULL, NULL, NULL } +\t{ NULL, NULL, NULL, NULL, NULL } }; ''' % { 'name' : name.replace('.', '_'), diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index feb51011309..a31e5e5ed10 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -20,6 +20,11 @@ class Config: common_meta = [ Config('app_metadata', '', r''' application-owned metadata for this object'''), + Config('collator', 'none', r''' + configure custom collation for keys. Permitted values are + \c "none" or a custom collator name created with + WT_CONNECTION::add_collator''', + func='__wt_collator_confchk'), Config('columns', '', r''' list of the column names. Comma-separated list of the form <code>(column[,...])</code>. For tables, the number of entries @@ -52,7 +57,7 @@ format_meta = common_meta + [ raw byte arrays. By default, records are stored in row-store files: keys of type \c 'r' are record numbers and records referenced by record number are stored in column-store files''', - type='format'), + type='format', func='__wt_struct_confchk'), Config('value_format', 'u', r''' the format of the data packed into value items. See @ref schema_format_types for details. By default, the value_format @@ -60,7 +65,7 @@ format_meta = common_meta + [ manipulate raw byte arrays. Value items of type 't' are bitfields, and when configured with record number type keys, will be stored using a fixed-length store''', - type='format'), + type='format', func='__wt_struct_confchk'), ] lsm_config = [ @@ -131,11 +136,12 @@ file_config = format_meta + [ requirements from the operating system or storage device''', min='512B', max='128MB'), Config('block_compressor', 'none', r''' - configure a compressor for file blocks. Permitted values are - \c "none" or custom compression engine name created with - WT_CONNECTION::add_compressor. If WiredTiger has builtin support - for \c "snappy" or \c "zlib" compression, these names are also - available. See @ref compression for more information'''), + configure a compressor for file blocks. Permitted values are \c "none" + or custom compression engine name created with + WT_CONNECTION::add_compressor. If WiredTiger has builtin support for + \c "bzip2", \c "snappy", \c "lz4" or \c "zlib" compression, these names + are also available. See @ref compression for more information''', + func='__wt_compressor_confchk'), Config('cache_resident', 'false', r''' do not ever evict the object's pages; see @ref tuning_cache_resident for more information''', @@ -148,10 +154,6 @@ file_config = format_meta + [ applications which can rely on decompression to fail if a block has been corrupted''', choices=['on', 'off', 'uncompressed']), - Config('collator', 'none', r''' - configure custom collation for keys. Permitted values are - \c "none" or a custom collator name created with - WT_CONNECTION::add_collator'''), Config('dictionary', '0', r''' the maximum number of unique values remembered in the Btree row-store leaf page value dictionary; see @@ -163,11 +165,13 @@ file_config = format_meta + [ Config('huffman_key', 'none', r''' configure Huffman encoding for keys. Permitted values are \c "none", \c "english", \c "utf8<file>" or \c "utf16<file>". - See @ref huffman for more information'''), + See @ref huffman for more information''', + func='__wt_huffman_confchk'), Config('huffman_value', 'none', r''' configure Huffman encoding for values. Permitted values are \c "none", \c "english", \c "utf8<file>" or \c "utf16<file>". - See @ref huffman for more information'''), + See @ref huffman for more information''', + func='__wt_huffman_confchk'), Config('internal_key_truncate', 'true', r''' configure internal key truncation, discarding unnecessary trailing bytes on internal keys (ignored for custom @@ -288,7 +292,8 @@ index_only_config = [ Config('extractor', 'none', r''' configure custom extractor for indices. Permitted values are \c "none" or an extractor name created with - WT_CONNECTION::add_extractor'''), + WT_CONNECTION::add_extractor''', + func='__wt_extractor_confchk'), Config('immutable', 'false', r''' configure the index to be immutable - that is an index is not changed by any update to a record in the table''', type='boolean'), @@ -313,7 +318,7 @@ connection_runtime_config = [ type='boolean'), Config('ops_max', '1024', r''' maximum number of expected simultaneous asynchronous - operations''', min='10', max='4096'), + operations''', min='1', max='4096'), Config('threads', '2', r''' the number of worker threads to service asynchronous requests''', @@ -364,6 +369,16 @@ connection_runtime_config = [ Config('eviction_trigger', '95', r''' trigger eviction when the cache is using this much memory, as a percentage of the total cache size''', min=10, max=99), + Config('file_manager', '', r''' + control how file handles are managed''', + type='category', subconfig=[ + Config('close_idle_time', '30', r''' + amount of time in seconds a file handle needs to be idle + before attempting to close it''', min=1, max=1000), + Config('close_scan_interval', '10', r''' + interval in seconds at which to check for files that are + inactive and close them''', min=1, max=1000) + ]), Config('lsm_manager', '', r''' configure database wide options for LSM tree management''', type='category', subconfig=[ @@ -383,9 +398,9 @@ connection_runtime_config = [ eviction configuration options.''', type='category', subconfig=[ Config('threads_max', '1', r''' - maximum number of threads WiredTiger will start to help evict - pages from cache. The number of threads started will vary - depending on the current eviction load''', + maximum number of threads WiredTiger will start to help evict + pages from cache. The number of threads started will vary + depending on the current eviction load''', min=1, max=20), Config('threads_min', '1', r''' minimum number of threads WiredTiger will start to help evict @@ -491,8 +506,9 @@ session_config = [ common_wiredtiger_open = [ Config('buffer_alignment', '-1', r''' in-memory alignment (in bytes) for buffers used for I/O. The - default value of -1 indicates a platform-specific alignment - value should be used (4KB on Linux systems, zero elsewhere)''', + default value of -1 indicates a platform-specific alignment value + should be used (4KB on Linux systems when direct I/O is configured, + zero elsewhere)''', min='-1', max='1MB'), Config('checkpoint_sync', 'true', r''' flush files to stable storage when closing or writing @@ -534,9 +550,11 @@ common_wiredtiger_open = [ type='boolean'), Config('compressor', 'none', r''' configure a compressor for log records. Permitted values are - \c "none" or \c "bzip2", \c "snappy" or custom compression - engine \c "name" created with WT_CONNECTION::add_compressor. - See @ref compression for more information'''), + \c "none" or custom compression engine name created with + WT_CONNECTION::add_compressor. If WiredTiger has builtin support + for \c "bzip2", \c "snappy", \c "lz4" or \c "zlib" compression, + these names are also available. See @ref compression for more + information'''), Config('enabled', 'false', r''' enable logging subsystem''', type='boolean'), @@ -574,13 +592,13 @@ common_wiredtiger_open = [ how to sync log records when the transaction commits''', type='category', subconfig=[ Config('enabled', 'false', r''' - whether to sync the log on every commit by default, can - be overridden by the \c sync setting to - WT_SESSION::begin_transaction''', + whether to sync the log on every commit by default, can be + overridden by the \c sync setting to + WT_SESSION::begin_transaction''', type='boolean'), Config('method', 'fsync', r''' - the method used to ensure log records are stable on disk, - see @ref tune_durability for more information''', + the method used to ensure log records are stable on disk, see + @ref tune_durability for more information''', choices=['dsync', 'fsync', 'none']), ]), ] diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index c2346caf1ea..ee70ccf765e 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -123,6 +123,7 @@ src/os_posix/os_remove.c src/os_posix/os_rename.c src/os_posix/os_rw.c src/os_posix/os_sleep.c +src/os_posix/os_stdio.c src/os_posix/os_strtouq.c src/os_posix/os_thread.c src/os_posix/os_time.c diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index f1eb6b24968..34f3ab3e02f 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -54,6 +54,7 @@ flags = { 'TXN_LOG_CKPT_PREPARE', 'TXN_LOG_CKPT_START', 'TXN_LOG_CKPT_STOP', + 'TXN_LOG_CKPT_SYNC', ], 'verbose' : [ 'VERB_API', diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 8b0335a6480..e2c6380660d 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -126,6 +126,7 @@ Filesystems FindFirstFile Fk FlushFileBuffers +Fprintf FreeBSD FreeBSD's FreeLibrary @@ -197,10 +198,12 @@ LNO LOGREC LOGSCAN LRU +LRVv LSB LSM LSN LSNs +LZ LZO LeafGreen LevelDB @@ -359,6 +362,7 @@ VLDB VMSG Vanishingly Vc +Vfprintf Vixie Vo Vv @@ -419,6 +423,7 @@ autoheader bInheritHandle basecfg bdb +beginthreadex bigram bitcnt bitfield @@ -494,6 +499,7 @@ compat concat cond conf +confchk config conn connectionp @@ -537,6 +543,7 @@ decl decr decrement decrementing +deflateCopy deflateEnd deflateInit defno @@ -913,6 +920,7 @@ qsort quartile qup rS +rb rbrace rbracket rdlock @@ -1144,6 +1152,7 @@ variable's vcell verrx versa +vfprintf vlcs vmsg vpack @@ -1156,6 +1165,7 @@ vtype vunpack vupdate walk's +wb wiredtiger workFactor wrapup diff --git a/src/third_party/wiredtiger/dist/s_style b/src/third_party/wiredtiger/dist/s_style index 50d00ff1d7f..ccee9404441 100755 --- a/src/third_party/wiredtiger/dist/s_style +++ b/src/third_party/wiredtiger/dist/s_style @@ -64,6 +64,15 @@ for f in \ cat $t fi + if ! expr "$f" : 'src/os_posix/.*' > /dev/null && + ! expr "$f" : 'src/os_win/.*' > /dev/null && + ! expr "$f" : 'src/include/extern.h' > /dev/null && + ! expr "$f" : 'src/include/os.h' > /dev/null && + grep '__wt_errno' $f > $t; then + echo "$f: upper-level code should not call __wt_errno" + cat $t + fi + if ! expr "$f" : 'examples/c/.*' > /dev/null && ! expr "$f" : 'ext/datasources/helium/helium.c' > /dev/null && ! expr "$f" : 'src/include/os.h' > /dev/null && diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index dd4d292c8b6..f133ab899ea 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -8,7 +8,7 @@ # NOTE: All statistics descriptions must have a prefix string followed by ':'. # # Optional configuration flags: -# no_clear Value ignored by the statistics refresh function +# no_clear Value not cleared when statistics cleared # no_aggregate Ignore the value when aggregating statistics # max_aggregate Take the maximum value when aggregating statistics # no_scale Don't scale value per second in the logging tool script @@ -145,11 +145,20 @@ connection_stats = [ # Cache and eviction statistics ########################################## CacheStat('cache_bytes_dirty', - 'tracked dirty bytes in the cache', 'no_scale'), + 'tracked dirty bytes in the cache', 'no_clear,no_scale'), CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale'), + CacheStat('cache_bytes_internal', + 'tracked bytes belonging to internal pages in the cache', + 'no_clear,no_scale'), + CacheStat('cache_bytes_leaf', + 'tracked bytes belonging to leaf pages in the cache', + 'no_clear,no_scale'), CacheStat('cache_bytes_max', 'maximum bytes configured', 'no_clear,no_scale'), + CacheStat('cache_bytes_overflow', + 'tracked bytes belonging to overflow pages in the cache', + 'no_clear,no_scale'), CacheStat('cache_bytes_read', 'bytes read into cache'), CacheStat('cache_bytes_write', 'bytes written from cache'), CacheStat('cache_eviction_app', 'pages evicted by application threads'), @@ -187,7 +196,7 @@ connection_stats = [ CacheStat('cache_inmem_split', 'in-memory page splits'), CacheStat('cache_overhead', 'percentage overhead', 'no_clear,no_scale'), CacheStat('cache_pages_dirty', - 'tracked dirty pages in the cache', 'no_scale'), + 'tracked dirty pages in the cache', 'no_clear,no_scale'), CacheStat('cache_pages_inuse', 'pages currently held in the cache', 'no_clear,no_scale'), CacheStat('cache_read', 'pages read into cache'), @@ -218,7 +227,8 @@ connection_stats = [ LogStat('log_compress_write_fails', 'log records not compressed'), LogStat('log_max_filesize', 'maximum log file size', 'no_clear,no_scale'), LogStat('log_prealloc_files', 'pre-allocated log files prepared'), - LogStat('log_prealloc_max', 'number of pre-allocated log files to create'), + LogStat('log_prealloc_max', + 'number of pre-allocated log files to create', 'no_clear,no_scale'), LogStat('log_prealloc_used', 'pre-allocated log files used'), LogStat('log_reads', 'log read operations'), LogStat('log_release_write_lsn', 'log release advances write LSN'), @@ -256,6 +266,8 @@ connection_stats = [ ########################################## TxnStat('txn_begin', 'transaction begins'), TxnStat('txn_checkpoint', 'transaction checkpoints'), + TxnStat('txn_checkpoint_generation', + 'transaction checkpoint generation', 'no_clear,no_scale'), TxnStat('txn_checkpoint_time_max', 'transaction checkpoint max time (msecs)', 'no_aggregate,no_clear,no_scale'), @@ -271,6 +283,9 @@ connection_stats = [ TxnStat('txn_checkpoint_running', 'transaction checkpoint currently running', 'no_aggregate,no_clear,no_scale'), + TxnStat('txn_pinned_checkpoint_range', + 'transaction range of IDs currently pinned by a checkpoint', + 'no_aggregate,no_clear,no_scale'), TxnStat('txn_pinned_range', 'transaction range of IDs currently pinned', 'no_aggregate,no_clear,no_scale'), @@ -360,6 +375,8 @@ dsrc_stats = [ ########################################## # Btree statistics ########################################## + BtreeStat('btree_checkpoint_generation', + 'btree checkpoint generation', 'no_clear,no_scale'), BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale'), BtreeStat('btree_column_fix', diff --git a/src/third_party/wiredtiger/ext/compressors/lz4/Makefile.am b/src/third_party/wiredtiger/ext/compressors/lz4/Makefile.am new file mode 100644 index 00000000000..c800a5adb15 --- /dev/null +++ b/src/third_party/wiredtiger/ext/compressors/lz4/Makefile.am @@ -0,0 +1,11 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include + +if HAVE_BUILTIN_EXTENSION_LZ4 +noinst_LTLIBRARIES = libwiredtiger_lz4.la +else +lib_LTLIBRARIES = libwiredtiger_lz4.la +libwiredtiger_lz4_la_LDFLAGS = -avoid-version -module +endif + +libwiredtiger_lz4_la_SOURCES = lz4_compress.c +libwiredtiger_lz4_la_LIBADD = -llz4 diff --git a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c new file mode 100644 index 00000000000..e6b8219aafb --- /dev/null +++ b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c @@ -0,0 +1,251 @@ +/*- + * Public Domain 2014-2015 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <lz4.h> +#include <errno.h> +#include <stdlib.h> +#include <string.h> + +#include <wiredtiger.h> +#include <wiredtiger_ext.h> + +/* + * We need to include the configuration file to detect whether this extension + * is being built into the WiredTiger library. + */ +#include "wiredtiger_config.h" + +/* Local compressor structure. */ +typedef struct { + WT_COMPRESSOR compressor; /* Must come first */ + WT_EXTENSION_API *wt_api; /* Extension API */ +} LZ4_COMPRESSOR; + +/* + * wt_lz4_error -- + * Output an error message, and return a standard error code. + */ +static int +wt_lz4_error( + WT_COMPRESSOR *compressor, WT_SESSION *session, const char *call, int zret) +{ + WT_EXTENSION_API *wt_api; + + wt_api = ((LZ4_COMPRESSOR *)compressor)->wt_api; + + (void)wt_api->err_printf(wt_api, + session, "lz4 error: %s: %d", call, zret); + return (WT_ERROR); +} + +/* + * wt_lz4_compress -- + * WiredTiger LZ4 compression. + */ +static int +wt_lz4_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp, int *compression_failed) +{ + char *lz4buf; + size_t lz4_len; + + /* + * The buffer should always be large enough due to the lz4_pre_size + * call, but be paranoid and error if it isn't. + */ + if (dst_len < src_len + sizeof(size_t)) + return (wt_lz4_error(compressor, session, + "LZ4 compress buffer too small", 0)); + + /* Store the length of the compressed block in the first 8 bytes. */ + lz4buf = (char *)dst + sizeof(size_t); + lz4_len = (size_t)LZ4_compress((const char *)src, lz4buf, (int)src_len); + + /* + * Flag no-compression if the result was larger than the original + * size or compression failed. + */ + if (lz4_len == 0 || lz4_len + sizeof(size_t) >= src_len) + *compression_failed = 1; + else { + /* + * On decompression, lz4 requires the exact compressed byte + * count (the current value of lz4_len). WiredTiger does not + * preserve that value, so save lz4_len at the beginning of the + * destination buffer. + */ + *(size_t *)dst = lz4_len; + *result_lenp = lz4_len + sizeof(size_t); + *compression_failed = 0; + } + + return (0); +} + +/* + * wt_lz4_decompress -- + * WiredTiger LZ4 decompression. + */ +static int +wt_lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp) +{ + WT_EXTENSION_API *wt_api; + char *compressed_data; + int decoded; + size_t src_data_len; + + wt_api = ((LZ4_COMPRESSOR *)compressor)->wt_api; + + /* Retrieve compressed length from start of the data buffer. */ + src_data_len = *(size_t *)src; + if (src_data_len + sizeof(size_t) > src_len) { + (void)wt_api->err_printf(wt_api, + session, + "wt_lz4_decompress: stored size exceeds buffer size"); + return (WT_ERROR); + } + + /* Skip over the data size to the start of compressed data. */ + compressed_data = (char *)src + sizeof(size_t); + + /* + * The destination buffer length should always be sufficient because + * wiredtiger keeps track of the byte count before compression. Use + * safe decompression: we may be relying on decompression to detect + * corruption. + */ + decoded = LZ4_decompress_safe( + compressed_data, (char *)dst, (int)src_data_len, (int)dst_len); + + if (decoded < 0) + return (wt_lz4_error(compressor, session, + "LZ4 decompress error", decoded)); + + /* return the uncompressed data length */ + *result_lenp = dst_len; + + return (0); +} + +/* + * wt_lz4_pre_size -- + * WiredTiger LZ4 destination buffer sizing for compression. + */ +static int +wt_lz4_pre_size(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + size_t *result_lenp) +{ + (void)compressor; + (void)session; + (void)src; + + /* + * LZ4 can use more space than the input data size, use the library + * calculation of that overhead (plus our overhead) to be safe. + */ + *result_lenp = LZ4_COMPRESSBOUND(src_len) + sizeof(size_t); + return (0); +} + +/* + * wt_lz4_terminate -- + * WiredTiger LZ4 compression termination. + */ +static int +wt_lz4_terminate(WT_COMPRESSOR *compressor, WT_SESSION *session) +{ + (void)session; + + /* Free the allocated memory. */ + free(compressor); + + return (0); +} + +int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); + +/* + * lz4_extension_init -- + * A simple shared library compression example. + */ +int +lz4_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) +{ + LZ4_COMPRESSOR *lz4_compressor; + + (void)config; /* Unused parameters */ + + if ((lz4_compressor = calloc(1, sizeof(LZ4_COMPRESSOR))) == NULL) + return (errno); + + /* + * Allocate a local compressor structure, with a WT_COMPRESSOR structure + * as the first field, allowing us to treat references to either type of + * structure as a reference to the other type. + * + * This could be simplified if only a single database is opened in the + * application, we could use a static WT_COMPRESSOR structure, and a + * static reference to the WT_EXTENSION_API methods, then we don't need + * to allocate memory when the compressor is initialized or free it when + * the compressor is terminated. However, this approach is more general + * purpose and supports multiple databases per application. + */ + lz4_compressor->compressor.compress = wt_lz4_compress; + lz4_compressor->compressor.compress_raw = NULL; + lz4_compressor->compressor.decompress = wt_lz4_decompress; + lz4_compressor->compressor.pre_size = wt_lz4_pre_size; + lz4_compressor->compressor.terminate = wt_lz4_terminate; + + lz4_compressor->wt_api = connection->get_extension_api(connection); + + /* Load the compressor */ + return (connection->add_compressor( + connection, "lz4", (WT_COMPRESSOR *)lz4_compressor, NULL)); +} + +/* + * We have to remove this symbol when building as a builtin extension otherwise + * it will conflict with other builtin libraries. + */ +#ifndef HAVE_BUILTIN_EXTENSION_LZ4 +/* + * wiredtiger_extension_init -- + * WiredTiger LZ4 compression extension. + */ +int +wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) +{ + return lz4_extension_init(connection, config); +} +#endif diff --git a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c index 63f6f87f7f6..286637f91c3 100644 --- a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c @@ -39,9 +39,6 @@ * is being built into the WiredTiger library. */ #include "wiredtiger_config.h" -#ifdef _MSC_VER -#define inline __inline -#endif /* Local compressor structure. */ typedef struct { @@ -63,6 +60,7 @@ wt_snappy_error(WT_COMPRESSOR *compressor, wt_api = ((SNAPPY_COMPRESSOR *)compressor)->wt_api; + msg = "unknown snappy status error"; switch (snret) { case SNAPPY_BUFFER_TOO_SMALL: msg = "SNAPPY_BUFFER_TOO_SMALL"; @@ -70,9 +68,8 @@ wt_snappy_error(WT_COMPRESSOR *compressor, case SNAPPY_INVALID_INPUT: msg = "SNAPPY_INVALID_INPUT"; break; - default: - msg = "unknown error"; - break; + case SNAPPY_OK: + return (0); } (void)wt_api->err_printf(wt_api, diff --git a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c index 8cf8ab4792e..381bf0d5070 100644 --- a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c @@ -143,7 +143,7 @@ zlib_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, zs.avail_out = (uint32_t)dst_len; if (deflate(&zs, Z_FINISH) == Z_STREAM_END) { *compression_failed = 0; - *result_lenp = zs.total_out; + *result_lenp = (size_t)zs.total_out; } else *compression_failed = 1; @@ -158,7 +158,7 @@ zlib_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, * Find the slot containing the target offset (binary search). */ static inline uint32_t -zlib_find_slot(uint32_t target, uint32_t *offsets, uint32_t slots) +zlib_find_slot(uint64_t target, uint32_t *offsets, uint32_t slots) { uint32_t base, indx, limit; @@ -180,6 +180,48 @@ zlib_find_slot(uint32_t target, uint32_t *offsets, uint32_t slots) } /* + * zlib_decompress -- + * WiredTiger zlib decompression. + */ +static int +zlib_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp) +{ + ZLIB_OPAQUE opaque; + z_stream zs; + int ret, tret; + + memset(&zs, 0, sizeof(zs)); + zs.zalloc = zalloc; + zs.zfree = zfree; + opaque.compressor = compressor; + opaque.session = session; + zs.opaque = &opaque; + + if ((ret = inflateInit(&zs)) != Z_OK) + return (zlib_error(compressor, session, "inflateInit", ret)); + + zs.next_in = src; + zs.avail_in = (uint32_t)src_len; + zs.next_out = dst; + zs.avail_out = (uint32_t)dst_len; + while ((ret = inflate(&zs, Z_FINISH)) == Z_OK) + ; + if (ret == Z_STREAM_END) { + *result_lenp = (size_t)zs.total_out; + ret = Z_OK; + } + + if ((tret = inflateEnd(&zs)) != Z_OK && ret == Z_OK) + ret = tret; + + return (ret == Z_OK ? + 0 : zlib_error(compressor, session, "inflate", ret)); +} + +/* * zlib_compress_raw -- * Pack records into a specified on-disk page size. */ @@ -192,7 +234,7 @@ zlib_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, { ZLIB_COMPRESSOR *zlib_compressor; ZLIB_OPAQUE opaque; - z_stream last_zs, zs; + z_stream *best_zs, last_zs, zs; uint32_t curr_slot, last_slot; int ret; @@ -210,8 +252,7 @@ zlib_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, opaque.session = session; zs.opaque = &opaque; - if ((ret = deflateInit(&zs, - zlib_compressor->zlib_level)) != Z_OK) + if ((ret = deflateInit(&zs, zlib_compressor->zlib_level)) != Z_OK) return (zlib_error(compressor, session, "deflateInit", ret)); zs.next_in = src; @@ -222,51 +263,79 @@ zlib_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, * inefficient. */ #define WT_ZLIB_RESERVED 24 - zs.avail_out = (uint32_t)(page_max - extra - WT_ZLIB_RESERVED); - last_zs = zs; + zs.avail_out = (uint32_t)(page_max - (extra + WT_ZLIB_RESERVED)); + + /* Save the stream state in case the chosen data doesn't fit. */ + if ((ret = deflateCopy(&last_zs, &zs)) != Z_OK) + return (zlib_error( + compressor, session, "deflateCopy", ret)); /* * Strategy: take the available output size and compress that much * input. Continue until there is no input small enough or the * compression fails to fit. - * - * Don't let the compression ratio become insanely good (which can - * happen with synthetic workloads). Once we hit a limit, stop so that - * the in-memory size of pages isn't totally different to the on-disk - * size. Otherwise we can get into trouble where every update to a - * page results in forced eviction based on in-memory size, even though - * the data fits into a single on-disk block. */ - while (zs.avail_out > 0 && zs.total_in <= zs.total_out * 20) { - /* Find the slot we will try to compress up to. */ + for (best_zs = NULL;;) { + /* Find the next slot we will try to compress up to. */ if ((curr_slot = zlib_find_slot( - zs.total_in + zs.avail_out, offsets, slots)) <= last_slot) - break; - - zs.avail_in = offsets[curr_slot] - offsets[last_slot]; - /* Save the stream state in case the chosen data doesn't fit. */ - last_zs = zs; + zs.total_in + zs.avail_out, offsets, slots)) > last_slot) { + zs.avail_in = offsets[curr_slot] - offsets[last_slot]; + while (zs.avail_in > 0 && zs.avail_out > 0) + if ((ret = deflate(&zs, Z_SYNC_FLUSH)) != Z_OK) + return (zlib_error(compressor, + session, "deflate", ret)); + } - while (zs.avail_in > 0 && zs.avail_out > 0) - if ((ret = deflate(&zs, Z_SYNC_FLUSH)) != Z_OK) + /* + * We didn't do a deflate, or it didn't work: use the last saved + * position. + */ + if (curr_slot <= last_slot || zs.avail_in > 0) { + if ((ret = deflateEnd(&zs)) != Z_OK && + ret != Z_DATA_ERROR) return (zlib_error( - compressor, session, "deflate", ret)); + compressor, session, "deflateEnd", ret)); - /* Roll back if the last deflate didn't complete. */ - if (zs.avail_in > 0) { - zs = last_zs; + best_zs = &last_zs; break; - } else - last_slot = curr_slot; + } + + /* The last deflation succeeded, discard the saved one. */ + if ((ret = deflateEnd(&last_zs)) != Z_OK && ret != Z_DATA_ERROR) + return (zlib_error( + compressor, session, "deflateEnd", ret)); + + /* + * If there's more compression to do, save a snapshot and keep + * going, otherwise, use the current compression. + * + * Don't let the compression ratio become insanely good (which + * can happen with synthetic workloads). Once we hit a limit, + * stop so the in-memory size of pages isn't hugely larger than + * the on-disk size, otherwise we can get into trouble where + * every update to a page results in forced eviction based on + * the in-memory size, even though the data fits into a single + * on-disk block. + */ + last_slot = curr_slot; + if (zs.avail_out > 0 && zs.total_in <= zs.total_out * 20) { + if ((ret = deflateCopy(&last_zs, &zs)) != Z_OK) + return (zlib_error( + compressor, session, "deflateCopy", ret)); + continue; + } + + best_zs = &zs; + break; } - zs.avail_out += WT_ZLIB_RESERVED; - ret = deflate(&zs, Z_FINISH); + best_zs->avail_out += WT_ZLIB_RESERVED; + ret = deflate(best_zs, Z_FINISH); /* - * If the end marker didn't fit, report that we got no work done. WT - * will compress the (possibly large) page image using ordinary - * compression instead. + * If the end marker didn't fit, report that we got no work done, + * WiredTiger will compress the (possibly large) page image using + * ordinary compression instead. */ if (ret == Z_OK || ret == Z_BUF_ERROR) last_slot = 0; @@ -274,12 +343,12 @@ zlib_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, return ( zlib_error(compressor, session, "deflate end block", ret)); - if ((ret = deflateEnd(&zs)) != Z_OK && ret != Z_DATA_ERROR) + if ((ret = deflateEnd(best_zs)) != Z_OK && ret != Z_DATA_ERROR) return (zlib_error(compressor, session, "deflateEnd", ret)); if (last_slot > 0) { *result_slotsp = last_slot; - *result_lenp = zs.total_out; + *result_lenp = (size_t)best_zs->total_out; } else { /* We didn't manage to compress anything: don't retry. */ *result_slotsp = 0; @@ -287,6 +356,28 @@ zlib_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, } #if 0 + /* Decompress the result and confirm it matches the original source. */ + if (last_slot > 0) { + void *decomp; + size_t result_len; + + if ((decomp = zalloc( + &opaque, 1, (uint32_t)best_zs->total_in + 100)) == NULL) + return (ENOMEM); + if ((ret = zlib_decompress( + compressor, session, dst, (size_t)best_zs->total_out, + decomp, (size_t)best_zs->total_in + 100, &result_len)) == 0) + if (memcmp(src, decomp, result_len) != 0) + ret = zlib_error(compressor, session, + "deflate compare with original source", + Z_DATA_ERROR); + zfree(&opaque, decomp); + if (ret != 0) + return (ret); + } +#endif + +#if 0 fprintf(stderr, "zlib_compress_raw (%s): page_max %" PRIuMAX ", slots %" PRIu32 ", take %" PRIu32 ": %" PRIu32 " -> %" PRIuMAX "\n", @@ -297,48 +388,6 @@ zlib_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, } /* - * zlib_decompress -- - * WiredTiger zlib decompression. - */ -static int -zlib_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, - uint8_t *src, size_t src_len, - uint8_t *dst, size_t dst_len, - size_t *result_lenp) -{ - ZLIB_OPAQUE opaque; - z_stream zs; - int ret, tret; - - memset(&zs, 0, sizeof(zs)); - zs.zalloc = zalloc; - zs.zfree = zfree; - opaque.compressor = compressor; - opaque.session = session; - zs.opaque = &opaque; - - if ((ret = inflateInit(&zs)) != Z_OK) - return (zlib_error(compressor, session, "inflateInit", ret)); - - zs.next_in = src; - zs.avail_in = (uint32_t)src_len; - zs.next_out = dst; - zs.avail_out = (uint32_t)dst_len; - while ((ret = inflate(&zs, Z_FINISH)) == Z_OK) - ; - if (ret == Z_STREAM_END) { - *result_lenp = zs.total_out; - ret = Z_OK; - } - - if ((tret = inflateEnd(&zs)) != Z_OK && ret == Z_OK) - ret = tret; - - return (ret == Z_OK ? - 0 : zlib_error(compressor, session, "inflate", ret)); -} - -/* * zlib_terminate -- * WiredTiger zlib compression termination. */ diff --git a/src/third_party/wiredtiger/lang/java/Makefile.am b/src/third_party/wiredtiger/lang/java/Makefile.am index 94a7cb2702d..e6e6f748837 100644 --- a/src/third_party/wiredtiger/lang/java/Makefile.am +++ b/src/third_party/wiredtiger/lang/java/Makefile.am @@ -1,4 +1,4 @@ -AM_CPPFLAGS = -I$(top_srcdir) +AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/src/include JAVADEST = src/com/wiredtiger/db JAVADESTFULL = $(srcdir)/$(JAVADEST) diff --git a/src/third_party/wiredtiger/lang/python/setup.py b/src/third_party/wiredtiger/lang/python/setup.py index 28bbe4d07e8..9eb57d55b5c 100644 --- a/src/third_party/wiredtiger/lang/python/setup.py +++ b/src/third_party/wiredtiger/lang/python/setup.py @@ -36,7 +36,7 @@ if not 'ARCHFLAGS' in os.environ: os.environ['ARCHFLAGS'] = '' # Suppress warnings building SWIG generated code -extra_cflags = [ '-w' ] +extra_cflags = [ '-w', '-I../../src/include'] dir = os.path.dirname(__file__) diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c index e211f2352a4..44e492cb0e5 100644 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ b/src/third_party/wiredtiger/src/async/async_api.c @@ -181,11 +181,13 @@ __async_config(WT_SESSION_IMPL *session, *runp = cval.val != 0; /* - * Even if async is turned off, we want to parse and store the - * default values so that reconfigure can just enable them. + * Even if async is turned off, we want to parse and store the default + * values so that reconfigure can just enable them. + * + * Bound the minimum maximum operations at 10. */ WT_RET(__wt_config_gets(session, cfg, "async.ops_max", &cval)); - conn->async_size = (uint32_t)cval.val; + conn->async_size = (uint32_t)WT_MAX(cval.val, 10); WT_RET(__wt_config_gets(session, cfg, "async.threads", &cval)); conn->async_workers = (uint32_t)cval.val; diff --git a/src/third_party/wiredtiger/src/async/async_op.c b/src/third_party/wiredtiger/src/async/async_op.c index 86797af635b..d0c58f584cc 100644 --- a/src/third_party/wiredtiger/src/async/async_op.c +++ b/src/third_party/wiredtiger/src/async/async_op.c @@ -267,11 +267,17 @@ __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) conn = S2C(session); async = conn->async; + /* - * Enqueue op at the tail of the work queue. + * If an application re-uses a WT_ASYNC_OP, we end up here with an + * invalid object. */ - WT_ASSERT(session, op->state == WT_ASYNCOP_READY); + if (op->state != WT_ASYNCOP_READY) + WT_RET_MSG(session, EINVAL, + "application error: WT_ASYNC_OP already in use"); + /* + * Enqueue op at the tail of the work queue. * We get our slot in the ring buffer to use. */ my_alloc = WT_ATOMIC_ADD8(async->alloc_head, 1); diff --git a/src/third_party/wiredtiger/src/async/async_worker.c b/src/third_party/wiredtiger/src/async/async_worker.c index c68d0e8a838..543046f7a0c 100644 --- a/src/third_party/wiredtiger/src/async/async_worker.c +++ b/src/third_party/wiredtiger/src/async/async_worker.c @@ -18,8 +18,8 @@ __async_op_dequeue(WT_CONNECTION_IMPL *conn, WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL **op) { WT_ASYNC *async; - long sleep_usec; uint64_t cur_tail, last_consume, my_consume, my_slot, prev_slot; + uint64_t sleep_usec; uint32_t tries; async = conn->async; @@ -75,7 +75,8 @@ retry: */ my_slot = my_consume % async->async_qsize; prev_slot = last_consume % async->async_qsize; - *op = WT_ATOMIC_STORE8(async->async_queue[my_slot], NULL); + *op = (WT_ASYNC_OP_IMPL*)WT_ATOMIC_STORE8( + async->async_queue[my_slot], NULL); WT_ASSERT(session, async->cur_queue > 0); WT_ASSERT(session, *op != NULL); @@ -278,10 +279,10 @@ __async_worker_op(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op, } /* - * __async_worker -- + * __wt_async_worker -- * The async worker threads. */ -void * +WT_THREAD_RET __wt_async_worker(void *arg) { WT_ASYNC *async; @@ -354,5 +355,5 @@ err: WT_PANIC_MSG(session, ret, "async worker error"); __wt_free(session, ac); ac = acnext; } - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index bb0a0543053..61f8a90bac5 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -224,6 +224,9 @@ __wt_block_checkpoint(WT_SESSION_IMPL *session, ci = &block->live; + /* Switch to first-fit allocation. */ + __wt_block_configure_first_fit(block, 1); + /* * Write the root page: it's possible for there to be a checkpoint of * an empty tree, in which case, we store an illegal root offset. @@ -237,7 +240,7 @@ __wt_block_checkpoint(WT_SESSION_IMPL *session, ci->root_offset = WT_BLOCK_INVALID_OFFSET; ci->root_size = ci->root_cksum = 0; } else - WT_RET(__wt_block_write_off(session, block, buf, + WT_ERR(__wt_block_write_off(session, block, buf, &ci->root_offset, &ci->root_size, &ci->root_cksum, data_cksum, 0)); @@ -245,7 +248,7 @@ __wt_block_checkpoint(WT_SESSION_IMPL *session, * Checkpoints are potentially reading/writing/merging lots of blocks, * pre-allocate structures for this thread's use. */ - WT_RET(__wt_block_ext_prealloc(session, 250)); + WT_ERR(__wt_block_ext_prealloc(session, 250)); /* Process the checkpoint list, deleting and updating as required. */ ret = __ckpt_process(session, block, ckptbase); @@ -253,6 +256,9 @@ __wt_block_checkpoint(WT_SESSION_IMPL *session, /* Discard any excessive memory we've allocated. */ WT_TRET(__wt_block_ext_discard(session, 250)); + /* Restore the original allocation plan. */ +err: __wt_block_configure_first_fit(block, 0); + return (ret); } diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index a28b6d7a71e..4fcf6ea24ff 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -19,15 +19,10 @@ __wt_block_compact_start(WT_SESSION_IMPL *session, WT_BLOCK *block) { WT_UNUSED(session); - /* - * Save the current allocation plan, switch to first-fit allocation. - * We don't need the lock, but it's not a performance question and - * might avoid bugs in the future. - */ - __wt_spin_lock(session, &block->live_lock); - block->allocfirst_save = block->allocfirst; - block->allocfirst = 1; - __wt_spin_unlock(session, &block->live_lock); + /* Switch to first-fit allocation. */ + __wt_block_configure_first_fit(block, 1); + + block->compact_pct_tenths = 0; return (0); } @@ -41,14 +36,8 @@ __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) { WT_UNUSED(session); - /* - * Restore the previous allocation plan. - * We don't need the lock, but it's not a performance question and - * might avoid bugs in the future. - */ - __wt_spin_lock(session, &block->live_lock); - block->allocfirst = block->allocfirst_save; - __wt_spin_unlock(session, &block->live_lock); + /* Restore the original allocation plan. */ + __wt_block_configure_first_fit(block, 0); block->compact_pct_tenths = 0; diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index 5c824b59ec9..83df5556ac4 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -37,7 +37,7 @@ __wt_block_manager_truncate( WT_ERR(__wt_fsync(session, fh)); /* Close the file handle. */ -err: WT_TRET(__wt_close(session, fh)); +err: WT_TRET(__wt_close(session, &fh)); return (ret); } @@ -67,7 +67,7 @@ __wt_block_manager_create( WT_TRET(__wt_fsync(session, fh)); /* Close the file handle. */ - WT_TRET(__wt_close(session, fh)); + WT_TRET(__wt_close(session, &fh)); /* * If checkpoint syncing is enabled, some filesystems require that we @@ -105,7 +105,7 @@ __block_destroy(WT_SESSION_IMPL *session, WT_BLOCK *block) __wt_free(session, block->name); if (block->fh != NULL) - WT_TRET(__wt_close(session, block->fh)); + WT_TRET(__wt_close(session, &block->fh)); __wt_spin_destroy(session, &block->live_lock); @@ -115,6 +115,25 @@ __block_destroy(WT_SESSION_IMPL *session, WT_BLOCK *block) } /* + * __wt_block_configure_first_fit -- + * Configure first-fit allocation. + */ +void +__wt_block_configure_first_fit(WT_BLOCK *block, int on) +{ + /* + * Switch to first-fit allocation so we rewrite blocks at the start of + * the file; use atomic instructions because checkpoints also configure + * first-fit allocation, and this way we stay on first-fit allocation + * as long as any operation wants it. + */ + if (on) + (void)WT_ATOMIC_ADD4(block->allocfirst, 1); + else + (void)WT_ATOMIC_SUB4(block->allocfirst, 1); +} + +/* * __wt_block_open -- * Open a block handle. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index e84a63695f9..fa7cff35e5f 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -99,8 +99,7 @@ __debug_config(WT_SESSION_IMPL *session, WT_DBG *ds, const char *ofile) return (__wt_scr_alloc(session, 512, &ds->msg)); /* If we're using a file, flush on each line. */ - if ((ds->fp = fopen(ofile, "w")) == NULL) - WT_RET_MSG(session, __wt_errno(), "%s", ofile); + WT_RET(__wt_fopen(session, ofile, WT_FHANDLE_WRITE, 0, &ds->fp)); (void)setvbuf(ds->fp, NULL, _IOLBF, 0); return (0); @@ -132,8 +131,7 @@ __dmsg_wrapup(WT_DBG *ds) } /* Close any file we opened. */ - if (ds->fp != NULL) - (void)fclose(ds->fp); + (void)__wt_fclose(session, &ds->fp, WT_FHANDLE_WRITE); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index 479f6547e42..8cca6328f21 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -264,6 +264,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) WT_PAGE *page; WT_PAGE_DELETED *page_del; WT_UPDATE **upd_array, *upd; + size_t size; uint32_t i; btree = S2BT(session); @@ -323,7 +324,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) * structures, fill in the per-page update array with references to * deleted items. */ - for (i = 0; i < page->pg_row_entries; ++i) { + for (i = 0, size = 0; i < page->pg_row_entries; ++i) { WT_ERR(__wt_calloc_one(session, &upd)); WT_UPDATE_DELETED_SET(upd); @@ -336,10 +337,11 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) upd->next = upd_array[i]; upd_array[i] = upd; + + size += sizeof(WT_UPDATE *) + WT_UPDATE_MEMSIZE(upd); } - __wt_cache_page_inmem_incr(session, page, - page->pg_row_entries * (sizeof(WT_UPDATE *) + sizeof(WT_UPDATE))); + __wt_cache_page_inmem_incr(session, page, size); return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 5b3624a4a2d..1e0f95d3131 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -185,15 +185,12 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) { WT_BTREE *btree; WT_CONFIG_ITEM cval, metadata; - WT_CONNECTION_IMPL *conn; - WT_NAMED_COMPRESSOR *ncomp; int64_t maj_version, min_version; uint32_t bitcnt; int fixed; const char **cfg; btree = S2BT(session); - conn = S2C(session); cfg = btree->dhandle->cfg; /* Dump out format information. */ @@ -212,7 +209,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) /* Validate file types and check the data format plan. */ WT_RET(__wt_config_gets(session, cfg, "key_format", &cval)); - WT_RET(__wt_struct_check(session, cval.str, cval.len, NULL, NULL)); + WT_RET(__wt_struct_confchk(session, &cval)); if (WT_STRING_MATCH("r", cval.str, cval.len)) btree->type = BTREE_COL_VAR; else @@ -220,18 +217,19 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_RET(__wt_strndup(session, cval.str, cval.len, &btree->key_format)); WT_RET(__wt_config_gets(session, cfg, "value_format", &cval)); - WT_RET(__wt_struct_check(session, cval.str, cval.len, NULL, NULL)); + WT_RET(__wt_struct_confchk(session, &cval)); WT_RET(__wt_strndup(session, cval.str, cval.len, &btree->value_format)); /* Row-store key comparison and key gap for prefix compression. */ if (btree->type == BTREE_ROW) { - WT_RET( - __wt_config_gets(session, cfg, "app_metadata", &metadata)); WT_RET(__wt_config_gets_none(session, cfg, "collator", &cval)); - if (cval.len != 0) + if (cval.len != 0) { + WT_RET(__wt_config_gets( + session, cfg, "app_metadata", &metadata)); WT_RET(__wt_collator_config( session, btree->dhandle->name, &cval, &metadata, &btree->collator, &btree->collator_owned)); + } WT_RET(__wt_config_gets(session, cfg, "key_gap", &cval)); btree->key_gap = (uint32_t)cval.val; @@ -307,17 +305,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) } WT_RET(__wt_config_gets_none(session, cfg, "block_compressor", &cval)); - if (cval.len > 0) { - TAILQ_FOREACH(ncomp, &conn->compqh, q) - if (WT_STRING_MATCH(ncomp->name, cval.str, cval.len)) { - btree->compressor = ncomp->compressor; - break; - } - if (btree->compressor == NULL) - WT_RET_MSG(session, EINVAL, - "unknown block compressor '%.*s'", - (int)cval.len, cval.str); - } + WT_RET(__wt_compressor_config(session, &cval, &btree->compressor)); /* Initialize locks. */ WT_RET(__wt_rwlock_alloc( @@ -376,9 +364,9 @@ __wt_btree_tree_open( * the page steals it. */ WT_ERR(__wt_bt_read(session, &dsk, addr, addr_size)); - WT_ERR(__wt_page_inmem(session, NULL, dsk.data, + WT_ERR(__wt_page_inmem(session, NULL, dsk.data, dsk.memsize, WT_DATA_IN_ITEM(&dsk) ? - WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED , &page)); + WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED, &page)); dsk.mem = NULL; /* Finish initializing the root, root reference links. */ @@ -623,10 +611,10 @@ __btree_page_sizes(WT_SESSION_IMPL *session) * When a page is forced to split, we want at least 50 entries on its * parent. * - * Don't let pages grow to more than half the cache size. Otherwise, - * with very small caches, we can end up in a situation where nothing - * can be evicted. Take care getting the cache size: with a shared - * cache, it may not have been set. + * Don't let pages grow larger than a quarter of the cache, with too- + * small caches, we can end up in a situation where nothing can be + * evicted. Take care getting the cache size: with a shared cache, + * it may not have been set. */ WT_RET(__wt_config_gets(session, cfg, "memory_page_max", &cval)); btree->maxmempage = diff --git a/src/third_party/wiredtiger/src/btree/bt_huffman.c b/src/third_party/wiredtiger/src/btree/bt_huffman.c index c31b3f2fdf1..6604bd9c9e1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_huffman.c +++ b/src/third_party/wiredtiger/src/btree/bt_huffman.c @@ -128,28 +128,66 @@ static const struct __wt_huffman_table __wt_huffman_nytenglish[] = { static int __wt_huffman_read(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, struct __wt_huffman_table **, u_int *, u_int *); -#define WT_HUFFMAN_CONFIG_VALID(str, len) \ - (WT_STRING_CASE_MATCH("english", (str), (len)) || \ - WT_PREFIX_MATCH((str), "utf8") || WT_PREFIX_MATCH((str), "utf16")) - /* - * __btree_huffman_config -- - * Verify the key or value strings passed in. + * __huffman_confchk_file -- + * Check for a Huffman configuration file and return the file name. */ static int -__btree_huffman_config(WT_SESSION_IMPL *session, - WT_CONFIG_ITEM *key_conf, WT_CONFIG_ITEM *value_conf) +__huffman_confchk_file( + WT_SESSION_IMPL *session, WT_CONFIG_ITEM *v, int *is_utf8p, FILE **fpp) { - if (key_conf->len != 0 && - !WT_HUFFMAN_CONFIG_VALID(key_conf->str, key_conf->len)) - WT_RET_MSG( - session, EINVAL, "illegal Huffman key configuration"); - if (value_conf->len != 0 && - !WT_HUFFMAN_CONFIG_VALID(value_conf->str, value_conf->len)) - WT_RET_MSG( - session, EINVAL, "illegal Huffman value configuration"); - return (0); + FILE *fp; + WT_DECL_RET; + size_t len; + char *fname; + + /* Look for a prefix and file name. */ + len = 0; + if (is_utf8p != NULL) + *is_utf8p = 0; + if (WT_PREFIX_MATCH(v->str, "utf8")) { + if (is_utf8p != NULL) + *is_utf8p = 1; + len = strlen("utf8"); + } else if (WT_PREFIX_MATCH(v->str, "utf16")) + len = strlen("utf16"); + if (len == 0 || len >= v->len) + WT_RET_MSG(session, EINVAL, + "illegal Huffman configuration: %.*s", (int)v->len, v->str); + + /* Check the file exists. */ + WT_RET(__wt_strndup(session, v->str + len, v->len - len, &fname)); + WT_ERR(__wt_fopen(session, + fname, WT_FHANDLE_READ, WT_FOPEN_FIXED, &fp)); + + /* Optionally return the file handle. */ + if (fpp == NULL) + (void)__wt_fclose(session, &fp, WT_FHANDLE_READ); + else + *fpp = fp; + +err: __wt_free(session, fname); + + return (ret); +} + +/* + * __wt_huffman_confchk -- + * Verify Huffman configuration. + */ +int +__wt_huffman_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *v) +{ + if (v->len == 0) + return (0); + + /* Standard Huffman encodings, no work to be done. */ + if (WT_STRING_MATCH("english", v->str, v->len)) + return (0); + if (WT_STRING_MATCH("none", v->str, v->len)) + return (0); + return (__huffman_confchk_file(session, v, NULL, NULL)); } /* @@ -170,11 +208,12 @@ __wt_btree_huffman_open(WT_SESSION_IMPL *session) cfg = btree->dhandle->cfg; WT_RET(__wt_config_gets_none(session, cfg, "huffman_key", &key_conf)); + WT_RET(__wt_huffman_confchk(session, &key_conf)); WT_RET( __wt_config_gets_none(session, cfg, "huffman_value", &value_conf)); + WT_RET(__wt_huffman_confchk(session, &value_conf)); if (key_conf.len == 0 && value_conf.len == 0) return (0); - WT_RET(__btree_huffman_config(session, &key_conf, &value_conf)); switch (btree->type) { /* Check file type compatibility. */ case BTREE_COL_FIX: @@ -193,7 +232,7 @@ __wt_btree_huffman_open(WT_SESSION_IMPL *session) if (key_conf.len == 0) { ; - } else if (strncasecmp(key_conf.str, "english", key_conf.len) == 0) { + } else if (strncmp(key_conf.str, "english", key_conf.len) == 0) { struct __wt_huffman_table copy[WT_ELEMENTS(__wt_huffman_nytenglish)]; @@ -204,7 +243,7 @@ __wt_btree_huffman_open(WT_SESSION_IMPL *session) 1, &btree->huffman_key)); /* Check for a shared key/value table. */ - if (value_conf.len != 0 && strncasecmp( + if (value_conf.len != 0 && strncmp( value_conf.str, "english", value_conf.len) == 0) { btree->huffman_value = btree->huffman_key; return (0); @@ -228,8 +267,7 @@ __wt_btree_huffman_open(WT_SESSION_IMPL *session) if (value_conf.len == 0) { ; - } else if ( - strncasecmp(value_conf.str, "english", value_conf.len) == 0) { + } else if (strncmp(value_conf.str, "english", value_conf.len) == 0) { struct __wt_huffman_table copy[WT_ELEMENTS(__wt_huffman_nytenglish)]; @@ -262,54 +300,37 @@ __wt_huffman_read(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *ip, struct __wt_huffman_table *table, *tp; FILE *fp; WT_DECL_RET; - uint64_t symbol, frequency; + int64_t symbol, frequency; u_int entries, lineno; - char *file; + int is_utf8; *tablep = NULL; *entriesp = *numbytesp = 0; fp = NULL; - file = NULL; table = NULL; /* + * Try and open the backing file. + */ + WT_RET(__huffman_confchk_file(session, ip, &is_utf8, &fp)); + + /* * UTF-8 table is 256 bytes, with a range of 0-255. * UTF-16 is 128KB (2 * 65536) bytes, with a range of 0-65535. */ - if (strncasecmp(ip->str, "utf8", 4) == 0) { + if (is_utf8) { entries = UINT8_MAX; *numbytesp = 1; WT_ERR(__wt_calloc_def(session, entries, &table)); - - if (ip->len == 4) - WT_ERR_MSG(session, EINVAL, - "no Huffman table file name specified"); - WT_ERR(__wt_calloc_def(session, ip->len, &file)); - memcpy(file, ip->str + 4, ip->len - 4); - } else if (strncasecmp(ip->str, "utf16", 5) == 0) { + } else { entries = UINT16_MAX; *numbytesp = 2; WT_ERR(__wt_calloc_def(session, entries, &table)); - - if (ip->len == 5) - WT_ERR_MSG(session, EINVAL, - "no Huffman table file name specified"); - WT_ERR(__wt_calloc_def(session, ip->len, &file)); - memcpy(file, ip->str + 5, ip->len - 5); - } else { - WT_ERR_MSG(session, EINVAL, - "unknown Huffman configuration value %.*s", - (int)ip->len, ip->str); } - if ((fp = fopen(file, "r")) == NULL) - WT_ERR_MSG(session, __wt_errno(), - "unable to read Huffman table file %.*s", - (int)ip->len, ip->str); - for (tp = table, lineno = 1; (ret = - fscanf(fp, "%" SCNu64 " %" SCNu64, &symbol, &frequency)) != EOF; + fscanf(fp, "%" SCNi64 " %" SCNi64, &symbol, &frequency)) != EOF; ++tp, ++lineno) { if (lineno > entries) WT_ERR_MSG(session, EINVAL, @@ -321,16 +342,19 @@ __wt_huffman_read(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *ip, "line %u of Huffman table file %.*s is corrupted: " "expected two unsigned integral values", lineno, (int)ip->len, ip->str); - if (symbol > entries) + if (symbol < 0 || symbol > entries) WT_ERR_MSG(session, EINVAL, - "line %u of Huffman table file %.*s is corrupted: " - "symbol larger than maximum value of %u", - lineno, (int)ip->len, ip->str, entries); - if (frequency > UINT32_MAX) + "line %u of Huffman file %.*s is corrupted; " + "symbol %" PRId64 " not in range, maximum " + "value is %u", + lineno, (int)ip->len, ip->str, symbol, entries); + if (frequency < 0 || frequency > UINT32_MAX) WT_ERR_MSG(session, EINVAL, - "line %u of Huffman table file %.*s is corrupted: " - "frequency larger than maximum value of %" PRIu32, - lineno, (int)ip->len, ip->str, UINT32_MAX); + "line %u of Huffman file %.*s is corrupted; " + "frequency %" PRId64 " not in range, maximum " + "value is %" PRIu32, + lineno, (int)ip->len, ip->str, frequency, + (uint32_t)UINT32_MAX); tp->symbol = (uint32_t)symbol; tp->frequency = (uint32_t)frequency; @@ -344,9 +368,7 @@ __wt_huffman_read(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *ip, if (0) { err: __wt_free(session, table); } - if (fp != NULL) - (void)fclose(fp); - __wt_free(session, file); + (void)__wt_fclose(session, &fp, WT_FHANDLE_READ); return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index e177b05cd24..0b93cc981d7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -304,8 +304,8 @@ err: if ((pindex = WT_INTL_INDEX_COPY(page)) != NULL) { * Build in-memory page information. */ int -__wt_page_inmem(WT_SESSION_IMPL *session, - WT_REF *ref, const void *image, uint32_t flags, WT_PAGE **pagep) +__wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, + const void *image, size_t memsize, uint32_t flags, WT_PAGE **pagep) { WT_DECL_RET; WT_PAGE *page; @@ -372,9 +372,10 @@ __wt_page_inmem(WT_SESSION_IMPL *session, /* * Track the memory allocated to build this page so we can update the - * cache statistics in a single call. + * cache statistics in a single call. If the disk image is in allocated + * memory, start with that. */ - size = LF_ISSET(WT_PAGE_DISK_ALLOC) ? dsk->mem_size : 0; + size = LF_ISSET(WT_PAGE_DISK_ALLOC) ? memsize : 0; switch (page->type) { case WT_PAGE_COL_FIX: diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 57c482604bb..e27f7c3398c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -61,7 +61,7 @@ __wt_cache_read(WT_SESSION_IMPL *session, WT_REF *ref) * image on return, the page steals it. */ WT_ERR(__wt_bt_read(session, &tmp, addr, addr_size)); - WT_ERR(__wt_page_inmem(session, ref, tmp.data, + WT_ERR(__wt_page_inmem(session, ref, tmp.data, tmp.memsize, WT_DATA_IN_ITEM(&tmp) ? WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED, &page)); tmp.mem = NULL; diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index d6c20556a9a..ba1802116d0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -124,7 +124,7 @@ static int __slvg_col_range_overlap( WT_SESSION_IMPL *, uint32_t, uint32_t, WT_STUFF *); static void __slvg_col_trk_update_start(uint32_t, WT_STUFF *); static int __slvg_merge_block_free(WT_SESSION_IMPL *, WT_STUFF *); -static int __slvg_ovfl_compare(const void *, const void *); +static int WT_CDECL __slvg_ovfl_compare(const void *, const void *); static int __slvg_ovfl_discard(WT_SESSION_IMPL *, WT_STUFF *); static int __slvg_ovfl_reconcile(WT_SESSION_IMPL *, WT_STUFF *); static int __slvg_ovfl_ref(WT_SESSION_IMPL *, WT_TRACK *, int); @@ -140,9 +140,9 @@ static int __slvg_row_range_overlap( WT_SESSION_IMPL *, uint32_t, uint32_t, WT_STUFF *); static int __slvg_row_trk_update_start( WT_SESSION_IMPL *, WT_ITEM *, uint32_t, WT_STUFF *); -static int __slvg_trk_compare_addr(const void *, const void *); -static int __slvg_trk_compare_gen(const void *, const void *); -static int __slvg_trk_compare_key(const void *, const void *); +static int WT_CDECL __slvg_trk_compare_addr(const void *, const void *); +static int WT_CDECL __slvg_trk_compare_gen(const void *, const void *); +static int WT_CDECL __slvg_trk_compare_key(const void *, const void *); static int __slvg_trk_free(WT_SESSION_IMPL *, WT_TRACK **, int); static void __slvg_trk_free_addr(WT_SESSION_IMPL *, WT_TRACK *); static int __slvg_trk_init(WT_SESSION_IMPL *, uint8_t *, @@ -612,7 +612,7 @@ __slvg_trk_leaf(WT_SESSION_IMPL *session, * on every leaf page, and if you need to speed up the salvage, * it's probably a great place to start. */ - WT_ERR(__wt_page_inmem(session, NULL, dsk, 0, &page)); + WT_ERR(__wt_page_inmem(session, NULL, dsk, 0, 0, &page)); WT_ERR(__wt_row_leaf_key_copy(session, page, &page->pg_row_d[0], &trk->row_start)); WT_ERR(__wt_row_leaf_key_copy(session, page, @@ -1744,7 +1744,7 @@ __slvg_row_trk_update_start( */ WT_RET(__wt_scr_alloc(session, trk->trk_size, &dsk)); WT_ERR(__wt_bt_read(session, dsk, trk->trk_addr, trk->trk_addr_size)); - WT_ERR(__wt_page_inmem(session, NULL, dsk->mem, 0, &page)); + WT_ERR(__wt_page_inmem(session, NULL, dsk->mem, 0, 0, &page)); /* * Walk the page, looking for a key sorting greater than the specified @@ -2098,7 +2098,7 @@ __slvg_row_ovfl(WT_SESSION_IMPL *session, * __slvg_trk_compare_addr -- * Compare two WT_TRACK array entries by address cookie. */ -static int +static int WT_CDECL __slvg_trk_compare_addr(const void *a, const void *b) { WT_DECL_RET; @@ -2124,7 +2124,7 @@ __slvg_trk_compare_addr(const void *a, const void *b) * __slvg_ovfl_compare -- * Bsearch comparison routine for the overflow array. */ -static int +static int WT_CDECL __slvg_ovfl_compare(const void *a, const void *b) { WT_ADDR *addr; @@ -2163,6 +2163,7 @@ __slvg_ovfl_reconcile(WT_SESSION_IMPL *session, WT_STUFF *ss) * with the lowest LSNs until overflow pages are only referenced once. * * This requires sorting the page list by LSN, and the overflow array + * by address cookie. */ qsort(ss->pages, @@ -2246,7 +2247,7 @@ err: __wt_free(session, slot); * __slvg_trk_compare_key -- * Compare two WT_TRACK array entries by key, and secondarily, by LSN. */ -static int +static int WT_CDECL __slvg_trk_compare_key(const void *a, const void *b) { WT_SESSION_IMPL *session; @@ -2303,7 +2304,7 @@ __slvg_trk_compare_key(const void *a, const void *b) * __slvg_trk_compare_gen -- * Compare two WT_TRACK array entries by LSN. */ -static int +static int WT_CDECL __slvg_trk_compare_gen(const void *a, const void *b) { WT_TRACK *a_trk, *b_trk; diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 95fb9c68a86..9fc567f02c1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -185,7 +185,7 @@ __split_should_deepen( * maximum size for a page in memory (presumably putting eviction * pressure on the cache). */ - if (page->memory_footprint < S2BT(session)->maxmempage) + if (page->memory_footprint < btree->maxmempage) return (0); /* @@ -199,16 +199,19 @@ __split_should_deepen( } /* - * The root is a special-case: if it's putting cache pressure on the - * system, split it even if there are only a few entries, we can't - * push it out of memory. Sanity check: if the root page is too big - * with less than 100 keys, there are huge keys and/or a too-small - * cache, there's not much to do. + * Don't allow a single page to put pressure on cache usage. The root + * page cannot be evicted, so if it's larger than the maximum, or if + * and page has a quarter of the cache, let it split, a deep tree is + * better than making no progress at all. Sanity check for 100 on-page + * keys, nothing helps in the case of large keys and a too-small cache. */ - if (__wt_ref_is_root(ref) && pindex->entries > 100) { + if (pindex->entries >= 100 && + (__wt_ref_is_root(ref) || + page->memory_footprint >= S2C(session)->cache_size / 4)) { *childrenp = pindex->entries / 10; return (1); } + return (0); } @@ -515,6 +518,7 @@ __split_deepen(WT_SESSION_IMPL *session, WT_PAGE *parent, uint32_t children) * footprint. From now on we've modified the parent page, attention * needs to be paid. */ + WT_ASSERT(session, WT_INTL_INDEX_COPY(parent) == pindex); WT_INTL_INDEX_SET(parent, alloc_index); split_gen = WT_ATOMIC_ADD8(S2C(session)->split_gen, 1); panic = 1; @@ -642,8 +646,10 @@ __split_multi_inmem( * when discarding the original page, and our caller will discard the * allocated page on error, when discarding the allocated WT_REF. */ - WT_RET(__wt_page_inmem( - session, ref, multi->skip_dsk, WT_PAGE_DISK_ALLOC, &page)); + + WT_RET(__wt_page_inmem(session, ref, + multi->skip_dsk, ((WT_PAGE_HEADER *)multi->skip_dsk)->mem_size, + WT_PAGE_DISK_ALLOC, &page)); multi->skip_dsk = NULL; if (orig->type == WT_PAGE_ROW_LEAF) @@ -785,16 +791,15 @@ __wt_multi_to_ref(WT_SESSION_IMPL *session, * Resolve a multi-page split, inserting new information into the parent. */ static int -__split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, - uint32_t new_entries, size_t parent_decr, size_t parent_incr, - int exclusive, int ref_discard, uint64_t *split_genp) +__split_parent(WT_SESSION_IMPL *session, WT_REF *ref, + WT_REF **ref_new, uint32_t new_entries, size_t parent_incr, int exclusive) { WT_DECL_RET; WT_IKEY *ikey; WT_PAGE *parent; WT_PAGE_INDEX *alloc_index, *pindex; WT_REF **alloc_refp, *next_ref, *parent_ref; - size_t size; + size_t parent_decr, size; uint64_t split_gen; uint32_t children, i, j; uint32_t deleted_entries, parent_entries, result_entries; @@ -803,8 +808,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, parent = NULL; /* -Wconditional-uninitialized */ alloc_index = pindex = NULL; parent_ref = NULL; - complete = hazard = 0; + parent_decr = 0; parent_entries = 0; + complete = hazard = 0; /* * Get a page-level lock on the parent to single-thread splits into the @@ -849,9 +855,10 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, * Remove any refs to deleted pages while we are splitting, we have * the internal page locked down, and are copying the refs into a new * array anyway. Switch them to the special split state, so that any - * reading thread will restart. + * reading thread will restart. Include the ref we are splitting in + * the count to be deleted. */ - for (i = 0, deleted_entries = 0; i < parent_entries; ++i) { + for (i = 0, deleted_entries = 1; i < parent_entries; ++i) { next_ref = pindex->index[i]; WT_ASSERT(session, next_ref->state != WT_REF_SPLIT); if (next_ref->state == WT_REF_DELETED && @@ -863,10 +870,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, /* * The final entry count consists of: The original count, plus any - * new pages, less any refs we are removing because they only - * contained deleted items, less 1 for the page being replaced. + * new pages, less any refs we are removing. */ - result_entries = (parent_entries + new_entries) - (deleted_entries + 1); + result_entries = (parent_entries + new_entries) - deleted_entries; /* * Allocate and initialize a new page index array for the parent, then @@ -901,8 +907,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, * Update the parent page's index: this update makes the split visible * to threads descending the tree. */ + WT_ASSERT(session, WT_INTL_INDEX_COPY(parent) == pindex); WT_INTL_INDEX_SET(parent, alloc_index); - split_gen = *split_genp = WT_ATOMIC_ADD8(S2C(session)->split_gen, 1); + split_gen = WT_ATOMIC_ADD8(S2C(session)->split_gen, 1); alloc_index = NULL; #ifdef HAVE_DIAGNOSTIC @@ -929,20 +936,36 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, */ complete = 1; + WT_ERR(__wt_verbose(session, WT_VERB_SPLIT, + "%s split into parent %" PRIu32 " -> %" PRIu32 + " (%" PRIu32 ")", + __wt_page_type_string(ref->page->type), parent_entries, + result_entries, result_entries - parent_entries)); + /* - * The new page index is in place, free any deleted WT_REFs we found, - * modulo the usual safe free semantics. Ignore the WT_REF we're - * replacing, our caller is responsible for freeing it. + * The new page index is in place, free the WT_REF we were splitting + * and any deleted WT_REFs we found, modulo the usual safe free + * semantics. */ for (i = 0; deleted_entries > 0 && i < parent_entries; ++i) { next_ref = pindex->index[i]; - if (next_ref == ref || next_ref->state != WT_REF_SPLIT) + if (next_ref->state != WT_REF_SPLIT) continue; --deleted_entries; /* * We set the WT_REF to split, discard it, freeing any resources * it holds. + * + * Row-store trees where the old version of the page is being + * discarded: the previous parent page's key for this child page + * may have been an on-page overflow key. In that case, if the + * key hasn't been deleted, delete it now, including its backing + * blocks. We are exchanging the WT_REF that referenced it for + * the split page WT_REFs and their keys, and there's no longer + * any reference to it. Done after completing the split (if we + * failed, we'd leak the underlying blocks, but the parent page + * would be unaffected). */ if (parent->type == WT_PAGE_ROW_INT) { WT_TRET(__split_ovfl_key_cleanup( @@ -972,6 +995,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, parent_decr += sizeof(WT_REF); } + /* We freed the reference that was split in the loop above. */ + ref = NULL; + /* * We can't free the previous page index, there may be threads using it. * Add it to the session discard list, to be freed when it's safe. @@ -981,30 +1007,11 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, parent_decr += size; /* - * Row-store trees where the old version of the page is being discarded: - * the previous parent page's key for this child page may have been an - * on-page overflow key. In that case, if the key hasn't been deleted, - * delete it now, including its backing blocks. We are exchanging the - * WT_REF that referenced it for the split page WT_REFs and their keys, - * and there's no longer any reference to it. Done after completing the - * split (if we failed, we'd leak the underlying blocks, but the parent - * page would be unaffected). - */ - if (ref_discard && parent->type == WT_PAGE_ROW_INT) - WT_TRET(__split_ovfl_key_cleanup(session, parent, ref)); - - /* * Adjust the parent's memory footprint. */ __wt_cache_page_inmem_incr(session, parent, parent_incr); __wt_cache_page_inmem_decr(session, parent, parent_decr); - WT_ERR(__wt_verbose(session, WT_VERB_SPLIT, - "%s split into parent %" PRIu32 " -> %" PRIu32 - " (%" PRIu32 ")", - __wt_page_type_string(ref->page->type), parent_entries, - result_entries, result_entries - parent_entries)); - /* * Simple page splits trickle up the tree, that is, as leaf pages grow * large enough and are evicted, they'll split into their parent. And, @@ -1071,23 +1078,20 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) { WT_BTREE *btree; WT_DECL_RET; - WT_IKEY *ikey; WT_DECL_ITEM(key); WT_INSERT *ins, **insp, *moved_ins, *prev_ins; WT_INSERT_HEAD *ins_head; WT_PAGE *page, *right; WT_REF *child, *split_ref[2] = { NULL, NULL }; - size_t page_decr, parent_decr, parent_incr, right_incr; - uint64_t split_gen; + size_t page_decr, parent_incr, right_incr; int i; *splitp = 0; btree = S2BT(session); page = ref->page; - ikey = NULL; right = NULL; - page_decr = parent_decr = parent_incr = right_incr = 0; + page_decr = parent_incr = right_incr = 0; /* * Check for pages with append-only workloads. A common application @@ -1198,16 +1202,6 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) parent_incr += sizeof(WT_REF) + sizeof(WT_IKEY) + WT_INSERT_KEY_SIZE(moved_ins); - /* - * After the split, we're going to discard the WT_REF, account for the - * change in memory footprint. Row store pages have keys that may be - * instantiated, check for that. - */ - parent_decr += sizeof(WT_REF); - if (page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT) - if ((ikey = __wt_ref_key_instantiated(ref)) != NULL) - parent_decr += sizeof(WT_IKEY) + ikey->size; - /* The new page is dirty by definition. */ WT_ERR(__wt_page_modify_init(session, right)); __wt_page_modify_set(session, right); @@ -1338,8 +1332,8 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) * longer locked, so we cannot safely look at it. */ page = NULL; - if ((ret = __split_parent(session, ref, split_ref, 2, - parent_decr, parent_incr, 0, 0, &split_gen)) != 0) { + if ((ret = __split_parent( + session, ref, split_ref, 2, parent_incr, 0)) != 0) { /* * Move the insert list element back to the original page list. * For simplicity, the previous skip list pointers originally @@ -1368,26 +1362,7 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) WT_STAT_FAST_CONN_INCR(session, cache_inmem_split); WT_STAT_FAST_DATA_INCR(session, cache_inmem_split); - /* - * We may not be able to immediately free the page's original WT_REF - * structure and instantiated key, there may be threads using them. - * Add them to the session discard list, to be freed once we know it's - * safe. - */ - if (ikey != NULL) - WT_TRET(__split_safe_free( - session, split_gen, 0, ikey, sizeof(WT_IKEY) + ikey->size)); - WT_TRET(__split_safe_free(session, split_gen, 0, ref, sizeof(WT_REF))); - - /* - * A note on error handling: if we completed the split, return success, - * nothing really bad can have happened, and our caller has to proceed - * with the split. - */ - if (ret != 0 && ret != WT_PANIC) - __wt_err(session, ret, - "ignoring not-fatal error during insert page split"); - return (ret == WT_PANIC ? WT_PANIC : 0); + return (0); err: if (split_ref[0] != NULL) { __wt_free(session, split_ref[0]->key.ikey); @@ -1454,21 +1429,18 @@ __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref) int __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) { - WT_IKEY *ikey; WT_DECL_RET; WT_PAGE *page; WT_PAGE_MODIFY *mod; WT_REF **ref_new; - size_t parent_decr, parent_incr; - uint64_t split_gen; + size_t parent_incr; uint32_t i, new_entries; page = ref->page; mod = page->modify; new_entries = mod->mod_multi_entries; - ikey = NULL; - parent_decr = parent_incr = 0; + parent_incr = 0; /* * Convert the split page's multiblock reconciliation information into @@ -1479,19 +1451,9 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) WT_ERR(__wt_multi_to_ref(session, page, &mod->mod_multi[i], &ref_new[i], &parent_incr)); - /* - * After the split, we're going to discard the WT_REF, account for the - * change in memory footprint. Row store pages have keys that may be - * instantiated, check for that. - */ - parent_decr += sizeof(WT_REF); - if (page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT) - if ((ikey = __wt_ref_key_instantiated(ref)) != NULL) - parent_decr += sizeof(WT_IKEY) + ikey->size; - /* Split into the parent. */ - WT_ERR(__split_parent(session, ref, ref_new, new_entries, - parent_decr, parent_incr, exclusive, 1, &split_gen)); + WT_ERR(__split_parent( + session, ref, ref_new, new_entries, parent_incr, exclusive)); WT_STAT_FAST_CONN_INCR(session, cache_eviction_split); WT_STAT_FAST_DATA_INCR(session, cache_eviction_split); @@ -1508,29 +1470,9 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) mod->write_gen = 0; __wt_cache_dirty_decr(session, page); } - __wt_ref_out(session, ref); + __wt_page_out(session, &page); - /* - * We may not be able to immediately free the page's original WT_REF - * structure and instantiated key, there may be threads using them. - * Add them to the session discard list, to be freed once we know it's - * safe. - */ - if (ikey != NULL) - WT_TRET(__split_safe_free(session, split_gen, exclusive, - ikey, sizeof(WT_IKEY) + ikey->size)); - WT_TRET(__split_safe_free(session, split_gen, exclusive, - ref, sizeof(WT_REF))); - - /* - * A note on error handling: if we completed the split, return success, - * nothing really bad can have happened, and our caller has to proceed - * with the split. - */ - if (ret != 0 && ret != WT_PANIC) - __wt_err(session, ret, - "ignoring not-fatal error during multi-page split"); - return (ret == WT_PANIC ? WT_PANIC : 0); + return (0); err: /* * A note on error handling: in the case of evicting a page that has diff --git a/src/third_party/wiredtiger/src/btree/bt_stat.c b/src/third_party/wiredtiger/src/btree/bt_stat.c index b550158a5a9..6285edde217 100644 --- a/src/third_party/wiredtiger/src/btree/bt_stat.c +++ b/src/third_party/wiredtiger/src/btree/bt_stat.c @@ -73,8 +73,6 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) static int __stat_page(WT_SESSION_IMPL *session, WT_PAGE *page, WT_DSRC_STATS *stats) { - WT_PAGE_INDEX *pindex; - /* * All internal pages and overflow pages are trivial, all we track is * a count of the page type. @@ -86,8 +84,6 @@ __stat_page(WT_SESSION_IMPL *session, WT_PAGE *page, WT_DSRC_STATS *stats) break; case WT_PAGE_COL_INT: WT_STAT_INCR(stats, btree_column_internal); - pindex = WT_INTL_INDEX_COPY(page); - WT_STAT_INCRV(stats, btree_entries, pindex->entries); break; case WT_PAGE_COL_VAR: __stat_page_col_var(page, stats); @@ -139,7 +135,10 @@ __stat_page_col_var(WT_PAGE *page, WT_DSRC_STATS *stats) } else { orig_deleted = 0; __wt_cell_unpack(cell, unpack); - entry_cnt += __wt_cell_rle(unpack); + if (unpack->type == WT_CELL_ADDR_DEL) + orig_deleted = 1; + else + entry_cnt += __wt_cell_rle(unpack); if (unpack->ovfl) ++ovfl_cnt; } @@ -165,7 +164,10 @@ __stat_page_col_var(WT_PAGE *page, WT_DSRC_STATS *stats) /* Walk any append list. */ WT_SKIP_FOREACH(ins, WT_COL_APPEND(page)) - ++entry_cnt; + if (WT_UPDATE_DELETED_ISSET(ins->upd)) + ++deleted_cnt; + else + ++entry_cnt; WT_STAT_INCRV(stats, btree_column_deleted, deleted_cnt); WT_STAT_INCRV(stats, btree_entries, entry_cnt); @@ -183,7 +185,6 @@ __stat_page_row_int( WT_BTREE *btree; WT_CELL *cell; WT_CELL_UNPACK unpack; - WT_PAGE_INDEX *pindex; uint32_t i, ovfl_cnt; btree = S2BT(session); @@ -192,13 +193,6 @@ __stat_page_row_int( WT_STAT_INCR(stats, btree_row_internal); /* - * The number of entries tells us the number of items on row-store - * internal page. - */ - pindex = WT_INTL_INDEX_COPY(page); - WT_STAT_INCRV(stats, btree_entries, pindex->entries); - - /* * Overflow keys are hard: we have to walk the disk image to count them, * the in-memory representation of the page doesn't necessarily contain * a reference to the original cell. diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index bc5d1051b1e..dae2dd8d480 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -109,6 +109,17 @@ __sync_file(WT_SESSION_IMPL *session, int syncop) /* Write all dirty in-cache pages. */ flags |= WT_READ_NO_EVICT; for (walk = NULL;;) { + /* + * If we have a page, and it was ever modified, track + * the highest transaction ID in the tree. We do this + * here because we want the value after reconciling + * dirty pages. + */ + if (walk != NULL && walk->page != NULL && + (mod = walk->page->modify) != NULL && + TXNID_LT(btree->rec_max_txn, mod->rec_max_txn)) + btree->rec_max_txn = mod->rec_max_txn; + WT_ERR(__wt_tree_walk(session, &walk, NULL, flags)); if (walk == NULL) break; @@ -145,8 +156,7 @@ __sync_file(WT_SESSION_IMPL *session, int syncop) } if (WT_PAGE_IS_INTERNAL(page)) { - internal_bytes += - page->memory_footprint; + internal_bytes += page->memory_footprint; ++internal_pages; } else { leaf_bytes += page->memory_footprint; diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index 917e0c54a30..98411ce548d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -252,7 +252,7 @@ ascend: /* * new page. This works because we never acquire * a hazard pointer on a leaf page we're not * going to return to our caller, this will quit - * work if that ever changes. + * working if that ever changes. */ WT_ASSERT(session, couple == couple_orig || diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index 85efdc08358..0e351682e9e 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -278,7 +278,7 @@ __wt_update_alloc( } *updp = upd; - *sizep = sizeof(WT_UPDATE) + size; + *sizep = WT_UPDATE_MEMSIZE(upd); return (0); } diff --git a/src/third_party/wiredtiger/src/config/config.c b/src/third_party/wiredtiger/src/config/config.c index 751bbbd289b..fbb5a528ec7 100644 --- a/src/third_party/wiredtiger/src/config/config.c +++ b/src/third_party/wiredtiger/src/config/config.c @@ -500,11 +500,10 @@ __config_process_value(WT_CONFIG *conf, WT_CONFIG_ITEM *value) return (0); if (value->type == WT_CONFIG_ITEM_ID) { - if (WT_STRING_CASE_MATCH("false", value->str, value->len)) { + if (WT_STRING_MATCH("false", value->str, value->len)) { value->type = WT_CONFIG_ITEM_BOOL; value->val = 0; - } else if ( - WT_STRING_CASE_MATCH("true", value->str, value->len)) { + } else if (WT_STRING_MATCH("true", value->str, value->len)) { value->type = WT_CONFIG_ITEM_BOOL; value->val = 1; } @@ -593,12 +592,11 @@ __config_getraw( if (k.type != WT_CONFIG_ITEM_STRING && k.type != WT_CONFIG_ITEM_ID) continue; - if (k.len == key->len && - strncasecmp(key->str, k.str, k.len) == 0) { + if (k.len == key->len && strncmp(key->str, k.str, k.len) == 0) { *value = v; found = 1; } else if (k.len < key->len && key->str[k.len] == '.' && - strncasecmp(key->str, k.str, k.len) == 0) { + strncmp(key->str, k.str, k.len) == 0) { subk.str = key->str + k.len + 1; subk.len = (key->len - k.len) - 1; WT_RET(__wt_config_initn( @@ -665,7 +663,7 @@ __wt_config_gets_none(WT_SESSION_IMPL *session, const char **cfg, const char *key, WT_CONFIG_ITEM *value) { WT_RET(__wt_config_gets(session, cfg, key, value)); - if (WT_STRING_CASE_MATCH("none", value->str, value->len)) + if (WT_STRING_MATCH("none", value->str, value->len)) value->len = 0; return (0); } @@ -710,7 +708,7 @@ __wt_config_getones_none(WT_SESSION_IMPL *session, const char *config, const char *key, WT_CONFIG_ITEM *value) { WT_RET(__wt_config_getones(session, config, key, value)); - if (WT_STRING_CASE_MATCH("none", value->str, value->len)) + if (WT_STRING_MATCH("none", value->str, value->len)) value->len = 0; return (0); } diff --git a/src/third_party/wiredtiger/src/config/config_check.c b/src/third_party/wiredtiger/src/config/config_check.c index 97f46e4211c..391209104c7 100644 --- a/src/third_party/wiredtiger/src/config/config_check.c +++ b/src/third_party/wiredtiger/src/config/config_check.c @@ -303,6 +303,9 @@ config_check(WT_SESSION_IMPL *session, "Invalid value for key '%.*s': expected a %s", (int)k.len, k.str, checks[i].type); + if (checks[i].checkf != NULL) + WT_RET(checks[i].checkf(session, &v)); + if (checks[i].checks == NULL) continue; diff --git a/src/third_party/wiredtiger/src/config/config_collapse.c b/src/third_party/wiredtiger/src/config/config_collapse.c index f54e4fc2074..23cb03c9b3a 100644 --- a/src/third_party/wiredtiger/src/config/config_collapse.c +++ b/src/third_party/wiredtiger/src/config/config_collapse.c @@ -314,7 +314,7 @@ err: __wt_scr_free(session, &build); * __config_merge_cmp -- * Qsort function: sort the config merge array. */ -static int +static int WT_CDECL __config_merge_cmp(const void *a, const void *b) { WT_CONFIG_MERGE_ENTRY *ae, *be; diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 43d87c518e4..57b5b25ce36 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -3,531 +3,624 @@ #include "wt_internal.h" static const WT_CONFIG_CHECK confchk_colgroup_meta[] = { - { "app_metadata", "string", NULL, NULL }, - { "columns", "list", NULL, NULL }, - { "source", "string", NULL, NULL }, - { "type", "string", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "app_metadata", "string", NULL, NULL, NULL }, + { "collator", "string", __wt_collator_confchk, NULL, NULL }, + { "columns", "list", NULL, NULL, NULL }, + { "source", "string", NULL, NULL, NULL }, + { "type", "string", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_connection_async_new_op[] = { - { "append", "boolean", NULL, NULL }, - { "overwrite", "boolean", NULL, NULL }, - { "raw", "boolean", NULL, NULL }, - { "timeout", "int", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "append", "boolean", NULL, NULL, NULL }, + { "overwrite", "boolean", NULL, NULL, NULL }, + { "raw", "boolean", NULL, NULL, NULL }, + { "timeout", "int", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_connection_close[] = { - { "leak_memory", "boolean", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "leak_memory", "boolean", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_connection_load_extension[] = { - { "config", "string", NULL, NULL }, - { "entry", "string", NULL, NULL }, - { "terminate", "string", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "config", "string", NULL, NULL, NULL }, + { "entry", "string", NULL, NULL, NULL }, + { "terminate", "string", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_connection_open_session[] = { { "isolation", "string", - "choices=[\"read-uncommitted\",\"read-committed\",\"snapshot\"]", + NULL, "choices=[\"read-uncommitted\",\"read-committed\"," + "\"snapshot\"]", NULL }, - { NULL, NULL, NULL, NULL } + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_async_subconfigs[] = { - { "enabled", "boolean", NULL, NULL }, - { "ops_max", "int", "min=10,max=4096", NULL }, - { "threads", "int", "min=1,max=20", NULL }, - { NULL, NULL, NULL, NULL } + { "enabled", "boolean", NULL, NULL, NULL }, + { "ops_max", "int", NULL, "min=1,max=4096", NULL }, + { "threads", "int", NULL, "min=1,max=20", NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_checkpoint_subconfigs[] = { - { "log_size", "int", "min=0,max=2GB", NULL }, - { "name", "string", NULL, NULL }, - { "wait", "int", "min=0,max=100000", NULL }, - { NULL, NULL, NULL, NULL } + { "log_size", "int", NULL, "min=0,max=2GB", NULL }, + { "name", "string", NULL, NULL, NULL }, + { "wait", "int", NULL, "min=0,max=100000", NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_eviction_subconfigs[] = { - { "threads_max", "int", "min=1,max=20", NULL }, - { "threads_min", "int", "min=1,max=20", NULL }, - { NULL, NULL, NULL, NULL } + { "threads_max", "int", NULL, "min=1,max=20", NULL }, + { "threads_min", "int", NULL, "min=1,max=20", NULL }, + { NULL, NULL, NULL, NULL, NULL } +}; + +static const WT_CONFIG_CHECK confchk_file_manager_subconfigs[] = { + { "close_idle_time", "int", NULL, "min=1,max=1000", NULL }, + { "close_scan_interval", "int", NULL, "min=1,max=1000", NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_lsm_manager_subconfigs[] = { - { "merge", "boolean", NULL, NULL }, - { "worker_thread_max", "int", "min=3,max=20", NULL }, - { NULL, NULL, NULL, NULL } + { "merge", "boolean", NULL, NULL, NULL }, + { "worker_thread_max", "int", NULL, "min=3,max=20", NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_shared_cache_subconfigs[] = { - { "chunk", "int", "min=1MB,max=10TB", NULL }, - { "name", "string", NULL, NULL }, - { "reserve", "int", NULL, NULL }, - { "size", "int", "min=1MB,max=10TB", NULL }, - { NULL, NULL, NULL, NULL } + { "chunk", "int", NULL, "min=1MB,max=10TB", NULL }, + { "name", "string", NULL, NULL, NULL }, + { "reserve", "int", NULL, NULL, NULL }, + { "size", "int", NULL, "min=1MB,max=10TB", NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_statistics_log_subconfigs[] = { - { "on_close", "boolean", NULL, NULL }, - { "path", "string", NULL, NULL }, - { "sources", "list", NULL, NULL }, - { "timestamp", "string", NULL, NULL }, - { "wait", "int", "min=0,max=100000", NULL }, - { NULL, NULL, NULL, NULL } + { "on_close", "boolean", NULL, NULL, NULL }, + { "path", "string", NULL, NULL, NULL }, + { "sources", "list", NULL, NULL, NULL }, + { "timestamp", "string", NULL, NULL, NULL }, + { "wait", "int", NULL, "min=0,max=100000", NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_connection_reconfigure[] = { - { "async", "category", NULL, confchk_async_subconfigs }, - { "cache_overhead", "int", "min=0,max=30", NULL }, - { "cache_size", "int", "min=1MB,max=10TB", NULL }, - { "checkpoint", "category", NULL, - confchk_checkpoint_subconfigs }, - { "error_prefix", "string", NULL, NULL }, - { "eviction", "category", NULL, confchk_eviction_subconfigs }, - { "eviction_dirty_target", "int", "min=10,max=99", NULL }, - { "eviction_target", "int", "min=10,max=99", NULL }, - { "eviction_trigger", "int", "min=10,max=99", NULL }, - { "lsm_manager", "category", NULL, - confchk_lsm_manager_subconfigs }, - { "lsm_merge", "boolean", NULL, NULL }, - { "shared_cache", "category", NULL, - confchk_shared_cache_subconfigs }, + { "async", "category", NULL, NULL, confchk_async_subconfigs }, + { "cache_overhead", "int", NULL, "min=0,max=30", NULL }, + { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL }, + { "checkpoint", "category", + NULL, NULL, + confchk_checkpoint_subconfigs }, + { "error_prefix", "string", NULL, NULL, NULL }, + { "eviction", "category", + NULL, NULL, + confchk_eviction_subconfigs }, + { "eviction_dirty_target", "int", + NULL, "min=10,max=99", + NULL }, + { "eviction_target", "int", NULL, "min=10,max=99", NULL }, + { "eviction_trigger", "int", NULL, "min=10,max=99", NULL }, + { "file_manager", "category", + NULL, NULL, + confchk_file_manager_subconfigs }, + { "lsm_manager", "category", + NULL, NULL, + confchk_lsm_manager_subconfigs }, + { "lsm_merge", "boolean", NULL, NULL, NULL }, + { "shared_cache", "category", + NULL, NULL, + confchk_shared_cache_subconfigs }, { "statistics", "list", - "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", NULL }, - { "statistics_log", "category", NULL, - confchk_statistics_log_subconfigs }, + { "statistics_log", "category", + NULL, NULL, + confchk_statistics_log_subconfigs }, { "verbose", "list", - "choices=[\"api\",\"block\",\"checkpoint\",\"compact\",\"evict\"" - ",\"evictserver\",\"fileops\",\"log\",\"lsm\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"reconcile\",\"recovery\"," - "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," + "\"evict\",\"evictserver\",\"fileops\",\"log\",\"lsm\"," + "\"metadata\",\"mutex\",\"overflow\",\"read\",\"reconcile\"," + "\"recovery\",\"salvage\",\"shared_cache\",\"split\"," + "\"temporary\",\"transaction\",\"verify\",\"version\",\"write\"]", NULL }, - { NULL, NULL, NULL, NULL } + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_cursor_reconfigure[] = { - { "append", "boolean", NULL, NULL }, - { "overwrite", "boolean", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "append", "boolean", NULL, NULL, NULL }, + { "overwrite", "boolean", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_file_meta[] = { - { "allocation_size", "int", "min=512B,max=128MB", NULL }, - { "app_metadata", "string", NULL, NULL }, + { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL }, + { "app_metadata", "string", NULL, NULL, NULL }, { "block_allocation", "string", - "choices=[\"first\",\"best\"]", + NULL, "choices=[\"first\",\"best\"]", NULL }, - { "block_compressor", "string", NULL, NULL }, - { "cache_resident", "boolean", NULL, NULL }, - { "checkpoint", "string", NULL, NULL }, - { "checkpoint_lsn", "string", NULL, NULL }, + { "block_compressor", "string", + __wt_compressor_confchk, NULL, + NULL }, + { "cache_resident", "boolean", NULL, NULL, NULL }, + { "checkpoint", "string", NULL, NULL, NULL }, + { "checkpoint_lsn", "string", NULL, NULL, NULL }, { "checksum", "string", - "choices=[\"on\",\"off\",\"uncompressed\"]", - NULL }, - { "collator", "string", NULL, NULL }, - { "columns", "list", NULL, NULL }, - { "dictionary", "int", "min=0", NULL }, - { "format", "string", "choices=[\"btree\"]", NULL }, - { "huffman_key", "string", NULL, NULL }, - { "huffman_value", "string", NULL, NULL }, - { "id", "string", NULL, NULL }, - { "internal_item_max", "int", "min=0", NULL }, - { "internal_key_max", "int", "min=0", NULL }, - { "internal_key_truncate", "boolean", NULL, NULL }, - { "internal_page_max", "int", "min=512B,max=512MB", NULL }, - { "key_format", "format", NULL, NULL }, - { "key_gap", "int", "min=0", NULL }, - { "leaf_item_max", "int", "min=0", NULL }, - { "leaf_key_max", "int", "min=0", NULL }, - { "leaf_page_max", "int", "min=512B,max=512MB", NULL }, - { "leaf_value_max", "int", "min=0", NULL }, - { "memory_page_max", "int", "min=512B,max=10TB", NULL }, - { "os_cache_dirty_max", "int", "min=0", NULL }, - { "os_cache_max", "int", "min=0", NULL }, - { "prefix_compression", "boolean", NULL, NULL }, - { "prefix_compression_min", "int", "min=0", NULL }, - { "split_deepen_min_child", "int", NULL, NULL }, - { "split_deepen_per_child", "int", NULL, NULL }, - { "split_pct", "int", "min=25,max=100", NULL }, - { "value_format", "format", NULL, NULL }, - { "version", "string", NULL, NULL }, - { NULL, NULL, NULL, NULL } + NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", + NULL }, + { "collator", "string", __wt_collator_confchk, NULL, NULL }, + { "columns", "list", NULL, NULL, NULL }, + { "dictionary", "int", NULL, "min=0", NULL }, + { "format", "string", NULL, "choices=[\"btree\"]", NULL }, + { "huffman_key", "string", __wt_huffman_confchk, NULL, NULL }, + { "huffman_value", "string", + __wt_huffman_confchk, NULL, + NULL }, + { "id", "string", NULL, NULL, NULL }, + { "internal_item_max", "int", NULL, "min=0", NULL }, + { "internal_key_max", "int", NULL, "min=0", NULL }, + { "internal_key_truncate", "boolean", NULL, NULL, NULL }, + { "internal_page_max", "int", + NULL, "min=512B,max=512MB", + NULL }, + { "key_format", "format", __wt_struct_confchk, NULL, NULL }, + { "key_gap", "int", NULL, "min=0", NULL }, + { "leaf_item_max", "int", NULL, "min=0", NULL }, + { "leaf_key_max", "int", NULL, "min=0", NULL }, + { "leaf_page_max", "int", NULL, "min=512B,max=512MB", NULL }, + { "leaf_value_max", "int", NULL, "min=0", NULL }, + { "memory_page_max", "int", NULL, "min=512B,max=10TB", NULL }, + { "os_cache_dirty_max", "int", NULL, "min=0", NULL }, + { "os_cache_max", "int", NULL, "min=0", NULL }, + { "prefix_compression", "boolean", NULL, NULL, NULL }, + { "prefix_compression_min", "int", NULL, "min=0", NULL }, + { "split_deepen_min_child", "int", NULL, NULL, NULL }, + { "split_deepen_per_child", "int", NULL, NULL, NULL }, + { "split_pct", "int", NULL, "min=25,max=100", NULL }, + { "value_format", "format", __wt_struct_confchk, NULL, NULL }, + { "version", "string", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_index_meta[] = { - { "app_metadata", "string", NULL, NULL }, - { "columns", "list", NULL, NULL }, - { "extractor", "string", NULL, NULL }, - { "immutable", "boolean", NULL, NULL }, - { "index_key_columns", "int", NULL, NULL }, - { "key_format", "format", NULL, NULL }, - { "source", "string", NULL, NULL }, - { "type", "string", NULL, NULL }, - { "value_format", "format", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "app_metadata", "string", NULL, NULL, NULL }, + { "collator", "string", __wt_collator_confchk, NULL, NULL }, + { "columns", "list", NULL, NULL, NULL }, + { "extractor", "string", __wt_extractor_confchk, NULL, NULL }, + { "immutable", "boolean", NULL, NULL, NULL }, + { "index_key_columns", "int", NULL, NULL, NULL }, + { "key_format", "format", __wt_struct_confchk, NULL, NULL }, + { "source", "string", NULL, NULL, NULL }, + { "type", "string", NULL, NULL, NULL }, + { "value_format", "format", __wt_struct_confchk, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_session_begin_transaction[] = { { "isolation", "string", - "choices=[\"read-uncommitted\",\"read-committed\",\"snapshot\"]", + NULL, "choices=[\"read-uncommitted\",\"read-committed\"," + "\"snapshot\"]", NULL }, - { "name", "string", NULL, NULL }, - { "priority", "int", "min=-100,max=100", NULL }, - { "sync", "boolean", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "name", "string", NULL, NULL, NULL }, + { "priority", "int", NULL, "min=-100,max=100", NULL }, + { "sync", "boolean", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_session_checkpoint[] = { - { "drop", "list", NULL, NULL }, - { "force", "boolean", NULL, NULL }, - { "name", "string", NULL, NULL }, - { "target", "list", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "drop", "list", NULL, NULL, NULL }, + { "force", "boolean", NULL, NULL, NULL }, + { "name", "string", NULL, NULL, NULL }, + { "target", "list", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_session_compact[] = { - { "timeout", "int", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "timeout", "int", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_lsm_subconfigs[] = { - { "auto_throttle", "boolean", NULL, NULL }, - { "bloom", "boolean", NULL, NULL }, - { "bloom_bit_count", "int", "min=2,max=1000", NULL }, - { "bloom_config", "string", NULL, NULL }, - { "bloom_hash_count", "int", "min=2,max=100", NULL }, - { "bloom_oldest", "boolean", NULL, NULL }, - { "chunk_count_limit", "int", NULL, NULL }, - { "chunk_max", "int", "min=100MB,max=10TB", NULL }, - { "chunk_size", "int", "min=512K,max=500MB", NULL }, - { "merge_max", "int", "min=2,max=100", NULL }, - { "merge_min", "int", "max=100", NULL }, - { NULL, NULL, NULL, NULL } + { "auto_throttle", "boolean", NULL, NULL, NULL }, + { "bloom", "boolean", NULL, NULL, NULL }, + { "bloom_bit_count", "int", NULL, "min=2,max=1000", NULL }, + { "bloom_config", "string", NULL, NULL, NULL }, + { "bloom_hash_count", "int", NULL, "min=2,max=100", NULL }, + { "bloom_oldest", "boolean", NULL, NULL, NULL }, + { "chunk_count_limit", "int", NULL, NULL, NULL }, + { "chunk_max", "int", NULL, "min=100MB,max=10TB", NULL }, + { "chunk_size", "int", NULL, "min=512K,max=500MB", NULL }, + { "merge_max", "int", NULL, "min=2,max=100", NULL }, + { "merge_min", "int", NULL, "max=100", NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_session_create[] = { - { "allocation_size", "int", "min=512B,max=128MB", NULL }, - { "app_metadata", "string", NULL, NULL }, + { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL }, + { "app_metadata", "string", NULL, NULL, NULL }, { "block_allocation", "string", - "choices=[\"first\",\"best\"]", + NULL, "choices=[\"first\",\"best\"]", + NULL }, + { "block_compressor", "string", + __wt_compressor_confchk, NULL, NULL }, - { "block_compressor", "string", NULL, NULL }, - { "cache_resident", "boolean", NULL, NULL }, + { "cache_resident", "boolean", NULL, NULL, NULL }, { "checksum", "string", - "choices=[\"on\",\"off\",\"uncompressed\"]", - NULL }, - { "colgroups", "list", NULL, NULL }, - { "collator", "string", NULL, NULL }, - { "columns", "list", NULL, NULL }, - { "dictionary", "int", "min=0", NULL }, - { "exclusive", "boolean", NULL, NULL }, - { "extractor", "string", NULL, NULL }, - { "format", "string", "choices=[\"btree\"]", NULL }, - { "huffman_key", "string", NULL, NULL }, - { "huffman_value", "string", NULL, NULL }, - { "immutable", "boolean", NULL, NULL }, - { "internal_item_max", "int", "min=0", NULL }, - { "internal_key_max", "int", "min=0", NULL }, - { "internal_key_truncate", "boolean", NULL, NULL }, - { "internal_page_max", "int", "min=512B,max=512MB", NULL }, - { "key_format", "format", NULL, NULL }, - { "key_gap", "int", "min=0", NULL }, - { "leaf_item_max", "int", "min=0", NULL }, - { "leaf_key_max", "int", "min=0", NULL }, - { "leaf_page_max", "int", "min=512B,max=512MB", NULL }, - { "leaf_value_max", "int", "min=0", NULL }, - { "lsm", "category", NULL, confchk_lsm_subconfigs }, - { "memory_page_max", "int", "min=512B,max=10TB", NULL }, - { "os_cache_dirty_max", "int", "min=0", NULL }, - { "os_cache_max", "int", "min=0", NULL }, - { "prefix_compression", "boolean", NULL, NULL }, - { "prefix_compression_min", "int", "min=0", NULL }, - { "source", "string", NULL, NULL }, - { "split_deepen_min_child", "int", NULL, NULL }, - { "split_deepen_per_child", "int", NULL, NULL }, - { "split_pct", "int", "min=25,max=100", NULL }, - { "type", "string", NULL, NULL }, - { "value_format", "format", NULL, NULL }, - { NULL, NULL, NULL, NULL } + NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", + NULL }, + { "colgroups", "list", NULL, NULL, NULL }, + { "collator", "string", __wt_collator_confchk, NULL, NULL }, + { "columns", "list", NULL, NULL, NULL }, + { "dictionary", "int", NULL, "min=0", NULL }, + { "exclusive", "boolean", NULL, NULL, NULL }, + { "extractor", "string", __wt_extractor_confchk, NULL, NULL }, + { "format", "string", NULL, "choices=[\"btree\"]", NULL }, + { "huffman_key", "string", __wt_huffman_confchk, NULL, NULL }, + { "huffman_value", "string", + __wt_huffman_confchk, NULL, + NULL }, + { "immutable", "boolean", NULL, NULL, NULL }, + { "internal_item_max", "int", NULL, "min=0", NULL }, + { "internal_key_max", "int", NULL, "min=0", NULL }, + { "internal_key_truncate", "boolean", NULL, NULL, NULL }, + { "internal_page_max", "int", + NULL, "min=512B,max=512MB", + NULL }, + { "key_format", "format", __wt_struct_confchk, NULL, NULL }, + { "key_gap", "int", NULL, "min=0", NULL }, + { "leaf_item_max", "int", NULL, "min=0", NULL }, + { "leaf_key_max", "int", NULL, "min=0", NULL }, + { "leaf_page_max", "int", NULL, "min=512B,max=512MB", NULL }, + { "leaf_value_max", "int", NULL, "min=0", NULL }, + { "lsm", "category", NULL, NULL, confchk_lsm_subconfigs }, + { "memory_page_max", "int", NULL, "min=512B,max=10TB", NULL }, + { "os_cache_dirty_max", "int", NULL, "min=0", NULL }, + { "os_cache_max", "int", NULL, "min=0", NULL }, + { "prefix_compression", "boolean", NULL, NULL, NULL }, + { "prefix_compression_min", "int", NULL, "min=0", NULL }, + { "source", "string", NULL, NULL, NULL }, + { "split_deepen_min_child", "int", NULL, NULL, NULL }, + { "split_deepen_per_child", "int", NULL, NULL, NULL }, + { "split_pct", "int", NULL, "min=25,max=100", NULL }, + { "type", "string", NULL, NULL, NULL }, + { "value_format", "format", __wt_struct_confchk, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_session_drop[] = { - { "force", "boolean", NULL, NULL }, - { "remove_files", "boolean", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "force", "boolean", NULL, NULL, NULL }, + { "remove_files", "boolean", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_session_open_cursor[] = { - { "append", "boolean", NULL, NULL }, - { "bulk", "string", NULL, NULL }, - { "checkpoint", "string", NULL, NULL }, + { "append", "boolean", NULL, NULL, NULL }, + { "bulk", "string", NULL, NULL, NULL }, + { "checkpoint", "string", NULL, NULL, NULL }, { "dump", "string", - "choices=[\"hex\",\"json\",\"print\"]", + NULL, "choices=[\"hex\",\"json\",\"print\"]", NULL }, - { "next_random", "boolean", NULL, NULL }, - { "overwrite", "boolean", NULL, NULL }, - { "raw", "boolean", NULL, NULL }, - { "readonly", "boolean", NULL, NULL }, - { "skip_sort_check", "boolean", NULL, NULL }, + { "next_random", "boolean", NULL, NULL, NULL }, + { "overwrite", "boolean", NULL, NULL, NULL }, + { "raw", "boolean", NULL, NULL, NULL }, + { "readonly", "boolean", NULL, NULL, NULL }, + { "skip_sort_check", "boolean", NULL, NULL, NULL }, { "statistics", "list", - "choices=[\"all\",\"fast\",\"clear\"]", + NULL, "choices=[\"all\",\"fast\",\"clear\"]", NULL }, - { "target", "list", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "target", "list", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_session_reconfigure[] = { { "isolation", "string", - "choices=[\"read-uncommitted\",\"read-committed\",\"snapshot\"]", + NULL, "choices=[\"read-uncommitted\",\"read-committed\"," + "\"snapshot\"]", NULL }, - { NULL, NULL, NULL, NULL } + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_session_salvage[] = { - { "force", "boolean", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "force", "boolean", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_session_verify[] = { - { "dump_address", "boolean", NULL, NULL }, - { "dump_blocks", "boolean", NULL, NULL }, - { "dump_offsets", "list", NULL, NULL }, - { "dump_pages", "boolean", NULL, NULL }, - { "dump_shape", "boolean", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "dump_address", "boolean", NULL, NULL, NULL }, + { "dump_blocks", "boolean", NULL, NULL, NULL }, + { "dump_offsets", "list", NULL, NULL, NULL }, + { "dump_pages", "boolean", NULL, NULL, NULL }, + { "dump_shape", "boolean", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_table_meta[] = { - { "app_metadata", "string", NULL, NULL }, - { "colgroups", "list", NULL, NULL }, - { "columns", "list", NULL, NULL }, - { "key_format", "format", NULL, NULL }, - { "value_format", "format", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "app_metadata", "string", NULL, NULL, NULL }, + { "colgroups", "list", NULL, NULL, NULL }, + { "collator", "string", __wt_collator_confchk, NULL, NULL }, + { "columns", "list", NULL, NULL, NULL }, + { "key_format", "format", __wt_struct_confchk, NULL, NULL }, + { "value_format", "format", __wt_struct_confchk, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_log_subconfigs[] = { - { "archive", "boolean", NULL, NULL }, - { "compressor", "string", NULL, NULL }, - { "enabled", "boolean", NULL, NULL }, - { "file_max", "int", "min=100KB,max=2GB", NULL }, - { "path", "string", NULL, NULL }, - { "prealloc", "boolean", NULL, NULL }, - { "recover", "string", "choices=[\"error\",\"on\"]", NULL }, - { NULL, NULL, NULL, NULL } + { "archive", "boolean", NULL, NULL, NULL }, + { "compressor", "string", NULL, NULL, NULL }, + { "enabled", "boolean", NULL, NULL, NULL }, + { "file_max", "int", NULL, "min=100KB,max=2GB", NULL }, + { "path", "string", NULL, NULL, NULL }, + { "prealloc", "boolean", NULL, NULL, NULL }, + { "recover", "string", + NULL, "choices=[\"error\",\"on\"]", + NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_transaction_sync_subconfigs[] = { - { "enabled", "boolean", NULL, NULL }, + { "enabled", "boolean", NULL, NULL, NULL }, { "method", "string", - "choices=[\"dsync\",\"fsync\",\"none\"]", + NULL, "choices=[\"dsync\",\"fsync\",\"none\"]", NULL }, - { NULL, NULL, NULL, NULL } + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { - { "async", "category", NULL, confchk_async_subconfigs }, - { "buffer_alignment", "int", "min=-1,max=1MB", NULL }, - { "cache_overhead", "int", "min=0,max=30", NULL }, - { "cache_size", "int", "min=1MB,max=10TB", NULL }, - { "checkpoint", "category", NULL, - confchk_checkpoint_subconfigs }, - { "checkpoint_sync", "boolean", NULL, NULL }, - { "config_base", "boolean", NULL, NULL }, - { "create", "boolean", NULL, NULL }, + { "async", "category", NULL, NULL, confchk_async_subconfigs }, + { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL }, + { "cache_overhead", "int", NULL, "min=0,max=30", NULL }, + { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL }, + { "checkpoint", "category", + NULL, NULL, + confchk_checkpoint_subconfigs }, + { "checkpoint_sync", "boolean", NULL, NULL, NULL }, + { "config_base", "boolean", NULL, NULL, NULL }, + { "create", "boolean", NULL, NULL, NULL }, { "direct_io", "list", - "choices=[\"checkpoint\",\"data\",\"log\"]", - NULL }, - { "error_prefix", "string", NULL, NULL }, - { "eviction", "category", NULL, confchk_eviction_subconfigs }, - { "eviction_dirty_target", "int", "min=10,max=99", NULL }, - { "eviction_target", "int", "min=10,max=99", NULL }, - { "eviction_trigger", "int", "min=10,max=99", NULL }, - { "exclusive", "boolean", NULL, NULL }, - { "extensions", "list", NULL, NULL }, - { "file_extend", "list", "choices=[\"data\",\"log\"]", NULL }, - { "hazard_max", "int", "min=15", NULL }, - { "log", "category", NULL, confchk_log_subconfigs }, - { "lsm_manager", "category", NULL, - confchk_lsm_manager_subconfigs }, - { "lsm_merge", "boolean", NULL, NULL }, - { "mmap", "boolean", NULL, NULL }, - { "multiprocess", "boolean", NULL, NULL }, - { "session_max", "int", "min=1", NULL }, - { "session_scratch_max", "int", NULL, NULL }, - { "shared_cache", "category", NULL, - confchk_shared_cache_subconfigs }, + NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", + NULL }, + { "error_prefix", "string", NULL, NULL, NULL }, + { "eviction", "category", + NULL, NULL, + confchk_eviction_subconfigs }, + { "eviction_dirty_target", "int", + NULL, "min=10,max=99", + NULL }, + { "eviction_target", "int", NULL, "min=10,max=99", NULL }, + { "eviction_trigger", "int", NULL, "min=10,max=99", NULL }, + { "exclusive", "boolean", NULL, NULL, NULL }, + { "extensions", "list", NULL, NULL, NULL }, + { "file_extend", "list", + NULL, "choices=[\"data\",\"log\"]", + NULL }, + { "file_manager", "category", + NULL, NULL, + confchk_file_manager_subconfigs }, + { "hazard_max", "int", NULL, "min=15", NULL }, + { "log", "category", NULL, NULL, confchk_log_subconfigs }, + { "lsm_manager", "category", + NULL, NULL, + confchk_lsm_manager_subconfigs }, + { "lsm_merge", "boolean", NULL, NULL, NULL }, + { "mmap", "boolean", NULL, NULL, NULL }, + { "multiprocess", "boolean", NULL, NULL, NULL }, + { "session_max", "int", NULL, "min=1", NULL }, + { "session_scratch_max", "int", NULL, NULL, NULL }, + { "shared_cache", "category", + NULL, NULL, + confchk_shared_cache_subconfigs }, { "statistics", "list", - "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", NULL }, - { "statistics_log", "category", NULL, - confchk_statistics_log_subconfigs }, - { "transaction_sync", "category", NULL, - confchk_transaction_sync_subconfigs }, - { "use_environment_priv", "boolean", NULL, NULL }, + { "statistics_log", "category", + NULL, NULL, + confchk_statistics_log_subconfigs }, + { "transaction_sync", "category", + NULL, NULL, + confchk_transaction_sync_subconfigs }, + { "use_environment_priv", "boolean", NULL, NULL, NULL }, { "verbose", "list", - "choices=[\"api\",\"block\",\"checkpoint\",\"compact\",\"evict\"" - ",\"evictserver\",\"fileops\",\"log\",\"lsm\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"reconcile\",\"recovery\"," - "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," + "\"evict\",\"evictserver\",\"fileops\",\"log\",\"lsm\"," + "\"metadata\",\"mutex\",\"overflow\",\"read\",\"reconcile\"," + "\"recovery\",\"salvage\",\"shared_cache\",\"split\"," + "\"temporary\",\"transaction\",\"verify\",\"version\",\"write\"]", NULL }, - { NULL, NULL, NULL, NULL } + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { - { "async", "category", NULL, confchk_async_subconfigs }, - { "buffer_alignment", "int", "min=-1,max=1MB", NULL }, - { "cache_overhead", "int", "min=0,max=30", NULL }, - { "cache_size", "int", "min=1MB,max=10TB", NULL }, - { "checkpoint", "category", NULL, - confchk_checkpoint_subconfigs }, - { "checkpoint_sync", "boolean", NULL, NULL }, - { "config_base", "boolean", NULL, NULL }, - { "create", "boolean", NULL, NULL }, + { "async", "category", NULL, NULL, confchk_async_subconfigs }, + { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL }, + { "cache_overhead", "int", NULL, "min=0,max=30", NULL }, + { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL }, + { "checkpoint", "category", + NULL, NULL, + confchk_checkpoint_subconfigs }, + { "checkpoint_sync", "boolean", NULL, NULL, NULL }, + { "config_base", "boolean", NULL, NULL, NULL }, + { "create", "boolean", NULL, NULL, NULL }, { "direct_io", "list", - "choices=[\"checkpoint\",\"data\",\"log\"]", - NULL }, - { "error_prefix", "string", NULL, NULL }, - { "eviction", "category", NULL, confchk_eviction_subconfigs }, - { "eviction_dirty_target", "int", "min=10,max=99", NULL }, - { "eviction_target", "int", "min=10,max=99", NULL }, - { "eviction_trigger", "int", "min=10,max=99", NULL }, - { "exclusive", "boolean", NULL, NULL }, - { "extensions", "list", NULL, NULL }, - { "file_extend", "list", "choices=[\"data\",\"log\"]", NULL }, - { "hazard_max", "int", "min=15", NULL }, - { "log", "category", NULL, confchk_log_subconfigs }, - { "lsm_manager", "category", NULL, - confchk_lsm_manager_subconfigs }, - { "lsm_merge", "boolean", NULL, NULL }, - { "mmap", "boolean", NULL, NULL }, - { "multiprocess", "boolean", NULL, NULL }, - { "session_max", "int", "min=1", NULL }, - { "session_scratch_max", "int", NULL, NULL }, - { "shared_cache", "category", NULL, - confchk_shared_cache_subconfigs }, + NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", + NULL }, + { "error_prefix", "string", NULL, NULL, NULL }, + { "eviction", "category", + NULL, NULL, + confchk_eviction_subconfigs }, + { "eviction_dirty_target", "int", + NULL, "min=10,max=99", + NULL }, + { "eviction_target", "int", NULL, "min=10,max=99", NULL }, + { "eviction_trigger", "int", NULL, "min=10,max=99", NULL }, + { "exclusive", "boolean", NULL, NULL, NULL }, + { "extensions", "list", NULL, NULL, NULL }, + { "file_extend", "list", + NULL, "choices=[\"data\",\"log\"]", + NULL }, + { "file_manager", "category", + NULL, NULL, + confchk_file_manager_subconfigs }, + { "hazard_max", "int", NULL, "min=15", NULL }, + { "log", "category", NULL, NULL, confchk_log_subconfigs }, + { "lsm_manager", "category", + NULL, NULL, + confchk_lsm_manager_subconfigs }, + { "lsm_merge", "boolean", NULL, NULL, NULL }, + { "mmap", "boolean", NULL, NULL, NULL }, + { "multiprocess", "boolean", NULL, NULL, NULL }, + { "session_max", "int", NULL, "min=1", NULL }, + { "session_scratch_max", "int", NULL, NULL, NULL }, + { "shared_cache", "category", + NULL, NULL, + confchk_shared_cache_subconfigs }, { "statistics", "list", - "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", NULL }, - { "statistics_log", "category", NULL, - confchk_statistics_log_subconfigs }, - { "transaction_sync", "category", NULL, - confchk_transaction_sync_subconfigs }, - { "use_environment_priv", "boolean", NULL, NULL }, + { "statistics_log", "category", + NULL, NULL, + confchk_statistics_log_subconfigs }, + { "transaction_sync", "category", + NULL, NULL, + confchk_transaction_sync_subconfigs }, + { "use_environment_priv", "boolean", NULL, NULL, NULL }, { "verbose", "list", - "choices=[\"api\",\"block\",\"checkpoint\",\"compact\",\"evict\"" - ",\"evictserver\",\"fileops\",\"log\",\"lsm\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"reconcile\",\"recovery\"," - "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," + "\"evict\",\"evictserver\",\"fileops\",\"log\",\"lsm\"," + "\"metadata\",\"mutex\",\"overflow\",\"read\",\"reconcile\"," + "\"recovery\",\"salvage\",\"shared_cache\",\"split\"," + "\"temporary\",\"transaction\",\"verify\",\"version\",\"write\"]", NULL }, - { "version", "string", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "version", "string", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { - { "async", "category", NULL, confchk_async_subconfigs }, - { "buffer_alignment", "int", "min=-1,max=1MB", NULL }, - { "cache_overhead", "int", "min=0,max=30", NULL }, - { "cache_size", "int", "min=1MB,max=10TB", NULL }, - { "checkpoint", "category", NULL, - confchk_checkpoint_subconfigs }, - { "checkpoint_sync", "boolean", NULL, NULL }, + { "async", "category", NULL, NULL, confchk_async_subconfigs }, + { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL }, + { "cache_overhead", "int", NULL, "min=0,max=30", NULL }, + { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL }, + { "checkpoint", "category", + NULL, NULL, + confchk_checkpoint_subconfigs }, + { "checkpoint_sync", "boolean", NULL, NULL, NULL }, { "direct_io", "list", - "choices=[\"checkpoint\",\"data\",\"log\"]", - NULL }, - { "error_prefix", "string", NULL, NULL }, - { "eviction", "category", NULL, confchk_eviction_subconfigs }, - { "eviction_dirty_target", "int", "min=10,max=99", NULL }, - { "eviction_target", "int", "min=10,max=99", NULL }, - { "eviction_trigger", "int", "min=10,max=99", NULL }, - { "extensions", "list", NULL, NULL }, - { "file_extend", "list", "choices=[\"data\",\"log\"]", NULL }, - { "hazard_max", "int", "min=15", NULL }, - { "log", "category", NULL, confchk_log_subconfigs }, - { "lsm_manager", "category", NULL, - confchk_lsm_manager_subconfigs }, - { "lsm_merge", "boolean", NULL, NULL }, - { "mmap", "boolean", NULL, NULL }, - { "multiprocess", "boolean", NULL, NULL }, - { "session_max", "int", "min=1", NULL }, - { "session_scratch_max", "int", NULL, NULL }, - { "shared_cache", "category", NULL, - confchk_shared_cache_subconfigs }, + NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", + NULL }, + { "error_prefix", "string", NULL, NULL, NULL }, + { "eviction", "category", + NULL, NULL, + confchk_eviction_subconfigs }, + { "eviction_dirty_target", "int", + NULL, "min=10,max=99", + NULL }, + { "eviction_target", "int", NULL, "min=10,max=99", NULL }, + { "eviction_trigger", "int", NULL, "min=10,max=99", NULL }, + { "extensions", "list", NULL, NULL, NULL }, + { "file_extend", "list", + NULL, "choices=[\"data\",\"log\"]", + NULL }, + { "file_manager", "category", + NULL, NULL, + confchk_file_manager_subconfigs }, + { "hazard_max", "int", NULL, "min=15", NULL }, + { "log", "category", NULL, NULL, confchk_log_subconfigs }, + { "lsm_manager", "category", + NULL, NULL, + confchk_lsm_manager_subconfigs }, + { "lsm_merge", "boolean", NULL, NULL, NULL }, + { "mmap", "boolean", NULL, NULL, NULL }, + { "multiprocess", "boolean", NULL, NULL, NULL }, + { "session_max", "int", NULL, "min=1", NULL }, + { "session_scratch_max", "int", NULL, NULL, NULL }, + { "shared_cache", "category", + NULL, NULL, + confchk_shared_cache_subconfigs }, { "statistics", "list", - "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", NULL }, - { "statistics_log", "category", NULL, - confchk_statistics_log_subconfigs }, - { "transaction_sync", "category", NULL, - confchk_transaction_sync_subconfigs }, + { "statistics_log", "category", + NULL, NULL, + confchk_statistics_log_subconfigs }, + { "transaction_sync", "category", + NULL, NULL, + confchk_transaction_sync_subconfigs }, { "verbose", "list", - "choices=[\"api\",\"block\",\"checkpoint\",\"compact\",\"evict\"" - ",\"evictserver\",\"fileops\",\"log\",\"lsm\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"reconcile\",\"recovery\"," - "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," + "\"evict\",\"evictserver\",\"fileops\",\"log\",\"lsm\"," + "\"metadata\",\"mutex\",\"overflow\",\"read\",\"reconcile\"," + "\"recovery\",\"salvage\",\"shared_cache\",\"split\"," + "\"temporary\",\"transaction\",\"verify\",\"version\",\"write\"]", NULL }, - { "version", "string", NULL, NULL }, - { NULL, NULL, NULL, NULL } + { "version", "string", NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { - { "async", "category", NULL, confchk_async_subconfigs }, - { "buffer_alignment", "int", "min=-1,max=1MB", NULL }, - { "cache_overhead", "int", "min=0,max=30", NULL }, - { "cache_size", "int", "min=1MB,max=10TB", NULL }, - { "checkpoint", "category", NULL, - confchk_checkpoint_subconfigs }, - { "checkpoint_sync", "boolean", NULL, NULL }, + { "async", "category", NULL, NULL, confchk_async_subconfigs }, + { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL }, + { "cache_overhead", "int", NULL, "min=0,max=30", NULL }, + { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL }, + { "checkpoint", "category", + NULL, NULL, + confchk_checkpoint_subconfigs }, + { "checkpoint_sync", "boolean", NULL, NULL, NULL }, { "direct_io", "list", - "choices=[\"checkpoint\",\"data\",\"log\"]", - NULL }, - { "error_prefix", "string", NULL, NULL }, - { "eviction", "category", NULL, confchk_eviction_subconfigs }, - { "eviction_dirty_target", "int", "min=10,max=99", NULL }, - { "eviction_target", "int", "min=10,max=99", NULL }, - { "eviction_trigger", "int", "min=10,max=99", NULL }, - { "extensions", "list", NULL, NULL }, - { "file_extend", "list", "choices=[\"data\",\"log\"]", NULL }, - { "hazard_max", "int", "min=15", NULL }, - { "log", "category", NULL, confchk_log_subconfigs }, - { "lsm_manager", "category", NULL, - confchk_lsm_manager_subconfigs }, - { "lsm_merge", "boolean", NULL, NULL }, - { "mmap", "boolean", NULL, NULL }, - { "multiprocess", "boolean", NULL, NULL }, - { "session_max", "int", "min=1", NULL }, - { "session_scratch_max", "int", NULL, NULL }, - { "shared_cache", "category", NULL, - confchk_shared_cache_subconfigs }, + NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", + NULL }, + { "error_prefix", "string", NULL, NULL, NULL }, + { "eviction", "category", + NULL, NULL, + confchk_eviction_subconfigs }, + { "eviction_dirty_target", "int", + NULL, "min=10,max=99", + NULL }, + { "eviction_target", "int", NULL, "min=10,max=99", NULL }, + { "eviction_trigger", "int", NULL, "min=10,max=99", NULL }, + { "extensions", "list", NULL, NULL, NULL }, + { "file_extend", "list", + NULL, "choices=[\"data\",\"log\"]", + NULL }, + { "file_manager", "category", + NULL, NULL, + confchk_file_manager_subconfigs }, + { "hazard_max", "int", NULL, "min=15", NULL }, + { "log", "category", NULL, NULL, confchk_log_subconfigs }, + { "lsm_manager", "category", + NULL, NULL, + confchk_lsm_manager_subconfigs }, + { "lsm_merge", "boolean", NULL, NULL, NULL }, + { "mmap", "boolean", NULL, NULL, NULL }, + { "multiprocess", "boolean", NULL, NULL, NULL }, + { "session_max", "int", NULL, "min=1", NULL }, + { "session_scratch_max", "int", NULL, NULL, NULL }, + { "shared_cache", "category", + NULL, NULL, + confchk_shared_cache_subconfigs }, { "statistics", "list", - "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", NULL }, - { "statistics_log", "category", NULL, - confchk_statistics_log_subconfigs }, - { "transaction_sync", "category", NULL, - confchk_transaction_sync_subconfigs }, + { "statistics_log", "category", + NULL, NULL, + confchk_statistics_log_subconfigs }, + { "transaction_sync", "category", + NULL, NULL, + confchk_transaction_sync_subconfigs }, { "verbose", "list", - "choices=[\"api\",\"block\",\"checkpoint\",\"compact\",\"evict\"" - ",\"evictserver\",\"fileops\",\"log\",\"lsm\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"reconcile\",\"recovery\"," - "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," + "\"evict\",\"evictserver\",\"fileops\",\"log\",\"lsm\"," + "\"metadata\",\"mutex\",\"overflow\",\"read\",\"reconcile\"," + "\"recovery\",\"salvage\",\"shared_cache\",\"split\"," + "\"temporary\",\"transaction\",\"verify\",\"version\",\"write\"]", NULL }, - { NULL, NULL, NULL, NULL } + { NULL, NULL, NULL, NULL, NULL } }; static const WT_CONFIG_ENTRY config_entries[] = { { "colgroup.meta", - "app_metadata=,columns=,source=,type=file", + "app_metadata=,collator=,columns=,source=,type=file", confchk_colgroup_meta }, { "connection.add_collator", @@ -568,10 +661,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "cache_size=100MB,checkpoint=(log_size=0," "name=\"WiredTigerCheckpoint\",wait=0),error_prefix=," "eviction=(threads_max=1,threads_min=1),eviction_dirty_target=80," - "eviction_target=80,eviction_trigger=95,lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,shared_cache=(chunk=10MB,name=," - "reserve=0,size=500MB),statistics=none,statistics_log=(on_close=0" - ",path=\"WiredTigerStat.%d.%H\",sources=," + "eviction_target=80,eviction_trigger=95," + "file_manager=(close_idle_time=30,close_scan_interval=10)," + "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=," + "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," + "statistics=none,statistics_log=(on_close=0," + "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", confchk_connection_reconfigure }, @@ -598,8 +693,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_file_meta }, { "index.meta", - "app_metadata=,columns=,extractor=,immutable=0,index_key_columns=" - ",key_format=u,source=,type=file,value_format=u", + "app_metadata=,collator=,columns=,extractor=,immutable=0," + "index_key_columns=,key_format=u,source=,type=file,value_format=u", confchk_index_meta }, { "session.begin_transaction", @@ -686,7 +781,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_session_verify }, { "table.meta", - "app_metadata=,colgroups=,columns=,key_format=u,value_format=u", + "app_metadata=,colgroups=,collator=,columns=,key_format=u," + "value_format=u", confchk_table_meta }, { "wiredtiger_open", @@ -696,15 +792,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { "config_base=,create=0,direct_io=,error_prefix=," "eviction=(threads_max=1,threads_min=1),eviction_dirty_target=80," "eviction_target=80,eviction_trigger=95,exclusive=0,extensions=," - "file_extend=,hazard_max=1000,log=(archive=,compressor=,enabled=0" - ",file_max=100MB,path=,prealloc=,recover=on),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," - "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" - ",name=,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," - "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),use_environment_priv=0" - ",verbose=", + "file_extend=,file_manager=(close_idle_time=30," + "close_scan_interval=10),hazard_max=1000,log=(archive=," + "compressor=,enabled=0,file_max=100MB,path=,prealloc=,recover=on)" + ",lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," + "multiprocess=0,session_max=100,session_scratch_max=2MB," + "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," + "statistics=none,statistics_log=(on_close=0," + "path=\"WiredTigerStat.%d.%H\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" + ",method=fsync),use_environment_priv=0,verbose=", confchk_wiredtiger_open }, { "wiredtiger_open_all", @@ -714,15 +811,17 @@ static const WT_CONFIG_ENTRY config_entries[] = { "config_base=,create=0,direct_io=,error_prefix=," "eviction=(threads_max=1,threads_min=1),eviction_dirty_target=80," "eviction_target=80,eviction_trigger=95,exclusive=0,extensions=," - "file_extend=,hazard_max=1000,log=(archive=,compressor=,enabled=0" - ",file_max=100MB,path=,prealloc=,recover=on),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," - "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" - ",name=,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," - "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),use_environment_priv=0" - ",verbose=,version=(major=0,minor=0)", + "file_extend=,file_manager=(close_idle_time=30," + "close_scan_interval=10),hazard_max=1000,log=(archive=," + "compressor=,enabled=0,file_max=100MB,path=,prealloc=,recover=on)" + ",lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," + "multiprocess=0,session_max=100,session_scratch_max=2MB," + "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," + "statistics=none,statistics_log=(on_close=0," + "path=\"WiredTigerStat.%d.%H\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" + ",method=fsync),use_environment_priv=0,verbose=,version=(major=0," + "minor=0)", confchk_wiredtiger_open_all }, { "wiredtiger_open_basecfg", @@ -731,7 +830,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "name=\"WiredTigerCheckpoint\",wait=0),checkpoint_sync=," "direct_io=,error_prefix=,eviction=(threads_max=1,threads_min=1)," "eviction_dirty_target=80,eviction_target=80,eviction_trigger=95," - "extensions=,file_extend=,hazard_max=1000,log=(archive=," + "extensions=,file_extend=,file_manager=(close_idle_time=30," + "close_scan_interval=10),hazard_max=1000,log=(archive=," "compressor=,enabled=0,file_max=100MB,path=,prealloc=,recover=on)" ",lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," "multiprocess=0,session_max=100,session_scratch_max=2MB," @@ -748,7 +848,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "name=\"WiredTigerCheckpoint\",wait=0),checkpoint_sync=," "direct_io=,error_prefix=,eviction=(threads_max=1,threads_min=1)," "eviction_dirty_target=80,eviction_target=80,eviction_trigger=95," - "extensions=,file_extend=,hazard_max=1000,log=(archive=," + "extensions=,file_extend=,file_manager=(close_idle_time=30," + "close_scan_interval=10),hazard_max=1000,log=(archive=," "compressor=,enabled=0,file_max=100MB,path=,prealloc=,recover=on)" ",lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," "multiprocess=0,session_max=100,session_scratch_max=2MB," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 6b9824fc415..b41cad25914 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -65,6 +65,44 @@ ext_collator_config(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, } /* + * __collator_confchk -- + * Check for a valid custom collator. + */ +static int +__collator_confchk( + WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cname, WT_COLLATOR **collatorp) +{ + WT_CONNECTION_IMPL *conn; + WT_NAMED_COLLATOR *ncoll; + + if (collatorp != NULL) + *collatorp = NULL; + + if (cname->len == 0 || WT_STRING_MATCH("none", cname->str, cname->len)) + return (0); + + conn = S2C(session); + TAILQ_FOREACH(ncoll, &conn->collqh, q) + if (WT_STRING_MATCH(ncoll->name, cname->str, cname->len)) { + if (collatorp != NULL) + *collatorp = ncoll->collator; + return (0); + } + WT_RET_MSG(session, EINVAL, + "unknown collator '%.*s'", (int)cname->len, cname->str); +} + +/* + * __wt_collator_confchk -- + * Check for a valid custom collator (public). + */ +int +__wt_collator_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cname) +{ + return (__collator_confchk(session, cname, NULL)); +} + +/* * __wt_collator_config -- * Configure a custom collator. */ @@ -74,23 +112,14 @@ __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_COLLATOR **collatorp, int *ownp) { WT_COLLATOR *collator; - WT_CONNECTION_IMPL *conn; - WT_NAMED_COLLATOR *ncoll; *collatorp = NULL; *ownp = 0; - conn = S2C(session); - - TAILQ_FOREACH(ncoll, &conn->collqh, q) - if (WT_STRING_MATCH(ncoll->name, cname->str, cname->len)) - break; - - if (ncoll == NULL) - WT_RET_MSG(session, EINVAL, - "unknown collator '%.*s'", (int)cname->len, cname->str); + WT_RET(__collator_confchk(session, cname, &collator)); + if (collator == NULL) + return (0); - collator = ncoll->collator; if (collator->customize != NULL) WT_RET(collator->customize(collator, &session->iface, uri, metadata, collatorp)); @@ -148,6 +177,9 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) #ifdef HAVE_BUILTIN_EXTENSION_ZLIB extern int zlib_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif +#ifdef HAVE_BUILTIN_EXTENSION_LZ4 + extern int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +#endif /* * __conn_load_default_extensions -- @@ -163,6 +195,9 @@ __conn_load_default_extensions(WT_CONNECTION_IMPL *conn) #ifdef HAVE_BUILTIN_EXTENSION_ZLIB WT_RET(zlib_extension_init(&conn->iface, NULL)); #endif +#ifdef HAVE_BUILTIN_EXTENSION_LZ4 + WT_RET(lz4_extension_init(&conn->iface, NULL)); +#endif return (0); } @@ -340,6 +375,55 @@ __wt_conn_remove_collator(WT_SESSION_IMPL *session) } /* + * __compressor_confchk -- + * Validate the compressor. + */ +static int +__compressor_confchk( + WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_COMPRESSOR **compressorp) +{ + WT_CONNECTION_IMPL *conn; + WT_NAMED_COMPRESSOR *ncomp; + + if (compressorp != NULL) + *compressorp = NULL; + + if (cval->len == 0 || WT_STRING_MATCH("none", cval->str, cval->len)) + return (0); + + conn = S2C(session); + TAILQ_FOREACH(ncomp, &conn->compqh, q) + if (WT_STRING_MATCH(ncomp->name, cval->str, cval->len)) { + if (compressorp != NULL) + *compressorp = ncomp->compressor; + return (0); + } + WT_RET_MSG(session, EINVAL, + "unknown compressor '%.*s'", (int)cval->len, cval->str); +} + +/* + * __wt_compressor_confchk -- + * Validate the compressor (public). + */ +int +__wt_compressor_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval) +{ + return (__compressor_confchk(session, cval, NULL)); +} + +/* + * __wt_compressor_config -- + * Given a configuration, configure the compressor. + */ +int +__wt_compressor_config( + WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_COMPRESSOR **compressorp) +{ + return (__compressor_confchk(session, cval, compressorp)); +} + +/* * __conn_add_compressor -- * WT_CONNECTION->add_compressor method. */ @@ -516,6 +600,44 @@ err: if (nextractor != NULL) { } /* + * __extractor_confchk -- + * Check for a valid custom extractor. + */ +static int +__extractor_confchk( + WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cname, WT_EXTRACTOR **extractorp) +{ + WT_CONNECTION_IMPL *conn; + WT_NAMED_EXTRACTOR *nextractor; + + if (extractorp != NULL) + *extractorp = NULL; + + if (cname->len == 0 || WT_STRING_MATCH("none", cname->str, cname->len)) + return (0); + + conn = S2C(session); + TAILQ_FOREACH(nextractor, &conn->extractorqh, q) + if (WT_STRING_MATCH(nextractor->name, cname->str, cname->len)) { + if (extractorp != NULL) + *extractorp = nextractor->extractor; + return (0); + } + WT_RET_MSG(session, EINVAL, + "unknown extractor '%.*s'", (int)cname->len, cname->str); +} + +/* + * __wt_extractor_confchk -- + * Check for a valid custom extractor (public). + */ +int +__wt_extractor_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cname) +{ + return (__extractor_confchk(session, cname, NULL)); +} + +/* * __wt_extractor_config -- * Given a configuration, configure the extractor. */ @@ -523,38 +645,30 @@ int __wt_extractor_config(WT_SESSION_IMPL *session, const char *config, WT_EXTRACTOR **extractorp, int *ownp) { - WT_CONNECTION_IMPL *conn; - WT_CONFIG_ITEM cval; - WT_NAMED_EXTRACTOR *nextractor; + WT_CONFIG_ITEM cname; + WT_EXTRACTOR *extractor; *extractorp = NULL; *ownp = 0; - conn = S2C(session); - WT_RET_NOTFOUND_OK( - __wt_config_getones_none(session, config, "extractor", &cval)); - if (cval.len == 0) + __wt_config_getones_none(session, config, "extractor", &cname)); + if (cname.len == 0) return (0); - TAILQ_FOREACH(nextractor, &conn->extractorqh, q) - if (WT_STRING_MATCH(nextractor->name, cval.str, cval.len)) - break; - - if (nextractor == NULL) - WT_RET_MSG(session, EINVAL, - "unknown extractor '%.*s'", (int)cval.len, cval.str); + WT_RET(__extractor_confchk(session, &cname, &extractor)); + if (extractor == NULL) + return (0); - if (nextractor->extractor->customize != NULL) { + if (extractor->customize != NULL) { WT_RET(__wt_config_getones(session, - config, "app_metadata", &cval)); - WT_RET(nextractor->extractor->customize( - nextractor->extractor, &session->iface, - session->dhandle->name, &cval, extractorp)); + config, "app_metadata", &cname)); + WT_RET(extractor->customize(extractor, &session->iface, + session->dhandle->name, &cname, extractorp)); } if (*extractorp == NULL) - *extractorp = nextractor->extractor; + *extractorp = extractor; else *ownp = 1; @@ -766,6 +880,7 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) WT_ERR(__wt_checkpoint_server_create(session, config_cfg)); WT_ERR(__wt_lsm_manager_reconfig(session, config_cfg)); WT_ERR(__wt_statlog_create(session, config_cfg)); + WT_ERR(__wt_sweep_config(session, cfg)); WT_ERR(__wt_verbose_config(session, config_cfg)); WT_ERR(__wt_config_merge(session, config_cfg, &p)); @@ -854,7 +969,6 @@ static int __conn_config_file(WT_SESSION_IMPL *session, const char *filename, int is_user, const char **cfg, WT_ITEM *cbuf) { - WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_FH *fh; size_t len; @@ -862,7 +976,6 @@ __conn_config_file(WT_SESSION_IMPL *session, int exist, quoted; char *p, *t; - conn = S2C(session); fh = NULL; /* Configuration files are always optional. */ @@ -870,14 +983,6 @@ __conn_config_file(WT_SESSION_IMPL *session, if (!exist) return (0); - /* - * The base configuration should not exist if we are creating this - * database. - */ - if (!is_user && conn->is_new) - WT_RET_MSG(session, EINVAL, - "%s exists before database creation", filename); - /* Open the configuration file. */ WT_RET(__wt_open(session, filename, 0, 0, 0, &fh)); WT_ERR(__wt_filesize(session, fh, &size)); @@ -990,8 +1095,7 @@ __conn_config_file(WT_SESSION_IMPL *session, /* Append it to the stack. */ __conn_config_append(cfg, cbuf->data); -err: if (fh != NULL) - WT_TRET(__wt_close(session, fh)); +err: WT_TRET(__wt_close(session, &fh)); return (ret); } @@ -1203,6 +1307,7 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) len = (size_t)snprintf(buf, sizeof(buf), "%s\n%s\n", WT_WIREDTIGER, WIREDTIGER_VERSION_STRING); WT_ERR(__wt_write(session, fh, (wt_off_t)0, len, buf)); + WT_ERR(__wt_fsync(session, fh)); conn->is_new = 1; } else { @@ -1219,8 +1324,7 @@ err: /* * We ignore the connection's lock file handle on error, it will be * closed when the connection structure is destroyed. */ - if (fh != NULL) - WT_TRET(__wt_close(session, fh)); + WT_TRET(__wt_close(session, &fh)); __wt_spin_unlock(session, &__wt_process.spinlock); return (ret); @@ -1352,45 +1456,46 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) } /* - * __conn_write_config -- - * Save the configuration used to create a database. + * __conn_write_base_config -- + * Save the base configuration used to create a database. */ static int -__conn_write_config( - WT_SESSION_IMPL *session, const char *filename, const char *cfg[]) +__conn_write_base_config(WT_SESSION_IMPL *session, const char *cfg[]) { FILE *fp; WT_CONFIG parser; - WT_CONFIG_ITEM k, v; + WT_CONFIG_ITEM cval, k, v; WT_DECL_RET; - char *path; + int exist; + + fp = NULL; /* - * We were passed an array of configuration strings where slot 0 is all - * all possible values and the second and subsequent slots are changes - * specified by the application during open (using the wiredtiger_open - * configuration string, an environment variable, or user-configuration - * file). The base configuration file contains all changes to default - * settings made at create, and we include the user-configuration file - * in that list, even though we don't expect it to change. Of course, - * an application could leave that file as it is right now and not - * remove a configuration we need, but applications can also guarantee - * all database users specify consistent environment variables and - * wiredtiger_open configuration arguments, and if we protect against - * those problems, might as well include the application's configuration - * file as well. - * - * If there is no configuration, don't bother creating an empty file. + * Discard any base configuration setup file left-over from previous + * runs. This doesn't matter for correctness, it's just cleaning up + * random files. + */ + WT_RET(__wt_remove_if_exists(session, WT_BASECONFIG_SET)); + + /* The base configuration file is optional, check the configuration. */ + WT_RET(__wt_config_gets(session, cfg, "config_base", &cval)); + if (!cval.val) + return (0); + + /* + * We don't test separately if we're creating the database in this run + * as we might have crashed between creating the "WiredTiger" file and + * creating the base configuration file. If configured, there's always + * a base configuration file, and we rename it into place, so it can + * only NOT exist if we crashed before it was created; in other words, + * if the base configuration file exists, we're done. */ - if (cfg[1] == NULL) + WT_RET(__wt_exist(session, WT_BASECONFIG, &exist)); + if (exist) return (0); - WT_RET(__wt_filename(session, filename, &path)); - if ((fp = fopen(path, "w")) == NULL) - ret = __wt_errno(); - __wt_free(session, path); - if (fp == NULL) - return (ret); + WT_RET(__wt_fopen(session, + WT_BASECONFIG_SET, WT_FHANDLE_WRITE, 0, &fp)); fprintf(fp, "%s\n\n", "# Do not modify this file.\n" @@ -1404,6 +1509,20 @@ __conn_write_config( WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR); /* + * We were passed an array of configuration strings where slot 0 is all + * possible values and the second and subsequent slots are changes + * specified by the application during open (using the wiredtiger_open + * configuration string, an environment variable, or user-configuration + * file). The base configuration file contains all changes to default + * settings made at create, and we include the user-configuration file + * in that list, even though we don't expect it to change. Of course, + * an application could leave that file as it is right now and not + * remove a configuration we need, but applications can also guarantee + * all database users specify consistent environment variables and + * wiredtiger_open configuration arguments, and if we protect against + * those problems, might as well include the application's configuration + * file as well. + * * We want the list of defaults that have been changed, that is, if the * application didn't somehow configure a setting, we don't write out a * default value, so future releases may silently migrate to new default @@ -1428,11 +1547,13 @@ __conn_write_config( WT_ERR_NOTFOUND_OK(ret); } -err: WT_TRET(fclose(fp)); + /* Flush the handle and rename the file into place. */ + return (__wt_sync_and_rename_fp( + session, &fp, WT_BASECONFIG_SET, WT_BASECONFIG)); - /* Don't leave a damaged file in place. */ - if (ret != 0) - (void)__wt_remove(session, filename); + /* Close any file handle left open, remove any temporary file. */ +err: WT_TRET(__wt_fclose(session, &fp, WT_FHANDLE_WRITE)); + WT_TRET(__wt_remove_if_exists(session, WT_BASECONFIG_SET)); return (ret); } @@ -1583,17 +1704,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, if (cval.val) F_SET(conn, WT_CONN_CKPT_SYNC); - WT_ERR(__wt_config_gets(session, cfg, "buffer_alignment", &cval)); - if (cval.val == -1) - conn->buffer_alignment = WT_BUFFER_ALIGNMENT_DEFAULT; - else - conn->buffer_alignment = (size_t)cval.val; -#ifndef HAVE_POSIX_MEMALIGN - if (conn->buffer_alignment != 0) - WT_ERR_MSG(session, EINVAL, - "buffer_alignment requires posix_memalign"); -#endif - WT_ERR(__wt_config_gets(session, cfg, "direct_io", &cval)); for (ft = file_types; ft->name != NULL; ft++) { ret = __wt_config_subgets(session, &cval, ft->name, &sval); @@ -1604,6 +1714,22 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, goto err; } + /* + * If buffer alignment is not configured, use zero unless direct I/O is + * also configured, in which case use the build-time default. + */ + WT_ERR(__wt_config_gets(session, cfg, "buffer_alignment", &cval)); + if (cval.val == -1) + conn->buffer_alignment = + (conn->direct_io == 0) ? 0 : WT_BUFFER_ALIGNMENT_DEFAULT; + else + conn->buffer_alignment = (size_t)cval.val; +#ifndef HAVE_POSIX_MEMALIGN + if (conn->buffer_alignment != 0) + WT_ERR_MSG(session, EINVAL, + "buffer_alignment requires posix_memalign"); +#endif + WT_ERR(__wt_config_gets(session, cfg, "file_extend", &cval)); for (ft = file_types; ft->name != NULL; ft++) { ret = __wt_config_subgets(session, &cval, ft->name, &sval); @@ -1625,6 +1751,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__conn_statistics_config(session, cfg)); WT_ERR(__wt_lsm_manager_config(session, cfg)); + WT_ERR(__wt_sweep_config(session, cfg)); WT_ERR(__wt_verbose_config(session, cfg)); /* Now that we know if verbose is configured, output the version. */ @@ -1655,15 +1782,10 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__conn_load_extensions(session, cfg)); /* - * We've completed configuration, write the base configuration file if - * we're creating the database. + * Configuration completed; optionally write the base configuration file + * if it doesn't already exist. */ - if (conn->is_new) { - WT_ERR(__wt_config_gets(session, cfg, "config_base", &cval)); - if (cval.val) - WT_ERR( - __conn_write_config(session, WT_BASECONFIG, cfg)); - } + WT_ERR(__conn_write_base_config(session, cfg)); /* * Start the worker threads last. diff --git a/src/third_party/wiredtiger/src/conn/conn_cache.c b/src/third_party/wiredtiger/src/conn/conn_cache.c index 4a7e15044de..a99e6d3ad20 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache.c @@ -190,6 +190,12 @@ __wt_cache_stats_update(WT_SESSION_IMPL *session) WT_STAT_SET(stats, cache_eviction_maximum_page_size, cache->evict_max_page_size); WT_STAT_SET(stats, cache_pages_dirty, cache->pages_dirty); + + /* Figure out internal, leaf and overflow stats */ + WT_STAT_SET(stats, cache_bytes_internal, cache->bytes_internal); + WT_STAT_SET(stats, cache_bytes_leaf, + conn->cache_size - (cache->bytes_internal + cache->bytes_overflow)); + WT_STAT_SET(stats, cache_bytes_overflow, cache->bytes_overflow); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c index 7bf090496a8..488864ce351 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c @@ -596,7 +596,7 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, * __wt_cache_pool_server -- * Thread to manage cache pool among connections. */ -void * +WT_THREAD_RET __wt_cache_pool_server(void *arg) { WT_CACHE *cache; @@ -642,5 +642,5 @@ __wt_cache_pool_server(void *arg) if (0) { err: WT_PANIC_MSG(session, ret, "cache pool manager server error"); } - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/src/conn/conn_ckpt.c b/src/third_party/wiredtiger/src/conn/conn_ckpt.c index 6eb134c1765..74f27d8bd18 100644 --- a/src/third_party/wiredtiger/src/conn/conn_ckpt.c +++ b/src/third_party/wiredtiger/src/conn/conn_ckpt.c @@ -31,7 +31,7 @@ __ckpt_server_config(WT_SESSION_IMPL *session, const char **cfg, int *startp) * Checkpoints based on log size also require logging be enabled. */ WT_RET(__wt_config_gets(session, cfg, "checkpoint.wait", &cval)); - conn->ckpt_usecs = (long)cval.val * 1000000; + conn->ckpt_usecs = (uint64_t)cval.val * 1000000; WT_RET(__wt_config_gets(session, cfg, "checkpoint.log_size", &cval)); conn->ckpt_logsize = (wt_off_t)cval.val; __wt_log_written_reset(session); @@ -69,7 +69,7 @@ err: __wt_scr_free(session, &tmp); * __ckpt_server -- * The checkpoint server thread. */ -static void * +static WT_THREAD_RET __ckpt_server(void *arg) { WT_CONNECTION_IMPL *conn; @@ -112,7 +112,7 @@ __ckpt_server(void *arg) if (0) { err: WT_PANIC_MSG(session, ret, "checkpoint server error"); } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 7756158594c..63180d64019 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -234,7 +234,7 @@ err: WT_TRET(__wt_rwlock_destroy(session, &dhandle->rwlock)); * Sync and close the underlying btree handle. */ int -__wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, int force) +__wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, int final, int force) { WT_BTREE *btree; WT_DATA_HANDLE *dhandle; @@ -273,7 +273,7 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, int force) */ if (!F_ISSET(btree, WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) - WT_ERR(__wt_checkpoint_close(session, force)); + WT_ERR(__wt_checkpoint_close(session, final, force)); if (dhandle->checkpoint == NULL) --S2C(session)->open_btree_count; @@ -361,8 +361,7 @@ err: __wt_free(session, metaconf); * Open the current btree handle. */ static int -__conn_btree_open( - WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) +__conn_btree_open(WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) { WT_BTREE *btree; WT_DATA_HANDLE *dhandle; @@ -392,7 +391,7 @@ __conn_btree_open( * in the tree that can block the close. */ if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) - WT_RET(__wt_conn_btree_sync_and_close(session, 0)); + WT_RET(__wt_conn_btree_sync_and_close(session, 0, 0)); /* Discard any previous configuration, set up the new configuration. */ __conn_btree_config_clear(session); @@ -424,7 +423,7 @@ __conn_btree_open( err: F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); /* If the open failed, close the handle. */ if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) - WT_TRET(__wt_conn_btree_sync_and_close(session, 0)); + WT_TRET(__wt_conn_btree_sync_and_close(session, 0, 0)); } return (ret); @@ -670,7 +669,7 @@ __wt_conn_dhandle_close_all( if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { if ((ret = __wt_meta_track_sub_on(session)) == 0) ret = __wt_conn_btree_sync_and_close( - session, force); + session, 0, force); /* * If the close succeeded, drop any locks it acquired. @@ -732,7 +731,7 @@ __wt_conn_dhandle_discard_single(WT_SESSION_IMPL *session, int final) dhandle = session->dhandle; if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { - tret = __wt_conn_btree_sync_and_close(session, 0); + tret = __wt_conn_btree_sync_and_close(session, final, 0); if (final && tret != 0) { __wt_err(session, tret, "Final close of %s failed", dhandle->name); @@ -745,7 +744,8 @@ __wt_conn_dhandle_discard_single(WT_SESSION_IMPL *session, int final) * Kludge: interrupt the eviction server in case it is holding the * handle list lock. */ - F_SET(S2C(session)->cache, WT_CACHE_CLEAR_WALKS); + if (!F_ISSET(session, WT_SESSION_HANDLE_LIST_LOCKED)) + F_SET(S2C(session)->cache, WT_CACHE_CLEAR_WALKS); /* Try to remove the handle, protected by the data handle lock. */ WT_WITH_DHANDLE_LOCK(session, diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 63bb9af5842..b212409be80 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -116,8 +116,7 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) * underlying file-close code uses the mutex to guard lists of * open files. */ - if (conn->lock_fh != NULL) - WT_TRET(__wt_close(session, conn->lock_fh)); + WT_TRET(__wt_close(session, &conn->lock_fh)); /* Remove from the list of connections. */ __wt_spin_lock(session, &__wt_process.spinlock); diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 315e93c1875..a6d53134ec2 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -46,7 +46,6 @@ __logmgr_config(WT_SESSION_IMPL *session, const char **cfg, int *runp) { WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; - WT_NAMED_COMPRESSOR *ncomp; conn = S2C(session); @@ -62,17 +61,7 @@ __logmgr_config(WT_SESSION_IMPL *session, const char **cfg, int *runp) */ conn->log_compressor = NULL; WT_RET(__wt_config_gets_none(session, cfg, "log.compressor", &cval)); - if (cval.len > 0) { - TAILQ_FOREACH(ncomp, &conn->compqh, q) - if (WT_STRING_MATCH(ncomp->name, cval.str, cval.len)) { - conn->log_compressor = ncomp->compressor; - break; - } - if (conn->log_compressor == NULL) - WT_RET_MSG(session, EINVAL, - "unknown log compressor '%.*s'", - (int)cval.len, cval.str); - } + WT_RET(__wt_compressor_config(session, &cval, &conn->log_compressor)); WT_RET(__wt_config_gets(session, cfg, "log.path", &cval)); WT_RET(__wt_strndup(session, cval.str, cval.len, &conn->log_path)); @@ -284,7 +273,7 @@ err: * __log_close_server -- * The log close server thread. */ -static void * +static WT_THREAD_RET __log_close_server(void *arg) { WT_CONNECTION_IMPL *conn; @@ -327,7 +316,7 @@ __log_close_server(void *arg) WT_ERR(__wt_fsync(session, close_fh)); __wt_spin_lock(session, &log->log_sync_lock); locked = 1; - WT_ERR(__wt_close(session, close_fh)); + WT_ERR(__wt_close(session, &close_fh)); log->sync_lsn = close_end_lsn; WT_ERR(__wt_cond_signal(session, log->log_sync_cond)); locked = 0; @@ -343,7 +332,7 @@ err: __wt_err(session, ret, "log close server error"); } if (locked) __wt_spin_unlock(session, &log->log_sync_lock); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -358,7 +347,7 @@ typedef struct { * __log_wrlsn_cmp -- * The log wrlsn comparison function for qsort. */ -static int +static int WT_CDECL __log_wrlsn_cmp(const void *a, const void *b) { WT_LOG_WRLSN_ENTRY *ae, *be; @@ -372,7 +361,7 @@ __log_wrlsn_cmp(const void *a, const void *b) * __log_wrlsn_server -- * The log wrlsn server thread. */ -static void * +static WT_THREAD_RET __log_wrlsn_server(void *arg) { WT_CONNECTION_IMPL *conn; @@ -461,14 +450,14 @@ __log_wrlsn_server(void *arg) if (0) err: __wt_err(session, ret, "log wrlsn server error"); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* * __log_server -- * The log server thread. */ -static void * +static WT_THREAD_RET __log_server(void *arg) { WT_CONNECTION_IMPL *conn; @@ -513,7 +502,7 @@ err: __wt_err(session, ret, "log server error"); } if (locked) (void)__wt_writeunlock(session, log->log_archive_lock); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 0a3d35ac0b1..86f62e176ad 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -149,7 +149,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) __wt_errx(session, "Connection has open file handles: %s", fh->name); - WT_TRET(__wt_close(session, fh)); + WT_TRET(__wt_close(session, &fh)); fh = SLIST_FIRST(&conn->fhlh); } diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index c38e0ef125f..0d008939d8c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -67,7 +67,7 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp) WT_RET(__wt_config_gets(session, cfg, "statistics_log.wait", &cval)); /* Only start the server if wait time is non-zero */ *runp = (cval.val == 0) ? 0 : 1; - conn->stat_usecs = (long)cval.val * 1000000; + conn->stat_usecs = (uint64_t)cval.val * 1000000; WT_RET(__wt_config_gets( session, cfg, "statistics_log.on_close", &cval)); @@ -166,10 +166,10 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, int conn_stats) sizeof(WT_DSRC_STATS) / sizeof(WT_STATS); for (i = 0, stats = WT_CURSOR_STATS(cursor); i < max; ++i, ++stats) - WT_ERR_TEST((fprintf(conn->stat_fp, + WT_ERR(__wt_fprintf(session, conn->stat_fp, "%s %" PRIu64 " %s %s\n", conn->stat_stamp, - stats->v, name, stats->desc) < 0), __wt_errno()); + stats->v, name, stats->desc)); WT_ERR(cursor->close(cursor)); break; case EBUSY: @@ -300,13 +300,11 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) if ((log_file = conn->stat_fp) == NULL || path == NULL || strcmp(tmp->mem, path->mem) != 0) { conn->stat_fp = NULL; - if (log_file != NULL) - WT_RET(fclose(log_file) == 0 ? 0 : __wt_errno()); - + WT_RET(__wt_fclose(session, &log_file, WT_FHANDLE_APPEND)); if (path != NULL) (void)strcpy(path->mem, tmp->mem); - WT_RET_TEST((log_file = - fopen(tmp->mem, "a")) == NULL, __wt_errno()); + WT_RET(__wt_fopen(session, + tmp->mem, WT_FHANDLE_APPEND, WT_FOPEN_FIXED, &log_file)); } conn->stat_fp = log_file; @@ -346,9 +344,7 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) WT_RET(__statlog_lsm_apply(session)); /* Flush. */ - WT_RET(fflush(conn->stat_fp) == 0 ? 0 : __wt_errno()); - - return (0); + return (__wt_fflush(session, conn->stat_fp)); } /* @@ -384,7 +380,7 @@ err: __wt_scr_free(session, &tmp); * __statlog_server -- * The statistics server thread. */ -static void * +static WT_THREAD_RET __statlog_server(void *arg) { WT_CONNECTION_IMPL *conn; @@ -423,7 +419,7 @@ err: WT_PANIC_MSG(session, ret, "statistics log server error"); } __wt_buf_free(session, &path); __wt_buf_free(session, &tmp); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -533,10 +529,7 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, int is_close) conn->stat_session = NULL; conn->stat_tid_set = 0; conn->stat_format = NULL; - if (conn->stat_fp != NULL) { - WT_TRET(fclose(conn->stat_fp) == 0 ? 0 : __wt_errno()); - conn->stat_fp = NULL; - } + WT_TRET(__wt_fclose(session, &conn->stat_fp, WT_FHANDLE_APPEND)); conn->stat_path = NULL; conn->stat_sources = NULL; conn->stat_stamp = NULL; diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index 5145583fc7d..90773a621e2 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -9,31 +9,86 @@ #include "wt_internal.h" /* + * __sweep_remove_handles -- + * Remove closed dhandles from the connection list. + */ +static int +__sweep_remove_handles(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DATA_HANDLE *dhandle, *dhandle_next; + WT_DECL_RET; + + conn = S2C(session); + dhandle = SLIST_FIRST(&conn->dhlh); + + for (; dhandle != NULL; dhandle = dhandle_next) { + dhandle_next = SLIST_NEXT(dhandle, l); + if (WT_IS_METADATA(dhandle)) + continue; + if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) + continue; + + /* Make sure we get exclusive access. */ + if ((ret = + __wt_try_writelock(session, dhandle->rwlock)) == EBUSY) + continue; + WT_RET(ret); + + /* + * If there are no longer any references to the handle in any + * sessions, attempt to discard it. + */ + if (F_ISSET(dhandle, WT_DHANDLE_OPEN) || + dhandle->session_inuse != 0 || dhandle->session_ref != 0) { + WT_RET(__wt_writeunlock(session, dhandle->rwlock)); + continue; + } + + WT_WITH_DHANDLE(session, dhandle, + ret = __wt_conn_dhandle_discard_single(session, 0)); + + /* If the handle was not successfully discarded, unlock it. */ + if (ret != 0) + WT_TRET(__wt_writeunlock(session, dhandle->rwlock)); + WT_RET_BUSY_OK(ret); + WT_STAT_FAST_CONN_INCR(session, dh_conn_ref); + } + + return (ret == EBUSY ? 0 : ret); +} + +/* * __sweep -- * Close unused dhandles on the connection dhandle list. */ static int __sweep(WT_SESSION_IMPL *session) { + WT_BTREE *btree; WT_CONNECTION_IMPL *conn; - WT_DATA_HANDLE *dhandle, *dhandle_next; + WT_DATA_HANDLE *dhandle; WT_DECL_RET; time_t now; - int locked; + int closed_handles; conn = S2C(session); + closed_handles = 0; /* Don't discard handles that have been open recently. */ WT_RET(__wt_seconds(session, &now)); WT_STAT_FAST_CONN_INCR(session, dh_conn_sweeps); - dhandle = SLIST_FIRST(&conn->dhlh); - for (; dhandle != NULL; dhandle = dhandle_next) { - dhandle_next = SLIST_NEXT(dhandle, l); + SLIST_FOREACH(dhandle, &conn->dhlh, l) { if (WT_IS_METADATA(dhandle)) continue; + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) && + dhandle->session_inuse == 0 && dhandle->session_ref == 0) { + ++closed_handles; + continue; + } if (dhandle->session_inuse != 0 || - now <= dhandle->timeofdeath + WT_DHANDLE_SWEEP_WAIT) + now <= dhandle->timeofdeath + conn->sweep_idle_time) continue; if (dhandle->timeofdeath == 0) { dhandle->timeofdeath = now; @@ -43,15 +98,13 @@ __sweep(WT_SESSION_IMPL *session) /* * We have a candidate for closing; if it's open, acquire an - * exclusive lock on the handle and close it. We might be - * blocking opens for a long time (over disk I/O), but the - * handle was quiescent for awhile. + * exclusive lock on the handle and close it. * - * The close can fail if an update cannot be written (updates - * in a no-longer-referenced file might not yet be globally - * visible if sessions have disjoint sets of files open). If - * the handle is busy, skip it, we'll retry the close the next - * time, after the transaction state has progressed. + * The close would require I/O if an update cannot be written + * (updates in a no-longer-referenced file might not yet be + * globally visible if sessions have disjoint sets of files + * open). In that case, skip it: we'll retry the close the + * next time, after the transaction state has progressed. * * We don't set WT_DHANDLE_EXCLUSIVE deliberately, we want * opens to block on us rather than returning an EBUSY error to @@ -61,41 +114,37 @@ __sweep(WT_SESSION_IMPL *session) __wt_try_writelock(session, dhandle->rwlock)) == EBUSY) continue; WT_RET(ret); - locked = 1; + + /* Only sweep clean trees where all updates are visible. */ + btree = dhandle->handle; + if (btree->modified || + !__wt_txn_visible_all(session, btree->rec_max_txn)) + goto unlock; /* If the handle is open, try to close it. */ if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { - WT_WITH_DHANDLE(session, dhandle, - ret = __wt_conn_btree_sync_and_close(session, 0)); - if (ret != 0) - goto unlock; + WT_WITH_DHANDLE(session, dhandle, ret = + __wt_conn_btree_sync_and_close(session, 0, 0)); /* We closed the btree handle, bump the statistic. */ - WT_STAT_FAST_CONN_INCR(session, dh_conn_handles); + if (ret == 0) + WT_STAT_FAST_CONN_INCR( + session, dh_conn_handles); } - /* - * If there are no longer any references to the handle in any - * sessions, attempt to discard it. The called function - * re-checks that the handle is not in use, which is why we - * don't do any special handling of EBUSY returns above. - */ - if (dhandle->session_inuse == 0 && dhandle->session_ref == 0) { - WT_WITH_DHANDLE(session, dhandle, - ret = __wt_conn_dhandle_discard_single(session, 0)); - if (ret != 0) - goto unlock; - - /* If the handle was discarded, it isn't locked. */ - locked = 0; - } else - WT_STAT_FAST_CONN_INCR(session, dh_conn_ref); - -unlock: if (locked) - WT_TRET(__wt_writeunlock(session, dhandle->rwlock)); + if (dhandle->session_inuse == 0 && dhandle->session_ref == 0) + ++closed_handles; +unlock: WT_TRET(__wt_writeunlock(session, dhandle->rwlock)); WT_RET_BUSY_OK(ret); } + + if (closed_handles) { + WT_WITH_DHANDLE_LOCK(session, + ret = __sweep_remove_handles(session)); + WT_RET(ret); + } + return (0); } @@ -103,7 +152,7 @@ unlock: if (locked) * __sweep_server -- * The handle sweep server thread. */ -static void * +static WT_THREAD_RET __sweep_server(void *arg) { WT_CONNECTION_IMPL *conn; @@ -119,8 +168,8 @@ __sweep_server(void *arg) while (F_ISSET(conn, WT_CONN_SERVER_RUN) && F_ISSET(conn, WT_CONN_SERVER_SWEEP)) { /* Wait until the next event. */ - WT_ERR(__wt_cond_wait(session, - conn->sweep_cond, WT_DHANDLE_SWEEP_PERIOD * WT_MILLION)); + WT_ERR(__wt_cond_wait(session, conn->sweep_cond, + (uint64_t)conn->sweep_interval * WT_MILLION)); /* Sweep the handles. */ WT_ERR(__sweep(session)); @@ -129,7 +178,31 @@ __sweep_server(void *arg) if (0) { err: WT_PANIC_MSG(session, ret, "handle sweep server error"); } - return (NULL); + return (WT_THREAD_RET_VALUE); +} + +/* + * __wt_sweep_config -- + * Pull out sweep configuration settings + */ +int +__wt_sweep_config(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + /* Pull out the sweep configurations. */ + WT_RET(__wt_config_gets(session, + cfg, "file_manager.close_idle_time", &cval)); + conn->sweep_idle_time = (time_t)cval.val; + + WT_RET(__wt_config_gets(session, + cfg, "file_manager.close_scan_interval", &cval)); + conn->sweep_interval = (time_t)cval.val; + + return (0); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index c85e9e24fb2..5b47c2c1a73 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -11,7 +11,6 @@ static int __backup_all(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); static int __backup_cleanup_handles(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); static int __backup_file_create(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, int); -static int __backup_file_remove(WT_SESSION_IMPL *); static int __backup_list_all_append(WT_SESSION_IMPL *, const char *[]); static int __backup_list_append( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *); @@ -249,10 +248,7 @@ __backup_start( * Close any hot backup file. * We're about to open the incremental backup file. */ - if (cb->bfp != NULL) { - WT_TRET(fclose(cb->bfp) == 0 ? 0 : __wt_errno()); - cb->bfp = NULL; - } + WT_TRET(__wt_fclose(session, &cb->bfp, WT_FHANDLE_WRITE)); WT_ERR(__backup_file_create(session, cb, log_only)); WT_ERR(__backup_list_append( session, cb, WT_INCREMENTAL_BACKUP)); @@ -270,10 +266,7 @@ __backup_start( } err: /* Close the hot backup file. */ - if (cb->bfp != NULL) { - WT_TRET(fclose(cb->bfp) == 0 ? 0 : __wt_errno()); - cb->bfp = NULL; - } + WT_TRET(__wt_fclose(session, &cb->bfp, WT_FHANDLE_WRITE)); if (ret != 0) { WT_TRET(__backup_cleanup_handles(session, cb)); WT_TRET(__backup_stop(session)); @@ -322,7 +315,7 @@ __backup_stop(WT_SESSION_IMPL *session) conn = S2C(session); /* Remove any backup specific file. */ - ret = __backup_file_remove(session); + ret = __wt_backup_file_remove(session); /* Checkpoint deletion can proceed, as can the next hot backup. */ __wt_spin_lock(session, &conn->hot_backup_lock); @@ -354,8 +347,7 @@ __backup_all(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) while ((ret = cursor->next(cursor)) == 0) { WT_ERR(cursor->get_key(cursor, &key)); WT_ERR(cursor->get_value(cursor, &value)); - WT_ERR_TEST((fprintf( - cb->bfp, "%s\n%s\n", key, value) < 0), __wt_errno()); + WT_ERR(__wt_fprintf(session, cb->bfp, "%s\n%s\n", key, value)); /* * While reading the metadata file, check there are no "sources" @@ -462,37 +454,22 @@ static int __backup_file_create( WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, int incremental) { - WT_DECL_RET; - char *path; - - /* Open the hot backup file. */ - if (incremental) - WT_RET(__wt_filename(session, WT_INCREMENTAL_BACKUP, &path)); - else - WT_RET(__wt_filename(session, WT_METADATA_BACKUP, &path)); - WT_ERR_TEST((cb->bfp = fopen(path, "w")) == NULL, __wt_errno()); - -err: __wt_free(session, path); - return (ret); + return (__wt_fopen(session, + incremental ? WT_INCREMENTAL_BACKUP : WT_METADATA_BACKUP, + WT_FHANDLE_WRITE, 0, &cb->bfp)); } /* - * __backup_file_remove -- - * Remove the meta-data backup file. + * __wt_backup_file_remove -- + * Remove the incremental and meta-data backup files. */ -static int -__backup_file_remove(WT_SESSION_IMPL *session) +int +__wt_backup_file_remove(WT_SESSION_IMPL *session) { WT_DECL_RET; - int exist; - - WT_ERR(__wt_exist(session, WT_INCREMENTAL_BACKUP, &exist)); - if (exist) - WT_ERR(__wt_remove(session, WT_INCREMENTAL_BACKUP)); - WT_ERR(__wt_exist(session, WT_METADATA_BACKUP, &exist)); - if (exist) - WT_ERR(__wt_remove(session, WT_METADATA_BACKUP)); -err: + + WT_TRET(__wt_remove_if_exists(session, WT_INCREMENTAL_BACKUP)); + WT_TRET(__wt_remove_if_exists(session, WT_METADATA_BACKUP)); return (ret); } @@ -518,8 +495,7 @@ __wt_backup_list_uri_append( /* Add the metadata entry to the backup file. */ WT_RET(__wt_metadata_search(session, name, &value)); - WT_RET_TEST( - (fprintf(cb->bfp, "%s\n%s\n", name, value) < 0), __wt_errno()); + WT_RET(__wt_fprintf(session, cb->bfp, "%s\n%s\n", name, value)); __wt_free(session, value); /* Add file type objects to the list of files to be copied. */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index bf086bcc813..7b234eab280 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -61,6 +61,35 @@ err: cursor->saved_err = ret; } /* + * __curindex_compare -- + * WT_CURSOR->compare method for the index cursor type. + */ +static int +__curindex_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) +{ + WT_CURSOR_INDEX *cindex; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + cindex = (WT_CURSOR_INDEX *)a; + CURSOR_API_CALL(a, session, compare, NULL); + + /* Check both cursors are "index:" type. */ + if (!WT_PREFIX_MATCH(a->uri, "index:") || + !WT_PREFIX_MATCH(b->uri, "index:")) + WT_ERR_MSG(session, EINVAL, + "Cursors must reference the same object"); + + WT_CURSOR_CHECKKEY(a); + WT_CURSOR_CHECKKEY(b); + + ret = __wt_compare( + session, cindex->index->collator, &a->key, &b->key, cmpp); + +err: API_END_RET(session, ret); +} + +/* * __curindex_move -- * When an index cursor changes position, set the primary key in the * associated column groups and update their positions to match. @@ -189,32 +218,46 @@ __curindex_search(WT_CURSOR *cursor) WT_CURSOR *child; WT_CURSOR_INDEX *cindex; WT_DECL_RET; + WT_ITEM found_key; WT_SESSION_IMPL *session; - int exact; + int cmp; cindex = (WT_CURSOR_INDEX *)cursor; child = cindex->child; CURSOR_API_CALL(cursor, session, search, NULL); /* - * We expect partial matches, but we want the smallest item that - * matches the prefix. Fail if there is no matching item. + * We are searching using the application-specified key, which + * (usually) doesn't contain the primary key, so it is just a prefix of + * any matching index key. Do a search_near, step to the next entry if + * we land on one that is too small, then check that the prefix + * matches. */ __wt_cursor_set_raw_key(child, &cursor->key); - WT_ERR(child->search_near(child, &exact)); + WT_ERR(child->search_near(child, &cmp)); + + if (cmp < 0) + WT_ERR(child->next(child)); /* * We expect partial matches, and want the smallest record with a key - * greater than or equal to the search key. The only way for the key - * to be equal is if there is an index on the primary key, because - * otherwise the primary key columns will be appended to the index key, - * but we don't disallow that (odd) case. + * greater than or equal to the search key. + * + * If the key we find is shorter than the search key, it can't possibly + * match. + * + * The only way for the key to be exactly equal is if there is an index + * on the primary key, because otherwise the primary key columns will + * be appended to the index key, but we don't disallow that (odd) case. */ - if (exact < 0) - WT_ERR(child->next(child)); - - if (child->key.size < cursor->key.size || - memcmp(child->key.data, cursor->key.data, cursor->key.size) != 0) { + found_key = child->key; + if (found_key.size < cursor->key.size) + WT_ERR(WT_NOTFOUND); + found_key.size = cursor->key.size; + + WT_ERR(__wt_compare( + session, cindex->index->collator, &cursor->key, &found_key, &cmp)); + if (cmp != 0) { ret = WT_NOTFOUND; goto err; } @@ -342,8 +385,8 @@ __wt_curindex_open(WT_SESSION_IMPL *session, __curindex_get_value, /* get-value */ __wt_cursor_set_key, /* set-key */ __curindex_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ + __curindex_compare, /* compare */ + __wt_cursor_equals, /* equals */ __curindex_next, /* next */ __curindex_prev, /* prev */ __curindex_reset, /* reset */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index 74b998876c2..85442592c39 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -8,9 +8,6 @@ #include "wt_internal.h" -static int __curstat_next(WT_CURSOR *cursor); -static int __curstat_prev(WT_CURSOR *cursor); - /* * The statistics identifier is an offset from a base to ensure the integer ID * values don't overlap (the idea is if they overlap it's easy for application @@ -40,6 +37,22 @@ __curstat_print_value(WT_SESSION_IMPL *session, uint64_t v, WT_ITEM *buf) } /* + * __curstat_free_config -- + * Free the saved configuration string stack + */ +static void +__curstat_free_config(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) +{ + size_t i; + + if (cst->cfg != NULL) { + for (i = 0; cst->cfg[i] != NULL; ++i) + __wt_free(session, cst->cfg[i]); + __wt_free(session, cst->cfg); + } +} + +/* * __curstat_get_key -- * WT_CURSOR->get_key for statistics cursors. */ @@ -185,6 +198,13 @@ __curstat_next(WT_CURSOR *cursor) cst = (WT_CURSOR_STAT *)cursor; CURSOR_API_CALL(cursor, session, next, NULL); + /* Initialize on demand. */ + if (cst->notinitialized) { + WT_ERR(__wt_curstat_init( + session, cursor->internal_uri, cst->cfg, cst)); + cst->notinitialized = 0; + } + /* Move to the next item. */ if (cst->notpositioned) { cst->notpositioned = 0; @@ -216,6 +236,13 @@ __curstat_prev(WT_CURSOR *cursor) cst = (WT_CURSOR_STAT *)cursor; CURSOR_API_CALL(cursor, session, prev, NULL); + /* Initialize on demand. */ + if (cst->notinitialized) { + WT_ERR(__wt_curstat_init( + session, cursor->internal_uri, cst->cfg, cst)); + cst->notinitialized = 0; + } + /* Move to the previous item. */ if (cst->notpositioned) { cst->notpositioned = 0; @@ -248,7 +275,7 @@ __curstat_reset(WT_CURSOR *cursor) cst = (WT_CURSOR_STAT *)cursor; CURSOR_API_CALL(cursor, session, reset, NULL); - cst->notpositioned = 1; + cst->notinitialized = cst->notpositioned = 1; F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); err: API_END_RET(session, ret); @@ -271,6 +298,13 @@ __curstat_search(WT_CURSOR *cursor) WT_CURSOR_NEEDKEY(cursor); F_CLR(cursor, WT_CURSTD_VALUE_SET | WT_CURSTD_VALUE_SET); + /* Initialize on demand. */ + if (cst->notinitialized) { + WT_ERR(__wt_curstat_init( + session, cursor->internal_uri, cst->cfg, cst)); + cst->notinitialized = 0; + } + if (cst->key < WT_STAT_KEY_MIN(cst) || cst->key > WT_STAT_KEY_MAX(cst)) WT_ERR(WT_NOTFOUND); @@ -295,6 +329,8 @@ __curstat_close(WT_CURSOR *cursor) cst = (WT_CURSOR_STAT *)cursor; CURSOR_API_CALL(cursor, session, close, NULL); + __curstat_free_config(session, cst); + __wt_buf_free(session, &cst->pv); WT_ERR(__wt_cursor_close(cursor)); @@ -381,8 +417,6 @@ __wt_curstat_init(WT_SESSION_IMPL *session, { const char *dsrc_uri; - cst->notpositioned = 1; - if (strcmp(uri, "statistics:") == 0) { __curstat_conn_init(session, cst); return (0); @@ -439,6 +473,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session, WT_CURSOR *cursor; WT_CURSOR_STAT *cst; WT_DECL_RET; + size_t i; WT_STATIC_ASSERT(offsetof(WT_CURSOR_STAT, iface) == 0); @@ -497,7 +532,28 @@ __wt_curstat_open(WT_SESSION_IMPL *session, */ cursor->key_format = "i"; cursor->value_format = "SSq"; - WT_ERR(__wt_curstat_init(session, uri, cfg, cst)); + + /* + * WT_CURSOR.reset on a statistics cursor refreshes the cursor, save + * the cursor's configuration for that. + */ + for (i = 0; cfg[i] != NULL; ++i) + ; + WT_ERR(__wt_calloc_def(session, i + 1, &cst->cfg)); + for (i = 0; cfg[i] != NULL; ++i) + WT_ERR(__wt_strdup(session, cfg[i], &cst->cfg[i])); + + /* + * Do the initial statistics snapshot: there won't be cursor operations + * to trigger initialization when aggregating statistics for upper-level + * objects like tables, we need to a valid set of statistics when before + * the open returns. + */ + WT_ERR(__wt_curstat_init(session, uri, cst->cfg, cst)); + cst->notinitialized = 0; + + /* The cursor isn't yet positioned. */ + cst->notpositioned = 1; /* __wt_cursor_init is last so we don't have to clean up on error. */ WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); @@ -509,7 +565,8 @@ config_err: WT_ERR_MSG(session, EINVAL, } if (0) { -err: __wt_free(session, cst); +err: __curstat_free_config(session, cst); + __wt_free(session, cst); } return (ret); diff --git a/src/third_party/wiredtiger/src/evict/evict_file.c b/src/third_party/wiredtiger/src/evict/evict_file.c index 1030c0aa818..864c116a380 100644 --- a/src/third_party/wiredtiger/src/evict/evict_file.c +++ b/src/third_party/wiredtiger/src/evict/evict_file.c @@ -72,28 +72,16 @@ __wt_evict_file(WT_SESSION_IMPL *session, int syncop) WT_READ_CACHE | WT_READ_NO_EVICT)); switch (syncop) { - case WT_SYNC_DISCARD: - /* - * Check that the page is clean: if we see a dirty page - * (including a dirty parent page after evicting a - * child), give up. The higher level can try to - * checkpoint, but during discard we aren't set up to - * manage checkpoints. - */ - if (__wt_page_is_modified(page)) - WT_ERR(EBUSY); - /* FALLTHROUGH */ case WT_SYNC_CLOSE: /* * Evict the page. - * Do not attempt to evict pages expected to be merged - * into their parents, with the exception that the root - * page can't be merged, it must be written. */ - if (__wt_ref_is_root(ref) || - page->modify == NULL || - !F_ISSET(page->modify, WT_PM_REC_EMPTY)) - WT_ERR(__wt_evict(session, ref, 1)); + WT_ERR(__wt_evict(session, ref, 1)); + break; + case WT_SYNC_DISCARD: + WT_ASSERT(session, + __wt_page_can_evict(session, page, 0)); + __wt_evict_page_clean_update(session, ref); break; case WT_SYNC_DISCARD_FORCE: /* diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 640c9b0541d..62326015d2c 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -10,13 +10,13 @@ static int __evict_clear_walks(WT_SESSION_IMPL *); static int __evict_has_work(WT_SESSION_IMPL *, uint32_t *); -static int __evict_lru_cmp(const void *, const void *); +static int WT_CDECL __evict_lru_cmp(const void *, const void *); static int __evict_lru_pages(WT_SESSION_IMPL *, int); static int __evict_lru_walk(WT_SESSION_IMPL *, uint32_t); static int __evict_pass(WT_SESSION_IMPL *); static int __evict_walk(WT_SESSION_IMPL *, uint32_t); static int __evict_walk_file(WT_SESSION_IMPL *, u_int *, uint32_t); -static void *__evict_worker(void *); +static WT_THREAD_RET __evict_worker(void *); static int __evict_server_work(WT_SESSION_IMPL *); /* @@ -54,7 +54,7 @@ __evict_read_gen(const WT_EVICT_ENTRY *entry) * __evict_lru_cmp -- * Qsort function: sort the eviction array. */ -static int +static int WT_CDECL __evict_lru_cmp(const void *a, const void *b) { uint64_t a_lru, b_lru; @@ -94,7 +94,7 @@ __wt_evict_list_clear_page(WT_SESSION_IMPL *session, WT_REF *ref) WT_EVICT_ENTRY *evict; uint32_t i, elem; - WT_ASSERT(session, + WT_ASSERT(session, __wt_ref_is_root(ref) || ref->state == WT_REF_LOCKED); /* Fast path: if the page isn't on the queue, don't bother searching. */ @@ -150,7 +150,7 @@ __wt_evict_server_wake(WT_SESSION_IMPL *session) * __evict_server -- * Thread to evict pages from the cache. */ -static void * +static WT_THREAD_RET __evict_server(void *arg) { WT_CACHE *cache; @@ -232,7 +232,7 @@ __evict_server(void *arg) if (0) { err: WT_PANIC_MSG(session, ret, "cache eviction server error"); } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -340,6 +340,20 @@ __wt_evict_destroy(WT_SESSION_IMPL *session) F_CLR(conn, WT_CONN_EVICTION_RUN); + /* + * Wait for the main eviction thread to exit before waiting on the + * helpers. The eviction server spawns helper threads, so we can't + * safely know how many helpers are running until the main thread is + * done. + */ + WT_TRET(__wt_verbose( + session, WT_VERB_EVICTSERVER, "waiting for main thread")); + if (conn->evict_tid_set) { + WT_TRET(__wt_evict_server_wake(session)); + WT_TRET(__wt_thread_join(session, conn->evict_tid)); + conn->evict_tid_set = 0; + } + WT_TRET(__wt_verbose( session, WT_VERB_EVICTSERVER, "waiting for helper threads")); for (i = 0; i < conn->evict_workers; i++) { @@ -356,12 +370,6 @@ __wt_evict_destroy(WT_SESSION_IMPL *session) __wt_free(session, conn->evict_workctx); } - if (conn->evict_tid_set) { - WT_TRET(__wt_evict_server_wake(session)); - WT_TRET(__wt_thread_join(session, conn->evict_tid)); - conn->evict_tid_set = 0; - } - if (conn->evict_session != NULL) { wt_session = &conn->evict_session->iface; WT_TRET(wt_session->close(wt_session, NULL)); @@ -376,7 +384,7 @@ __wt_evict_destroy(WT_SESSION_IMPL *session) * __evict_worker -- * Thread to help evict pages from the cache. */ -static void * +static WT_THREAD_RET __evict_worker(void *arg) { WT_CACHE *cache; @@ -405,7 +413,7 @@ __evict_worker(void *arg) if (0) { err: WT_PANIC_MSG(session, ret, "cache eviction worker error"); } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -538,14 +546,20 @@ __evict_pass(WT_SESSION_IMPL *session) * that can free space in cache, such as LSM discarding * handles. */ - __wt_sleep(0, 1000 * (long)loop); + __wt_sleep(0, 1000 * (uint64_t)loop); if (loop == 100) { - F_SET(cache, WT_CACHE_STUCK); - WT_STAT_FAST_CONN_INCR( - session, cache_eviction_slow); - WT_RET(__wt_verbose( - session, WT_VERB_EVICTSERVER, - "unable to reach eviction goal")); + /* + * Mark the cache as stuck if we need space + * and aren't evicting any pages. + */ + if (!LF_ISSET(WT_EVICT_PASS_WOULD_BLOCK)) { + F_SET(cache, WT_CACHE_STUCK); + WT_STAT_FAST_CONN_INCR( + session, cache_eviction_slow); + WT_RET(__wt_verbose( + session, WT_VERB_EVICTSERVER, + "unable to reach eviction goal")); + } break; } } else { @@ -989,6 +1003,11 @@ retry: while (slot < max_entries && ret == 0) { !LF_ISSET(WT_EVICT_PASS_AGGRESSIVE)) continue; + /* Skip files if we have used all available hazard pointers. */ + if (btree->evict_ref == NULL && session->nhazard >= + conn->hazard_max - WT_MIN(conn->hazard_max / 2, 10)) + continue; + /* * If we are filling the queue, skip files that haven't been * useful in the past. @@ -1103,6 +1122,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp, uint32_t flags) WT_EVICT_ENTRY *end, *evict, *start; WT_PAGE *page; WT_PAGE_MODIFY *mod; + WT_REF *ref; uint64_t pages_walked; uint32_t walk_flags; int enough, internal_pages, modified, restarts; @@ -1137,16 +1157,17 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp, uint32_t flags) ret = __wt_tree_walk( session, &btree->evict_ref, &pages_walked, walk_flags)) { enough = (pages_walked > WT_EVICT_MAX_PER_FILE); - if (btree->evict_ref == NULL) { + if ((ref = btree->evict_ref) == NULL) { if (++restarts == 2 || enough) break; continue; } /* Ignore root pages entirely. */ - if (__wt_ref_is_root(btree->evict_ref)) + if (__wt_ref_is_root(ref)) continue; - page = btree->evict_ref->page; + + page = ref->page; modified = __wt_page_is_modified(page); /* @@ -1190,7 +1211,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp, uint32_t flags) } fast: /* If the page can't be evicted, give up. */ - if (!__wt_page_can_evict(session, page, 0)) + if (!__wt_page_can_evict(session, page, 1)) continue; /* @@ -1226,13 +1247,27 @@ fast: /* If the page can't be evicted, give up. */ continue; WT_ASSERT(session, evict->ref == NULL); - __evict_init_candidate(session, evict, btree->evict_ref); + __evict_init_candidate(session, evict, ref); ++evict; WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, "select: %p, size %" PRIu64, page, page->memory_footprint)); } + /* + * If we happen to end up on the root page, clear it. We have to track + * hazard pointers, and the root page complicates that calculation. + * + * Also clear the walk if we land on a page requiring forced eviction. + * The eviction server may go to sleep, and we want this page evicted + * as quickly as possible. + */ + if ((ref = btree->evict_ref) != NULL && (__wt_ref_is_root(ref) || + ref->page->read_gen == WT_READGEN_OLDEST)) { + btree->evict_ref = NULL; + __wt_page_release(session, ref, 0); + } + /* If the walk was interrupted by a locked page, that's okay. */ if (ret == WT_NOTFOUND) ret = 0; diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 892d5b4ac60..37612bda7e6 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -8,15 +8,45 @@ #include "wt_internal.h" -static int __evict_exclusive(WT_SESSION_IMPL *, WT_REF *, int); static int __evict_page_dirty_update(WT_SESSION_IMPL *, WT_REF *, int); -static int __evict_review(WT_SESSION_IMPL *, WT_REF *, int, int, int *, int *); -static void __evict_discard_tree(WT_SESSION_IMPL *, WT_REF *, int, int); -static void __evict_excl_clear(WT_SESSION_IMPL *); +static int __evict_review(WT_SESSION_IMPL *, WT_REF *, int, int *); + +/* + * __evict_exclusive_clear -- + * Release exclusive access to a page. + */ +static inline void +__evict_exclusive_clear(WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_ASSERT(session, ref->state == WT_REF_LOCKED && ref->page != NULL); + + ref->state = WT_REF_MEM; +} + +/* + * __evict_exclusive -- + * Acquire exclusive access to a page. + */ +static inline int +__evict_exclusive(WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_ASSERT(session, ref->state == WT_REF_LOCKED); + + /* + * Check for a hazard pointer indicating another thread is using the + * page, meaning the page cannot be evicted. + */ + if (__wt_page_hazard_check(session, ref->page) == NULL) + return (0); + + WT_STAT_FAST_DATA_INCR(session, cache_eviction_hazard); + WT_STAT_FAST_CONN_INCR(session, cache_eviction_hazard); + return (EBUSY); +} /* * __wt_evict -- - * Eviction. + * Evict a page. */ int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) @@ -25,38 +55,25 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) WT_DECL_RET; WT_PAGE *page; WT_PAGE_MODIFY *mod; - WT_TXN_STATE *txn_state; - int forced_eviction, inmem_split, istree; + int forced_eviction, inmem_split; conn = S2C(session); page = ref->page; forced_eviction = (page->read_gen == WT_READGEN_OLDEST); - inmem_split = istree = 0; + inmem_split = 0; WT_RET(__wt_verbose(session, WT_VERB_EVICT, "page %p (%s)", page, __wt_page_type_string(page->type))); /* - * Pin the oldest transaction ID: eviction looks at page structures - * that are freed when no transaction in the system needs them. - */ - txn_state = WT_SESSION_TXN_STATE(session); - if (txn_state->snap_min == WT_TXN_NONE) - txn_state->snap_min = conn->txn_global.oldest_id; - else - txn_state = NULL; - - /* - * Get exclusive access to the page and review the page and its subtree - * for conditions that would block our eviction of the page. If the - * check fails (for example, we find a child page that can't be merged), - * we're done. We have to make this check for clean pages, too: while - * unlikely eviction would choose an internal page with children, it's - * not disallowed anywhere. + * Get exclusive access to the page and review it for conditions that + * would block our eviction of the page. If the check fails (for + * example, we find a page with active children), we're done. We have + * to make this check for clean pages, too: while unlikely eviction + * would choose an internal page with children, it's not disallowed. */ - WT_ERR( - __evict_review(session, ref, exclusive, 1, &inmem_split, &istree)); + WT_ERR(__evict_review(session, ref, exclusive, &inmem_split)); /* * If there was an in-memory split, the tree has been left in the state @@ -72,8 +89,7 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) mod = page->modify; /* Count evictions of internal pages during normal operation. */ - if (!exclusive && - (page->type == WT_PAGE_COL_INT || page->type == WT_PAGE_ROW_INT)) { + if (!exclusive && WT_PAGE_IS_INTERNAL(page)) { WT_STAT_FAST_CONN_INCR(session, cache_eviction_internal); WT_STAT_FAST_DATA_INCR(session, cache_eviction_internal); } @@ -86,15 +102,8 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) if (page->memory_footprint > conn->cache->evict_max_page_size) conn->cache->evict_max_page_size = page->memory_footprint; - /* Discard any subtree rooted in this page. */ - if (istree) - WT_WITH_PAGE_INDEX(session, - __evict_discard_tree(session, ref, exclusive, 1)); - /* Update the reference and discard the page. */ if (mod == NULL || !F_ISSET(mod, WT_PM_REC_MASK)) { - WT_ASSERT(session, exclusive || ref->state == WT_REF_LOCKED); - if (__wt_ref_is_root(ref)) __wt_ref_out(session, ref); else @@ -114,22 +123,14 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) } if (0) { -err: /* - * If unable to evict this page, release exclusive reference(s) - * we've acquired. - */ - if (!exclusive) - __evict_excl_clear(session); +err: if (!exclusive) + __evict_exclusive_clear(session, ref); WT_STAT_FAST_CONN_INCR(session, cache_eviction_fail); WT_STAT_FAST_DATA_INCR(session, cache_eviction_fail); } -done: session->excl_next = 0; - if (txn_state != NULL) - txn_state->snap_min = WT_TXN_NONE; - - if ((inmem_split || (forced_eviction && ret == EBUSY)) && +done: if ((inmem_split || (forced_eviction && ret == EBUSY)) && !F_ISSET(conn->cache, WT_CACHE_WOULD_BLOCK)) { F_SET(conn->cache, WT_CACHE_WOULD_BLOCK); WT_TRET(__wt_evict_server_wake(session)); @@ -242,69 +243,21 @@ __evict_page_dirty_update(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) } /* - * __evict_discard_tree -- - * Discard the tree rooted a page (that is, any pages merged into it), - * then the page itself. - */ -static void -__evict_discard_tree( - WT_SESSION_IMPL *session, WT_REF *ref, int exclusive, int top) -{ - WT_REF *child; - - switch (ref->page->type) { - case WT_PAGE_COL_INT: - case WT_PAGE_ROW_INT: - /* For each entry in the page... */ - WT_INTL_FOREACH_BEGIN(session, ref->page, child) { - if (child->state == WT_REF_DISK || - child->state == WT_REF_DELETED) - continue; - WT_ASSERT(session, - exclusive || child->state == WT_REF_LOCKED); - __evict_discard_tree(session, child, exclusive, 0); - } WT_INTL_FOREACH_END; - /* FALLTHROUGH */ - default: - if (!top) - __wt_ref_out(session, ref); - break; - } -} - -/* - * __evict_review_subtree -- - * Review a subtree for conditions that would block its eviction. + * __evict_child_check -- + * Review an internal page for active children. */ static int -__evict_review_subtree(WT_SESSION_IMPL *session, - WT_REF *ref, int exclusive, int *inmem_splitp, int *istreep) +__evict_child_check(WT_SESSION_IMPL *session, WT_REF *parent) { - WT_PAGE *page; WT_REF *child; - page = ref->page; - - WT_INTL_FOREACH_BEGIN(session, page, child) { + WT_INTL_FOREACH_BEGIN(session, parent->page, child) { switch (child->state) { case WT_REF_DISK: /* On-disk */ case WT_REF_DELETED: /* On-disk, deleted */ break; - case WT_REF_MEM: /* In-memory */ - /* - * Tell our caller if there's a subtree so we - * know to do a full walk when discarding the - * page. - */ - *istreep = 1; - WT_RET(__evict_review(session, child, exclusive, - 0, inmem_splitp, istreep)); - break; - case WT_REF_LOCKED: /* Being evicted */ - case WT_REF_READING: /* Being read */ - case WT_REF_SPLIT: /* Being split */ + default: return (EBUSY); - WT_ILLEGAL_VALUE(session); } } WT_INTL_FOREACH_END; @@ -317,22 +270,20 @@ __evict_review_subtree(WT_SESSION_IMPL *session, * for conditions that would block its eviction. */ static int -__evict_review(WT_SESSION_IMPL *session, WT_REF *ref, - int exclusive, int top, int *inmem_splitp, int *istreep) +__evict_review( + WT_SESSION_IMPL *session, WT_REF *ref, int exclusive, int *inmem_splitp) { WT_DECL_RET; WT_PAGE *page; WT_PAGE_MODIFY *mod; uint32_t flags; - flags = WT_EVICTING; - /* * Get exclusive access to the page if our caller doesn't have the tree * locked down. */ if (!exclusive) { - WT_RET(__evict_exclusive(session, ref, top)); + WT_RET(__evict_exclusive(session, ref)); /* * Now the page is locked, remove it from the LRU eviction @@ -349,46 +300,18 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, mod = page->modify; /* - * Recurse through the page's subtree: this happens first because we - * have to write pages in depth-first order, otherwise we'll dirty - * pages after we've written them. + * Fail if an internal has active children, the children must be evicted + * first. The test is necessary but shouldn't fire much: the eviction + * code is biased for leaf pages, an internal page shouldn't be selected + * for eviction until all children have been evicted. */ if (WT_PAGE_IS_INTERNAL(page)) { - /* - * Quit if we're trying to push out a "tree", an internal page - * with live internal pages as children, it's not likely to - * succeed. - */ - if (!top && !exclusive) - return (EBUSY); - - WT_WITH_PAGE_INDEX(session, ret = __evict_review_subtree( - session, ref, exclusive, inmem_splitp, istreep)); + WT_WITH_PAGE_INDEX(session, + ret = __evict_child_check(session, ref)); WT_RET(ret); } - /* - * Check whether the page can be evicted. - * - * If the file is being checkpointed, we can't evict dirty pages: - * if we write a page and free the previous version of the page, that - * previous version might be referenced by an internal page already - * been written in the checkpoint, leaving the checkpoint inconsistent. - * - * Don't rely on new updates being skipped by the transaction used - * for transaction reads: (1) there are paths that dirty pages for - * artificial reasons; (2) internal pages aren't transactional; and - * (3) if an update was skipped during the checkpoint (leaving the page - * dirty), then rolled back, we could still successfully overwrite a - * page and corrupt the checkpoint. - * - * Further, we can't race with the checkpoint's reconciliation of - * an internal page as we evict a clean child from the page's subtree. - * This works in the usual way: eviction locks the page and then checks - * for existing hazard pointers, the checkpoint thread reconciling an - * internal page acquires hazard pointers on child pages it reads, and - * is blocked by the exclusive lock. - */ + /* Check if the page can be evicted. */ if (!exclusive && !__wt_page_can_evict(session, page, 0)) return (EBUSY); @@ -402,51 +325,36 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, * If an in-memory split completes, the page stays in memory and the * tree is left in the desired state: avoid the usual cleanup. */ - if (top && !exclusive) { + if (!exclusive) { WT_RET(__wt_split_insert(session, ref, inmem_splitp)); if (*inmem_splitp) return (0); } /* - * Fail if any page in the top-level page's subtree won't be merged into - * its parent, the page that cannot be merged must be evicted first. - * The test is necessary but should not fire much: the eviction code is - * biased for leaf pages, an internal page shouldn't be selected for - * eviction until its children have been evicted. - * - * We have to write dirty pages to know their final state, a page marked - * empty may have had records added since reconciliation. Writing the - * page is expensive, do a cheap test first: if it doesn't seem likely a - * subtree page can be merged, quit. - */ - if (!top && (mod == NULL || !F_ISSET(mod, WT_PM_REC_EMPTY))) - return (EBUSY); - - /* - * If the page is dirty and can possibly change state, write it so we - * know the final state. + * If the page is dirty and can possibly change state, reconcile it to + * determine the final state. * * If we have an exclusive lock (we're discarding the tree), assert * there are no updates we cannot read. * - * Otherwise, if the top-level page we're evicting is a leaf page - * marked for forced eviction, set the update-restore flag, so - * reconciliation will write blocks it can write and create a list of - * skipped updates for blocks it cannot write. This is how forced - * eviction of huge pages works: we take a big page and reconcile it - * into blocks, some of which we write and discard, the rest of which - * we re-create as smaller in-memory pages, (restoring the updates that - * stopped us from writing the block), and inserting the whole mess - * into the page's parent. + * Otherwise, if the page we're evicting is a leaf page marked for + * forced eviction, set the update-restore flag, so reconciliation will + * write blocks it can write and create a list of skipped updates for + * blocks it cannot write. This is how forced eviction of active, huge + * pages works: we take a big page and reconcile it into blocks, some of + * which we write and discard, the rest of which we re-create as smaller + * in-memory pages, (restoring the updates that stopped us from writing + * the block), and inserting the whole mess into the page's parent. * - * Don't set the update-restore flag for internal pages, they don't - * have updates that can be saved and restored. + * Don't set the update-restore flag for internal pages, they don't have + * updates that can be saved and restored. */ + flags = WT_EVICTING; if (__wt_page_is_modified(page)) { if (exclusive) LF_SET(WT_SKIP_UPDATE_ERR); - else if (top && !WT_PAGE_IS_INTERNAL(page) && + else if (!WT_PAGE_IS_INTERNAL(page) && page->read_gen == WT_READGEN_OLDEST) LF_SET(WT_SKIP_UPDATE_RESTORE); WT_RET(__wt_reconcile(session, ref, NULL, flags)); @@ -464,68 +372,5 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, !LF_ISSET(WT_SKIP_UPDATE_RESTORE)) return (EBUSY); - /* - * Repeat the test: fail if any page in the top-level page's subtree - * won't be merged into its parent. - */ - if (!top && (mod == NULL || !F_ISSET(mod, WT_PM_REC_EMPTY))) - return (EBUSY); - return (0); } - -/* - * __evict_excl_clear -- - * Discard exclusive access and return a page's subtree to availability. - */ -static void -__evict_excl_clear(WT_SESSION_IMPL *session) -{ - WT_REF *ref; - uint32_t i; - - for (i = 0; i < session->excl_next; ++i) { - if ((ref = session->excl[i]) == NULL) - break; - WT_ASSERT(session, - ref->state == WT_REF_LOCKED && ref->page != NULL); - ref->state = WT_REF_MEM; - } -} - -/* - * __evict_exclusive -- - * Request exclusive access to a page. - */ -static int -__evict_exclusive(WT_SESSION_IMPL *session, WT_REF *ref, int top) -{ - /* - * Make sure there is space to track exclusive access so we can unlock - * to clean up. - */ - WT_RET(__wt_realloc_def(session, &session->excl_allocated, - session->excl_next + 1, &session->excl)); - - /* - * Request exclusive access to the page. The top-level page should - * already be in the locked state, lock child pages in memory. - * If another thread already has this page, give up. - */ - if (!top && !WT_ATOMIC_CAS4(ref->state, WT_REF_MEM, WT_REF_LOCKED)) - return (EBUSY); /* We couldn't change the state. */ - WT_ASSERT(session, ref->state == WT_REF_LOCKED); - - session->excl[session->excl_next++] = ref; - - /* Check for a matching hazard pointer. */ - if (__wt_page_hazard_check(session, ref->page) == NULL) - return (0); - - WT_STAT_FAST_DATA_INCR(session, cache_eviction_hazard); - WT_STAT_FAST_CONN_INCR(session, cache_eviction_hazard); - - WT_RET(__wt_verbose(session, WT_VERB_EVICT, - "page %p hazard request failed", ref->page)); - return (EBUSY); -} diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h index 65c83afc696..4ef1b9da4ec 100644 --- a/src/third_party/wiredtiger/src/include/block.h +++ b/src/third_party/wiredtiger/src/include/block.h @@ -218,7 +218,7 @@ struct __wt_block { SLIST_ENTRY(__wt_block) hashl; /* Hashed list of handles */ /* Configuration information, set when the file is opened. */ - int allocfirst; /* Allocation is first-fit */ + uint32_t allocfirst; /* Allocation is first-fit */ uint32_t allocsize; /* Allocation size */ size_t os_cache; /* System buffer cache flush max */ size_t os_cache_max; @@ -239,7 +239,6 @@ struct __wt_block { int ckpt_inprogress;/* Live checkpoint in progress */ /* Compaction support */ - int allocfirst_save; /* Saved: allocation is first-fit */ int compact_pct_tenths; /* Percent to compact */ /* Salvage support */ diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 101fd450fc7..cda672bc7b4 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -847,13 +847,20 @@ WT_PACKED_STRUCT_BEGIN(__wt_update) */ #define WT_UPDATE_DELETED_ISSET(upd) ((upd)->size == UINT32_MAX) #define WT_UPDATE_DELETED_SET(upd) ((upd)->size = UINT32_MAX) -#define WT_UPDATE_MEMSIZE(upd) \ - (sizeof(WT_UPDATE) + (WT_UPDATE_DELETED_ISSET(upd) ? 0 : (upd)->size)) uint32_t size; /* update length */ /* The untyped value immediately follows the WT_UPDATE structure. */ #define WT_UPDATE_DATA(upd) \ ((void *)((uint8_t *)(upd) + sizeof(WT_UPDATE))) + + /* + * The memory size of an update: include some padding because this is + * such a common case that overhead of tiny allocations can swamp our + * cache overhead calculation. + */ +#define WT_UPDATE_MEMSIZE(upd) \ + WT_ALIGN(sizeof(WT_UPDATE) + \ + (WT_UPDATE_DELETED_ISSET(upd) ? 0 : (upd)->size), 32) }; /* diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index dd3acf6940d..cc571124207 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -123,6 +123,8 @@ struct __wt_btree { WT_BM *bm; /* Block manager reference */ u_int block_header; /* WT_PAGE_HEADER_BYTE_SIZE */ + uint64_t checkpoint_gen; /* Checkpoint generation */ + uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ uint64_t write_gen; /* Write generation */ WT_REF *evict_ref; /* Eviction thread's location */ diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 56fb66abaef..7d9a3095a0c 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -55,6 +55,11 @@ __wt_cache_page_inmem_incr(WT_SESSION_IMPL *session, WT_PAGE *page, size_t size) (void)WT_ATOMIC_ADD8(cache->bytes_dirty, size); (void)WT_ATOMIC_ADD8(page->modify->bytes_dirty, size); } + /* Track internal and overflow size in cache. */ + if (WT_PAGE_IS_INTERNAL(page)) + (void)WT_ATOMIC_ADD8(cache->bytes_internal, size); + else if (page->type == WT_PAGE_OVFL) + (void)WT_ATOMIC_ADD8(cache->bytes_overflow, size); } /* @@ -148,6 +153,11 @@ __wt_cache_page_inmem_decr(WT_SESSION_IMPL *session, WT_PAGE *page, size_t size) WT_CACHE_DECR(session, page->memory_footprint, size); if (__wt_page_is_modified(page)) __wt_cache_page_byte_dirty_decr(session, page, size); + /* Track internal and overflow size in cache. */ + if (WT_PAGE_IS_INTERNAL(page)) + WT_CACHE_DECR(session, cache->bytes_internal, size); + else if (page->type == WT_PAGE_OVFL) + WT_CACHE_DECR(session, cache->bytes_overflow, size); } /* @@ -970,7 +980,7 @@ __wt_page_can_evict(WT_SESSION_IMPL *session, WT_PAGE *page, int check_splits) * a transaction value, once that's globally visible, we know we can * evict the created page. */ - if (WT_PAGE_IS_INTERNAL(page) && + if (check_splits && WT_PAGE_IS_INTERNAL(page) && !__wt_txn_visible_all(session, mod->mod_split_txn)) return (0); @@ -991,6 +1001,20 @@ __wt_page_can_evict(WT_SESSION_IMPL *session, WT_PAGE *page, int check_splits) /* * If we aren't (potentially) doing eviction that can restore updates * and the updates on this page are too recent, give up. + * + * Don't rely on new updates being skipped by the transaction used + * for transaction reads: (1) there are paths that dirty pages for + * artificial reasons; (2) internal pages aren't transactional; and + * (3) if an update was skipped during the checkpoint (leaving the page + * dirty), then rolled back, we could still successfully overwrite a + * page and corrupt the checkpoint. + * + * Further, we can't race with the checkpoint's reconciliation of + * an internal page as we evict a clean child from the page's subtree. + * This works in the usual way: eviction locks the page and then checks + * for existing hazard pointers, the checkpoint thread reconciling an + * internal page acquires hazard pointers on child pages it reads, and + * is blocked by the exclusive lock. */ if (page->read_gen != WT_READGEN_OLDEST && !__wt_txn_visible_all(session, __wt_page_is_modified(page) ? @@ -999,7 +1023,7 @@ __wt_page_can_evict(WT_SESSION_IMPL *session, WT_PAGE *page, int check_splits) /* * If the page was recently split in-memory, don't force it out: we - * hope eviction will find it first. + * hope an eviction thread will find it first. */ if (check_splits && !__wt_txn_visible_all(session, mod->inmem_split_txn)) @@ -1115,6 +1139,13 @@ __wt_page_swap_func(WT_SESSION_IMPL *session, WT_REF *held, int acquired; /* + * In rare cases when walking the tree, we try to swap to the same + * page. Fast-path that to avoid thinking about error handling. + */ + if (held == want) + return (0); + + /* * This function is here to simplify the error handling during hazard * pointer coupling so we never leave a hazard pointer dangling. The * assumption is we're holding a hazard pointer on "held", and want to diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 8ed3176492f..11f631416af 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -57,6 +57,8 @@ struct __wt_cache { */ uint64_t bytes_inmem; /* Bytes/pages in memory */ uint64_t pages_inmem; + uint64_t bytes_internal; /* Bytes of internal pages */ + uint64_t bytes_overflow; /* Bytes of overflow pages */ uint64_t bytes_evict; /* Bytes/pages discarded by eviction */ uint64_t pages_evict; uint64_t bytes_dirty; /* Bytes/pages currently dirty */ diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 046f515188c..1f30667b8d6 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -19,6 +19,7 @@ struct __wt_config { struct __wt_config_check { const char *name; const char *type; + int (*checkf)(WT_SESSION_IMPL *, WT_CONFIG_ITEM *); const char *checks; const WT_CONFIG_CHECK *subconfigs; }; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 78b2949ab98..533b9ea8bbe 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -242,7 +242,7 @@ struct __wt_connection_impl { #define WT_CKPT_LOGSIZE(conn) ((conn)->ckpt_logsize != 0) wt_off_t ckpt_logsize; /* Checkpoint log size period */ uint32_t ckpt_signalled;/* Checkpoint signalled */ - long ckpt_usecs; /* Checkpoint period */ + uint64_t ckpt_usecs; /* Checkpoint period */ int compact_in_memory_pass; /* Compaction serialization */ @@ -304,7 +304,7 @@ struct __wt_connection_impl { char *stat_path; /* Statistics log path format */ char **stat_sources; /* Statistics log list of objects */ const char *stat_stamp; /* Statistics log entry timestamp */ - long stat_usecs; /* Statistics log period */ + uint64_t stat_usecs; /* Statistics log period */ #define WT_CONN_LOG_ARCHIVE 0x01 /* Archive is enabled */ #define WT_CONN_LOG_ENABLED 0x02 /* Logging is enabled */ @@ -335,6 +335,8 @@ struct __wt_connection_impl { wt_thread_t sweep_tid; /* Handle sweep thread */ int sweep_tid_set; /* Handle sweep thread set */ WT_CONDVAR *sweep_cond; /* Handle sweep wait mutex */ + time_t sweep_idle_time;/* Handle sweep idle time */ + time_t sweep_interval;/* Handle sweep interval */ /* Locked: collator list */ TAILQ_HEAD(__wt_coll_qh, __wt_named_collator) collqh; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index ba53bd23c78..d23deee8c98 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -289,6 +289,7 @@ struct __wt_cursor_metadata { struct __wt_cursor_stat { WT_CURSOR iface; + int notinitialized; /* Cursor not initialized */ int notpositioned; /* Cursor not positioned */ WT_STATS *stats; /* Stats owned by the cursor */ @@ -301,6 +302,8 @@ struct __wt_cursor_stat { WT_CONNECTION_STATS conn_stats; } u; + const char **cfg; /* Original cursor configuration */ + int key; /* Current stats key */ uint64_t v; /* Current stats value */ WT_ITEM pv; /* Current stats value (string) */ diff --git a/src/third_party/wiredtiger/src/include/cursor.i b/src/third_party/wiredtiger/src/include/cursor.i index f6a5fc8d8a9..dd38a5746c1 100644 --- a/src/third_party/wiredtiger/src/include/cursor.i +++ b/src/third_party/wiredtiger/src/include/cursor.i @@ -177,22 +177,14 @@ static inline int __cursor_func_init(WT_CURSOR_BTREE *cbt, int reenter) { WT_SESSION_IMPL *session; - WT_TXN *txn; session = (WT_SESSION_IMPL *)cbt->iface.session; - txn = &session->txn; if (reenter) WT_RET(__curfile_leave(cbt)); - /* - * If there is no transaction active in this thread and we haven't - * checked if the cache is full, do it now. If we have to block for - * eviction, this is the best time to do it. - */ - if (F_ISSET(txn, TXN_RUNNING) && - !F_ISSET(txn, TXN_HAS_ID) && !F_ISSET(txn, TXN_HAS_SNAPSHOT)) - WT_RET(__wt_cache_full_check(session)); + /* If the transaction is idle, check that the cache isn't full. */ + WT_RET(__wt_txn_idle_cache_check(session)); if (!F_ISSET(cbt, WT_CBT_ACTIVE)) WT_RET(__curfile_enter(cbt)); diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index efc1617fcd3..fcb96b16361 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -28,28 +28,17 @@ __wt_err(session, ret, __VA_ARGS__); \ goto err; \ } while (0) -#define WT_ERR_BUSY_OK(a) do { \ - if ((ret = (a)) != 0) { \ - if (ret == EBUSY) \ - ret = 0; \ - else \ - goto err; \ - } \ -} while (0) -#define WT_ERR_NOTFOUND_OK(a) do { \ - if ((ret = (a)) != 0) { \ - if (ret == WT_NOTFOUND) \ - ret = 0; \ - else \ - goto err; \ - } \ -} while (0) #define WT_ERR_TEST(a, v) do { \ if (a) { \ ret = (v); \ goto err; \ - } \ + } else \ + ret = 0; \ } while (0) +#define WT_ERR_BUSY_OK(a) \ + WT_ERR_TEST((ret = (a)) != 0 && ret != EBUSY, ret) +#define WT_ERR_NOTFOUND_OK(a) \ + WT_ERR_TEST((ret = (a)) != 0 && ret != WT_NOTFOUND, ret) /* Return tests. */ #define WT_RET(a) do { \ @@ -57,24 +46,22 @@ if ((__ret = (a)) != 0) \ return (__ret); \ } while (0) -#define WT_RET_TEST(a, v) do { \ - if (a) \ - return (v); \ -} while (0) #define WT_RET_MSG(session, v, ...) do { \ int __ret = (v); \ __wt_err(session, __ret, __VA_ARGS__); \ return (__ret); \ } while (0) +#define WT_RET_TEST(a, v) do { \ + if (a) \ + return (v); \ +} while (0) #define WT_RET_BUSY_OK(a) do { \ - int __ret; \ - if ((__ret = (a)) != 0 && __ret != EBUSY) \ - return (__ret); \ + int __ret = (a); \ + WT_RET_TEST(__ret != 0 && __ret != EBUSY, __ret); \ } while (0) #define WT_RET_NOTFOUND_OK(a) do { \ - int __ret; \ - if ((__ret = (a)) != 0 && __ret != WT_NOTFOUND) \ - return (__ret); \ + int __ret = (a); \ + WT_RET_TEST(__ret != 0 && __ret != WT_NOTFOUND, __ret); \ } while (0) /* Set "ret" if not already set. */ #define WT_TRET(a) do { \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index bddbb5e01eb..86c8f501eb2 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -8,7 +8,7 @@ extern int __wt_async_flush(WT_SESSION_IMPL *session); extern int __wt_async_new_op(WT_SESSION_IMPL *session, const char *uri, const char *config, const char *cfg[], WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP_IMPL **opp); extern int __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op); extern int __wt_async_op_init(WT_SESSION_IMPL *session); -extern void *__wt_async_worker(void *arg); +extern WT_THREAD_RET __wt_async_worker(void *arg); extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, wt_off_t offset, uint32_t size, uint32_t cksum); extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, wt_off_t *offsetp, uint32_t *sizep, uint32_t *cksump); extern int __wt_block_addr_valid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, int live); @@ -45,6 +45,7 @@ extern int __wt_block_unmap( WT_SESSION_IMPL *session, WT_BLOCK *block, void *ma extern int __wt_block_manager_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[], int forced_salvage, int readonly, uint32_t allocsize, WT_BM **bmp); extern int __wt_block_manager_truncate( WT_SESSION_IMPL *session, const char *filename, uint32_t allocsize); extern int __wt_block_manager_create( WT_SESSION_IMPL *session, const char *filename, uint32_t allocsize); +extern void __wt_block_configure_first_fit(WT_BLOCK *block, int on); extern int __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[], int forced_salvage, int readonly, uint32_t allocsize, WT_BLOCK **blockp); extern int __wt_block_close(WT_SESSION_IMPL *session, WT_BLOCK *block); extern int __wt_desc_init(WT_SESSION_IMPL *session, WT_FH *fh, uint32_t allocsize); @@ -123,6 +124,7 @@ extern void __wt_root_ref_init(WT_REF *root_ref, WT_PAGE *root, int is_recno); extern int __wt_btree_tree_open( WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size); extern int __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep); extern void __wt_btree_evictable(WT_SESSION_IMPL *session, int on); +extern int __wt_huffman_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *v); extern int __wt_btree_huffman_open(WT_SESSION_IMPL *session); extern void __wt_btree_huffman_close(WT_SESSION_IMPL *session); extern int __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size); @@ -141,7 +143,7 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags #endif ); extern int __wt_page_alloc(WT_SESSION_IMPL *session, uint8_t type, uint64_t recno, uint32_t alloc_entries, int alloc_refs, WT_PAGE **pagep); -extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, uint32_t flags, WT_PAGE **pagep); +extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, size_t memsize, uint32_t flags, WT_PAGE **pagep); extern int __wt_cache_read(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd); extern int __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]); @@ -200,10 +202,14 @@ extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_ extern int __wt_ext_config_get(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, const char *key, WT_CONFIG_ITEM *cval); extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf); extern const char *__wt_wiredtiger_error(int error); +extern int __wt_collator_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cname); extern int __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_CONFIG_ITEM *cname, WT_CONFIG_ITEM *metadata, WT_COLLATOR **collatorp, int *ownp); extern int __wt_conn_remove_collator(WT_SESSION_IMPL *session); +extern int __wt_compressor_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval); +extern int __wt_compressor_config( WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_COMPRESSOR **compressorp); extern int __wt_conn_remove_compressor(WT_SESSION_IMPL *session); extern int __wt_conn_remove_data_source(WT_SESSION_IMPL *session); +extern int __wt_extractor_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cname); extern int __wt_extractor_config(WT_SESSION_IMPL *session, const char *config, WT_EXTRACTOR **extractorp, int *ownp); extern int __wt_conn_remove_extractor(WT_SESSION_IMPL *session); extern int __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]); @@ -214,12 +220,12 @@ extern int __wt_cache_destroy(WT_SESSION_IMPL *session); extern int __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg); extern int __wt_conn_cache_pool_open(WT_SESSION_IMPL *session); extern int __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session); -extern void *__wt_cache_pool_server(void *arg); +extern WT_THREAD_RET __wt_cache_pool_server(void *arg); extern int __wt_checkpoint_server_create(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session); extern int __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize); extern int __wt_conn_dhandle_find(WT_SESSION_IMPL *session, const char *name, const char *ckpt, uint32_t flags); -extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, int force); +extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, int final, int force); extern int __wt_conn_btree_get(WT_SESSION_IMPL *session, const char *name, const char *ckpt, const char *cfg[], uint32_t flags); extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, int apply_checkpoints, const char *uri, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); extern int __wt_conn_btree_apply_single_ckpt(WT_SESSION_IMPL *session, const char *uri, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); @@ -240,9 +246,11 @@ extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); extern int __wt_statlog_log_one(WT_SESSION_IMPL *session); extern int __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_statlog_destroy(WT_SESSION_IMPL *session, int is_close); +extern int __wt_sweep_config(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_sweep_create(WT_SESSION_IMPL *session); extern int __wt_sweep_destroy(WT_SESSION_IMPL *session); extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); +extern int __wt_backup_file_remove(WT_SESSION_IMPL *session); extern int __wt_backup_list_uri_append( WT_SESSION_IMPL *session, const char *name, int *skip); extern int __wt_curbulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, int bitmap, int skip_sort_check); extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); @@ -417,7 +425,7 @@ extern int __wt_metadata_remove(WT_SESSION_IMPL *session, const char *key); extern int __wt_metadata_search( WT_SESSION_IMPL *session, const char *key, char **valuep); extern void __wt_meta_track_discard(WT_SESSION_IMPL *session); extern int __wt_meta_track_on(WT_SESSION_IMPL *session); -extern int __wt_meta_track_off(WT_SESSION_IMPL *session, int unroll); +extern int __wt_meta_track_off(WT_SESSION_IMPL *session, int need_sync, int unroll); extern int __wt_meta_track_sub_on(WT_SESSION_IMPL *session); extern int __wt_meta_track_sub_off(WT_SESSION_IMPL *session); extern int __wt_meta_track_checkpoint(WT_SESSION_IMPL *session); @@ -460,7 +468,7 @@ extern int __wt_mmap_preload(WT_SESSION_IMPL *session, const void *p, size_t siz extern int __wt_mmap_discard(WT_SESSION_IMPL *session, void *p, size_t size); extern int __wt_munmap(WT_SESSION_IMPL *session, WT_FH *fh, void *map, size_t len, void **mappingcookie); extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, int is_signalled, WT_CONDVAR **condp); -extern int __wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, long usecs); +extern int __wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs); extern int __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); extern int __wt_rwlock_alloc( WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp, const char *name); @@ -473,7 +481,7 @@ extern int __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); extern int __wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp); extern int __wt_once(void (*init_routine)(void)); extern int __wt_open(WT_SESSION_IMPL *session, const char *name, int ok_create, int exclusive, int dio_type, WT_FH **fhp); -extern int __wt_close(WT_SESSION_IMPL *session, WT_FH *fh); +extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp); extern int __wt_absolute_path(const char *path); extern const char *__wt_path_separator(void); extern int __wt_has_priv(void); @@ -481,9 +489,14 @@ extern int __wt_remove(WT_SESSION_IMPL *session, const char *name); extern int __wt_rename(WT_SESSION_IMPL *session, const char *from, const char *to); extern int __wt_read( WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, size_t len, void *buf); extern int __wt_write(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, size_t len, const void *buf); -extern void __wt_sleep(long seconds, long micro_seconds); +extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); +extern int __wt_fopen(WT_SESSION_IMPL *session, const char *name, WT_FHANDLE_MODE mode_flag, u_int flags, FILE **fpp); +extern int __wt_vfprintf(WT_SESSION_IMPL *session, FILE *fp, const char *fmt, va_list ap); +extern int __wt_fprintf(WT_SESSION_IMPL *session, FILE *fp, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); +extern int __wt_fflush(WT_SESSION_IMPL *session, FILE *fp); +extern int __wt_fclose(WT_SESSION_IMPL *session, FILE **fpp, WT_FHANDLE_MODE mode_flag); extern uint64_t __wt_strtouq(const char *nptr, char **endptr, int base); -extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, void *(*func)(void *), void *arg); +extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg); extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid); extern void __wt_thread_id(char *buf, size_t buflen); extern int __wt_seconds(WT_SESSION_IMPL *session, time_t *timep); @@ -493,6 +506,7 @@ extern int __wt_ext_struct_pack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session extern int __wt_ext_struct_size(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, size_t *sizep, const char *fmt, ...); extern int __wt_ext_struct_unpack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_check(WT_SESSION_IMPL *session, const char *fmt, size_t len, int *fixedp, uint32_t *fixed_lenp); +extern int __wt_struct_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *v); extern int __wt_struct_size(WT_SESSION_IMPL *session, size_t *sizep, const char *fmt, ...); extern int __wt_struct_pack(WT_SESSION_IMPL *session, void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, ...); @@ -585,6 +599,9 @@ extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri); extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri); extern int __wt_filename(WT_SESSION_IMPL *session, const char *name, char **path); extern int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path); +extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name); +extern int __wt_sync_and_rename_fh( WT_SESSION_IMPL *session, WT_FH **fhp, const char *from, const char *to); +extern int __wt_sync_and_rename_fp( WT_SESSION_IMPL *session, FILE **fpp, const char *from, const char *to); extern int __wt_library_init(void); extern int __wt_breakpoint(void); extern void __wt_attach(WT_SESSION_IMPL *session); @@ -637,7 +654,7 @@ extern void __wt_stat_refresh_dsrc_stats(void *stats_arg); extern void __wt_stat_aggregate_dsrc_stats(const void *child, const void *parent); extern void __wt_stat_init_connection_stats(WT_CONNECTION_STATS *stats); extern void __wt_stat_refresh_connection_stats(void *stats_arg); -extern int __wt_txnid_cmp(const void *v1, const void *v2); +extern int WT_CDECL __wt_txnid_cmp(const void *v1, const void *v2); extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); extern void __wt_txn_update_oldest(WT_SESSION_IMPL *session); extern void __wt_txn_refresh(WT_SESSION_IMPL *session, int get_snapshot); @@ -655,7 +672,7 @@ extern int __wt_checkpoint_list(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]); -extern int __wt_checkpoint_close(WT_SESSION_IMPL *session, int force); +extern int __wt_checkpoint_close(WT_SESSION_IMPL *session, int final, int force); extern uint64_t __wt_ext_transaction_id(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session); extern int __wt_ext_transaction_isolation_level( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session); extern int __wt_ext_transaction_notify( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_TXN_NOTIFY *notify); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h index 30b2ab1c0e3..99c77c94f49 100644 --- a/src/third_party/wiredtiger/src/include/flags.h +++ b/src/third_party/wiredtiger/src/include/flags.h @@ -65,6 +65,7 @@ #define WT_TXN_LOG_CKPT_PREPARE 0x00000002 #define WT_TXN_LOG_CKPT_START 0x00000004 #define WT_TXN_LOG_CKPT_STOP 0x00000008 +#define WT_TXN_LOG_CKPT_SYNC 0x00000010 #define WT_VERB_API 0x00000001 #define WT_VERB_BLOCK 0x00000002 #define WT_VERB_CHECKPOINT 0x00000004 diff --git a/src/third_party/wiredtiger/src/include/gcc.h b/src/third_party/wiredtiger/src/include/gcc.h index 805838eb84b..2efbb20b39a 100644 --- a/src/third_party/wiredtiger/src/include/gcc.h +++ b/src/third_party/wiredtiger/src/include/gcc.h @@ -6,6 +6,8 @@ * See the file LICENSE for redistribution information. */ +#define WT_SIZET_FMT "zu" /* size_t format string */ + /* Add GCC-specific attributes to types and function declarations. */ #define WT_COMPILER_TYPE_ALIGN(x) __attribute__((aligned(x))) diff --git a/src/third_party/wiredtiger/src/include/lint.h b/src/third_party/wiredtiger/src/include/lint.h index 5668abc6dab..631f00cb5cd 100644 --- a/src/third_party/wiredtiger/src/include/lint.h +++ b/src/third_party/wiredtiger/src/include/lint.h @@ -6,6 +6,8 @@ * See the file LICENSE for redistribution information. */ +#define WT_SIZET_FMT "zu" /* size_t format string */ + #define WT_COMPILER_TYPE_ALIGN(x) #define WT_PACKED_STRUCT_BEGIN(name) \ diff --git a/src/third_party/wiredtiger/src/include/lsm.h b/src/third_party/wiredtiger/src/include/lsm.h index 7dd8c9f97af..aa1d797e3b5 100644 --- a/src/third_party/wiredtiger/src/include/lsm.h +++ b/src/third_party/wiredtiger/src/include/lsm.h @@ -182,8 +182,8 @@ struct __wt_lsm_tree { uint64_t dsk_gen; - long ckpt_throttle; /* Rate limiting due to checkpoints */ - long merge_throttle; /* Rate limiting due to merges */ + uint64_t ckpt_throttle; /* Rate limiting due to checkpoints */ + uint64_t merge_throttle; /* Rate limiting due to merges */ uint64_t chunk_fill_ms; /* Estimate of time to fill a chunk */ struct timespec last_flush_ts; /* Timestamp last flush finished */ struct timespec work_push_ts; /* Timestamp last work unit added */ diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index d40b2d836bf..66547262417 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -9,18 +9,20 @@ #define WT_WIREDTIGER "WiredTiger" /* Version file */ #define WT_SINGLETHREAD "WiredTiger.lock" /* Locking file */ -#define WT_BASECONFIG "WiredTiger.basecfg" /* Configuration */ -#define WT_USERCONFIG "WiredTiger.config" /* Configuration */ +#define WT_BASECONFIG "WiredTiger.basecfg" /* Base configuration */ +#define WT_BASECONFIG_SET "WiredTiger.basecfg.set"/* Base config temp */ + +#define WT_USERCONFIG "WiredTiger.config" /* User configuration */ -#define WT_INCREMENTAL_BACKUP \ - "WiredTiger.ibackup" /* Incremental backup file */ #define WT_METADATA_BACKUP "WiredTiger.backup" /* Hot backup file */ +#define WT_INCREMENTAL_BACKUP "WiredTiger.ibackup" /* Incremental backup */ #define WT_METADATA_TURTLE "WiredTiger.turtle" /* Metadata metadata */ #define WT_METADATA_TURTLE_SET "WiredTiger.turtle.set" /* Turtle temp file */ #define WT_METADATA_URI "metadata:" /* Metadata alias */ #define WT_METAFILE_URI "file:WiredTiger.wt" /* Metadata file URI */ + /* * Pre computed hash for the metadata file. Used to optimize comparisons * against the metafile URI. The validity is checked on connection open diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 71d6e802fa8..12cf2dec375 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -24,14 +24,6 @@ #define WT_EXABYTE ((uint64_t)1152921504606846976) /* - * Number of directory entries can grow dynamically. - */ -#define WT_DIR_ENTRY 32 - -#define WT_DIRLIST_EXCLUDE 0x1 /* Exclude files matching prefix */ -#define WT_DIRLIST_INCLUDE 0x2 /* Include files matching prefix */ - -/* * Sizes that cannot be larger than 2**32 are stored in uint32_t fields in * common structures to save space. To minimize conversions from size_t to * uint32_t through the code, we use the following macros. @@ -182,10 +174,6 @@ #define WT_STRING_MATCH(str, bytes, len) \ (((const char *)str)[0] == ((const char *)bytes)[0] && \ strncmp(str, bytes, len) == 0 && (str)[(len)] == '\0') -#define WT_STRING_CASE_MATCH(str, bytes, len) \ - (tolower(((const char *)str)[0]) == \ - tolower(((const char *)bytes)[0]) && \ - strncasecmp(str, bytes, len) == 0 && (str)[(len)] == '\0') /* * Macro that produces a string literal that isn't wrapped in quotes, to avoid diff --git a/src/third_party/wiredtiger/src/include/msvc.h b/src/third_party/wiredtiger/src/include/msvc.h index 3ec74b2d629..fa5b2d848e8 100644 --- a/src/third_party/wiredtiger/src/include/msvc.h +++ b/src/third_party/wiredtiger/src/include/msvc.h @@ -13,6 +13,8 @@ #define inline __inline +#define WT_SIZET_FMT "Iu" /* size_t format string */ + /* * Add MSVC-specific attributes and pragmas to types and function declarations. */ diff --git a/src/third_party/wiredtiger/src/include/os.h b/src/third_party/wiredtiger/src/include/os.h index acbc77409c3..ba5d95657d5 100644 --- a/src/third_party/wiredtiger/src/include/os.h +++ b/src/third_party/wiredtiger/src/include/os.h @@ -6,6 +6,37 @@ * See the file LICENSE for redistribution information. */ +/* + * FILE handle close/open configuration. + */ +typedef enum { + WT_FHANDLE_APPEND, WT_FHANDLE_READ, WT_FHANDLE_WRITE +} WT_FHANDLE_MODE; + +#ifdef _WIN32 +/* + * Open in binary (untranslated) mode; translations involving carriage-return + * and linefeed characters are suppressed. + */ +#define WT_FOPEN_APPEND "ab" +#define WT_FOPEN_READ "rb" +#define WT_FOPEN_WRITE "wb" +#else +#define WT_FOPEN_APPEND "a" +#define WT_FOPEN_READ "r" +#define WT_FOPEN_WRITE "w" +#endif + +#define WT_FOPEN_FIXED 0x1 /* Path isn't relative to home */ + +/* + * Number of directory entries can grow dynamically. + */ +#define WT_DIR_ENTRY 32 + +#define WT_DIRLIST_EXCLUDE 0x1 /* Exclude files matching prefix */ +#define WT_DIRLIST_INCLUDE 0x2 /* Include files matching prefix */ + #define WT_SYSCALL_RETRY(call, ret) do { \ int __retry; \ for (__retry = 0; __retry < 10; ++__retry) { \ @@ -72,9 +103,3 @@ struct __wt_fh { WT_FALLOCATE_SYS } fallocate_available; int fallocate_requires_locking; }; - -#ifndef _WIN32 -#define WT_SIZET_FMT "zu" /* size_t format string */ -#else -#define WT_SIZET_FMT "Iu" /* size_t format string */ -#endif diff --git a/src/third_party/wiredtiger/src/include/os_windows.h b/src/third_party/wiredtiger/src/include/os_windows.h index 49dbc694895..de97143335f 100644 --- a/src/third_party/wiredtiger/src/include/os_windows.h +++ b/src/third_party/wiredtiger/src/include/os_windows.h @@ -14,13 +14,25 @@ typedef CONDITION_VARIABLE wt_cond_t; typedef CRITICAL_SECTION wt_mutex_t; typedef HANDLE wt_thread_t; +/* + * Thread callbacks need to match the return signature of _beginthreadex. + */ +#define WT_THREAD_CALLBACK(x) unsigned (__stdcall x) +#define WT_THREAD_RET unsigned __stdcall +#define WT_THREAD_RET_VALUE 0 + +/* + * WT declaration for calling convention type + */ +#define WT_CDECL __cdecl + +#if _MSC_VER < 1900 /* Timespec is a POSIX structure not defined in Windows */ struct timespec { time_t tv_sec; /* seconds */ long tv_nsec; /* nanoseconds */ }; - -#define strncasecmp _strnicmp +#endif /* * Windows Portability stuff @@ -29,7 +41,7 @@ struct timespec { */ typedef uint32_t u_int; typedef unsigned char u_char; -typedef unsigned long u_long; +typedef uint64_t u_long; /* <= VS 2013 is not C99 compat */ #if _MSC_VER < 1900 @@ -63,3 +75,6 @@ _Check_return_opt_ int __cdecl _wt_vsnprintf( /* Provide a custom version of localtime_r */ struct tm *localtime_r(const time_t* timer, struct tm* result); + +/* Windows does not provide fsync */ +#define fsync _commit diff --git a/src/third_party/wiredtiger/src/include/packing.i b/src/third_party/wiredtiger/src/include/packing.i index 9caa58ed2e1..b97b3a322ce 100644 --- a/src/third_party/wiredtiger/src/include/packing.i +++ b/src/third_party/wiredtiger/src/include/packing.i @@ -376,6 +376,11 @@ __pack_write( pad = pv->size - s; if (pv->type == 'U') { oldp = *pp; + /* + * Check that there is at least one byte available: the + * low-level routines treat zero length as unchecked. + */ + WT_SIZE_CHECK(1, maxlen); WT_RET(__wt_vpack_uint(pp, maxlen, s + pad)); maxlen -= (size_t)(*pp - oldp); } @@ -404,6 +409,11 @@ __pack_write( case 'i': case 'l': case 'q': + /* + * Check that there is at least one byte available: the + * low-level routines treat zero length as unchecked. + */ + WT_SIZE_CHECK(1, maxlen); WT_RET(__wt_vpack_int(pp, maxlen, pv->u.i)); break; case 'H': @@ -411,6 +421,11 @@ __pack_write( case 'L': case 'Q': case 'r': + /* + * Check that there is at least one byte available: the + * low-level routines treat zero length as unchecked. + */ + WT_SIZE_CHECK(1, maxlen); WT_RET(__wt_vpack_uint(pp, maxlen, pv->u.u)); break; case 'R': @@ -453,6 +468,11 @@ __unpack_read(WT_SESSION_IMPL *session, *pp += s; break; case 'U': + /* + * Check that there is at least one byte available: the + * low-level routines treat zero length as unchecked. + */ + WT_SIZE_CHECK(1, maxlen); WT_RET(__wt_vunpack_uint(pp, maxlen, &pv->u.u)); /* FALLTHROUGH */ case 'u': @@ -481,6 +501,11 @@ __unpack_read(WT_SESSION_IMPL *session, case 'i': case 'l': case 'q': + /* + * Check that there is at least one byte available: the + * low-level routines treat zero length as unchecked. + */ + WT_SIZE_CHECK(1, maxlen); WT_RET(__wt_vunpack_int(pp, maxlen, &pv->u.i)); break; case 'H': @@ -488,6 +513,11 @@ __unpack_read(WT_SESSION_IMPL *session, case 'L': case 'Q': case 'r': + /* + * Check that there is at least one byte available: the + * low-level routines treat zero length as unchecked. + */ + WT_SIZE_CHECK(1, maxlen); WT_RET(__wt_vunpack_uint(pp, maxlen, &pv->u.u)); break; case 'R': diff --git a/src/third_party/wiredtiger/src/include/posix.h b/src/third_party/wiredtiger/src/include/posix.h index 14249e3ed37..1aa629c98e7 100644 --- a/src/third_party/wiredtiger/src/include/posix.h +++ b/src/third_party/wiredtiger/src/include/posix.h @@ -26,3 +26,15 @@ typedef pthread_cond_t wt_cond_t; typedef pthread_mutex_t wt_mutex_t; typedef pthread_t wt_thread_t; + +/* + * Thread callbacks need to match the platform specific callback types + */ +#define WT_THREAD_CALLBACK(x) void* (x) +#define WT_THREAD_RET void* +#define WT_THREAD_RET_VALUE NULL + +/* + * WT declaration for calling convention type + */ +#define WT_CDECL diff --git a/src/third_party/wiredtiger/src/include/schema.h b/src/third_party/wiredtiger/src/include/schema.h index 4d722a0068b..25c1baae60f 100644 --- a/src/third_party/wiredtiger/src/include/schema.h +++ b/src/third_party/wiredtiger/src/include/schema.h @@ -28,6 +28,9 @@ struct __wt_index { WT_CONFIG_ITEM colconf; /* List of columns from config */ + WT_COLLATOR *collator; /* Custom collator */ + int collator_owned; /* Collator is owned by this index */ + WT_EXTRACTOR *extractor; /* Custom key extractor */ int extractor_owned; /* Extractor is owned by this index */ diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 36df35a104e..2c88727c662 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -68,8 +68,6 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { */ /* Session handle reference list */ SLIST_HEAD(__dhandles, __wt_data_handle_cache) dhandles; -#define WT_DHANDLE_SWEEP_WAIT 30 /* Idle wait before discarding */ -#define WT_DHANDLE_SWEEP_PERIOD 10 /* Sweep interim */ time_t last_sweep; /* Last sweep for dead handles */ WT_CURSOR *cursor; /* Current cursor */ @@ -117,10 +115,6 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { WT_TXN txn; /* Transaction state */ u_int ncursors; /* Count of active file cursors. */ - WT_REF **excl; /* Eviction exclusive list */ - u_int excl_next; /* Next empty slot */ - size_t excl_allocated; /* Bytes allocated */ - void *block_manager; /* Block-manager support */ int (*block_manager_cleanup)(WT_SESSION_IMPL *); diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 21eaff0677f..e04e645b3ea 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -153,8 +153,11 @@ struct __wt_connection_stats { WT_STATS block_read; WT_STATS block_write; WT_STATS cache_bytes_dirty; + WT_STATS cache_bytes_internal; WT_STATS cache_bytes_inuse; + WT_STATS cache_bytes_leaf; WT_STATS cache_bytes_max; + WT_STATS cache_bytes_overflow; WT_STATS cache_bytes_read; WT_STATS cache_bytes_write; WT_STATS cache_eviction_app; @@ -260,6 +263,7 @@ struct __wt_connection_stats { WT_STATS session_open; WT_STATS txn_begin; WT_STATS txn_checkpoint; + WT_STATS txn_checkpoint_generation; WT_STATS txn_checkpoint_running; WT_STATS txn_checkpoint_time_max; WT_STATS txn_checkpoint_time_min; @@ -267,6 +271,7 @@ struct __wt_connection_stats { WT_STATS txn_checkpoint_time_total; WT_STATS txn_commit; WT_STATS txn_fail_cache; + WT_STATS txn_pinned_checkpoint_range; WT_STATS txn_pinned_range; WT_STATS txn_rollback; WT_STATS write_io; @@ -294,6 +299,7 @@ struct __wt_dsrc_stats { WT_STATS bloom_page_evict; WT_STATS bloom_page_read; WT_STATS bloom_size; + WT_STATS btree_checkpoint_generation; WT_STATS btree_column_deleted; WT_STATS btree_column_fix; WT_STATS btree_column_internal; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index c1c4703316b..927ab09d5f9 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -48,6 +48,17 @@ struct __wt_txn_global { /* Count of scanning threads, or -1 for exclusive access. */ volatile int32_t scan_count; + /* + * Track information about the running checkpoint. The transaction IDs + * used when checkpointing are special. Checkpoints can run for a long + * time so we keep them out of regular visibility checks. Eviction and + * checkpoint operations know when they need to be aware of + * checkpoint IDs. + */ + volatile uint64_t checkpoint_gen; + volatile uint64_t checkpoint_id; + volatile uint64_t checkpoint_snap_min; + WT_TXN_STATE *states; /* Per-session transaction states */ }; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index f5f9b662e6f..3a3bdde2b73 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -91,14 +91,40 @@ __wt_txn_modify_ref(WT_SESSION_IMPL *session, WT_REF *ref) /* * __wt_txn_visible_all -- * Check if a given transaction ID is "globally visible". This is, if - * all sessions in the system will see the transaction ID. + * all sessions in the system will see the transaction ID including the + * ID that belongs to a running checkpoint. */ static inline int __wt_txn_visible_all(WT_SESSION_IMPL *session, uint64_t id) { - uint64_t oldest_id; + WT_BTREE *btree; + WT_TXN_GLOBAL *txn_global; + uint64_t checkpoint_snap_min, oldest_id; + + txn_global = &S2C(session)->txn_global; + btree = S2BT_SAFE(session); + + /* + * Take a local copy of ID in case they are updated while we are + * checking visibility. + */ + checkpoint_snap_min = txn_global->checkpoint_snap_min; + oldest_id = txn_global->oldest_id; + + /* + * If there is no active checkpoint or this handle is up to date with + * the active checkpoint it's safe to ignore the checkpoint ID in the + * visibility check. + */ + if (checkpoint_snap_min != WT_TXN_NONE && (btree == NULL || + btree->checkpoint_gen != txn_global->checkpoint_gen) && + TXNID_LT(checkpoint_snap_min, oldest_id)) + /* + * Use the checkpoint ID for the visibility check if it is the + * oldest ID in the system. + */ + oldest_id = checkpoint_snap_min; - oldest_id = S2C(session)->txn_global.oldest_id; return (TXNID_LT(id, oldest_id)); } @@ -179,7 +205,7 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_UPDATE *upd) /* * __wt_txn_autocommit_check -- - * If an auto-commit transaction is required, start one. + * If an auto-commit transaction is required, start one. */ static inline int __wt_txn_autocommit_check(WT_SESSION_IMPL *session) @@ -212,9 +238,35 @@ __wt_txn_new_id(WT_SESSION_IMPL *session) } /* + * __wt_txn_idle_cache_check -- + * If there is no transaction active in this thread and we haven't checked + * if the cache is full, do it now. If we have to block for eviction, + * this is the best time to do it. + */ +static inline int +__wt_txn_idle_cache_check(WT_SESSION_IMPL *session) +{ + WT_TXN *txn; + WT_TXN_STATE *txn_state; + + txn = &session->txn; + txn_state = &S2C(session)->txn_global.states[session->id]; + + /* + * Check the published snap_min because read-uncommitted never sets + * TXN_HAS_SNAPSHOT. + */ + if (F_ISSET(txn, TXN_RUNNING) && + !F_ISSET(txn, TXN_HAS_ID) && txn_state->snap_min == WT_TXN_NONE) + WT_RET(__wt_cache_full_check(session)); + + return (0); +} + +/* * __wt_txn_id_check -- * A transaction is going to do an update, start an auto commit - * transaction if required and allocate a transaction ID. + * transaction if required and allocate a transaction ID. */ static inline int __wt_txn_id_check(WT_SESSION_IMPL *session) @@ -228,14 +280,8 @@ __wt_txn_id_check(WT_SESSION_IMPL *session) WT_ASSERT(session, F_ISSET(txn, TXN_RUNNING)); - /* - * If there is no transaction active in this thread and we haven't - * checked if the cache is full, do it now. If we have to block for - * eviction, this is the best time to do it. - */ - if (F_ISSET(txn, TXN_RUNNING) && - !F_ISSET(txn, TXN_HAS_ID) && !F_ISSET(txn, TXN_HAS_SNAPSHOT)) - WT_RET(__wt_cache_full_check(session)); + /* If the transaction is idle, check that the cache isn't full. */ + WT_RET(__wt_txn_idle_cache_check(session)); if (!F_ISSET(txn, TXN_HAS_ID)) { conn = S2C(session); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index fed6042c67a..bfd2641785f 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -322,10 +322,13 @@ struct __wt_cursor { int __F(prev)(WT_CURSOR *cursor); /*! - * Reset the position of the cursor. Any resources held by the cursor - * are released, and the cursor's key and position are no longer valid. - * A subsequent iteration with WT_CURSOR::next will move to the first - * record, or with WT_CURSOR::prev will move to the last record. + * Reset the cursor. Any resources held by the cursor are released, + * and the cursor's key and position are no longer valid. Subsequent + * iterations with WT_CURSOR::next will move to the first record, or + * with WT_CURSOR::prev will move to the last record. + * + * In the case of a statistics cursor, resetting the cursor refreshes + * the statistics information returned. * * @snippet ex_all.c Reset the cursor * @@ -990,9 +993,9 @@ struct __wt_session { * @config{block_compressor, configure a compressor for file blocks. * Permitted values are \c "none" or custom compression engine name * created with WT_CONNECTION::add_compressor. If WiredTiger has - * builtin support for \c "snappy" or \c "zlib" compression\, these - * names are also available. See @ref compression for more - * information., a string; default \c none.} + * builtin support for \c "bzip2"\, \c "snappy"\, \c "lz4" or \c "zlib" + * compression\, these names are also available. See @ref compression + * for more information., a string; default \c none.} * @config{cache_resident, do not ever evict the object's pages; see * @ref tuning_cache_resident for more information., a boolean flag; * default \c false.} @@ -1549,11 +1552,11 @@ struct __wt_connection { * @config{ enabled, enable asynchronous * operation., a boolean flag; default \c false.} * @config{ ops_max, maximum number of expected - * simultaneous asynchronous operations., an integer between 10 and - * 4096; default \c 1024.} - * @config{ threads, the - * number of worker threads to service asynchronous requests., an - * integer between 1 and 20; default \c 2.} + * simultaneous asynchronous operations., an integer between 1 and 4096; + * default \c 1024.} + * @config{ threads, the number + * of worker threads to service asynchronous requests., an integer + * between 1 and 20; default \c 2.} * @config{ ),,} * @config{cache_overhead, assume the heap allocator overhead is the * specified percentage\, and adjust the cache usage by that amount (for @@ -1604,6 +1607,15 @@ struct __wt_connection { * @config{eviction_trigger, trigger eviction when the cache is using * this much memory\, as a percentage of the total cache size., an * integer between 10 and 99; default \c 95.} + * @config{file_manager = (, control how file handles are managed., a + * set of related configuration options defined below.} + * @config{ close_idle_time, amount of time in + * seconds a file handle needs to be idle before attempting to close + * it., an integer between 1 and 1000; default \c 30.} + * @config{ close_scan_interval, interval in + * seconds at which to check for files that are inactive and close + * them., an integer between 1 and 1000; default \c 10.} + * @config{ ),,} * @config{lsm_manager = (, configure database wide options for LSM tree * management., a set of related configuration options defined below.} * @config{ merge, merge LSM chunks where @@ -1883,14 +1895,14 @@ struct __wt_connection { * boolean flag; default \c false.} * @config{ ops_max, * maximum number of expected simultaneous asynchronous operations., an integer - * between 10 and 4096; default \c 1024.} + * between 1 and 4096; default \c 1024.} * @config{ threads, the number of worker threads to * service asynchronous requests., an integer between 1 and 20; default \c 2.} * @config{ ),,} * @config{buffer_alignment, in-memory alignment (in bytes) for buffers used for * I/O. The default value of -1 indicates a platform-specific alignment value - * should be used (4KB on Linux systems\, zero elsewhere)., an integer between - * -1 and 1MB; default \c -1.} + * should be used (4KB on Linux systems when direct I/O is configured\, zero + * elsewhere)., an integer between -1 and 1MB; default \c -1.} * @config{cache_overhead, assume the heap allocator overhead is the specified * percentage\, and adjust the cache usage by that amount (for example\, if * there is 10GB of data in cache\, a percentage of 10 means WiredTiger treats @@ -1966,6 +1978,15 @@ struct __wt_connection { * each new block is written. For example\, * <code>file_extend=(data=16MB)</code>., a list\, with values chosen from the * following options: \c "data"\, \c "log"; default empty.} + * @config{file_manager = (, control how file handles are managed., a set of + * related configuration options defined below.} + * @config{ close_idle_time, amount of time in seconds a + * file handle needs to be idle before attempting to close it., an integer + * between 1 and 1000; default \c 30.} + * @config{ close_scan_interval, interval in seconds at + * which to check for files that are inactive and close them., an integer + * between 1 and 1000; default \c 10.} + * @config{ ),,} * @config{hazard_max, maximum number of simultaneous hazard pointers per * session handle., an integer greater than or equal to 15; default \c 1000.} * @config{log = (, enable logging., a set of related configuration options @@ -1973,10 +1994,11 @@ struct __wt_connection { * @config{ archive, automatically * archive unneeded log files., a boolean flag; default \c true.} * @config{ compressor, configure a compressor for log - * records. Permitted values are \c "none" or \c "bzip2"\, \c "snappy" or - * custom compression engine \c "name" created with - * WT_CONNECTION::add_compressor. See @ref compression for more information., a - * string; default \c none.} + * records. Permitted values are \c "none" or custom compression engine name + * created with WT_CONNECTION::add_compressor. If WiredTiger has builtin + * support for \c "bzip2"\, \c "snappy"\, \c "lz4" or \c "zlib" compression\, + * these names are also available. See @ref compression for more information., + * a string; default \c none.} * @config{ enabled, enable * logging subsystem., a boolean flag; default \c false.} * @config{ file_max, the maximum size of log files., an @@ -2461,10 +2483,10 @@ struct __wt_config_item { * This API is outside the scope of a WiredTiger connection handle, since * applications may need to generate configuration strings prior to calling * ::wiredtiger_open. - * @param session the session handle to be used for error reporting. If NULL - * error messages will be written to stdout. + * @param session the session handle to be used for error reporting (if NULL, + * error messages will be written to stderr). * @param config the configuration string being parsed. The string must - * remain valid for the lifetime of the parser handle. + * remain valid for the lifetime of the parser handle. * @param len the number of valid bytes in \c config * @param[out] config_parserp A pointer to the newly opened handle * @errors @@ -3211,240 +3233,250 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_BLOCK_WRITE 1019 /*! cache: tracked dirty bytes in the cache */ #define WT_STAT_CONN_CACHE_BYTES_DIRTY 1020 +/*! cache: tracked bytes belonging to internal pages in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1021 /*! cache: bytes currently in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INUSE 1021 +#define WT_STAT_CONN_CACHE_BYTES_INUSE 1022 +/*! cache: tracked bytes belonging to leaf pages in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1023 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1022 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1024 +/*! cache: tracked bytes belonging to overflow pages in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_OVERFLOW 1025 /*! cache: bytes read into cache */ -#define WT_STAT_CONN_CACHE_BYTES_READ 1023 +#define WT_STAT_CONN_CACHE_BYTES_READ 1026 /*! cache: bytes written from cache */ -#define WT_STAT_CONN_CACHE_BYTES_WRITE 1024 +#define WT_STAT_CONN_CACHE_BYTES_WRITE 1027 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1025 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1028 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1026 +#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1029 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1027 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1030 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1028 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1031 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1029 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1032 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1030 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1033 /*! cache: pages evicted because they exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1031 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1034 /*! cache: pages evicted because they had chains of deleted items */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1032 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1035 /*! cache: failed eviction of pages that exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1033 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1036 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1034 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1037 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1035 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1038 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1036 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1039 /*! cache: eviction server candidate queue empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1037 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1040 /*! cache: eviction server candidate queue not empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1038 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1041 /*! cache: eviction server evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1039 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1042 /*! cache: eviction server populating queue, but not evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING 1040 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING 1043 /*! cache: eviction server unable to reach eviction goal */ -#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1041 +#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1044 /*! cache: pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT 1042 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT 1045 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1043 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1046 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1044 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1047 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1045 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1048 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1046 +#define WT_STAT_CONN_CACHE_OVERHEAD 1049 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1047 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1050 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1048 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1051 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1049 +#define WT_STAT_CONN_CACHE_READ 1052 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1050 +#define WT_STAT_CONN_CACHE_WRITE 1053 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1051 +#define WT_STAT_CONN_COND_WAIT 1054 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1052 +#define WT_STAT_CONN_CURSOR_CREATE 1055 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1053 +#define WT_STAT_CONN_CURSOR_INSERT 1056 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1054 +#define WT_STAT_CONN_CURSOR_NEXT 1057 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1055 +#define WT_STAT_CONN_CURSOR_PREV 1058 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1056 +#define WT_STAT_CONN_CURSOR_REMOVE 1059 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1057 +#define WT_STAT_CONN_CURSOR_RESET 1060 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1058 +#define WT_STAT_CONN_CURSOR_SEARCH 1061 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1059 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1062 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1060 +#define WT_STAT_CONN_CURSOR_UPDATE 1063 /*! data-handle: connection dhandles swept */ -#define WT_STAT_CONN_DH_CONN_HANDLES 1061 +#define WT_STAT_CONN_DH_CONN_HANDLES 1064 /*! data-handle: connection candidate referenced */ -#define WT_STAT_CONN_DH_CONN_REF 1062 +#define WT_STAT_CONN_DH_CONN_REF 1065 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_CONN_SWEEPS 1063 +#define WT_STAT_CONN_DH_CONN_SWEEPS 1066 /*! data-handle: connection time-of-death sets */ -#define WT_STAT_CONN_DH_CONN_TOD 1064 +#define WT_STAT_CONN_DH_CONN_TOD 1067 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1065 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1068 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1066 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1069 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1067 +#define WT_STAT_CONN_FILE_OPEN 1070 /*! log: log buffer size increases */ -#define WT_STAT_CONN_LOG_BUFFER_GROW 1068 +#define WT_STAT_CONN_LOG_BUFFER_GROW 1071 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1069 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1072 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1070 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1073 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1071 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1074 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1072 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1075 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1073 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1076 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1074 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1077 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1075 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1078 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1076 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1079 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1077 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1080 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1078 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1081 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1079 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1082 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1080 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1083 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1081 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1084 /*! log: log read operations */ -#define WT_STAT_CONN_LOG_READS 1082 +#define WT_STAT_CONN_LOG_READS 1085 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1083 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1086 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1084 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1087 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1085 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1088 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1086 +#define WT_STAT_CONN_LOG_SCANS 1089 /*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1087 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1090 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1088 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1091 /*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1089 +#define WT_STAT_CONN_LOG_SLOT_JOINS 1092 /*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1090 +#define WT_STAT_CONN_LOG_SLOT_RACES 1093 /*! log: slots selected for switching that were unavailable */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_FAILS 1091 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_FAILS 1094 /*! log: record size exceeded maximum */ -#define WT_STAT_CONN_LOG_SLOT_TOOBIG 1092 +#define WT_STAT_CONN_LOG_SLOT_TOOBIG 1095 /*! log: failed to find a slot large enough for record */ -#define WT_STAT_CONN_LOG_SLOT_TOOSMALL 1093 +#define WT_STAT_CONN_LOG_SLOT_TOOSMALL 1096 /*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1094 +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1097 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1095 +#define WT_STAT_CONN_LOG_SYNC 1098 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1096 +#define WT_STAT_CONN_LOG_SYNC_DIR 1099 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1097 +#define WT_STAT_CONN_LOG_WRITE_LSN 1100 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1098 +#define WT_STAT_CONN_LOG_WRITES 1101 /*! LSM: sleep for LSM checkpoint throttle */ -#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1099 +#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1102 /*! LSM: sleep for LSM merge throttle */ -#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1100 +#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1103 /*! LSM: rows merged in an LSM tree */ -#define WT_STAT_CONN_LSM_ROWS_MERGED 1101 +#define WT_STAT_CONN_LSM_ROWS_MERGED 1104 /*! LSM: application work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_APP 1102 +#define WT_STAT_CONN_LSM_WORK_QUEUE_APP 1105 /*! LSM: merge work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_MANAGER 1103 +#define WT_STAT_CONN_LSM_WORK_QUEUE_MANAGER 1106 /*! LSM: tree queue hit maximum */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1104 +#define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1107 /*! LSM: switch work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_SWITCH 1105 +#define WT_STAT_CONN_LSM_WORK_QUEUE_SWITCH 1108 /*! LSM: tree maintenance operations scheduled */ -#define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1106 +#define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1109 /*! LSM: tree maintenance operations discarded */ -#define WT_STAT_CONN_LSM_WORK_UNITS_DISCARDED 1107 +#define WT_STAT_CONN_LSM_WORK_UNITS_DISCARDED 1110 /*! LSM: tree maintenance operations executed */ -#define WT_STAT_CONN_LSM_WORK_UNITS_DONE 1108 +#define WT_STAT_CONN_LSM_WORK_UNITS_DONE 1111 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1109 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1112 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1110 +#define WT_STAT_CONN_MEMORY_FREE 1113 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1111 +#define WT_STAT_CONN_MEMORY_GROW 1114 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1112 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1115 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1113 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1116 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1114 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1117 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1115 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1118 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1116 +#define WT_STAT_CONN_PAGE_SLEEP 1119 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1117 +#define WT_STAT_CONN_READ_IO 1120 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1118 +#define WT_STAT_CONN_REC_PAGES 1121 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1119 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1122 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1120 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1123 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1121 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1124 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1122 +#define WT_STAT_CONN_RWLOCK_READ 1125 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1123 +#define WT_STAT_CONN_RWLOCK_WRITE 1126 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1124 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1127 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1125 +#define WT_STAT_CONN_SESSION_OPEN 1128 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1126 +#define WT_STAT_CONN_TXN_BEGIN 1129 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1127 +#define WT_STAT_CONN_TXN_CHECKPOINT 1130 +/*! transaction: transaction checkpoint generation */ +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1131 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1128 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1132 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1129 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1133 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1130 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1134 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1131 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1135 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1132 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1136 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1133 +#define WT_STAT_CONN_TXN_COMMIT 1137 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1134 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1138 +/*! transaction: transaction range of IDs currently pinned by a checkpoint */ +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1139 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1135 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1140 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1136 +#define WT_STAT_CONN_TXN_ROLLBACK 1141 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1137 +#define WT_STAT_CONN_WRITE_IO 1142 /*! * @} @@ -3486,152 +3518,154 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_BLOOM_PAGE_READ 2015 /*! LSM: total size of bloom filters */ #define WT_STAT_DSRC_BLOOM_SIZE 2016 +/*! btree: btree checkpoint generation */ +#define WT_STAT_DSRC_BTREE_CHECKPOINT_GENERATION 2017 /*! btree: column-store variable-size deleted values */ -#define WT_STAT_DSRC_BTREE_COLUMN_DELETED 2017 +#define WT_STAT_DSRC_BTREE_COLUMN_DELETED 2018 /*! btree: column-store fixed-size leaf pages */ -#define WT_STAT_DSRC_BTREE_COLUMN_FIX 2018 +#define WT_STAT_DSRC_BTREE_COLUMN_FIX 2019 /*! btree: column-store internal pages */ -#define WT_STAT_DSRC_BTREE_COLUMN_INTERNAL 2019 +#define WT_STAT_DSRC_BTREE_COLUMN_INTERNAL 2020 /*! btree: column-store variable-size leaf pages */ -#define WT_STAT_DSRC_BTREE_COLUMN_VARIABLE 2020 +#define WT_STAT_DSRC_BTREE_COLUMN_VARIABLE 2021 /*! btree: pages rewritten by compaction */ -#define WT_STAT_DSRC_BTREE_COMPACT_REWRITE 2021 +#define WT_STAT_DSRC_BTREE_COMPACT_REWRITE 2022 /*! btree: number of key/value pairs */ -#define WT_STAT_DSRC_BTREE_ENTRIES 2022 +#define WT_STAT_DSRC_BTREE_ENTRIES 2023 /*! btree: fixed-record size */ -#define WT_STAT_DSRC_BTREE_FIXED_LEN 2023 +#define WT_STAT_DSRC_BTREE_FIXED_LEN 2024 /*! btree: maximum tree depth */ -#define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2024 +#define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2025 /*! btree: maximum internal page key size */ -#define WT_STAT_DSRC_BTREE_MAXINTLKEY 2025 +#define WT_STAT_DSRC_BTREE_MAXINTLKEY 2026 /*! btree: maximum internal page size */ -#define WT_STAT_DSRC_BTREE_MAXINTLPAGE 2026 +#define WT_STAT_DSRC_BTREE_MAXINTLPAGE 2027 /*! btree: maximum leaf page key size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFKEY 2027 +#define WT_STAT_DSRC_BTREE_MAXLEAFKEY 2028 /*! btree: maximum leaf page size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFPAGE 2028 +#define WT_STAT_DSRC_BTREE_MAXLEAFPAGE 2029 /*! btree: maximum leaf page value size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFVALUE 2029 +#define WT_STAT_DSRC_BTREE_MAXLEAFVALUE 2030 /*! btree: overflow pages */ -#define WT_STAT_DSRC_BTREE_OVERFLOW 2030 +#define WT_STAT_DSRC_BTREE_OVERFLOW 2031 /*! btree: row-store internal pages */ -#define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2031 +#define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2032 /*! btree: row-store leaf pages */ -#define WT_STAT_DSRC_BTREE_ROW_LEAF 2032 +#define WT_STAT_DSRC_BTREE_ROW_LEAF 2033 /*! cache: bytes read into cache */ -#define WT_STAT_DSRC_CACHE_BYTES_READ 2033 +#define WT_STAT_DSRC_CACHE_BYTES_READ 2034 /*! cache: bytes written from cache */ -#define WT_STAT_DSRC_CACHE_BYTES_WRITE 2034 +#define WT_STAT_DSRC_CACHE_BYTES_WRITE 2035 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2035 +#define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2036 /*! cache: unmodified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2036 +#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2037 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2037 +#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2038 /*! cache: modified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2038 +#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2039 /*! cache: data source pages selected for eviction unable to be evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2039 +#define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2040 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2040 +#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2041 /*! cache: internal pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2041 +#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2042 /*! cache: pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT 2042 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT 2043 /*! cache: in-memory page splits */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2043 +#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2044 /*! cache: overflow values cached in memory */ -#define WT_STAT_DSRC_CACHE_OVERFLOW_VALUE 2044 +#define WT_STAT_DSRC_CACHE_OVERFLOW_VALUE 2045 /*! cache: pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ 2045 +#define WT_STAT_DSRC_CACHE_READ 2046 /*! cache: overflow pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2046 +#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2047 /*! cache: pages written from cache */ -#define WT_STAT_DSRC_CACHE_WRITE 2047 +#define WT_STAT_DSRC_CACHE_WRITE 2048 /*! compression: raw compression call failed, no additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2048 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2049 /*! compression: raw compression call failed, additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2049 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2050 /*! compression: raw compression call succeeded */ -#define WT_STAT_DSRC_COMPRESS_RAW_OK 2050 +#define WT_STAT_DSRC_COMPRESS_RAW_OK 2051 /*! compression: compressed pages read */ -#define WT_STAT_DSRC_COMPRESS_READ 2051 +#define WT_STAT_DSRC_COMPRESS_READ 2052 /*! compression: compressed pages written */ -#define WT_STAT_DSRC_COMPRESS_WRITE 2052 +#define WT_STAT_DSRC_COMPRESS_WRITE 2053 /*! compression: page written failed to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2053 +#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2054 /*! compression: page written was too small to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2054 +#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2055 /*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2055 +#define WT_STAT_DSRC_CURSOR_CREATE 2056 /*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2056 +#define WT_STAT_DSRC_CURSOR_INSERT 2057 /*! cursor: bulk-loaded cursor-insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2057 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2058 /*! cursor: cursor-insert key and value bytes inserted */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2058 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2059 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2059 +#define WT_STAT_DSRC_CURSOR_NEXT 2060 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2060 +#define WT_STAT_DSRC_CURSOR_PREV 2061 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2061 +#define WT_STAT_DSRC_CURSOR_REMOVE 2062 /*! cursor: cursor-remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2062 +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2063 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2063 +#define WT_STAT_DSRC_CURSOR_RESET 2064 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2064 +#define WT_STAT_DSRC_CURSOR_SEARCH 2065 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2065 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2066 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2066 +#define WT_STAT_DSRC_CURSOR_UPDATE 2067 /*! cursor: cursor-update value bytes updated */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2067 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2068 /*! LSM: sleep for LSM checkpoint throttle */ -#define WT_STAT_DSRC_LSM_CHECKPOINT_THROTTLE 2068 +#define WT_STAT_DSRC_LSM_CHECKPOINT_THROTTLE 2069 /*! LSM: chunks in the LSM tree */ -#define WT_STAT_DSRC_LSM_CHUNK_COUNT 2069 +#define WT_STAT_DSRC_LSM_CHUNK_COUNT 2070 /*! LSM: highest merge generation in the LSM tree */ -#define WT_STAT_DSRC_LSM_GENERATION_MAX 2070 +#define WT_STAT_DSRC_LSM_GENERATION_MAX 2071 /*! LSM: queries that could have benefited from a Bloom filter that did * not exist */ -#define WT_STAT_DSRC_LSM_LOOKUP_NO_BLOOM 2071 +#define WT_STAT_DSRC_LSM_LOOKUP_NO_BLOOM 2072 /*! LSM: sleep for LSM merge throttle */ -#define WT_STAT_DSRC_LSM_MERGE_THROTTLE 2072 +#define WT_STAT_DSRC_LSM_MERGE_THROTTLE 2073 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2073 +#define WT_STAT_DSRC_REC_DICTIONARY 2074 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2074 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2075 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2075 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2076 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2076 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2077 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2077 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2078 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2078 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2079 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2079 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2080 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2080 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2081 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2081 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2082 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2082 +#define WT_STAT_DSRC_REC_PAGES 2083 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2083 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2084 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2084 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2085 /*! reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2085 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2086 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2086 +#define WT_STAT_DSRC_SESSION_COMPACT 2087 /*! session: open cursor count */ -#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2087 +#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2088 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2088 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2089 /*! @} */ /* * Statistics section: END diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index f485f0a09e5..28483624b83 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -221,7 +221,7 @@ __wt_log_extract_lognum( if (id == NULL || name == NULL) return (WT_ERROR); if ((p = strrchr(name, '.')) == NULL || - sscanf(++p, "%" PRIu32, id) != 1) + sscanf(++p, "%" SCNu32, id) != 1) WT_RET_MSG(session, WT_ERROR, "Bad log file name '%s'", name); return (0); } @@ -529,8 +529,7 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) WT_ERR(__wt_scr_alloc(session, 0, &from_path)); WT_ERR(__wt_scr_alloc(session, 0, &to_path)); - WT_ERR(__log_filename(session, - from_num, WT_LOG_PREPNAME, from_path)); + WT_ERR(__log_filename(session, from_num, WT_LOG_PREPNAME, from_path)); WT_ERR(__log_filename(session, to_num, WT_LOG_FILENAME, to_path)); WT_ERR(__wt_verbose(session, WT_VERB_LOG, "log_alloc_prealloc: rename log %s to %s", @@ -585,7 +584,7 @@ __log_truncate(WT_SESSION_IMPL *session, tmp_fh = log_fh; log_fh = NULL; WT_ERR(__wt_fsync(session, tmp_fh)); - WT_ERR(__wt_close(session, tmp_fh)); + WT_ERR(__wt_close(session, &tmp_fh)); /* * If we just want to truncate the current log, return and skip @@ -609,11 +608,10 @@ __log_truncate(WT_SESSION_IMPL *session, tmp_fh = log_fh; log_fh = NULL; WT_ERR(__wt_fsync(session, tmp_fh)); - WT_ERR(__wt_close(session, tmp_fh)); + WT_ERR(__wt_close(session, &tmp_fh)); } } -err: if (log_fh != NULL) - WT_TRET(__wt_close(session, log_fh)); +err: WT_TRET(__wt_close(session, &log_fh)); if (logfiles != NULL) __wt_log_files_free(session, logfiles, logcount); return (ret); @@ -660,7 +658,7 @@ __wt_log_allocfile( tmp_fh = log_fh; log_fh = NULL; WT_ERR(__wt_fsync(session, tmp_fh)); - WT_ERR(__wt_close(session, tmp_fh)); + WT_ERR(__wt_close(session, &tmp_fh)); WT_ERR(__wt_verbose(session, WT_VERB_LOG, "log_prealloc: rename %s to %s", (char *)from_path->data, (char *)to_path->data)); @@ -671,8 +669,7 @@ __wt_log_allocfile( err: __wt_scr_free(session, &from_path); __wt_scr_free(session, &to_path); - if (log_fh != NULL) - WT_TRET(__wt_close(session, log_fh)); + WT_TRET(__wt_close(session, &log_fh)); return (ret); } @@ -805,20 +802,20 @@ __wt_log_close(WT_SESSION_IMPL *session) WT_RET(__wt_verbose(session, WT_VERB_LOG, "closing old log %s", log->log_close_fh->name)); WT_RET(__wt_fsync(session, log->log_close_fh)); - WT_RET(__wt_close(session, log->log_close_fh)); + WT_RET(__wt_close(session, &log->log_close_fh)); } if (log->log_fh != NULL) { WT_RET(__wt_verbose(session, WT_VERB_LOG, "closing log %s", log->log_fh->name)); WT_RET(__wt_fsync(session, log->log_fh)); - WT_RET(__wt_close(session, log->log_fh)); + WT_RET(__wt_close(session, &log->log_fh)); log->log_fh = NULL; } if (log->log_dir_fh != NULL) { WT_RET(__wt_verbose(session, WT_VERB_LOG, "closing log directory %s", log->log_dir_fh->name)); WT_RET(__wt_directory_sync_fh(session, log->log_dir_fh)); - WT_RET(__wt_close(session, log->log_dir_fh)); + WT_RET(__wt_close(session, &log->log_dir_fh)); log->log_dir_fh = NULL; } return (0); @@ -1230,7 +1227,7 @@ __log_read_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, record->size = logrec->len; WT_STAT_FAST_CONN_INCR(session, log_reads); err: - WT_TRET(__wt_close(session, log_fh)); + WT_TRET(__wt_close(session, &log_fh)); return (ret); } @@ -1353,7 +1350,7 @@ advance: /* * If we read the last record, go to the next file. */ - WT_ERR(__wt_close(session, log_fh)); + WT_ERR(__wt_close(session, &log_fh)); log_fh = NULL; eol = 1; /* @@ -1382,7 +1379,7 @@ advance: WT_ERR(__wt_read(session, log_fh, rd_lsn.offset, (size_t)allocsize, buf.mem)); /* - * First 8 bytes is the real record length. See if we + * First 4 bytes is the real record length. See if we * need to read more than the allocation size. We expect * that we rarely will have to read more. Most log records * will be fairly small. @@ -1397,6 +1394,7 @@ advance: */ if (reclen == 0) { /* This LSN is the end. */ + eol = 1; break; } rdup_len = __wt_rduppo2(reclen, allocsize); @@ -1480,8 +1478,7 @@ err: WT_STAT_FAST_CONN_INCR(session, log_scans); ret = WT_NOTFOUND; if (ret == ENOENT) ret = 0; - if (log_fh != NULL) - WT_TRET(__wt_close(session, log_fh)); + WT_TRET(__wt_close(session, &log_fh)); return (ret); } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index 8474b6e8b37..a18269baa28 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1288,13 +1288,13 @@ __clsm_put(WT_SESSION_IMPL *session, lsm_tree->merge_throttle + lsm_tree->ckpt_throttle > 0) { clsm->update_count = 0; WT_STAT_FAST_INCRV(session, &clsm->lsm_tree->stats, - lsm_checkpoint_throttle, (uint64_t)lsm_tree->ckpt_throttle); + lsm_checkpoint_throttle, lsm_tree->ckpt_throttle); WT_STAT_FAST_CONN_INCRV(session, - lsm_checkpoint_throttle, (uint64_t)lsm_tree->ckpt_throttle); + lsm_checkpoint_throttle, lsm_tree->ckpt_throttle); WT_STAT_FAST_INCRV(session, &clsm->lsm_tree->stats, - lsm_merge_throttle, (uint64_t)lsm_tree->merge_throttle); + lsm_merge_throttle, lsm_tree->merge_throttle); WT_STAT_FAST_CONN_INCRV(session, - lsm_merge_throttle, (uint64_t)lsm_tree->merge_throttle); + lsm_merge_throttle, lsm_tree->merge_throttle); __wt_sleep(0, lsm_tree->ckpt_throttle + lsm_tree->merge_throttle); } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_manager.c b/src/third_party/wiredtiger/src/lsm/lsm_manager.c index 75d3e8ef6e8..12b24984fcb 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_manager.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_manager.c @@ -11,7 +11,7 @@ static int __lsm_manager_aggressive_update(WT_SESSION_IMPL *, WT_LSM_TREE *); static int __lsm_manager_run_server(WT_SESSION_IMPL *); -static void * __lsm_worker_manager(void *); +static WT_THREAD_RET __lsm_worker_manager(void *); /* * __wt_lsm_manager_config -- @@ -500,7 +500,7 @@ err: if (dhandle_locked) { * A thread that manages all open LSM trees, and the shared LSM worker * threads. */ -static void * +static WT_THREAD_RET __lsm_worker_manager(void *arg) { WT_DECL_RET; @@ -518,7 +518,7 @@ __lsm_worker_manager(void *arg) err: WT_PANIC_MSG(session, ret, "LSM worker manager thread error"); } F_CLR(S2C(session), WT_CONN_SERVER_LSM); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/src/third_party/wiredtiger/src/lsm/lsm_meta.c b/src/third_party/wiredtiger/src/lsm/lsm_meta.c index 2fcced4d1c4..e4c95125428 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_meta.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_meta.c @@ -41,7 +41,7 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) cv.str, cv.len, &lsm_tree->value_format)); } else if (WT_STRING_MATCH("collator", ck.str, ck.len)) { if (cv.len == 0 || - WT_STRING_CASE_MATCH("none", cv.str, cv.len)) + WT_STRING_MATCH("none", cv.str, cv.len)) continue; /* * Extract the application-supplied metadata (if any) diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 1141710aa63..2bded10cb96 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -371,6 +371,8 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, cval.str++; cval.len -= 2; } + WT_ERR(__wt_config_check(session, + WT_CONFIG_REF(session, session_create), cval.str, cval.len)); WT_ERR(__wt_strndup( session, cval.str, cval.len, &lsm_tree->bloom_config)); @@ -728,7 +730,7 @@ __wt_lsm_tree_throttle( timediff = WT_TIMEDIFF(last_chunk->create_ts, ondisk->create_ts); lsm_tree->ckpt_throttle = - (long)((in_memory - 2) * timediff / (20 * record_count)); + (in_memory - 2) * timediff / (20 * record_count); /* * Get more aggressive as the number of in memory chunks @@ -827,8 +829,8 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) nchunks + 1, &lsm_tree->chunk)); WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Tree %s switch to: %" PRIu32 ", checkpoint throttle %ld, " - "merge throttle %ld", lsm_tree->name, + "Tree %s switch to: %" PRIu32 ", checkpoint throttle %" PRIu64 + ", merge throttle %" PRIu64, lsm_tree->name, new_id, lsm_tree->ckpt_throttle, lsm_tree->merge_throttle)); WT_ERR(__wt_calloc_one(session, &chunk)); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_worker.c b/src/third_party/wiredtiger/src/lsm/lsm_worker.c index a376a81f4eb..d1272df763d 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_worker.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_worker.c @@ -10,7 +10,7 @@ static int __lsm_worker_general_op( WT_SESSION_IMPL *, WT_LSM_WORKER_ARGS *, int *); -static void * __lsm_worker(void *); +static WT_THREAD_RET __lsm_worker(void *); /* * __wt_lsm_worker_start -- @@ -82,7 +82,7 @@ err: __wt_lsm_manager_free_work_unit(session, entry); * __lsm_worker -- * A thread that executes work units for all open LSM trees. */ -static void * +static WT_THREAD_RET __lsm_worker(void *arg) { WT_CONNECTION_IMPL *conn; @@ -166,5 +166,5 @@ err: __wt_lsm_manager_free_work_unit(session, entry); WT_PANIC_MSG(session, ret, "Error in LSM worker thread %d", cookie->id); } - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 0a8557c7a33..70c9bf8dfcd 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -230,7 +230,7 @@ err: __wt_free(session, namep); * __ckpt_compare_order -- * Qsort comparison routine for the checkpoint list. */ -static int +static int WT_CDECL __ckpt_compare_order(const void *a, const void *b) { WT_CKPT *ackpt, *bckpt; diff --git a/src/third_party/wiredtiger/src/meta/meta_track.c b/src/third_party/wiredtiger/src/meta/meta_track.c index 85ca1732586..3bc6a1f9d60 100644 --- a/src/third_party/wiredtiger/src/meta/meta_track.c +++ b/src/third_party/wiredtiger/src/meta/meta_track.c @@ -188,7 +188,7 @@ free: trk->op = WT_ST_EMPTY; * Turn off metadata operation tracking, unrolling on error. */ int -__wt_meta_track_off(WT_SESSION_IMPL *session, int unroll) +__wt_meta_track_off(WT_SESSION_IMPL *session, int need_sync, int unroll) { WT_DECL_RET; WT_META_TRACK *trk, *trk_orig; @@ -218,13 +218,28 @@ __wt_meta_track_off(WT_SESSION_IMPL *session, int unroll) WT_TRET(__meta_track_apply(session, trk, unroll)); /* - * If the operation succeeded and we aren't relying on the log for - * durability, checkpoint the metadata. + * Unroll operations don't need to flush the metadata. + * + * Also, if we don't have the metadata handle (e.g, we're in the + * process of creating the metadata), we can't sync it. */ - if (!unroll && ret == 0 && session->meta_dhandle != NULL && - !FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ENABLED)) + if (unroll || ret != 0 || !need_sync || session->meta_dhandle == NULL) + return (ret); + + /* If we're logging, make sure the metadata update was flushed. */ + if (FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ENABLED)) { + if (!FLD_ISSET(S2C(session)->txn_logsync, + WT_LOG_DSYNC | WT_LOG_FSYNC)) + WT_WITH_DHANDLE(session, session->meta_dhandle, + ret = __wt_txn_checkpoint_log(session, + 0, WT_TXN_LOG_CKPT_SYNC, NULL)); + } else { WT_WITH_DHANDLE(session, session->meta_dhandle, ret = __wt_checkpoint(session, NULL)); + WT_RET(ret); + WT_WITH_DHANDLE(session, session->meta_dhandle, + ret = __wt_checkpoint_sync(session, NULL)); + } return (ret); } diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 5983fb12e38..7e3ac3530f8 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -72,16 +72,14 @@ __metadata_load_hot_backup(WT_SESSION_IMPL *session) WT_DECL_ITEM(key); WT_DECL_ITEM(value); WT_DECL_RET; - char *path; - - path = NULL; + int exist; /* Look for a hot backup file: if we find it, load it. */ - WT_RET(__wt_filename(session, WT_METADATA_BACKUP, &path)); - fp = fopen(path, "r"); - __wt_free(session, path); - if (fp == NULL) + WT_RET(__wt_exist(session, WT_METADATA_BACKUP, &exist)); + if (!exist) return (0); + WT_RET(__wt_fopen(session, + WT_METADATA_BACKUP, WT_FHANDLE_READ, 0, &fp)); /* Read line pairs and load them into the metadata file. */ WT_ERR(__wt_scr_alloc(session, 512, &key)); @@ -98,7 +96,7 @@ __metadata_load_hot_backup(WT_SESSION_IMPL *session) F_SET(S2C(session), WT_CONN_WAS_BACKUP); -err: WT_TRET(fclose(fp) == 0 ? 0 : __wt_errno()); +err: WT_TRET(__wt_fclose(session, &fp, WT_FHANDLE_READ)); __wt_scr_free(session, &key); __wt_scr_free(session, &value); return (ret); @@ -166,9 +164,7 @@ __wt_turtle_init(WT_SESSION_IMPL *session) * Discard any turtle setup file left-over from previous runs. This * doesn't matter for correctness, it's just cleaning up random files. */ - WT_RET(__wt_exist(session, WT_METADATA_TURTLE_SET, &exist)); - if (exist) - WT_RET(__wt_remove(session, WT_METADATA_TURTLE_SET)); + WT_RET(__wt_remove_if_exists(session, WT_METADATA_TURTLE_SET)); /* * We could die after creating the turtle file and before creating the @@ -209,12 +205,8 @@ __wt_turtle_init(WT_SESSION_IMPL *session) WT_RET(__metadata_config(session, &metaconf)); WT_ERR(__wt_turtle_update(session, WT_METAFILE_URI, metaconf)); - /* Remove the backup file if it exists, we'll never read it again. */ - if (exist_incr) - WT_ERR(__wt_remove(session, WT_INCREMENTAL_BACKUP)); - WT_ERR(__wt_exist(session, WT_METADATA_BACKUP, &exist)); - if (exist) - WT_ERR(__wt_remove(session, WT_METADATA_BACKUP)); + /* Remove the backup files, we'll never read them again. */ + WT_ERR(__wt_backup_file_remove(session)); err: __wt_free(session, metaconf); return (ret); @@ -230,26 +222,22 @@ __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) FILE *fp; WT_DECL_ITEM(buf); WT_DECL_RET; - int match; - char *path; + int exist, match; *valuep = NULL; - path = NULL; - /* * Open the turtle file; there's one case where we won't find the turtle * file, yet still succeed. We create the metadata file before creating * the turtle file, and that means returning the default configuration * string for the metadata file. */ - WT_RET(__wt_filename(session, WT_METADATA_TURTLE, &path)); - if ((fp = fopen(path, "r")) == NULL) - ret = __wt_errno(); - __wt_free(session, path); - if (fp == NULL) + WT_RET(__wt_exist(session, WT_METADATA_TURTLE, &exist)); + if (!exist) return (strcmp(key, WT_METAFILE_URI) == 0 ? - __metadata_config(session, valuep) : ret); + __metadata_config(session, valuep) : WT_NOTFOUND); + WT_RET(__wt_fopen(session, + WT_METADATA_TURTLE, WT_FHANDLE_READ, 0, &fp)); /* Search for the key. */ WT_ERR(__wt_scr_alloc(session, 512, &buf)); @@ -271,7 +259,7 @@ __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) /* Copy the value for the caller. */ WT_ERR(__wt_strdup(session, buf->data, valuep)); -err: WT_TRET(fclose(fp) == 0 ? 0 : __wt_errno()); +err: WT_TRET(__wt_fclose(session, &fp, WT_FHANDLE_READ)); __wt_scr_free(session, &buf); return (ret); } @@ -300,34 +288,22 @@ __wt_turtle_update( session, WT_METADATA_TURTLE_SET, 1, 1, WT_FILE_TYPE_TURTLE, &fh)); version = wiredtiger_version(&vmajor, &vminor, &vpatch); - WT_ERR(__wt_scr_alloc(session, 1000, &buf)); + WT_ERR(__wt_scr_alloc(session, 2 * 1024, &buf)); WT_ERR(__wt_buf_fmt(session, buf, "%s\n%s\n%s\n" "major=%d,minor=%d,patch=%d\n%s\n%s\n", WT_METADATA_VERSION_STR, version, WT_METADATA_VERSION, vmajor, vminor, vpatch, key, value)); - WT_ERR(__wt_write(session, fh, 0, buf->size, buf->data)); - if (F_ISSET(S2C(session), WT_CONN_CKPT_SYNC)) - WT_ERR(__wt_fsync(session, fh)); + /* Flush the handle and rename the file into place. */ + ret = __wt_sync_and_rename_fh( + session, &fh, WT_METADATA_TURTLE_SET, WT_METADATA_TURTLE); - ret = __wt_close(session, fh); - fh = NULL; - WT_ERR(ret); - - WT_ERR( - __wt_rename(session, WT_METADATA_TURTLE_SET, WT_METADATA_TURTLE)); - - if (F_ISSET(S2C(session), WT_CONN_CKPT_SYNC)) - WT_ERR(__wt_directory_sync(session, NULL)); - - if (0) { -err: WT_TRET(__wt_remove(session, WT_METADATA_TURTLE_SET)); - } + /* Close any file handle left open, remove any temporary file. */ +err: WT_TRET(__wt_close(session, &fh)); + WT_TRET(__wt_remove_if_exists(session, WT_METADATA_TURTLE_SET)); - if (fh != NULL) - WT_TRET(__wt_close(session, fh)); __wt_scr_free(session, &buf); return (ret); } diff --git a/src/third_party/wiredtiger/src/os_posix/os_alloc.c b/src/third_party/wiredtiger/src/os_posix/os_alloc.c index 0795fe2fe0d..d3d4fd1dbcf 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_alloc.c +++ b/src/third_party/wiredtiger/src/os_posix/os_alloc.c @@ -134,6 +134,17 @@ __wt_realloc_aligned(WT_SESSION_IMPL *session, WT_ASSERT(session, bytes_to_allocate != 0); WT_ASSERT(session, bytes_allocated < bytes_to_allocate); + /* + * We are going to allocate an aligned buffer. When we do this + * repeatedly, the allocator is expected to start on a boundary + * each time, account for that additional space by never asking + * for less than a full alignment size. The primary use case + * for aligned buffers is Linux direct I/O, which requires that + * the size be a multiple of the alignment anyway. + */ + bytes_to_allocate = + WT_ALIGN(bytes_to_allocate, S2C(session)->buffer_alignment); + if (session != NULL) WT_STAT_FAST_CONN_INCR(session, memory_allocation); diff --git a/src/third_party/wiredtiger/src/os_posix/os_exist.c b/src/third_party/wiredtiger/src/os_posix/os_exist.c index 3e51a98766a..173021e6100 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_exist.c +++ b/src/third_party/wiredtiger/src/os_posix/os_exist.c @@ -19,6 +19,8 @@ __wt_exist(WT_SESSION_IMPL *session, const char *filename, int *existp) WT_DECL_RET; char *path; + *existp = 0; + WT_RET(__wt_filename(session, filename, &path)); WT_SYSCALL_RETRY(stat(path, &sb), ret); @@ -29,10 +31,8 @@ __wt_exist(WT_SESSION_IMPL *session, const char *filename, int *existp) *existp = 1; return (0); } - if (ret == ENOENT) { - *existp = 0; + if (ret == ENOENT) return (0); - } WT_RET_MSG(session, ret, "%s: fstat", filename); } diff --git a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c index cf43b630997..dfd72dd0cd2 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c @@ -45,14 +45,13 @@ err: __wt_free(session, cond); * Wait on a mutex, optionally timing out. */ int -__wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, long usecs) +__wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs) { struct timespec ts; WT_DECL_RET; int locked; locked = 0; - WT_ASSERT(session, usecs >= 0); /* Fast path if already signalled. */ if (WT_ATOMIC_ADD4(cond->waiters, 1) == 0) @@ -73,8 +72,10 @@ __wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, long usecs) if (usecs > 0) { WT_ERR(__wt_epoch(session, &ts)); - ts.tv_sec += (ts.tv_nsec + 1000 * usecs) / WT_BILLION; - ts.tv_nsec = (ts.tv_nsec + 1000 * usecs) % WT_BILLION; + ts.tv_sec += (time_t) + (((uint64_t)ts.tv_nsec + 1000 * usecs) / WT_BILLION); + ts.tv_nsec = (long) + (((uint64_t)ts.tv_nsec + 1000 * usecs) % WT_BILLION); ret = pthread_cond_timedwait(&cond->cond, &cond->mtx, &ts); } else ret = pthread_cond_wait(&cond->cond, &cond->mtx); diff --git a/src/third_party/wiredtiger/src/os_posix/os_open.c b/src/third_party/wiredtiger/src/os_posix/os_open.c index 1d8cb7023b6..33b07ed6a49 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_open.c +++ b/src/third_party/wiredtiger/src/os_posix/os_open.c @@ -207,14 +207,20 @@ err: if (fh != NULL) { * Close a file handle. */ int -__wt_close(WT_SESSION_IMPL *session, WT_FH *fh) +__wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_FH *fh; uint64_t bucket; conn = S2C(session); + if (*fhp == NULL) + return (0); + fh = *fhp; + *fhp = NULL; + __wt_spin_lock(session, &conn->fh_lock); if (fh == NULL || fh->ref == 0 || --fh->ref > 0) { __wt_spin_unlock(session, &conn->fh_lock); diff --git a/src/third_party/wiredtiger/src/os_posix/os_sleep.c b/src/third_party/wiredtiger/src/os_posix/os_sleep.c index 64a5f9e56af..f888e51bf7f 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_sleep.c +++ b/src/third_party/wiredtiger/src/os_posix/os_sleep.c @@ -13,11 +13,11 @@ * Pause the thread of control. */ void -__wt_sleep(long seconds, long micro_seconds) +__wt_sleep(uint64_t seconds, uint64_t micro_seconds) { struct timeval t; - t.tv_sec = seconds + micro_seconds / 1000000; + t.tv_sec = (time_t)(seconds + micro_seconds / 1000000); t.tv_usec = (suseconds_t)(micro_seconds % 1000000); (void)select(0, NULL, NULL, NULL, &t); diff --git a/src/third_party/wiredtiger/src/os_posix/os_stdio.c b/src/third_party/wiredtiger/src/os_posix/os_stdio.c new file mode 100644 index 00000000000..8d97fdd2880 --- /dev/null +++ b/src/third_party/wiredtiger/src/os_posix/os_stdio.c @@ -0,0 +1,133 @@ +/*- + * Copyright (c) 2014-2015 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_fopen -- + * Open a FILE handle. + */ +int +__wt_fopen(WT_SESSION_IMPL *session, + const char *name, WT_FHANDLE_MODE mode_flag, u_int flags, FILE **fpp) +{ + WT_DECL_RET; + const char *mode, *path; + char *pathbuf; + + WT_RET(__wt_verbose(session, WT_VERB_FILEOPS, "%s: fopen", name)); + + pathbuf = NULL; + if (LF_ISSET(WT_FOPEN_FIXED)) + path = name; + else { + WT_RET(__wt_filename(session, name, &pathbuf)); + path = pathbuf; + } + + mode = NULL; + switch (mode_flag) { + case WT_FHANDLE_APPEND: + mode = WT_FOPEN_APPEND; + break; + case WT_FHANDLE_READ: + mode = WT_FOPEN_READ; + break; + case WT_FHANDLE_WRITE: + mode = WT_FOPEN_WRITE; + break; + } + *fpp = fopen(path, mode); + if (*fpp == NULL) + ret = __wt_errno(); + + if (pathbuf != NULL) + __wt_free(session, pathbuf); + + if (ret == 0) + return (0); + WT_RET_MSG(session, ret, "%s: fopen", name); +} + +/* + * __wt_vfprintf -- + * Vfprintf for a FILE handle. + */ +int +__wt_vfprintf(WT_SESSION_IMPL *session, FILE *fp, const char *fmt, va_list ap) +{ + WT_DECL_RET; + + WT_UNUSED(session); + + return (vfprintf(fp, fmt, ap) < 0 ? __wt_errno() : ret); +} + +/* + * __wt_fprintf -- + * Fprintf for a FILE handle. + */ +int +__wt_fprintf(WT_SESSION_IMPL *session, FILE *fp, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) +{ + WT_DECL_RET; + va_list ap; + + va_start(ap, fmt); + ret = __wt_vfprintf(session, fp, fmt, ap); + va_end(ap); + + return (ret); +} + +/* + * __wt_fflush -- + * Flush a FILE handle. + */ +int +__wt_fflush(WT_SESSION_IMPL *session, FILE *fp) +{ + WT_UNUSED(session); + + /* Flush the handle. */ + return (fflush(fp) == 0 ? 0 : __wt_errno()); +} + +/* + * __wt_fclose -- + * Close a FILE handle. + */ +int +__wt_fclose(WT_SESSION_IMPL *session, FILE **fpp, WT_FHANDLE_MODE mode_flag) +{ + FILE *fp; + WT_DECL_RET; + + if (*fpp == NULL) + return (0); + + fp = *fpp; + *fpp = NULL; + + /* + * If the handle was opened for writing, flush the file to the backing + * OS buffers, then flush the OS buffers to the backing disk. + */ + if (mode_flag == WT_FHANDLE_APPEND || mode_flag == WT_FHANDLE_WRITE) { + ret = __wt_fflush(session, fp); + if (fsync(fileno(fp)) != 0) + WT_TRET(__wt_errno()); + } + + /* Close the handle. */ + if (fclose(fp) != 0) + WT_TRET(__wt_errno()); + + return (ret); +} diff --git a/src/third_party/wiredtiger/src/os_posix/os_thread.c b/src/third_party/wiredtiger/src/os_posix/os_thread.c index 392f997f1ac..c70a04c8df7 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_thread.c +++ b/src/third_party/wiredtiger/src/os_posix/os_thread.c @@ -14,7 +14,7 @@ */ int __wt_thread_create(WT_SESSION_IMPL *session, - wt_thread_t *tidret, void *(*func)(void *), void *arg) + wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) { WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/os_win/os_fallocate.c b/src/third_party/wiredtiger/src/os_win/os_fallocate.c index 9d0a86882c6..f01ef0e101a 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fallocate.c +++ b/src/third_party/wiredtiger/src/os_win/os_fallocate.c @@ -15,6 +15,8 @@ void __wt_fallocate_config(WT_SESSION_IMPL *session, WT_FH *fh) { + WT_UNUSED(session); + fh->fallocate_available = WT_FALLOCATE_AVAILABLE; /* diff --git a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c index 36de49d1aae..51f6d6533c8 100644 --- a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c @@ -41,13 +41,14 @@ __wt_cond_alloc(WT_SESSION_IMPL *session, * Wait on a mutex, optionally timing out. */ int -__wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, long usecs) +__wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs) { + DWORD milliseconds; WT_DECL_RET; + uint64_t milliseconds64; int locked; - int milliseconds; + locked = 0; - WT_ASSERT(session, usecs >= 0); /* Fast path if already signalled. */ if (WT_ATOMIC_ADD4(cond->waiters, 1) == 0) @@ -67,13 +68,23 @@ __wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, long usecs) locked = 1; if (usecs > 0) { - milliseconds = usecs / 1000; + milliseconds64 = usecs / 1000; + + /* + * Check for 32-bit unsigned integer overflow + * INFINITE is max unsigned int on Windows + */ + if (milliseconds64 >= INFINITE) + milliseconds64 = INFINITE - 1; + milliseconds = (DWORD)milliseconds64; + /* * 0 would mean the CV sleep becomes a TryCV which we do not * want */ if (milliseconds == 0) milliseconds = 1; + ret = SleepConditionVariableCS( &cond->cond, &cond->mtx, milliseconds); } else diff --git a/src/third_party/wiredtiger/src/os_win/os_once.c b/src/third_party/wiredtiger/src/os_win/os_once.c index 179c1bc97f7..bec8c08777c 100644 --- a/src/third_party/wiredtiger/src/os_win/os_once.c +++ b/src/third_party/wiredtiger/src/os_win/os_once.c @@ -19,6 +19,8 @@ BOOL CALLBACK _wt_init_once_callback( ) { void(*init_routine)(void) = Parameter; + WT_UNUSED(InitOnce); + WT_UNUSED(Context); init_routine(); diff --git a/src/third_party/wiredtiger/src/os_win/os_open.c b/src/third_party/wiredtiger/src/os_win/os_open.c index f146c732e85..4039bcb21e5 100644 --- a/src/third_party/wiredtiger/src/os_win/os_open.c +++ b/src/third_party/wiredtiger/src/os_win/os_open.c @@ -194,14 +194,20 @@ err: if (fh != NULL) { * Close a file handle. */ int -__wt_close(WT_SESSION_IMPL *session, WT_FH *fh) +__wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_FH *fh; uint64_t bucket; conn = S2C(session); + if (*fhp == NULL) + return (0); + fh = *fhp; + *fhp = NULL; + __wt_spin_lock(session, &conn->fh_lock); if (fh == NULL || fh->ref == 0 || --fh->ref > 0) { __wt_spin_unlock(session, &conn->fh_lock); diff --git a/src/third_party/wiredtiger/src/os_win/os_sleep.c b/src/third_party/wiredtiger/src/os_win/os_sleep.c index 9f8eafec903..484cf218f26 100644 --- a/src/third_party/wiredtiger/src/os_win/os_sleep.c +++ b/src/third_party/wiredtiger/src/os_win/os_sleep.c @@ -13,7 +13,13 @@ * Pause the thread of control. */ void -__wt_sleep(long seconds, long micro_seconds) +__wt_sleep(uint64_t seconds, uint64_t micro_seconds) { + /* + * If the caller wants a small pause, set to our + * smallest granularity. + */ + if (seconds == 0 && micro_seconds < 1000) + micro_seconds = 1000; Sleep(seconds * 1000 + micro_seconds / 1000); } diff --git a/src/third_party/wiredtiger/src/os_win/os_thread.c b/src/third_party/wiredtiger/src/os_win/os_thread.c index 05f7dc15914..b5f13aea4e9 100644 --- a/src/third_party/wiredtiger/src/os_win/os_thread.c +++ b/src/third_party/wiredtiger/src/os_win/os_thread.c @@ -14,14 +14,14 @@ */ int __wt_thread_create(WT_SESSION_IMPL *session, - wt_thread_t *tidret, void *(*func)(void *), void *arg) + wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) { /* Spawn a new thread of control. */ - *tidret = CreateThread(NULL, 0, func, arg, 0, NULL); - if (*tidret != NULL) + *tidret = (HANDLE)_beginthreadex(NULL, 0, func, arg, 0, NULL); + if (*tidret != 0) return (0); - WT_RET_MSG(session, __wt_errno(), "CreateThread"); + WT_RET_MSG(session, errno, "_beginthreadex"); } /* diff --git a/src/third_party/wiredtiger/src/os_win/os_time.c b/src/third_party/wiredtiger/src/os_win/os_time.c index 30fde045c54..c51db118ce1 100644 --- a/src/third_party/wiredtiger/src/os_win/os_time.c +++ b/src/third_party/wiredtiger/src/os_win/os_time.c @@ -32,8 +32,10 @@ int __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) { uint64_t ns100; - FILETIME time; + + WT_UNUSED(session); + GetSystemTimeAsFileTime(&time); ns100 = (((int64_t)time.dwHighDateTime << 32) + time.dwLowDateTime) diff --git a/src/third_party/wiredtiger/src/packing/pack_impl.c b/src/third_party/wiredtiger/src/packing/pack_impl.c index 0b699814fc1..c92325a4c23 100644 --- a/src/third_party/wiredtiger/src/packing/pack_impl.c +++ b/src/third_party/wiredtiger/src/packing/pack_impl.c @@ -44,6 +44,16 @@ __wt_struct_check(WT_SESSION_IMPL *session, } /* + * __wt_struct_confchk -- + * Check that the specified packing format is valid, configuration version. + */ +int +__wt_struct_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *v) +{ + return (__wt_struct_check(session, v->str, v->len, NULL, NULL)); +} + +/* * __wt_struct_size -- * Calculate the size of a packed byte string. */ diff --git a/src/third_party/wiredtiger/src/packing/pack_stream.c b/src/third_party/wiredtiger/src/packing/pack_stream.c index 739e803b88c..1f3449d79d3 100644 --- a/src/third_party/wiredtiger/src/packing/pack_stream.c +++ b/src/third_party/wiredtiger/src/packing/pack_stream.c @@ -83,6 +83,10 @@ wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item) session = ps->pack.session; + /* Lower-level packing routines treat a length of zero as unchecked. */ + if (ps->p >= ps->end) + return (ENOMEM); + WT_RET(__pack_next(&ps->pack, &pv)); switch (pv.type) { case 'U': @@ -110,6 +114,10 @@ wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i) session = ps->pack.session; + /* Lower-level packing routines treat a length of zero as unchecked. */ + if (ps->p >= ps->end) + return (ENOMEM); + WT_RET(__pack_next(&ps->pack, &pv)); switch (pv.type) { case 'b': @@ -139,6 +147,10 @@ wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s) session = ps->pack.session; + /* Lower-level packing routines treat a length of zero as unchecked. */ + if (ps->p >= ps->end) + return (ENOMEM); + WT_RET(__pack_next(&ps->pack, &pv)); switch (pv.type) { case 'S': @@ -165,6 +177,10 @@ wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u) session = ps->pack.session; + /* Lower-level packing routines treat a length of zero as unchecked. */ + if (ps->p >= ps->end) + return (ENOMEM); + WT_RET(__pack_next(&ps->pack, &pv)); switch (pv.type) { case 'B': @@ -197,6 +213,10 @@ wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item) session = ps->pack.session; + /* Lower-level packing routines treat a length of zero as unchecked. */ + if (ps->p >= ps->end) + return (ENOMEM); + WT_RET(__pack_next(&ps->pack, &pv)); switch (pv.type) { case 'U': @@ -224,6 +244,10 @@ wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip) session = ps->pack.session; + /* Lower-level packing routines treat a length of zero as unchecked. */ + if (ps->p >= ps->end) + return (ENOMEM); + WT_RET(__pack_next(&ps->pack, &pv)); switch (pv.type) { case 'b': @@ -252,6 +276,10 @@ wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp) session = ps->pack.session; + /* Lower-level packing routines treat a length of zero as unchecked. */ + if (ps->p >= ps->end) + return (ENOMEM); + WT_RET(__pack_next(&ps->pack, &pv)); switch (pv.type) { case 'S': @@ -277,6 +305,10 @@ wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) session = ps->pack.session; + /* Lower-level packing routines treat a length of zero as unchecked. */ + if (ps->p >= ps->end) + return (ENOMEM); + WT_RET(__pack_next(&ps->pack, &pv)); switch (pv.type) { case 'B': diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index c0f4f4bb57e..80e443d8a21 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -580,9 +580,9 @@ __create_data_source(WT_SESSION_IMPL *session, * data source doesn't have access to the functions that check. */ WT_RET(__wt_config_gets(session, cfg, "key_format", &cval)); - WT_RET(__wt_struct_check(session, cval.str, cval.len, NULL, NULL)); + WT_RET(__wt_struct_confchk(session, &cval)); WT_RET(__wt_config_gets(session, cfg, "value_format", &cval)); - WT_RET(__wt_struct_check(session, cval.str, cval.len, NULL, NULL)); + WT_RET(__wt_struct_confchk(session, &cval)); /* * User-specified collators aren't supported for data-source objects. @@ -637,7 +637,7 @@ __wt_schema_create( ret = __wt_bad_object_type(session, uri); session->dhandle = NULL; - WT_TRET(__wt_meta_track_off(session, ret != 0)); + WT_TRET(__wt_meta_track_off(session, 1, ret != 0)); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index dfac4a172ef..03097128ec2 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -18,7 +18,7 @@ __drop_file( { WT_CONFIG_ITEM cval; WT_DECL_RET; - int exist, remove_files; + int remove_files; const char *filename; WT_RET(__wt_config_gets(session, cfg, "remove_files", &cval)); @@ -38,16 +38,11 @@ __drop_file( if (!remove_files) return (ret); - /* Remove the underlying physical file. */ - exist = 0; - WT_TRET(__wt_exist(session, filename, &exist)); - if (exist) { - /* - * There is no point tracking this operation: there is no going - * back from here. - */ - WT_TRET(__wt_remove(session, filename)); - } + /* + * Remove the underlying physical file. There is no point tracking this + * operation: there is no going back from here. + */ + WT_TRET(__wt_remove_if_exists(session, filename)); return (ret); } @@ -197,7 +192,7 @@ __wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) /* Bump the schema generation so that stale data is ignored. */ ++S2C(session)->schema_gen; - WT_TRET(__wt_meta_track_off(session, ret != 0)); + WT_TRET(__wt_meta_track_off(session, 1, ret != 0)); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 9d70e5b943a..57ea3b96647 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -139,6 +139,15 @@ __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX *idx) { WT_DECL_RET; + /* If there is a custom collator configured, terminate it. */ + if (idx->collator != NULL && + idx->collator_owned && idx->collator->terminate != NULL) { + WT_TRET(idx->collator->terminate( + idx->collator, &session->iface)); + idx->collator = NULL; + idx->collator_owned = 0; + } + /* If there is a custom extractor configured, terminate it. */ if (idx->extractor != NULL && idx->extractor_owned && idx->extractor->terminate != NULL) { diff --git a/src/third_party/wiredtiger/src/schema/schema_open.c b/src/third_party/wiredtiger/src/schema/schema_open.c index fa655c7108b..5bc589f0781 100644 --- a/src/third_party/wiredtiger/src/schema/schema_open.c +++ b/src/third_party/wiredtiger/src/schema/schema_open.c @@ -130,7 +130,7 @@ static int __open_index(WT_SESSION_IMPL *session, WT_TABLE *table, WT_INDEX *idx) { WT_CONFIG colconf; - WT_CONFIG_ITEM ckey, cval; + WT_CONFIG_ITEM ckey, cval, metadata; WT_DECL_ITEM(buf); WT_DECL_ITEM(plan); WT_DECL_RET; @@ -147,6 +147,22 @@ __open_index(WT_SESSION_IMPL *session, WT_TABLE *table, WT_INDEX *idx) if (cval.val) F_SET(idx, WT_INDEX_IMMUTABLE); + /* + * Compatibility: we didn't always maintain collator information in + * index metadata, cope when it isn't found. + */ + WT_CLEAR(cval); + WT_ERR_NOTFOUND_OK(__wt_config_getones( + session, idx->config, "collator", &cval)); + if (cval.len != 0) { + WT_CLEAR(metadata); + WT_ERR_NOTFOUND_OK(__wt_config_getones( + session, idx->config, "app_metadata", &metadata)); + WT_ERR(__wt_collator_config( + session, idx->name, &cval, &metadata, + &idx->collator, &idx->collator_owned)); + } + WT_ERR(__wt_extractor_config( session, idx->config, &idx->extractor, &idx->extractor_owned)); diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index 38124754cd5..51281eccec5 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -274,7 +274,7 @@ __wt_schema_rename(WT_SESSION_IMPL *session, /* Bump the schema generation so that stale data is ignored. */ ++S2C(session)->schema_gen; - WT_TRET(__wt_meta_track_off(session, ret != 0)); + WT_TRET(__wt_meta_track_off(session, 1, ret != 0)); /* If we didn't find a metadata entry, map that error to ENOENT. */ return (ret == WT_NOTFOUND ? ENOENT : ret); diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index e95bea3f75b..599c7bdf44a 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -144,9 +144,6 @@ __session_close(WT_SESSION *wt_session, const char *config) if (session->reconcile_cleanup != NULL) WT_TRET(session->reconcile_cleanup(session)); - /* Free the eviction exclusive-lock information. */ - __wt_free(session, session->excl); - /* Destroy the thread's mutex. */ WT_TRET(__wt_cond_destroy(session, &session->cond)); @@ -608,9 +605,15 @@ __session_truncate(WT_SESSION *wt_session, "the truncate method should not specify any" "target after the log: URI prefix."); ret = __wt_log_truncate_files(session, start, cfg); - } else + } else { + /* Wait for checkpoints to avoid EBUSY errors. */ + __wt_spin_lock(session, + &S2C(session)->checkpoint_lock); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_truncate(session, uri, cfg)); + __wt_spin_unlock(session, + &S2C(session)->checkpoint_lock); + } goto done; } diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 13af615930b..0825f783ca3 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -168,7 +168,7 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) WT_ASSERT(session, F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE)); F_CLR(dhandle, WT_DHANDLE_DISCARD); - WT_TRET(__wt_conn_btree_sync_and_close(session, 0)); + WT_TRET(__wt_conn_btree_sync_and_close(session, 0, 0)); } if (F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE)) @@ -288,16 +288,19 @@ __wt_session_close_cache(WT_SESSION_IMPL *session) static int __session_dhandle_sweep(WT_SESSION_IMPL *session) { + WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DATA_HANDLE_CACHE *dhandle_cache, *dhandle_cache_next; time_t now; + conn = S2C(session); + /* * Periodically sweep for dead handles; if we've swept recently, don't * do it again. */ WT_RET(__wt_seconds(session, &now)); - if (now - session->last_sweep < WT_DHANDLE_SWEEP_PERIOD) + if (now - session->last_sweep < conn->sweep_interval) return (0); session->last_sweep = now; @@ -309,7 +312,7 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) dhandle = dhandle_cache->dhandle; if (dhandle != session->dhandle && dhandle->session_inuse == 0 && - now - dhandle->timeofdeath > WT_DHANDLE_SWEEP_WAIT) { + now - dhandle->timeofdeath > conn->sweep_idle_time) { WT_STAT_FAST_CONN_INCR(session, dh_session_handles); __session_discard_btree(session, dhandle_cache); } @@ -380,7 +383,20 @@ __wt_session_get_btree(WT_SESSION_IMPL *session, /* Try to lock the handle; if this succeeds, we're done. */ if ((ret = __wt_session_lock_dhandle(session, flags)) == 0) goto done; - WT_RET_NOTFOUND_OK(ret); + + /* Propagate errors we don't expect. */ + if (ret != WT_NOTFOUND && ret != EBUSY) + return (ret); + + /* + * Don't try harder to get the btree handle if our caller + * hasn't allowed us to take the schema lock - they do so on + * purpose and will handle error returns. + */ + if (!F_ISSET(session, WT_SESSION_SCHEMA_LOCKED) && + F_ISSET(session, + WT_SESSION_HANDLE_LIST_LOCKED | WT_SESSION_TABLE_LOCKED)) + return (ret); /* We found the data handle, don't try to get it again. */ LF_SET(WT_DHANDLE_HAVE_REF); diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 49a3891c58a..fc6569cb81e 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -14,14 +14,18 @@ */ static int __handle_error_default(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *errmsg) + WT_SESSION *wt_session, int error, const char *errmsg) { + WT_SESSION_IMPL *session; + WT_UNUSED(handler); - WT_UNUSED(session); WT_UNUSED(error); - return (fprintf(stderr, "%s\n", errmsg) >= 0 && - fflush(stderr) == 0 ? 0 : __wt_errno()); + session = (WT_SESSION_IMPL *)wt_session; + + WT_RET(__wt_fprintf(session, stderr, "%s\n", errmsg)); + WT_RET(__wt_fflush(session, stderr)); + return (0); } /* @@ -30,13 +34,17 @@ __handle_error_default(WT_EVENT_HANDLER *handler, */ static int __handle_message_default(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *message) + WT_SESSION *wt_session, const char *message) { + WT_SESSION_IMPL *session; + WT_UNUSED(handler); - WT_UNUSED(session); - return (printf("%s\n", message) >= 0 && - fflush(stdout) == 0 ? 0 : __wt_errno()); + session = (WT_SESSION_IMPL *)wt_session; + + WT_RET(__wt_fprintf(session, stdout, "%s\n", message)); + WT_RET(__wt_fflush(session, stdout)); + return (0); } /* @@ -45,10 +53,10 @@ __handle_message_default(WT_EVENT_HANDLER *handler, */ static int __handle_progress_default(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *operation, uint64_t progress) + WT_SESSION *wt_session, const char *operation, uint64_t progress) { WT_UNUSED(handler); - WT_UNUSED(session); + WT_UNUSED(wt_session); WT_UNUSED(operation); WT_UNUSED(progress); @@ -61,10 +69,10 @@ __handle_progress_default(WT_EVENT_HANDLER *handler, */ static int __handle_close_default(WT_EVENT_HANDLER *handler, - WT_SESSION *session, WT_CURSOR *cursor) + WT_SESSION *wt_session, WT_CURSOR *cursor) { WT_UNUSED(handler); - WT_UNUSED(session); + WT_UNUSED(wt_session); WT_UNUSED(cursor); return (0); @@ -97,7 +105,7 @@ __handler_failure(WT_SESSION_IMPL *session, (void)snprintf(s, sizeof(s), "application %s event handler failed: %s", - which, wiredtiger_strerror(error)); + which, __wt_strerror(session, error, NULL, 0)); /* * Use the error handler to report the failure, unless it was the error @@ -172,11 +180,15 @@ __wt_eventv(WT_SESSION_IMPL *session, int msg_event, int error, * first session, but if the allocation of the first session fails, for * example, we can end up here without a session.) */ - if (session == NULL) - return (fprintf(stderr, "WiredTiger Error%s%s\n", + if (session == NULL) { + WT_RET(__wt_fprintf(session, stderr, + "WiredTiger Error%s%s: ", error == 0 ? "" : ": ", - error == 0 ? "" : wiredtiger_strerror(error)) >= 0 && - fflush(stderr) == 0 ? 0 : __wt_errno()); + error == 0 ? "" : __wt_strerror(session, error, NULL, 0))); + WT_RET(__wt_vfprintf(session, stderr, fmt, ap)); + WT_RET(__wt_fprintf(session, stderr, "\n")); + return (__wt_fflush(session, stderr)); + } p = s; end = s + sizeof(s); @@ -247,7 +259,7 @@ __wt_eventv(WT_SESSION_IMPL *session, int msg_event, int error, * Use strcmp to compare: both strings are nul-terminated, and * we don't want to run past the end of the buffer. */ - err = wiredtiger_strerror(error); + err = __wt_strerror(session, error, NULL, 0); len = strlen(err); if (WT_PTRDIFF(p, s) < len || strcmp(p - len, err) != 0) { remain = WT_PTRDIFF(end, p); diff --git a/src/third_party/wiredtiger/src/support/filename.c b/src/third_party/wiredtiger/src/support/filename.c index 6d65cdf9ada..db466ed3bd4 100644 --- a/src/third_party/wiredtiger/src/support/filename.c +++ b/src/third_party/wiredtiger/src/support/filename.c @@ -48,3 +48,67 @@ __wt_nfilename( return (0); } + +/* + * __wt_remove_if_exists -- + * Remove a file if it exists. + */ +int +__wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name) +{ + int exist; + + WT_RET(__wt_exist(session, name, &exist)); + if (exist) + WT_RET(__wt_remove(session, name)); + return (0); +} + +/* + * __wt_sync_and_rename_fh -- + * Sync and close a file, and swap it into place. + */ +int +__wt_sync_and_rename_fh( + WT_SESSION_IMPL *session, WT_FH **fhp, const char *from, const char *to) +{ + WT_DECL_RET; + WT_FH *fh; + + fh = *fhp; + *fhp = NULL; + + /* Flush to disk and close the handle. */ + ret = __wt_fsync(session, fh); + WT_TRET(__wt_close(session, &fh)); + WT_RET(ret); + + /* Rename the source file to the target. */ + WT_RET(__wt_rename(session, from, to)); + + /* Flush the backing directory to guarantee the rename. */ + return (__wt_directory_sync(session, NULL)); +} + +/* + * __wt_sync_and_rename_fp -- + * Sync and close a file, and swap it into place. + */ +int +__wt_sync_and_rename_fp( + WT_SESSION_IMPL *session, FILE **fpp, const char *from, const char *to) +{ + FILE *fp; + + fp = *fpp; + *fpp = NULL; + + /* Flush to disk and close the handle. */ + WT_RET(__wt_fclose(session, &fp, WT_FHANDLE_WRITE)); + + /* Rename the source file to the target. */ + WT_RET(__wt_rename(session, from, to)); + + /* Flush the backing directory to guarantee the rename. */ + return (__wt_directory_sync(session, NULL)); +} diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index 4b8b916b121..bc44f7967a5 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -24,10 +24,12 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, int *busyp ) { WT_BTREE *btree; + WT_CONNECTION_IMPL *conn; WT_HAZARD *hp; int restarts = 0; btree = S2BT(session); + conn = S2C(session); *busyp = 0; /* If a file can never be evicted, hazard pointers aren't required. */ @@ -54,19 +56,27 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, int *busyp * for that case. */ for (hp = session->hazard + session->nhazard;; ++hp) { - /* Expand the number of hazard pointers if available.*/ + /* + * If we get to the end of the array, either: + * 1. If we know there are free slots somewhere, and this is + * the first time through, continue the search from the + * start. Don't actually continue the loop because that + * will skip the first slot. + * 2. If we have searched all the way through and we have + * allocated the maximum number of slots, give up. + * 3. Allocate another increment of slots, up to the maximum. + * The slot we are on should now be available. + */ if (hp >= session->hazard + session->hazard_size) { - if (session->hazard_size >= S2C(session)->hazard_max) - break; - /* Restart the search. */ if (session->nhazard < session->hazard_size && - restarts++ == 0) { + restarts++ == 0) hp = session->hazard; - continue; - } - WT_PUBLISH(session->hazard_size, - WT_MIN(session->hazard_size + WT_HAZARD_INCR, - S2C(session)->hazard_max)); + else if (session->hazard_size >= conn->hazard_max) + break; + else + WT_PUBLISH(session->hazard_size, WT_MIN( + session->hazard_size + WT_HAZARD_INCR, + conn->hazard_max)); } if (hp->page != NULL) diff --git a/src/third_party/wiredtiger/src/support/huffman.c b/src/third_party/wiredtiger/src/support/huffman.c index 48b31b5e299..48361551ba1 100644 --- a/src/third_party/wiredtiger/src/support/huffman.c +++ b/src/third_party/wiredtiger/src/support/huffman.c @@ -96,8 +96,8 @@ typedef struct __indexed_byte { uint32_t frequency; } INDEXED_SYMBOL; -static int indexed_freq_compare(const void *, const void *); -static int indexed_symbol_compare(const void *, const void *); +static int WT_CDECL indexed_freq_compare(const void *, const void *); +static int WT_CDECL indexed_symbol_compare(const void *, const void *); static void make_table( WT_SESSION_IMPL *, uint8_t *, uint16_t, WT_HUFFMAN_CODE *, u_int); static void node_queue_close(WT_SESSION_IMPL *, NODE_QUEUE *); @@ -117,7 +117,7 @@ static void set_codes(WT_FREQTREE_NODE *, WT_HUFFMAN_CODE *, uint16_t, uint8_t); * indexed_symbol_compare -- * Qsort comparator to order the table by symbol, lowest to highest. */ -static int +static int WT_CDECL indexed_symbol_compare(const void *a, const void *b) { return (((INDEXED_SYMBOL *)a)->symbol > @@ -131,7 +131,7 @@ indexed_symbol_compare(const void *a, const void *b) * Qsort comparator to order the table by frequency (the most frequent * symbols will be at the end of the array). */ -static int +static int WT_CDECL indexed_freq_compare(const void *a, const void *b) { return (((INDEXED_SYMBOL *)a)->frequency > @@ -302,8 +302,7 @@ __wt_huffman_open(WT_SESSION_IMPL *session, uint64_t w1, w2; uint16_t i; - indexed_freqs = symbol_frequency_array; - + indexed_freqs = NULL; combined_nodes = leaves = NULL; node = node2 = tempnode = NULL; @@ -330,26 +329,24 @@ __wt_huffman_open(WT_SESSION_IMPL *session, * Order the array by symbol and check for invalid symbols and * duplicates. */ - qsort((void *)indexed_freqs, - symcnt, sizeof(INDEXED_SYMBOL), indexed_symbol_compare); + sym = symbol_frequency_array; + qsort(sym, symcnt, sizeof(INDEXED_SYMBOL), indexed_symbol_compare); for (i = 0; i < symcnt; ++i) { - if (i > 0 && - indexed_freqs[i].symbol == indexed_freqs[i - 1].symbol) + if (i > 0 && sym[i].symbol == sym[i - 1].symbol) WT_ERR_MSG(session, EINVAL, - "duplicate symbol %" PRIx32 - " specified in a huffman table", - indexed_freqs[i].symbol); - if (indexed_freqs[i].symbol > huffman->numSymbols) + "duplicate symbol %" PRIu32 " (%#" PRIx32 ") " + "specified in a huffman table", + sym[i].symbol, sym[i].symbol); + if (sym[i].symbol > huffman->numSymbols) WT_ERR_MSG(session, EINVAL, - "illegal symbol %" PRIx32 - " specified in a huffman table", - indexed_freqs[i].symbol); + "out-of-range symbol %" PRIu32 " (%#" PRIx32 ") " + "specified in a huffman table", + sym[i].symbol, sym[i].symbol); } /* * Massage frequencies. */ - indexed_freqs = NULL; WT_ERR(__wt_calloc_def(session, 256, &indexed_freqs)); /* diff --git a/src/third_party/wiredtiger/src/support/mutex.c b/src/third_party/wiredtiger/src/support/mutex.c index c35656051f6..fa85cfc33d5 100644 --- a/src/third_party/wiredtiger/src/support/mutex.c +++ b/src/third_party/wiredtiger/src/support/mutex.c @@ -194,27 +194,24 @@ __wt_statlog_dump_spinlock(WT_CONNECTION_IMPL *conn, const char *tag) continue; } - WT_RET_TEST((fprintf(conn->stat_fp, + WT_RET(__wt_fprintf(session, conn->stat_fp, "%s %" PRIu64 " %s spinlock %s: acquisitions\n", conn->stat_stamp, spin->counter <= ignore ? 0 : spin->counter, - tag, spin->name) < 0), - __wt_errno()); + tag, spin->name)); if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_CLEAR)) spin->counter = 0; } - WT_RET_TEST((fprintf(conn->stat_fp, + WT_RET(__wt_fprintf(session, conn->stat_fp, "%s %" PRIu64 " %s spinlock %s: acquisitions\n", conn->stat_stamp, block_manager <= ignore ? 0 : block_manager, - tag, "block manager") < 0), - __wt_errno()); - WT_RET_TEST((fprintf(conn->stat_fp, + tag, "block manager")); + WT_RET(__wt_fprintf(session, conn->stat_fp, "%s %" PRIu64 " %s spinlock %s: acquisitions\n", conn->stat_stamp, btree_page <= ignore ? 0 : btree_page, - tag, "btree page") < 0), - __wt_errno()); + tag, "btree page")); /* * Output the number of times each location acquires its spinlock and @@ -225,12 +222,12 @@ __wt_statlog_dump_spinlock(WT_CONNECTION_IMPL *conn, const char *tag) if (p->name == NULL) continue; - WT_RET_TEST((fprintf(conn->stat_fp, + WT_RET(__wt_fprintf(session, conn->stat_fp, "%s %d %s spinlock %s acquired by %s(%d)\n", conn->stat_stamp, p->total <= ignore ? 0 : p->total, tag, - p->name, p->file, p->line) < 0), __wt_errno()); + p->name, p->file, p->line)); if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_CLEAR)) p->total = 0; @@ -239,13 +236,13 @@ __wt_statlog_dump_spinlock(WT_CONNECTION_IMPL *conn, const char *tag) if (t->name == NULL) continue; - WT_RET_TEST((fprintf(conn->stat_fp, + WT_RET(__wt_fprintf(session, conn->stat_fp, "%s %d %s spinlock %s: %s(%d) blocked by %s(%d)\n", conn->stat_stamp, p->blocked[j] <= ignore ? 0 : p->blocked[j], tag, p->name, p->file, p->line, - t->file, t->line) < 0), __wt_errno()); + t->file, t->line)); if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_CLEAR)) p->blocked[j] = 0; } diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 9d10c4d5ca6..824914bf8bf 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -21,6 +21,8 @@ __wt_stat_init_dsrc_stats(WT_DSRC_STATS *stats) stats->block_major.desc = "block-manager: file major version number"; stats->block_size.desc = "block-manager: file size in bytes"; stats->block_minor.desc = "block-manager: minor version number"; + stats->btree_checkpoint_generation.desc = + "btree: btree checkpoint generation"; stats->btree_column_fix.desc = "btree: column-store fixed-size leaf pages"; stats->btree_column_internal.desc = @@ -246,6 +248,7 @@ __wt_stat_aggregate_dsrc_stats(const void *child, const void *parent) p->block_checkpoint_size.v += c->block_checkpoint_size.v; p->block_reuse_bytes.v += c->block_reuse_bytes.v; p->block_size.v += c->block_size.v; + p->btree_checkpoint_generation.v += c->btree_checkpoint_generation.v; p->btree_column_fix.v += c->btree_column_fix.v; p->btree_column_internal.v += c->btree_column_internal.v; p->btree_column_deleted.v += c->btree_column_deleted.v; @@ -398,6 +401,12 @@ __wt_stat_init_connection_stats(WT_CONNECTION_STATS *stats) stats->cache_eviction_walk.desc = "cache: pages walked for eviction"; stats->cache_write.desc = "cache: pages written from cache"; stats->cache_overhead.desc = "cache: percentage overhead"; + stats->cache_bytes_internal.desc = + "cache: tracked bytes belonging to internal pages in the cache"; + stats->cache_bytes_leaf.desc = + "cache: tracked bytes belonging to leaf pages in the cache"; + stats->cache_bytes_overflow.desc = + "cache: tracked bytes belonging to overflow pages in the cache"; stats->cache_bytes_dirty.desc = "cache: tracked dirty bytes in the cache"; stats->cache_pages_dirty.desc = @@ -513,6 +522,8 @@ __wt_stat_init_connection_stats(WT_CONNECTION_STATS *stats) stats->txn_begin.desc = "transaction: transaction begins"; stats->txn_checkpoint_running.desc = "transaction: transaction checkpoint currently running"; + stats->txn_checkpoint_generation.desc = + "transaction: transaction checkpoint generation"; stats->txn_checkpoint_time_max.desc = "transaction: transaction checkpoint max time (msecs)"; stats->txn_checkpoint_time_min.desc = @@ -526,6 +537,8 @@ __wt_stat_init_connection_stats(WT_CONNECTION_STATS *stats) "transaction: transaction failures due to cache overflow"; stats->txn_pinned_range.desc = "transaction: transaction range of IDs currently pinned"; + stats->txn_pinned_checkpoint_range.desc = + "transaction: transaction range of IDs currently pinned by a checkpoint"; stats->txn_commit.desc = "transaction: transactions committed"; stats->txn_rollback.desc = "transaction: transactions rolled back"; } @@ -580,8 +593,6 @@ __wt_stat_refresh_connection_stats(void *stats_arg) stats->cache_eviction_split.v = 0; stats->cache_eviction_walk.v = 0; stats->cache_write.v = 0; - stats->cache_bytes_dirty.v = 0; - stats->cache_pages_dirty.v = 0; stats->cache_eviction_clean.v = 0; stats->memory_allocation.v = 0; stats->memory_free.v = 0; @@ -626,7 +637,6 @@ __wt_stat_refresh_connection_stats(void *stats_arg) stats->log_sync_dir.v = 0; stats->log_writes.v = 0; stats->log_slot_consolidated.v = 0; - stats->log_prealloc_max.v = 0; stats->log_prealloc_files.v = 0; stats->log_prealloc_used.v = 0; stats->log_slot_toobig.v = 0; diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 5b8f11a88a5..a1bec569ce7 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -12,7 +12,7 @@ * __wt_txnid_cmp -- * Compare transaction IDs for sorting / searching. */ -int +int WT_CDECL __wt_txnid_cmp(const void *v1, const void *v2) { uint64_t id1, id2; @@ -142,15 +142,20 @@ __wt_txn_refresh(WT_SESSION_IMPL *session, int get_snapshot) /* Walk the array of concurrent transactions. */ WT_ORDERED_READ(session_cnt, conn->session_cnt); for (i = n = 0, s = txn_global->states; i < session_cnt; i++, s++) { + /* Skip the checkpoint transaction; it is never read from. */ + if (txn_global->checkpoint_id != WT_TXN_NONE && + s->id == txn_global->checkpoint_id) + continue; + /* * Build our snapshot of any concurrent transaction IDs. * - * Ignore our own ID: we always read our own updates. - * - * Also ignore the ID if it is older than the oldest ID we saw. - * This can happen if we race with a thread that is allocating - * an ID -- the ID will not be used because the thread will - * keep spinning until it gets a valid one. + * Ignore: + * - Our own ID: we always read our own updates. + * - The ID if it is older than the oldest ID we saw. This + * can happen if we race with a thread that is allocating + * an ID -- the ID will not be used because the thread will + * keep spinning until it gets a valid one. */ if (s != txn_state && (id = s->id) != WT_TXN_NONE && @@ -216,6 +221,14 @@ __wt_txn_refresh(WT_SESSION_IMPL *session, int get_snapshot) WT_ATOMIC_CAS4(txn_global->scan_count, 1, -1)) { WT_ORDERED_READ(session_cnt, conn->session_cnt); for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { + /* + * Skip the checkpoint transaction; it is never read + * from. + */ + if (txn_global->checkpoint_id != WT_TXN_NONE && + s->id == txn_global->checkpoint_id) + continue; + if ((id = s->id) != WT_TXN_NONE && TXNID_LT(id, oldest_id)) oldest_id = id; @@ -499,13 +512,19 @@ __wt_txn_stats_update(WT_SESSION_IMPL *session) WT_TXN_GLOBAL *txn_global; WT_CONNECTION_IMPL *conn; WT_CONNECTION_STATS *stats; + uint64_t checkpoint_snap_min; conn = S2C(session); txn_global = &conn->txn_global; stats = &conn->stats; + checkpoint_snap_min = txn_global->checkpoint_snap_min; WT_STAT_SET(stats, txn_pinned_range, txn_global->current - txn_global->oldest_id); + + WT_STAT_SET(stats, txn_pinned_checkpoint_range, + checkpoint_snap_min == WT_TXN_NONE ? + 0 : txn_global->current - checkpoint_snap_min); } /* diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 87b85eb2d8d..7c1532390f9 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -301,16 +301,53 @@ __checkpoint_stats( } /* + * __checkpoint_verbose_track -- + * Output a verbose message with timing information + */ +static int +__checkpoint_verbose_track(WT_SESSION_IMPL *session, + const char *msg, struct timespec *start) +{ +#ifdef HAVE_VERBOSE + struct timespec stop; + uint64_t msec; + + if (!WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) + return (0); + + WT_RET(__wt_epoch(session, &stop)); + + /* + * Get time diff in microseconds. + */ + msec = WT_TIMEDIFF(stop, *start) / WT_MILLION; + WT_RET(__wt_verbose(session, + WT_VERB_CHECKPOINT, "time: %" PRIu64 " us, gen: %" PRIu64 + ": Full database checkpoint %s", + msec, S2C(session)->txn_global.checkpoint_gen, msg)); + + /* Update the timestamp so we are reporting intervals. */ + memcpy(start, &stop, sizeof(*start)); +#else + WT_UNUSED(session); + WT_UNUSED(msg); + WT_UNUSED(start); +#endif + return (0); +} + +/* * __wt_txn_checkpoint -- * Checkpoint a database or a list of objects in the database. */ int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) { - struct timespec start, stop; + struct timespec start, stop, verb_timer; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; WT_TXN_ISOLATION saved_isolation; const char *txn_cfg[] = { WT_CONFIG_BASE(session, session_begin_transaction), @@ -320,6 +357,7 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) u_int i; conn = S2C(session); + txn_global = &conn->txn_global; saved_isolation = session->isolation; txn = &session->txn; full = logging = tracking = 0; @@ -355,6 +393,10 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) /* Flush data-sources before we start the checkpoint. */ WT_ERR(__checkpoint_data_source(session, cfg)); + WT_ERR(__wt_epoch(session, &verb_timer)); + WT_ERR(__checkpoint_verbose_track(session, + "starting write leaves", &verb_timer)); + /* Flush dirty leaf pages before we start the checkpoint. */ session->isolation = txn->isolation = TXN_ISO_READ_COMMITTED; WT_ERR(__checkpoint_apply(session, cfg, __checkpoint_write_leaves)); @@ -381,6 +423,9 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_txn_checkpoint_log( session, full, WT_TXN_LOG_CKPT_PREPARE, NULL)); + WT_ERR(__checkpoint_verbose_track(session, + "starting transaction", &verb_timer)); + /* * Start a snapshot transaction for the checkpoint. * @@ -392,6 +437,24 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_epoch(session, &start)); WT_ERR(__wt_txn_begin(session, txn_cfg)); + /* Ensure a transaction ID is allocated prior to sharing it globally */ + WT_ERR(__wt_txn_id_check(session)); + /* + * Save a copy of the checkpoint transaction ID so that refresh can + * skip the checkpoint IDs. Save a copy of the snap min so that + * visibility checks for the checkpoint use the right ID. + */ + txn_global->checkpoint_id = session->txn.id; + txn_global->checkpoint_snap_min = session->txn.snap_min; + + /* + * No need for this to be atomic it is only written while holding the + * checkpoint lock. + */ + txn_global->checkpoint_gen += 1; + WT_STAT_FAST_CONN_SET(session, + txn_checkpoint_generation, txn_global->checkpoint_gen); + /* Tell logging that we have started a database checkpoint. */ if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) && full) { WT_ERR(__wt_txn_checkpoint_log( @@ -401,9 +464,23 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint)); + /* + * Clear the dhandle so the visibility check doesn't get confused about + * the snap min. Don't bother restoring the handle since it doesn't + * make sense to carry a handle across a checkpoint. + */ + session->dhandle = NULL; + /* Commit the transaction before syncing the file(s). */ WT_ERR(__wt_txn_commit(session, NULL)); + /* Clear the global checkpoint transaction IDs */ + txn_global->checkpoint_id = WT_TXN_NONE; + txn_global->checkpoint_snap_min = WT_TXN_NONE; + + WT_ERR(__checkpoint_verbose_track(session, + "committing transaction", &verb_timer)); + /* * Checkpoints have to hit disk (it would be reasonable to configure for * lazy checkpoints, but we don't support them yet). @@ -411,6 +488,9 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) if (F_ISSET(conn, WT_CONN_CKPT_SYNC)) WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); + WT_ERR(__checkpoint_verbose_track(session, + "sync completed", &verb_timer)); + /* * Disable metadata tracking during the metadata checkpoint. * @@ -430,6 +510,10 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) ret = __wt_checkpoint_sync(session, NULL)); WT_ERR(ret); } + + WT_ERR(__checkpoint_verbose_track(session, + "metadata sync completed", &verb_timer)); + if (full) { WT_ERR(__wt_epoch(session, &stop)); __checkpoint_stats(session, &start, &stop); @@ -450,10 +534,22 @@ err: /* */ session->isolation = txn->isolation = TXN_ISO_READ_UNCOMMITTED; if (tracking) - WT_TRET(__wt_meta_track_off(session, ret != 0)); + WT_TRET(__wt_meta_track_off(session, 0, ret != 0)); - if (F_ISSET(txn, TXN_RUNNING)) + if (F_ISSET(txn, TXN_RUNNING)) { + /* + * Clear the dhandle so the visibility check doesn't get + * confused about the snap min. Don't bother restoring the + * handle since it doesn't make sense to carry a handle across + * a checkpoint. + */ + session->dhandle = NULL; WT_TRET(__wt_txn_rollback(session, NULL)); + } + + /* Ensure the checkpoint IDs are cleared on the error path. */ + txn_global->checkpoint_id = WT_TXN_NONE; + txn_global->checkpoint_snap_min = WT_TXN_NONE; /* Tell logging that we have finished a database checkpoint. */ if (logging) @@ -926,7 +1022,15 @@ fake: /* WT_ERR(__wt_txn_checkpoint_log( session, 0, WT_TXN_LOG_CKPT_STOP, NULL)); -done: err: + /* + * Update the checkpoint generation for this handle so visible + * updates newer than the checkpoint can be evicted. + */ +done: btree->checkpoint_gen = conn->txn_global.checkpoint_gen; + WT_STAT_FAST_DATA_SET(session, + btree_checkpoint_generation, btree->checkpoint_gen); + +err: /* * If the checkpoint didn't complete successfully, make sure the * tree is marked dirty. @@ -986,30 +1090,50 @@ __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) * Checkpoint a single file as part of closing the handle. */ int -__wt_checkpoint_close(WT_SESSION_IMPL *session, int force) +__wt_checkpoint_close(WT_SESSION_IMPL *session, int final, int force) { + WT_BTREE *btree; WT_DECL_RET; + int bulk, need_tracking; + + btree = S2BT(session); + bulk = F_ISSET(btree, WT_BTREE_BULK) ? 1 : 0; /* Handle forced discard (when dropping a file). */ if (force) return (__wt_cache_op(session, NULL, WT_SYNC_DISCARD_FORCE)); - /* If closing an unmodified file, try to evict its pages. */ - if (!S2BT(session)->modified) { - ret = __wt_cache_op(session, NULL, WT_SYNC_DISCARD); - if (ret != EBUSY) - return (ret); + /* + * If closing an unmodified file, check that no update is required + * for active readers. + */ + if (!btree->modified && !bulk) { + __wt_txn_update_oldest(session); + return (__wt_txn_visible_all(session, btree->rec_max_txn) ? + __wt_cache_op(session, NULL, WT_SYNC_DISCARD) : EBUSY); } /* - * If closing a modified file, or closing an unmodified file was blocked - * for any reason, checkpoint the file and optionally flush the writes - * (the checkpoint call will discard the blocks, there's no additional - * step needed). + * If closing a modified file, checkpoint the file and optionally flush + * the writes (the checkpoint call will discard the blocks, there's no + * additional step needed). + * + * We should already have the schema lock unless we're finishing a bulk + * load -- the only other paths to closing files (sweep and LSM) have + * already checked for read-only trees. */ - WT_RET(__checkpoint_worker(session, NULL, 0)); - if (F_ISSET(S2C(session), WT_CONN_CKPT_SYNC)) - WT_RET(__wt_checkpoint_sync(session, NULL)); + if (!final) + WT_ASSERT(session, + bulk || F_ISSET(session, WT_SESSION_SCHEMA_LOCKED)); - return (0); + need_tracking = !bulk && !final && !WT_META_TRACKING(session); + if (need_tracking) + WT_RET(__wt_meta_track_on(session)); + + WT_TRET(__checkpoint_worker(session, NULL, 0)); + + if (need_tracking) + WT_RET(__wt_meta_track_off(session, 1, ret != 0)); + + return (ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index afe98d95c6f..d3e010cf401 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -221,11 +221,12 @@ __txn_log_file_sync(WT_SESSION_IMPL *session, uint32_t flags, WT_LSN *lsnp) WT_DECL_RET; size_t header_size; uint32_t rectype = WT_LOGREC_FILE_SYNC; - int start; + int start, need_sync; const char *fmt = WT_UNCHECKED_STRING(III); btree = S2BT(session); start = LF_ISSET(WT_TXN_LOG_CKPT_START); + need_sync = LF_ISSET(WT_TXN_LOG_CKPT_SYNC); WT_RET(__wt_struct_size( session, &header_size, fmt, rectype, btree->id, start)); @@ -236,7 +237,8 @@ __txn_log_file_sync(WT_SESSION_IMPL *session, uint32_t flags, WT_LSN *lsnp) fmt, rectype, btree->id, start)); logrec->size += (uint32_t)header_size; - WT_ERR(__wt_log_write(session, logrec, lsnp, 0)); + WT_ERR(__wt_log_write( + session, logrec, lsnp, need_sync ? WT_LOG_FSYNC : 0)); err: __wt_logrec_free(session, &logrec); return (ret); } @@ -360,6 +362,8 @@ __wt_txn_checkpoint_log( __wt_scr_free(session, &txn->ckpt_snapshot); txn->full_ckpt = 0; break; + + WT_ILLEGAL_VALUE_ERR(session); } err: __wt_logrec_free(session, &logrec); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 2c834083691..bc098875abf 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -304,6 +304,7 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) { WT_CONFIG_ITEM cval; WT_LSN lsn; + intmax_t offset; uint32_t fileid; WT_RET(__wt_config_getones(r->session, config, "id", &cval)); @@ -325,8 +326,10 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) /* If there is checkpoint logged for the file, apply everything. */ if (cval.type != WT_CONFIG_ITEM_STRUCT) WT_INIT_LSN(&lsn); - else if (sscanf(cval.str, "(%" PRIu32 ",%" PRIdMAX ")", - &lsn.file, (intmax_t*)&lsn.offset) != 2) + else if (sscanf(cval.str, + "(%" SCNu32 ",%" SCNdMAX ")", &lsn.file, &offset) == 2) + lsn.offset = offset; + else WT_RET_MSG(r->session, EINVAL, "Failed to parse checkpoint LSN '%.*s'", (int)cval.len, cval.str); diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h index aaf68159c68..692b02e63f2 100644 --- a/src/third_party/wiredtiger/src/utilities/util.h +++ b/src/third_party/wiredtiger/src/utilities/util.h @@ -27,25 +27,25 @@ extern int __wt_optreset; /* reset getopt */ extern char *__wt_optarg; /* argument associated with option */ int util_backup(WT_SESSION *, int, char *[]); -int util_cerr(const char *, const char *, int); +int util_cerr(WT_CURSOR *, const char *, int); int util_compact(WT_SESSION *, int, char *[]); void util_copyright(void); int util_create(WT_SESSION *, int, char *[]); int util_drop(WT_SESSION *, int, char *[]); int util_dump(WT_SESSION *, int, char *[]); -int util_err(int, const char *, ...); +int util_err(WT_SESSION *, int, const char *, ...); int util_flush(WT_SESSION *, const char *); int util_list(WT_SESSION *, int, char *[]); int util_load(WT_SESSION *, int, char *[]); int util_loadtext(WT_SESSION *, int, char *[]); -char *util_name(const char *, const char *); +char *util_name(WT_SESSION *, const char *, const char *); int util_printlog(WT_SESSION *, int, char *[]); int util_read(WT_SESSION *, int, char *[]); -int util_read_line(ULINE *, int, int *); +int util_read_line(WT_SESSION *, ULINE *, int, int *); int util_rename(WT_SESSION *, int, char *[]); int util_salvage(WT_SESSION *, int, char *[]); int util_stat(WT_SESSION *, int, char *[]); -int util_str2recno(const char *p, uint64_t *recnop); +int util_str2recno(WT_SESSION *, const char *p, uint64_t *recnop); int util_upgrade(WT_SESSION *, int, char *[]); int util_verify(WT_SESSION *, int, char *[]); int util_write(WT_SESSION *, int, char *[]); diff --git a/src/third_party/wiredtiger/src/utilities/util_backup.c b/src/third_party/wiredtiger/src/utilities/util_backup.c index cd81cd9bfb5..cad91591a61 100644 --- a/src/third_party/wiredtiger/src/utilities/util_backup.c +++ b/src/third_party/wiredtiger/src/utilities/util_backup.c @@ -19,7 +19,7 @@ static char *cbuf; * Build a list of comma-separated targets. */ static int -append_target(const char *target, char **bufp) +append_target(WT_SESSION *session, const char *target, char **bufp) { static int first = 1; static size_t len = 0, remain = 0; @@ -30,7 +30,7 @@ append_target(const char *target, char **bufp) len += strlen(target) + 512; remain += strlen(target) + 512; if ((buf = realloc(buf, len)) == NULL) - return (util_err(errno, NULL)); + return (util_err(session, errno, NULL)); *bufp = buf; } if (first) { @@ -59,7 +59,7 @@ util_backup(WT_SESSION *session, int argc, char *argv[]) while ((ch = __wt_getopt(progname, argc, argv, "t:")) != EOF) switch (ch) { case 't': - if (append_target(__wt_optarg, &config)) + if (append_target(session, __wt_optarg, &config)) return (1); break; case '?': @@ -78,7 +78,7 @@ util_backup(WT_SESSION *session, int argc, char *argv[]) if ((ret = session->open_cursor( session, "backup:", NULL, config, &cursor)) != 0) { fprintf(stderr, "%s: cursor open(backup:) failed: %s\n", - progname, wiredtiger_strerror(ret)); + progname, session->strerror(session, ret)); goto err; } @@ -93,7 +93,7 @@ util_backup(WT_SESSION *session, int argc, char *argv[]) if (ret != 0) { fprintf(stderr, "%s: cursor next(backup:) failed: %s\n", - progname, wiredtiger_strerror(ret)); + progname, session->strerror(session, ret)); goto err; } diff --git a/src/third_party/wiredtiger/src/utilities/util_compact.c b/src/third_party/wiredtiger/src/utilities/util_compact.c index 633a92df04a..bdfb2cd55d7 100644 --- a/src/third_party/wiredtiger/src/utilities/util_compact.c +++ b/src/third_party/wiredtiger/src/utilities/util_compact.c @@ -30,12 +30,12 @@ util_compact(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((uri = util_name(*argv, "table")) == NULL) + if ((uri = util_name(session, *argv, "table")) == NULL) return (1); if ((ret = session->compact(session, uri, NULL)) != 0) { fprintf(stderr, "%s: compact(%s): %s\n", - progname, uri, wiredtiger_strerror(ret)); + progname, uri, session->strerror(session, ret)); goto err; } diff --git a/src/third_party/wiredtiger/src/utilities/util_create.c b/src/third_party/wiredtiger/src/utilities/util_create.c index 94ef8f4e3cb..06ea5edd8cc 100644 --- a/src/third_party/wiredtiger/src/utilities/util_create.c +++ b/src/third_party/wiredtiger/src/utilities/util_create.c @@ -35,11 +35,11 @@ util_create(WT_SESSION *session, int argc, char *argv[]) if (argc != 1) return (usage()); - if ((uri = util_name(*argv, "table")) == NULL) + if ((uri = util_name(session, *argv, "table")) == NULL) return (1); if ((ret = session->create(session, uri, config)) != 0) - return (util_err(ret, "%s: session.create", uri)); + return (util_err(session, ret, "%s: session.create", uri)); return (0); } diff --git a/src/third_party/wiredtiger/src/utilities/util_drop.c b/src/third_party/wiredtiger/src/utilities/util_drop.c index 15f6ea1be0c..722ffc31738 100644 --- a/src/third_party/wiredtiger/src/utilities/util_drop.c +++ b/src/third_party/wiredtiger/src/utilities/util_drop.c @@ -30,7 +30,7 @@ util_drop(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the uri. */ if (argc != 1) return (usage()); - if ((name = util_name(*argv, "table")) == NULL) + if ((name = util_name(session, *argv, "table")) == NULL) return (1); ret = session->drop(session, name, "force"); diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index f7d14209fa4..b02186cee75 100644 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -9,21 +9,21 @@ #include "util.h" static int dump_config(WT_SESSION *, const char *, int); -static int dump_json_begin(void); -static int dump_json_end(void); -static int dump_json_separator(void); +static int dump_json_begin(WT_SESSION *); +static int dump_json_end(WT_SESSION *); +static int dump_json_separator(WT_SESSION *); static int dump_json_table_begin( WT_SESSION *, WT_CURSOR *, const char *, const char *); -static int dump_json_table_cg(WT_SESSION *, WT_CURSOR *, - const char *, const char *, const char *, const char *); +static int dump_json_table_cg( + WT_SESSION *, WT_CURSOR *, const char *, const char *, const char *); static int dump_json_table_config(WT_SESSION *, const char *); -static int dump_json_table_end(void); -static int dump_prefix(int); -static int dump_record(WT_CURSOR *, const char *, int, int); -static int dump_suffix(void); +static int dump_json_table_end(WT_SESSION *); +static int dump_prefix(WT_SESSION *, int); +static int dump_record(WT_CURSOR *, int, int); +static int dump_suffix(WT_SESSION *); static int dump_table_config(WT_SESSION *, WT_CURSOR *, const char *); -static int dump_table_config_type(WT_SESSION *, - WT_CURSOR *, WT_CURSOR *, const char *, const char *, const char *); +static int dump_table_config_type( + WT_SESSION *, WT_CURSOR *, WT_CURSOR *, const char *, const char *); static int dup_json_string(const char *, char **); static int print_config(WT_SESSION *, const char *, const char *, const char *); static int usage(void); @@ -46,8 +46,8 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) break; case 'f': /* output file */ if (freopen(__wt_optarg, "w", stdout) == NULL) - return ( - util_err(errno, "%s: reopen", __wt_optarg)); + return (util_err( + session, errno, "%s: reopen", __wt_optarg)); break; case 'j': json = 1; @@ -77,18 +77,18 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) if (argc < 1 || (argc != 1 && !json)) return (usage()); - if (json && (ret = dump_json_begin()) != 0) + if (json && (ret = dump_json_begin(session)) != 0) goto err; for (i = 0; i < argc; i++) { if (json && i > 0) - if ((ret = dump_json_separator()) != 0) + if ((ret = dump_json_separator(session)) != 0) goto err; if (name != NULL) { free(name); name = NULL; } - if ((name = util_name(argv[i], "table")) == NULL) + if ((name = util_name(session, argv[i], "table")) == NULL) goto err; if (json && dump_json_table_config(session, name) != 0) @@ -115,16 +115,16 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) if ((ret = session->open_cursor( session, name, NULL, config, &cursor)) != 0) { fprintf(stderr, "%s: cursor open(%s) failed: %s\n", - progname, name, wiredtiger_strerror(ret)); + progname, name, session->strerror(session, ret)); goto err; } - if ((ret = dump_record(cursor, name, reverse, json)) != 0) + if ((ret = dump_record(cursor, reverse, json)) != 0) goto err; - if (json && (ret = dump_json_table_end()) != 0) + if (json && (ret = dump_json_table_end(session)) != 0) goto err; } - if (json && ((ret = dump_json_end()) != 0)) + if (json && ((ret = dump_json_end(session)) != 0)) goto err; if (0) { @@ -154,7 +154,7 @@ dump_config(WT_SESSION *session, const char *uri, int hex) if ((ret = session->open_cursor( session, WT_METADATA_URI, NULL, NULL, &cursor)) != 0) { fprintf(stderr, "%s: %s: session.open_cursor: %s\n", - progname, WT_METADATA_URI, wiredtiger_strerror(ret)); + progname, WT_METADATA_URI, session->strerror(session, ret)); return (1); } /* @@ -164,17 +164,17 @@ dump_config(WT_SESSION *session, const char *uri, int hex) */ cursor->set_key(cursor, uri); if ((ret = cursor->search(cursor)) == 0) { - if (dump_prefix(hex) != 0 || + if (dump_prefix(session, hex) != 0 || dump_table_config(session, cursor, uri) != 0 || - dump_suffix() != 0) + dump_suffix(session) != 0) ret = 1; } else if (ret == WT_NOTFOUND) - ret = util_err(0, "%s: No such object exists", uri); + ret = util_err(session, 0, "%s: No such object exists", uri); else - ret = util_err(ret, "%s", uri); + ret = util_err(session, ret, "%s", uri); if ((tret = cursor->close(cursor)) != 0) { - tret = util_cerr(uri, "close", tret); + tret = util_cerr(cursor, "close", tret); if (ret == 0) ret = tret; } @@ -187,10 +187,10 @@ dump_config(WT_SESSION *session, const char *uri, int hex) * Output the dump file header prefix. */ static int -dump_json_begin(void) +dump_json_begin(WT_SESSION *session) { if (printf("{\n") < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); return (0); } @@ -199,10 +199,10 @@ dump_json_begin(void) * Output the dump file header suffix. */ static int -dump_json_end(void) +dump_json_end(WT_SESSION *session) { if (printf("\n}\n") < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); return (0); } @@ -211,10 +211,10 @@ dump_json_end(void) * Output the dump file header prefix. */ static int -dump_json_separator(void) +dump_json_separator(WT_SESSION *session) { if (printf(",\n") < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); return (0); } @@ -241,29 +241,29 @@ dump_json_table_begin( if ((ret = __wt_session_create_strip(session, config, NULL, &stripped)) != 0) - return (util_err(ret, NULL)); + return (util_err(session, ret, NULL)); ret = dup_json_string(stripped, &jsonconfig); free(stripped); if (ret != 0) - return (util_cerr(uri, "config dup", ret)); + return (util_cerr(cursor, "config dup", ret)); if (printf(" \"%s\" : [\n {\n", uri) < 0) goto eio; if (printf(" \"config\" : \"%s\",\n", jsonconfig) < 0) goto eio; if ((ret = dump_json_table_cg( - session, cursor, uri, name, "colgroup:", "colgroups")) == 0) { + session, cursor, name, "colgroup:", "colgroups")) == 0) { if (printf(",\n") < 0) goto eio; ret = dump_json_table_cg( - session, cursor, uri, name, "index:", "indices"); + session, cursor, name, "index:", "indices"); } if (printf("\n },\n {\n \"data\" : [") < 0) goto eio; if (0) { -eio: ret = util_err(EIO, NULL); +eio: ret = util_err(session, EIO, NULL); } free(jsonconfig); @@ -276,7 +276,7 @@ eio: ret = util_err(EIO, NULL); */ static int dump_json_table_cg(WT_SESSION *session, WT_CURSOR *cursor, - const char *uri, const char *name, const char *entry, const char *header) + const char *name, const char *entry, const char *header) { WT_DECL_RET; const char *key, *skip, *value; @@ -286,14 +286,14 @@ dump_json_table_cg(WT_SESSION *session, WT_CURSOR *cursor, once = 0; if (printf(" \"%s\" : [", header) < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); /* * For table dumps, we're done. */ if (cursor == NULL) { if (printf("]") < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); else return (0); } @@ -307,13 +307,13 @@ dump_json_table_cg(WT_SESSION *session, WT_CURSOR *cursor, if ((ret = cursor->search_near(cursor, &exact)) != 0) { if (ret == WT_NOTFOUND) return (0); - return (util_cerr(uri, "search_near", ret)); + return (util_cerr(cursor, "search_near", ret)); } if (exact >= 0) goto match; while ((ret = cursor->next(cursor)) == 0) { match: if ((ret = cursor->get_key(cursor, &key)) != 0) - return (util_cerr(uri, "get_key", ret)); + return (util_cerr(cursor, "get_key", ret)); /* Check if we've finished the list of entries. */ if (!WT_PREFIX_MATCH(key, entry)) @@ -327,15 +327,15 @@ match: if ((ret = cursor->get_key(cursor, &key)) != 0) /* Get the value. */ if ((ret = cursor->get_value(cursor, &value)) != 0) - return (util_cerr(uri, "get_value", ret)); + return (util_cerr(cursor, "get_value", ret)); if ((ret = __wt_session_create_strip( session, value, NULL, &stripped)) != 0) - return (util_err(ret, NULL)); + return (util_err(session, ret, NULL)); ret = dup_json_string(stripped, &jsonconfig); free(stripped); if (ret != 0) - return (util_cerr(uri, "config dup", ret)); + return (util_cerr(cursor, "config dup", ret)); ret = printf("%s\n" "%s{\n" "%s \"uri\" : \"%s\",\n" @@ -345,15 +345,15 @@ match: if ((ret = cursor->get_key(cursor, &key)) != 0) indent, indent, key, indent, jsonconfig, indent); free(jsonconfig); if (ret < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); once = 1; } if (printf("%s]", (once == 0 ? "" : "\n ")) < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); if (ret == 0 || ret == WT_NOTFOUND) return (0); - return (util_cerr(uri, "next", ret)); + return (util_cerr(cursor, "next", ret)); } /* @@ -376,7 +376,7 @@ dump_json_table_config(WT_SESSION *session, const char *uri) session, WT_METADATA_URI, NULL, NULL, &cursor)) != 0) { fprintf(stderr, "%s: %s: session.open_cursor: %s\n", progname, WT_METADATA_URI, - wiredtiger_strerror(ret)); + session->strerror(session, ret)); return (1); } @@ -389,17 +389,18 @@ dump_json_table_config(WT_SESSION *session, const char *uri) cursor->set_key(cursor, uri); if ((ret = cursor->search(cursor)) == 0) { if ((ret = cursor->get_value(cursor, &value)) != 0) - ret = util_cerr(uri, "get_value", ret); + ret = util_cerr(cursor, "get_value", ret); else if (dump_json_table_begin( session, cursor, uri, value) != 0) ret = 1; } else if (ret == WT_NOTFOUND) - ret = util_err(0, "%s: No such object exists", uri); + ret = util_err( + session, 0, "%s: No such object exists", uri); else - ret = util_err(ret, "%s", uri); + ret = util_err(session, ret, "%s", uri); if ((tret = cursor->close(cursor)) != 0) { - tret = util_cerr(uri, "close", tret); + tret = util_cerr(cursor, "close", tret); if (ret == 0) ret = tret; } @@ -421,9 +422,10 @@ dump_json_table_config(WT_SESSION *session, const char *uri) session, NULL, uri, value) != 0) ret = 1; } else if (ret == WT_NOTFOUND) - ret = util_err(0, "%s: No such object exists", uri); + ret = util_err( + session, 0, "%s: No such object exists", uri); else - ret = util_err(ret, "%s", uri); + ret = util_err(session, ret, "%s", uri); } return (ret); @@ -434,10 +436,10 @@ dump_json_table_config(WT_SESSION *session, const char *uri) * Output the JSON syntax that ends a table. */ static int -dump_json_table_end(void) +dump_json_table_end(WT_SESSION *session) { if (printf(" ]\n }\n ]") < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); return (0); } @@ -466,11 +468,11 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) */ cursor->set_key(cursor, uri); if ((ret = cursor->search(cursor)) != 0) - return (util_cerr(uri, "search", ret)); + return (util_cerr(cursor, "search", ret)); if ((ret = cursor->get_key(cursor, &key)) != 0) - return (util_cerr(uri, "get_key", ret)); + return (util_cerr(cursor, "get_key", ret)); if ((ret = cursor->get_value(cursor, &value)) != 0) - return (util_cerr(uri, "get_value", ret)); + return (util_cerr(cursor, "get_value", ret)); if (print_config(session, key, value, NULL) != 0) return (1); @@ -480,15 +482,15 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) */ if ((ret = session->open_cursor(session, NULL, cursor, NULL, &srch)) != 0) - return (util_cerr(uri, "open_cursor", ret)); + return (util_cerr(cursor, "open_cursor", ret)); if ((ret = dump_table_config_type( - session, cursor, srch, uri, name, "colgroup:")) == 0) + session, cursor, srch, name, "colgroup:")) == 0) ret = dump_table_config_type( - session, cursor, srch, uri, name, "index:"); + session, cursor, srch, name, "index:"); if ((tret = srch->close(srch)) != 0) { - tret = util_cerr(uri, "close", tret); + tret = util_cerr(cursor, "close", tret); if (ret == 0) ret = tret; } @@ -502,8 +504,7 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) */ static int dump_table_config_type(WT_SESSION *session, - WT_CURSOR *cursor, WT_CURSOR *srch, - const char *uri, const char *name, const char *entry) + WT_CURSOR *cursor, WT_CURSOR *srch, const char *name, const char *entry) { WT_CONFIG_ITEM cval; WT_DECL_RET; @@ -520,13 +521,13 @@ dump_table_config_type(WT_SESSION *session, if ((ret = cursor->search_near(cursor, &exact)) != 0) { if (ret == WT_NOTFOUND) return (0); - return (util_cerr(uri, "search_near", ret)); + return (util_cerr(cursor, "search_near", ret)); } if (exact >= 0) goto match; while ((ret = cursor->next(cursor)) == 0) { match: if ((ret = cursor->get_key(cursor, &key)) != 0) - return (util_cerr(uri, "get_key", ret)); + return (util_cerr(cursor, "get_key", ret)); /* Check if we've finished the list of entries. */ if (!WT_PREFIX_MATCH(key, entry)) @@ -540,39 +541,40 @@ match: if ((ret = cursor->get_key(cursor, &key)) != 0) /* Get the value. */ if ((ret = cursor->get_value(cursor, &value)) != 0) - return (util_cerr(uri, "get_value", ret)); + return (util_cerr(cursor, "get_value", ret)); /* Crack it and get the underlying source. */ if ((ret = __wt_config_getones( (WT_SESSION_IMPL *)session, value, "source", &cval)) != 0) - return (util_err(ret, "%s: source entry", key)); + return ( + util_err(session, ret, "%s: source entry", key)); /* Nul-terminate the source entry. */ if ((p = malloc(cval.len + 10)) == NULL) - return (util_err(errno, NULL)); + return (util_err(session, errno, NULL)); (void)strncpy(p, cval.str, cval.len); p[cval.len] = '\0'; srch->set_key(srch, p); if ((ret = srch->search(srch)) != 0) - ret = util_err(ret, "%s: %s", key, p); + ret = util_err(session, ret, "%s: %s", key, p); free(p); if (ret != 0) return (1); /* Get the source's value. */ if ((ret = srch->get_value(srch, &value_source)) != 0) - return (util_cerr(uri, "get_value", ret)); + return (util_cerr(cursor, "get_value", ret)); /* * The dumped configuration string is the original key plus the * source's configuration. */ if (print_config(session, key, value, value_source) != 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); } if (ret == 0 || ret == WT_NOTFOUND) return (0); - return (util_cerr(uri, "next", ret)); + return (util_cerr(cursor, "next", ret)); } /* @@ -580,7 +582,7 @@ match: if ((ret = cursor->get_key(cursor, &key)) != 0) * Output the dump file header prefix. */ static int -dump_prefix(int hex) +dump_prefix(WT_SESSION *session, int hex) { int vmajor, vminor, vpatch; @@ -591,7 +593,7 @@ dump_prefix(int hex) vmajor, vminor, vpatch) < 0 || printf("Format=%s\n", hex ? "hex" : "print") < 0 || printf("Header\n") < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); return (0); } @@ -601,12 +603,15 @@ dump_prefix(int hex) * with JSON formatting if needed. */ static int -dump_record(WT_CURSOR *cursor, const char *name, int reverse, int json) +dump_record(WT_CURSOR *cursor, int reverse, int json) { WT_DECL_RET; + WT_SESSION *session; const char *infix, *key, *prefix, *suffix, *value; int once; + session = cursor->session; + once = 0; if (json) { prefix = "\n{\n"; @@ -620,18 +625,18 @@ dump_record(WT_CURSOR *cursor, const char *name, int reverse, int json) while ((ret = (reverse ? cursor->prev(cursor) : cursor->next(cursor))) == 0) { if ((ret = cursor->get_key(cursor, &key)) != 0) - return (util_cerr(name, "get_key", ret)); + return (util_cerr(cursor, "get_key", ret)); if ((ret = cursor->get_value(cursor, &value)) != 0) - return (util_cerr(name, "get_value", ret)); + return (util_cerr(cursor, "get_value", ret)); if (printf("%s%s%s%s%s%s", (json && once) ? "," : "", prefix, key, infix, value, suffix) < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); once = 1; } if (json && once && printf("\n") < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); return (ret == WT_NOTFOUND ? 0 : - util_cerr(name, (reverse ? "prev" : "next"), ret)); + util_cerr(cursor, (reverse ? "prev" : "next"), ret)); } /* @@ -639,10 +644,10 @@ dump_record(WT_CURSOR *cursor, const char *name, int reverse, int json) * Output the dump file header suffix. */ static int -dump_suffix(void) +dump_suffix(WT_SESSION *session) { if (printf("Data\n") < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); return (0); } @@ -696,11 +701,11 @@ print_config(WT_SESSION *session, } if ((ret = __wt_session_create_strip(session, v1, v2, &value_ret)) != 0) - return (util_err(ret, NULL)); + return (util_err(session, ret, NULL)); ret = printf("%s\n%s\n", key, value_ret); free((char *)value_ret); if (ret < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); return (0); } diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index c7f05a5435f..270f06666a2 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -40,7 +40,7 @@ util_list(WT_SESSION *session, int argc, char *argv[]) case 0: break; case 1: - if ((name = util_name(*argv, "table")) == NULL) + if ((name = util_name(session, *argv, "table")) == NULL) return (1); break; default: @@ -78,7 +78,7 @@ list_print(WT_SESSION *session, const char *name, int cflag, int vflag) return (0); fprintf(stderr, "%s: %s: session.open_cursor: %s\n", - progname, WT_METADATA_URI, wiredtiger_strerror(ret)); + progname, WT_METADATA_URI, session->strerror(session, ret)); return (1); } @@ -86,7 +86,7 @@ list_print(WT_SESSION *session, const char *name, int cflag, int vflag) while ((ret = cursor->next(cursor)) == 0) { /* Get the key. */ if ((ret = cursor->get_key(cursor, &key)) != 0) - return (util_cerr("metadata", "get_key", ret)); + return (util_cerr(cursor, "get_key", ret)); /* * If a name is specified, only show objects that match. @@ -113,13 +113,12 @@ list_print(WT_SESSION *session, const char *name, int cflag, int vflag) return (ret); if (vflag) { if ((ret = cursor->get_value(cursor, &value)) != 0) - return ( - util_cerr("metadata", "get_value", ret)); + return (util_cerr(cursor, "get_value", ret)); printf("%s\n", value); } } if (ret != WT_NOTFOUND) - return (util_cerr("metadata", "next", ret)); + return (util_cerr(cursor, "next", ret)); if (!found) { fprintf(stderr, "%s: %s: not found\n", progname, name); return (1); diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c index 0083fab4bd8..994df10b70b 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.c +++ b/src/third_party/wiredtiger/src/utilities/util_load.c @@ -9,10 +9,10 @@ #include "util.h" #include "util_load.h" -static int config_read(char ***, int *); -static int config_rename(char **, const char *); +static int config_read(WT_SESSION *, char ***, int *); +static int config_rename(WT_SESSION *, char **, const char *); static void config_remove(char *, const char *); -static int format(void); +static int format(WT_SESSION *); static int insert(WT_CURSOR *, const char *); static int load_dump(WT_SESSION *); static int usage(void); @@ -41,7 +41,8 @@ util_load(WT_SESSION *session, int argc, char *argv[]) case 'f': /* input file */ if (freopen(__wt_optarg, "r", stdin) == NULL) return ( - util_err(errno, "%s: reopen", __wt_optarg)); + util_err(session, + errno, "%s: reopen", __wt_optarg)); else filename = __wt_optarg; break; @@ -63,7 +64,7 @@ util_load(WT_SESSION *session, int argc, char *argv[]) /* -a and -o are mutually exclusive. */ if (append == 1 && no_overwrite == 1) - return (util_err(EINVAL, + return (util_err(session, EINVAL, "the -a (append) and -n (no-overwrite) flags are mutually " "exclusive")); @@ -102,11 +103,11 @@ load_dump(WT_SESSION *session) uri = NULL; /* Read the metadata file. */ - if ((ret = config_read(&list, &hex)) != 0) + if ((ret = config_read(session, &list, &hex)) != 0) return (ret); /* Reorder and check the list. */ - if ((ret = config_reorder(list)) != 0) + if ((ret = config_reorder(session, list)) != 0) goto err; /* Update the config based on any command-line configuration. */ @@ -125,7 +126,7 @@ load_dump(WT_SESSION *session) append ? ",append" : "", no_overwrite ? ",overwrite=false" : ""); if ((ret = session->open_cursor( session, uri, NULL, config, &cursor)) != 0) { - ret = util_err(ret, "%s: session.open", uri); + ret = util_err(session, ret, "%s: session.open", uri); goto err; } @@ -148,7 +149,7 @@ err: /* * the close succeed, it's better to fail early when loading files. */ if (cursor != NULL && (tret = cursor->close(cursor)) != 0) { - tret = util_err(tret, "%s: cursor.close", uri); + tret = util_err(session, tret, "%s: cursor.close", uri); if (ret == 0) ret = tret; } @@ -173,22 +174,23 @@ config_exec(WT_SESSION *session, char **list) for (; *list != NULL; list += 2) if ((ret = session->create(session, list[0], list[1])) != 0) - return (util_err(ret, "%s: session.create", list[0])); + return (util_err( + session, ret, "%s: session.create", list[0])); return (0); } /* - * config_list_free -- + * config_list_add -- * Add a value to the config list. */ int -config_list_add(CONFIG_LIST *clp, char *val) +config_list_add(WT_SESSION *session, CONFIG_LIST *clp, char *val) { if (clp->entry + 1 >= clp->max_entry) if ((clp->list = realloc(clp->list, (size_t) (clp->max_entry += 100) * sizeof(char *))) == NULL) /* List already freed by realloc. */ - return (util_err(errno, NULL)); + return (util_err(session, errno, NULL)); clp->list[clp->entry++] = val; clp->list[clp->entry] = NULL; @@ -216,7 +218,7 @@ config_list_free(CONFIG_LIST *clp) * Read the config lines and do some basic validation. */ static int -config_read(char ***listp, int *hexp) +config_read(WT_SESSION *session, char ***listp, int *hexp) { ULINE l; WT_DECL_RET; @@ -228,31 +230,31 @@ config_read(char ***listp, int *hexp) memset(&l, 0, sizeof(l)); /* Header line #1: "WiredTiger Dump" and a WiredTiger version. */ - if (util_read_line(&l, 0, &eof)) + if (util_read_line(session, &l, 0, &eof)) return (1); s = "WiredTiger Dump "; if (strncmp(l.mem, s, strlen(s)) != 0) - return (format()); + return (format(session)); /* Header line #2: "Format={hex,print}". */ - if (util_read_line(&l, 0, &eof)) + if (util_read_line(session, &l, 0, &eof)) return (1); if (strcmp(l.mem, "Format=print") == 0) *hexp = 0; else if (strcmp(l.mem, "Format=hex") == 0) *hexp = 1; else - return (format()); + return (format(session)); /* Header line #3: "Header". */ - if (util_read_line(&l, 0, &eof)) + if (util_read_line(session, &l, 0, &eof)) return (1); if (strcmp(l.mem, "Header") != 0) - return (format()); + return (format(session)); /* Now, read in lines until we get to the end of the headers. */ for (entry = max_entry = 0, list = NULL;; ++entry) { - if ((ret = util_read_line(&l, 0, &eof)) != 0) + if ((ret = util_read_line(session, &l, 0, &eof)) != 0) goto err; if (strcmp(l.mem, "Data") == 0) break; @@ -264,7 +266,7 @@ config_read(char ***listp, int *hexp) if (entry + 1 >= max_entry) { if ((tlist = realloc(list, (size_t) (max_entry += 100) * sizeof(char *))) == NULL) { - ret = util_err(errno, NULL); + ret = util_err(session, errno, NULL); /* * List already freed by realloc, still use err @@ -276,7 +278,7 @@ config_read(char ***listp, int *hexp) list = tlist; } if ((list[entry] = strdup(l.mem)) == NULL) { - ret = util_err(errno, NULL); + ret = util_err(session, errno, NULL); goto err; } list[entry + 1] = NULL; @@ -284,7 +286,7 @@ config_read(char ***listp, int *hexp) /* Headers are required, and they're supposed to be in pairs. */ if (list == NULL || entry % 2 != 0) { - ret = format(); + ret = format(session); goto err; } *listp = list; @@ -304,7 +306,7 @@ err: if (list != NULL) { * For other dumps, make any needed checks. */ int -config_reorder(char **list) +config_reorder(WT_SESSION *session, char **list) { char **entry, *p; @@ -323,7 +325,7 @@ config_reorder(char **list) if ((list[0] == NULL || list[1] == NULL || list[2] != NULL) || (WT_PREFIX_MATCH(list[0], "file:") && WT_PREFIX_MATCH(list[0], "lsm:"))) - return (format()); + return (format(session)); entry = list; } @@ -366,7 +368,7 @@ config_update(WT_SESSION *session, char **list) WT_PREFIX_MATCH(*listp, "file:") || WT_PREFIX_MATCH(*listp, "index:") || WT_PREFIX_MATCH(*listp, "table:")) - if (config_rename(listp, cmdname)) + if (config_rename(session, listp, cmdname)) return (1); /* @@ -376,7 +378,7 @@ config_update(WT_SESSION *session, char **list) */ for (configp = cmdconfig; cmdconfig != NULL && *configp != NULL; configp += 2) - if (config_rename(configp, cmdname)) + if (config_rename(session, configp, cmdname)) return (1); } @@ -399,7 +401,7 @@ config_update(WT_SESSION *session, char **list) cmdconfig != NULL && *configp != NULL; configp += 2) if (strstr(configp[1], "key_format=") || strstr(configp[1], "value_format=")) - return (util_err(0, + return (util_err(session, 0, "the command line configuration string may not " "modify the object's key or value format")); @@ -436,13 +438,13 @@ config_update(WT_SESSION *session, char **list) } switch (found) { case 0: - return (util_err(0, + return (util_err(session, 0, "the command line object name %s was not matched " "by any loaded object name", *configp)); case 1: break; default: - return (util_err(0, + return (util_err(session, 0, "the command line object name %s was not unique, " "matching more than a single loaded object name", *configp)); @@ -458,7 +460,7 @@ config_update(WT_SESSION *session, char **list) * Update the URI name. */ static int -config_rename(char **urip, const char *name) +config_rename(WT_SESSION *session, char **urip, const char *name) { size_t len; char *buf, *p; @@ -466,7 +468,7 @@ config_rename(char **urip, const char *name) /* Allocate room. */ len = strlen(*urip) + strlen(name) + 10; if ((buf = malloc(len)) == NULL) - return (util_err(errno, NULL)); + return (util_err(session, errno, NULL)); /* * Find the separating colon characters, but not the trailing one may @@ -474,7 +476,7 @@ config_rename(char **urip, const char *name) */ if ((p = strchr(*urip, ':')) == NULL) { free(buf); - return (format()); + return (format(session)); } *p = '\0'; p = strchr(p + 1, ':'); @@ -529,9 +531,10 @@ config_remove(char *config, const char *ckey) * The input doesn't match the dump format. */ static int -format(void) +format(WT_SESSION *session) { - return (util_err(0, "input does not match WiredTiger dump format")); + return (util_err( + session, 0, "input does not match WiredTiger dump format")); } /* @@ -543,9 +546,12 @@ insert(WT_CURSOR *cursor, const char *name) { ULINE key, value; WT_DECL_RET; + WT_SESSION *session; uint64_t insert_count; int eof; + session = cursor->session; + memset(&key, 0, sizeof(key)); memset(&value, 0, sizeof(value)); @@ -557,19 +563,20 @@ insert(WT_CURSOR *cursor, const char *name) * and ignore it (a dump with "append" set), or not read it at * all (flat-text load). */ - if (util_read_line(&key, 1, &eof)) + if (util_read_line(session, &key, 1, &eof)) return (1); if (eof == 1) break; if (!append) cursor->set_key(cursor, key.mem); - if (util_read_line(&value, 0, &eof)) + if (util_read_line(session, &value, 0, &eof)) return (1); cursor->set_value(cursor, value.mem); if ((ret = cursor->insert(cursor)) != 0) - return (util_err(ret, "%s: cursor.insert", name)); + return ( + util_err(session, ret, "%s: cursor.insert", name)); /* Report on progress every 100 inserts. */ if (verbose && ++insert_count % 100 == 0) { diff --git a/src/third_party/wiredtiger/src/utilities/util_load.h b/src/third_party/wiredtiger/src/utilities/util_load.h index 43ae1ce63b0..ca359ce662b 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.h +++ b/src/third_party/wiredtiger/src/utilities/util_load.h @@ -16,9 +16,9 @@ typedef struct { } CONFIG_LIST; int config_exec(WT_SESSION *, char **); -int config_list_add(CONFIG_LIST *, char *); +int config_list_add(WT_SESSION *, CONFIG_LIST *, char *); void config_list_free(CONFIG_LIST *); -int config_reorder(char **); +int config_reorder(WT_SESSION *, char **); int config_update(WT_SESSION *, char **); /* Flags for util_load_json */ diff --git a/src/third_party/wiredtiger/src/utilities/util_load_json.c b/src/third_party/wiredtiger/src/utilities/util_load_json.c index 7ccabf87010..f498b821073 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load_json.c +++ b/src/third_party/wiredtiger/src/utilities/util_load_json.c @@ -45,8 +45,9 @@ static int json_data(WT_SESSION *, JSON_INPUT_STATE *, CONFIG_LIST *, uint32_t); static int json_expect(WT_SESSION *, JSON_INPUT_STATE *, int); static int json_peek(WT_SESSION *, JSON_INPUT_STATE *); static int json_skip(WT_SESSION *, JSON_INPUT_STATE *, const char **); -static int json_kvraw_append(JSON_INPUT_STATE *, const char *, size_t); -static int json_strdup(JSON_INPUT_STATE *, char **); +static int json_kvraw_append( + WT_SESSION *, JSON_INPUT_STATE *, const char *, size_t); +static int json_strdup(WT_SESSION *, JSON_INPUT_STATE *, char **); static int json_top_level(WT_SESSION *, JSON_INPUT_STATE *, uint32_t); #define JSON_STRING_MATCH(ins, match) \ @@ -66,8 +67,8 @@ static int json_top_level(WT_SESSION *, JSON_INPUT_STATE *, uint32_t); * Parse a column group or index entry from JSON input. */ static int -json_column_group_index(WT_SESSION *session, JSON_INPUT_STATE *ins, - CONFIG_LIST *clp, int idx) +json_column_group_index(WT_SESSION *session, + JSON_INPUT_STATE *ins, CONFIG_LIST *clp, int idx) { WT_DECL_RET; char *config, *p, *uri; @@ -85,8 +86,8 @@ json_column_group_index(WT_SESSION *session, JSON_INPUT_STATE *ins, JSON_EXPECT(session, ins, ':'); JSON_EXPECT(session, ins, 's'); - if ((ret = json_strdup(ins, &p)) != 0) { - ret = util_err(ret, NULL); + if ((ret = json_strdup(session, ins, &p)) != 0) { + ret = util_err(session, ret, NULL); goto err; } if (isconfig) @@ -102,8 +103,8 @@ json_column_group_index(WT_SESSION *session, JSON_INPUT_STATE *ins, JSON_EXPECT(session, ins, ':'); JSON_EXPECT(session, ins, 's'); - if ((ret = json_strdup(ins, &p)) != 0) { - ret = util_err(ret, NULL); + if ((ret = json_strdup(session, ins, &p)) != 0) { + ret = util_err(session, ret, NULL); goto err; } if (isconfig) @@ -113,12 +114,12 @@ json_column_group_index(WT_SESSION *session, JSON_INPUT_STATE *ins, JSON_EXPECT(session, ins, '}'); if ((idx && strncmp(uri, "index:", 6) != 0) || (!idx && strncmp(uri, "colgroup:", 9) != 0)) { - ret = util_err(EINVAL, + ret = util_err(session, EINVAL, "%s: misplaced colgroup or index", uri); goto err; } - if ((ret = config_list_add(clp, uri)) != 0 || - (ret = config_list_add(clp, config)) != 0) + if ((ret = config_list_add(session, clp, uri)) != 0 || + (ret = config_list_add(session, clp, config)) != 0) goto err; if (json_peek(session, ins) != ',') @@ -139,7 +140,9 @@ err: if (ret == 0) * Append to the kvraw buffer, which is used to collect all the * raw key/value pairs from JSON input. */ -static int json_kvraw_append(JSON_INPUT_STATE *ins, const char *str, size_t len) +static int +json_kvraw_append(WT_SESSION *session, + JSON_INPUT_STATE *ins, const char *str, size_t len) { char *tmp; size_t needsize; @@ -147,7 +150,7 @@ static int json_kvraw_append(JSON_INPUT_STATE *ins, const char *str, size_t len) if (len > 0) { needsize = strlen(ins->kvraw) + len + 2; if ((tmp = malloc(needsize)) == NULL) - return (util_err(errno, NULL)); + return (util_err(session, errno, NULL)); snprintf(tmp, needsize, "%s %.*s", ins->kvraw, (int)len, str); free(ins->kvraw); ins->kvraw = tmp; @@ -161,7 +164,7 @@ static int json_kvraw_append(JSON_INPUT_STATE *ins, const char *str, size_t len) * JSON string at the current input position. */ static int -json_strdup(JSON_INPUT_STATE *ins, char **resultp) +json_strdup(WT_SESSION *session, JSON_INPUT_STATE *ins, char **resultp) { WT_DECL_RET; char *result, *resultcpy; @@ -173,12 +176,12 @@ json_strdup(JSON_INPUT_STATE *ins, char **resultp) src = ins->tokstart + 1; /*strip "" from token */ srclen = ins->toklen - 2; if ((resultlen = __wt_json_strlen(src, srclen)) < 0) { - ret = util_err(EINVAL, "Invalid config string"); + ret = util_err(session, EINVAL, "Invalid config string"); goto err; } resultlen += 1; if ((result = (char *)malloc((size_t)resultlen)) == NULL) { - ret = util_err(errno, NULL); + ret = util_err(session, errno, NULL); goto err; } *resultp = result; @@ -186,7 +189,7 @@ json_strdup(JSON_INPUT_STATE *ins, char **resultp) if ((ret = __wt_json_strncpy(&resultcpy, (size_t)resultlen, src, srclen)) != 0) { - ret = util_err(ret, NULL); + ret = util_err(session, ret, NULL); goto err; } @@ -206,8 +209,8 @@ err: if (ret == 0) * values. */ static int -json_data(WT_SESSION *session, JSON_INPUT_STATE *ins, CONFIG_LIST *clp, - uint32_t flags) +json_data(WT_SESSION *session, + JSON_INPUT_STATE *ins, CONFIG_LIST *clp, uint32_t flags) { WT_CURSOR *cursor; WT_DECL_RET; @@ -222,7 +225,7 @@ json_data(WT_SESSION *session, JSON_INPUT_STATE *ins, CONFIG_LIST *clp, uri = NULL; /* Reorder and check the list. */ - if ((ret = config_reorder(clp->list)) != 0) + if ((ret = config_reorder(session, clp->list)) != 0) goto err; /* Update config based on command-line configuration. */ @@ -240,7 +243,7 @@ json_data(WT_SESSION *session, JSON_INPUT_STATE *ins, CONFIG_LIST *clp, LF_ISSET(LOAD_JSON_NO_OVERWRITE) ? ",overwrite=false" : ""); if ((ret = session->open_cursor( session, uri, NULL, config, &cursor)) != 0) { - ret = util_err(ret, "%s: session.open", uri); + ret = util_err(session, ret, "%s: session.open", uri); goto err; } keyformat = cursor->key_format; @@ -255,7 +258,7 @@ json_data(WT_SESSION *session, JSON_INPUT_STATE *ins, CONFIG_LIST *clp, JSON_EXPECT(session, ins, '{'); if (ins->kvraw == NULL) { if ((ins->kvraw = (char *)malloc(1)) == NULL) { - ret = util_err(errno, NULL); + ret = util_err(session, errno, NULL); goto err; } } @@ -274,14 +277,14 @@ json_data(WT_SESSION *session, JSON_INPUT_STATE *ins, CONFIG_LIST *clp, gotnolen = (endp - ins->tokstart); if (recno != gotno || ins->toklen != (size_t)gotnolen) { - ret = util_err(0, + ret = util_err(session, 0, "%s: recno out of order", uri); goto err; } } if (++nfield == nkeys) { size_t curpos = JSON_INPUT_POS(ins); - if ((ret = json_kvraw_append(ins, + if ((ret = json_kvraw_append(session, ins, (char *)ins->line.mem + ins->kvrawstart, curpos - ins->kvrawstart)) != 0) goto err; @@ -294,7 +297,8 @@ json_data(WT_SESSION *session, JSON_INPUT_STATE *ins, CONFIG_LIST *clp, if (json_peek(session, ins) != 's') goto err; } - if (json_kvraw_append(ins, ins->line.mem, JSON_INPUT_POS(ins))) + if (json_kvraw_append( + session, ins, ins->line.mem, JSON_INPUT_POS(ins))) goto err; ins->kvraw[keystrlen] = '\0'; @@ -303,7 +307,7 @@ json_data(WT_SESSION *session, JSON_INPUT_STATE *ins, CONFIG_LIST *clp, /* skip over inserted space and comma */ cursor->set_value(cursor, &ins->kvraw[keystrlen+2]); if ((ret = cursor->insert(cursor)) != 0) { - ret = util_err(ret, "%s: cursor.insert", uri); + ret = util_err(session, ret, "%s: cursor.insert", uri); goto err; } @@ -324,7 +328,7 @@ err: if (ret == 0) * the close succeed, it's better to fail early when loading files. */ if (cursor != NULL && (tret = cursor->close(cursor)) != 0) { - tret = util_err(tret, "%s: cursor.close", uri); + tret = util_err(session, tret, "%s: cursor.close", uri); if (ret == 0) ret = tret; } @@ -371,13 +375,16 @@ json_top_level(WT_SESSION *session, JSON_INPUT_STATE *ins, uint32_t flags) if (JSON_STRING_MATCH(ins, "config")) { JSON_EXPECT(session, ins, ':'); JSON_EXPECT(session, ins, 's'); - if ((ret = json_strdup(ins, &config)) != 0) { - ret = util_err(ret, NULL); + if ((ret = + json_strdup(session, ins, &config)) != 0) { + ret = util_err(session, ret, NULL); goto err; } - if ((ret = config_list_add(&cl, tableuri)) != 0) + if ((ret = config_list_add( + session, &cl, tableuri)) != 0) goto err; - if ((ret = config_list_add(&cl, config)) != 0) + if ((ret = config_list_add( + session, &cl, config)) != 0) goto err; tableuri = NULL; } else if (JSON_STRING_MATCH(ins, "colgroups")) { @@ -448,7 +455,7 @@ json_peek(WT_SESSION *session, JSON_INPUT_STATE *ins) if (*ins->p) break; if (ins->kvraw != NULL) { - if (json_kvraw_append(ins, + if (json_kvraw_append(session, ins, (char *)ins->line.mem + ins->kvrawstart, strlen(ins->line.mem) - ins->kvrawstart)) { ret = -1; @@ -456,8 +463,8 @@ json_peek(WT_SESSION *session, JSON_INPUT_STATE *ins) } ins->kvrawstart = 0; } - if (util_read_line(&ins->line, 1, - &ins->ateof)) { + if (util_read_line( + session, &ins->line, 1, &ins->ateof)) { ins->toktype = -1; ret = -1; goto err; @@ -527,7 +534,7 @@ json_skip(WT_SESSION *session, JSON_INPUT_STATE *ins, const char **matches) for (match = matches; *match != NULL; match++) if ((hit = strstr(ins->p, *match)) != NULL) goto out; - if (util_read_line(&ins->line, 1, &ins->ateof)) { + if (util_read_line(session, &ins->line, 1, &ins->ateof)) { ins->toktype = -1; return (1); } @@ -558,7 +565,7 @@ util_load_json(WT_SESSION *session, const char *filename, uint32_t flags) memset(&instate, 0, sizeof(instate)); instate.session = session; - if (util_read_line(&instate.line, 0, &instate.ateof)) + if (util_read_line(session, &instate.line, 0, &instate.ateof)) return (1); instate.p = (const char *)instate.line.mem; instate.linenum = 1; diff --git a/src/third_party/wiredtiger/src/utilities/util_loadtext.c b/src/third_party/wiredtiger/src/utilities/util_loadtext.c index cb84fb03d13..c200198a0a1 100644 --- a/src/third_party/wiredtiger/src/utilities/util_loadtext.c +++ b/src/third_party/wiredtiger/src/utilities/util_loadtext.c @@ -22,8 +22,8 @@ util_loadtext(WT_SESSION *session, int argc, char *argv[]) switch (ch) { case 'f': /* input file */ if (freopen(__wt_optarg, "r", stdin) == NULL) - return ( - util_err(errno, "%s: reopen", __wt_optarg)); + return (util_err( + session, errno, "%s: reopen", __wt_optarg)); break; case '?': default: @@ -35,7 +35,7 @@ util_loadtext(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the uri. */ if (argc != 1) return (usage()); - if ((uri = util_name(*argv, "table")) == NULL) + if ((uri = util_name(session, *argv, "table")) == NULL) return (1); return (text(session, uri)); @@ -60,7 +60,7 @@ text(WT_SESSION *session, const char *uri) */ if ((ret = session->open_cursor( session, uri, NULL, "append,overwrite", &cursor)) != 0) - return (util_err(ret, "%s: session.open", uri)); + return (util_err(session, ret, "%s: session.open", uri)); /* * We're about to load strings, make sure the formats match. @@ -71,7 +71,7 @@ text(WT_SESSION *session, const char *uri) if (strcmp(cursor->value_format, "S") != 0 || (strcmp(cursor->key_format, "S") != 0 && strcmp(cursor->key_format, "r") != 0)) - return (util_err(EINVAL, + return (util_err(session, EINVAL, "the loadtext command can only load objects configured " "for record number or string keys, and string values")); readkey = strcmp(cursor->key_format, "r") == 0 ? 0 : 1; @@ -85,7 +85,7 @@ text(WT_SESSION *session, const char *uri) * the close succeed, it's better to fail early when loading files. */ if ((tret = cursor->close(cursor)) != 0) { - tret = util_err(tret, "%s: cursor.close", uri); + tret = util_err(session, tret, "%s: cursor.close", uri); if (ret == 0) ret = tret; } @@ -104,9 +104,12 @@ insert(WT_CURSOR *cursor, const char *name, int readkey) { ULINE key, value; WT_DECL_RET; + WT_SESSION *session; uint64_t insert_count; int eof; + session = cursor->session; + memset(&key, 0, sizeof(key)); memset(&value, 0, sizeof(value)); @@ -119,20 +122,21 @@ insert(WT_CURSOR *cursor, const char *name, int readkey) * all (flat-text load). */ if (readkey) { - if (util_read_line(&key, 1, &eof)) + if (util_read_line(session, &key, 1, &eof)) return (1); if (eof == 1) break; cursor->set_key(cursor, key.mem); } - if (util_read_line(&value, readkey ? 0 : 1, &eof)) + if (util_read_line(session, &value, readkey ? 0 : 1, &eof)) return (1); if (eof == 1) break; cursor->set_value(cursor, value.mem); if ((ret = cursor->insert(cursor)) != 0) - return (util_err(ret, "%s: cursor.insert", name)); + return ( + util_err(session, ret, "%s: cursor.insert", name)); /* Report on progress every 100 inserts. */ if (verbose && ++insert_count % 100 == 0) { diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 51a11f82c43..79d768f5752 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -30,6 +30,7 @@ main(int argc, char *argv[]) WT_SESSION *session; size_t len; int ch, major_v, minor_v, tret, (*func)(WT_SESSION *, int, char *[]); + int logoff, recover; char *p; const char *cmd_config, *config, *rec_config; @@ -64,8 +65,9 @@ main(int argc, char *argv[]) * needed, the user can specify -R to run recovery. */ rec_config = REC_ERROR; + logoff = recover = 0; /* Check for standard options. */ - while ((ch = __wt_getopt(progname, argc, argv, "C:h:RVv")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "C:h:LRVv")) != EOF) switch (ch) { case 'C': /* wiredtiger_open config */ cmd_config = __wt_optarg; @@ -73,8 +75,13 @@ main(int argc, char *argv[]) case 'h': /* home directory */ home = __wt_optarg; break; + case 'L': /* no logging */ + rec_config = REC_LOGOFF; + logoff = 1; + break; case 'R': /* recovery */ rec_config = REC_RECOVER; + recover = 1; break; case 'V': /* version */ printf("%s\n", wiredtiger_version(NULL, NULL, NULL)); @@ -86,6 +93,10 @@ main(int argc, char *argv[]) default: return (usage()); } + if (logoff && recover) { + fprintf(stderr, "Only one of -L and -R is allowed.\n"); + return (EXIT_FAILURE); + } argc -= __wt_optind; argv += __wt_optind; @@ -177,7 +188,7 @@ main(int argc, char *argv[]) len += strlen(cmd_config); len += strlen(rec_config); if ((p = malloc(len)) == NULL) { - ret = util_err(errno, NULL); + ret = util_err(NULL, errno, NULL); goto err; } (void)snprintf(p, len, "%s,%s,%s", @@ -188,11 +199,11 @@ main(int argc, char *argv[]) /* Open the database and a session. */ if ((ret = wiredtiger_open(home, verbose ? verbose_handler : NULL, config, &conn)) != 0) { - ret = util_err(ret, NULL); + ret = util_err(NULL, ret, NULL); goto err; } if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { - ret = util_err(ret, NULL); + ret = util_err(NULL, ret, NULL); goto err; } @@ -218,11 +229,12 @@ usage(void) WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR); fprintf(stderr, "global options:\n" - "\t" "-C\twiredtiger_open configuration\n" - "\t" "-h\tdatabase directory\n" - "\t" "-R\trun recovery if configured\n" - "\t" "-V\tdisplay library version and exit\n" - "\t" "-v\tverbose\n"); + "\t" "-C\t" "wiredtiger_open configuration\n" + "\t" "-h\t" "database directory\n" + "\t" "-L\t" "turn logging off for debug-mode\n" + "\t" "-R\t" "run recovery if configured\n" + "\t" "-V\t" "display library version and exit\n" + "\t" "-v\t" "verbose\n"); fprintf(stderr, "commands:\n" "\t" "backup\t database backup\n" @@ -251,7 +263,7 @@ usage(void) * Build a name. */ char * -util_name(const char *s, const char *type) +util_name(WT_SESSION *session, const char *s, const char *type) { size_t len; char *name; @@ -267,7 +279,7 @@ util_name(const char *s, const char *type) len = strlen(type) + strlen(s) + 2; if ((name = calloc(len, 1)) == NULL) { - (void)util_err(errno, NULL); + (void)util_err(session, errno, NULL); return (NULL); } diff --git a/src/third_party/wiredtiger/src/utilities/util_misc.c b/src/third_party/wiredtiger/src/utilities/util_misc.c index 15605362b50..f14ad7119bd 100644 --- a/src/third_party/wiredtiger/src/utilities/util_misc.c +++ b/src/third_party/wiredtiger/src/utilities/util_misc.c @@ -9,9 +9,10 @@ #include "util.h" int -util_cerr(const char *uri, const char *op, int ret) +util_cerr(WT_CURSOR *cursor, const char *op, int ret) { - return (util_err(ret, "%s: cursor.%s", uri, op)); + return ( + util_err(cursor->session, ret, "%s: cursor.%s", cursor->uri, op)); } /* @@ -19,7 +20,7 @@ util_cerr(const char *uri, const char *op, int ret) * Report an error. */ int -util_err(int e, const char *fmt, ...) +util_err(WT_SESSION *session, int e, const char *fmt, ...) { va_list ap; @@ -32,7 +33,8 @@ util_err(int e, const char *fmt, ...) (void)fprintf(stderr, ": "); } if (e != 0) - (void)fprintf(stderr, "%s", wiredtiger_strerror(e)); + (void)fprintf(stderr, "%s", session == NULL ? + wiredtiger_strerror(e) : session->strerror(session, e)); (void)fprintf(stderr, "\n"); return (1); } @@ -42,7 +44,7 @@ util_err(int e, const char *fmt, ...) * Read a line from stdin into a ULINE. */ int -util_read_line(ULINE *l, int eof_expected, int *eofp) +util_read_line(WT_SESSION *session, ULINE *l, int eof_expected, int *eofp) { static uint64_t line = 0; size_t len; @@ -53,7 +55,7 @@ util_read_line(ULINE *l, int eof_expected, int *eofp) if (l->memsize == 0) { if ((l->mem = realloc(l->mem, l->memsize + 1024)) == NULL) - return (util_err(errno, NULL)); + return (util_err(session, errno, NULL)); l->memsize = 1024; } for (len = 0;; ++len) { @@ -63,11 +65,11 @@ util_read_line(ULINE *l, int eof_expected, int *eofp) *eofp = 1; return (0); } - return (util_err(0, + return (util_err(session, 0, "line %" PRIu64 ": unexpected end-of-file", line)); } - return (util_err(0, + return (util_err(session, 0, "line %" PRIu64 ": no newline terminator", line)); } if (ch == '\n') @@ -80,7 +82,7 @@ util_read_line(ULINE *l, int eof_expected, int *eofp) if (len >= l->memsize - 1) { if ((l->mem = realloc(l->mem, l->memsize + 1024)) == NULL) - return (util_err(errno, NULL)); + return (util_err(session, errno, NULL)); l->memsize += 1024; } ((uint8_t *)l->mem)[len] = (uint8_t)ch; @@ -96,7 +98,7 @@ util_read_line(ULINE *l, int eof_expected, int *eofp) * Convert a string to a record number. */ int -util_str2recno(const char *p, uint64_t *recnop) +util_str2recno(WT_SESSION *session, const char *p, uint64_t *recnop) { uint64_t recno; char *endptr; @@ -112,10 +114,12 @@ util_str2recno(const char *p, uint64_t *recnop) errno = 0; recno = __wt_strtouq(p, &endptr, 0); if (recno == ULLONG_MAX && errno == ERANGE) - return (util_err(ERANGE, "%s: invalid record number", p)); + return ( + util_err(session, ERANGE, "%s: invalid record number", p)); if (endptr[0] != '\0') -format: return (util_err(EINVAL, "%s: invalid record number", p)); +format: return ( + util_err(session, EINVAL, "%s: invalid record number", p)); *recnop = recno; return (0); @@ -134,11 +138,11 @@ util_flush(WT_SESSION *session, const char *uri) len = strlen(uri) + 100; if ((buf = malloc(len)) == NULL) - return (util_err(errno, NULL)); + return (util_err(session, errno, NULL)); (void)snprintf(buf, len, "target=(\"%s\")", uri); if ((ret = session->checkpoint(session, buf)) != 0) { - ret = util_err(ret, "%s: session.checkpoint", uri); + ret = util_err(session, ret, "%s: session.checkpoint", uri); (void)session->drop(session, uri, NULL); } diff --git a/src/third_party/wiredtiger/src/utilities/util_printlog.c b/src/third_party/wiredtiger/src/utilities/util_printlog.c index ff9210b575f..4ec2d0b7e1c 100644 --- a/src/third_party/wiredtiger/src/utilities/util_printlog.c +++ b/src/third_party/wiredtiger/src/utilities/util_printlog.c @@ -45,7 +45,7 @@ util_printlog(WT_SESSION *session, int argc, char *argv[]) if (ret != 0) { fprintf(stderr, "%s: printlog failed: %s\n", - progname, wiredtiger_strerror(ret)); + progname, session->strerror(session, ret)); goto err; } diff --git a/src/third_party/wiredtiger/src/utilities/util_read.c b/src/third_party/wiredtiger/src/utilities/util_read.c index de6b8adb5c2..fe9f5541d6e 100644 --- a/src/third_party/wiredtiger/src/utilities/util_read.c +++ b/src/third_party/wiredtiger/src/utilities/util_read.c @@ -31,13 +31,13 @@ util_read(WT_SESSION *session, int argc, char *argv[]) /* The remaining arguments are a uri followed by a list of keys. */ if (argc < 2) return (usage()); - if ((uri = util_name(*argv, "table")) == NULL) + if ((uri = util_name(session, *argv, "table")) == NULL) return (1); /* Open the object. */ if ((ret = session->open_cursor( session, uri, NULL, NULL, &cursor)) != 0) - return (util_err(ret, "%s: session.open", uri)); + return (util_err(session, ret, "%s: session.open", uri)); /* * A simple search only makes sense if the key format is a string or a @@ -66,7 +66,7 @@ util_read(WT_SESSION *session, int argc, char *argv[]) */ for (rval = 0; *++argv != NULL;) { if (rkey) { - if (util_str2recno(*argv, &recno)) + if (util_str2recno(session, *argv, &recno)) return (1); cursor->set_key(cursor, recno); } else @@ -75,16 +75,16 @@ util_read(WT_SESSION *session, int argc, char *argv[]) switch (ret = cursor->search(cursor)) { case 0: if ((ret = cursor->get_value(cursor, &value)) != 0) - return (util_cerr(uri, "get_value", ret)); + return (util_cerr(cursor, "get_value", ret)); if (printf("%s\n", value) < 0) - return (util_err(EIO, NULL)); + return (util_err(session, EIO, NULL)); break; case WT_NOTFOUND: - (void)util_err(0, "%s: not found", *argv); + (void)util_err(session, 0, "%s: not found", *argv); rval = 1; break; default: - return (util_cerr(uri, "search", ret)); + return (util_cerr(cursor, "search", ret)); } } diff --git a/src/third_party/wiredtiger/src/utilities/util_rename.c b/src/third_party/wiredtiger/src/utilities/util_rename.c index e5fda06b310..971e1376f26 100644 --- a/src/third_party/wiredtiger/src/utilities/util_rename.c +++ b/src/third_party/wiredtiger/src/utilities/util_rename.c @@ -30,13 +30,13 @@ util_rename(WT_SESSION *session, int argc, char *argv[]) /* The remaining arguments are the object uri and new name. */ if (argc != 2) return (usage()); - if ((uri = util_name(*argv, "table")) == NULL) + if ((uri = util_name(session, *argv, "table")) == NULL) return (1); newuri = argv[1]; if ((ret = session->rename(session, uri, newuri, NULL)) != 0) { fprintf(stderr, "%s: rename %s to %s: %s\n", - progname, uri, newuri, wiredtiger_strerror(ret)); + progname, uri, newuri, session->strerror(session, ret)); goto err; } diff --git a/src/third_party/wiredtiger/src/utilities/util_salvage.c b/src/third_party/wiredtiger/src/utilities/util_salvage.c index dc8affcb1d9..ffe413da732 100644 --- a/src/third_party/wiredtiger/src/utilities/util_salvage.c +++ b/src/third_party/wiredtiger/src/utilities/util_salvage.c @@ -35,12 +35,12 @@ util_salvage(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the file name. */ if (argc != 1) return (usage()); - if ((name = util_name(*argv, "file")) == NULL) + if ((name = util_name(session, *argv, "file")) == NULL) return (1); if ((ret = session->salvage(session, name, force)) != 0) { fprintf(stderr, "%s: salvage(%s): %s\n", - progname, name, wiredtiger_strerror(ret)); + progname, name, session->strerror(session, ret)); goto err; } diff --git a/src/third_party/wiredtiger/src/utilities/util_stat.c b/src/third_party/wiredtiger/src/utilities/util_stat.c index 2f99214e57a..77c420f8dee 100644 --- a/src/third_party/wiredtiger/src/utilities/util_stat.c +++ b/src/third_party/wiredtiger/src/utilities/util_stat.c @@ -54,7 +54,7 @@ util_stat(WT_SESSION *session, int argc, char *argv[]) objname = (char *)""; break; case 1: - if ((objname = util_name(*argv, "table")) == NULL) + if ((objname = util_name(session, *argv, "table")) == NULL) return (1); objname_free = 1; break; @@ -73,7 +73,7 @@ util_stat(WT_SESSION *session, int argc, char *argv[]) if ((ret = session->open_cursor(session, uri, NULL, config, &cursor)) != 0) { fprintf(stderr, "%s: cursor open(%s) failed: %s\n", - progname, uri, wiredtiger_strerror(ret)); + progname, uri, session->strerror(session, ret)); goto err; } @@ -90,7 +90,7 @@ util_stat(WT_SESSION *session, int argc, char *argv[]) if (ret != 0) { fprintf(stderr, "%s: cursor get(%s) failed: %s\n", - progname, objname, wiredtiger_strerror(ret)); + progname, objname, session->strerror(session, ret)); goto err; } diff --git a/src/third_party/wiredtiger/src/utilities/util_upgrade.c b/src/third_party/wiredtiger/src/utilities/util_upgrade.c index 6be11f77099..2f15069b6fc 100644 --- a/src/third_party/wiredtiger/src/utilities/util_upgrade.c +++ b/src/third_party/wiredtiger/src/utilities/util_upgrade.c @@ -30,12 +30,12 @@ util_upgrade(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((name = util_name(*argv, "table")) == NULL) + if ((name = util_name(session, *argv, "table")) == NULL) return (1); if ((ret = session->upgrade(session, name, NULL)) != 0) { fprintf(stderr, "%s: upgrade(%s): %s\n", - progname, name, wiredtiger_strerror(ret)); + progname, name, session->strerror(session, ret)); goto err; } diff --git a/src/third_party/wiredtiger/src/utilities/util_verify.c b/src/third_party/wiredtiger/src/utilities/util_verify.c index 796d24a8a6d..20a7060f729 100644 --- a/src/third_party/wiredtiger/src/utilities/util_verify.c +++ b/src/third_party/wiredtiger/src/utilities/util_verify.c @@ -54,7 +54,7 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((name = util_name(*argv, "table")) == NULL) + if ((name = util_name(session, *argv, "table")) == NULL) return (1); /* Build the configuration string as necessary. */ @@ -68,7 +68,7 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) strlen("dump_offsets[],") + (dump_offsets == NULL ? 0 : strlen(dump_offsets)) + 20; if ((config = malloc(size)) == NULL) { - (void)util_err(errno, NULL); + (void)util_err(session, errno, NULL); goto err; } snprintf(config, size, @@ -83,7 +83,7 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) } if ((ret = session->verify(session, name, config)) != 0) { fprintf(stderr, "%s: verify(%s): %s\n", - progname, name, wiredtiger_strerror(ret)); + progname, name, session->strerror(session, ret)); goto err; } diff --git a/src/third_party/wiredtiger/src/utilities/util_write.c b/src/third_party/wiredtiger/src/utilities/util_write.c index f9fb2d111fe..e5eafa5ab25 100644 --- a/src/third_party/wiredtiger/src/utilities/util_write.c +++ b/src/third_party/wiredtiger/src/utilities/util_write.c @@ -46,7 +46,7 @@ util_write(WT_SESSION *session, int argc, char *argv[]) } else if (argc < 3 || ((argc - 1) % 2 != 0)) return (usage()); - if ((uri = util_name(*argv, "table")) == NULL) + if ((uri = util_name(session, *argv, "table")) == NULL) return (1); /* Open the object. */ @@ -54,7 +54,7 @@ util_write(WT_SESSION *session, int argc, char *argv[]) append ? "append=true" : "", overwrite ? "overwrite=true" : ""); if ((ret = session->open_cursor( session, uri, NULL, config, &cursor)) != 0) - return (util_err(ret, "%s: session.open", uri)); + return (util_err(session, ret, "%s: session.open", uri)); /* * A simple search only makes sense if the key format is a string or a @@ -81,7 +81,7 @@ util_write(WT_SESSION *session, int argc, char *argv[]) while (*++argv != NULL) { if (!append) { if (rkey) { - if (util_str2recno(*argv, &recno)) + if (util_str2recno(session, *argv, &recno)) return (1); cursor->set_key(cursor, recno); } else @@ -91,7 +91,7 @@ util_write(WT_SESSION *session, int argc, char *argv[]) cursor->set_value(cursor, *argv); if ((ret = cursor->insert(cursor)) != 0) - return (util_cerr(uri, "search", ret)); + return (util_cerr(cursor, "search", ret)); } return (0); diff --git a/src/third_party/wiredtiger/tools/wtstats/stat_data.py b/src/third_party/wiredtiger/tools/wtstats/stat_data.py index da6fb3a7f7f..a4fc2fcc831 100644 --- a/src/third_party/wiredtiger/tools/wtstats/stat_data.py +++ b/src/third_party/wiredtiger/tools/wtstats/stat_data.py @@ -7,10 +7,14 @@ no_scale_per_second_list = [ 'cache: maximum page size at eviction', 'cache: pages currently held in the cache', 'cache: percentage overhead', + 'cache: tracked bytes belonging to internal pages in the cache', + 'cache: tracked bytes belonging to leaf pages in the cache', + 'cache: tracked bytes belonging to overflow pages in the cache', 'cache: tracked dirty bytes in the cache', 'cache: tracked dirty pages in the cache', 'connection: files currently open', 'log: maximum log file size', + 'log: number of pre-allocated log files to create', 'log: total log buffer size', 'LSM: application work units currently queued', 'LSM: merge work units currently queued', @@ -20,17 +24,20 @@ no_scale_per_second_list = [ 'session: open cursor count', 'session: open session count', 'transaction: transaction checkpoint currently running', + 'transaction: transaction checkpoint generation', 'transaction: transaction checkpoint max time (msecs)', 'transaction: transaction checkpoint min time (msecs)', 'transaction: transaction checkpoint most recent time (msecs)', 'transaction: transaction checkpoint total time (msecs)', 'transaction: transaction range of IDs currently pinned', + 'transaction: transaction range of IDs currently pinned by a checkpoint', 'block-manager: checkpoint size', 'block-manager: file allocation unit size', 'block-manager: file magic number', 'block-manager: file major version number', 'block-manager: file size in bytes', 'block-manager: minor version number', + 'btree: btree checkpoint generation', 'btree: column-store fixed-size leaf pages', 'btree: column-store internal pages', 'btree: column-store variable-size deleted values', @@ -59,8 +66,14 @@ no_clear_list = [ 'cache: maximum bytes configured', 'cache: pages currently held in the cache', 'cache: percentage overhead', + 'cache: tracked bytes belonging to internal pages in the cache', + 'cache: tracked bytes belonging to leaf pages in the cache', + 'cache: tracked bytes belonging to overflow pages in the cache', + 'cache: tracked dirty bytes in the cache', + 'cache: tracked dirty pages in the cache', 'connection: files currently open', 'log: maximum log file size', + 'log: number of pre-allocated log files to create', 'log: total log buffer size', 'LSM: application work units currently queued', 'LSM: merge work units currently queued', @@ -70,11 +83,14 @@ no_clear_list = [ 'session: open cursor count', 'session: open session count', 'transaction: transaction checkpoint currently running', + 'transaction: transaction checkpoint generation', 'transaction: transaction checkpoint max time (msecs)', 'transaction: transaction checkpoint min time (msecs)', 'transaction: transaction checkpoint most recent time (msecs)', 'transaction: transaction checkpoint total time (msecs)', 'transaction: transaction range of IDs currently pinned', + 'transaction: transaction range of IDs currently pinned by a checkpoint', + 'btree: btree checkpoint generation', 'session: open cursor count', ] prefix_list = [ |