diff options
Diffstat (limited to 'src/third_party/wiredtiger')
114 files changed, 2390 insertions, 1069 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 96012456b34..2646d51378e 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -28,6 +28,9 @@ AddOption("--enable-diagnostic", dest="diagnostic", action="store_true", default AddOption("--enable-lz4", dest="lz4", type="string", nargs=1, action="store", help="Use LZ4 compression") +AddOption("--enable-java", dest="lang-java", type="string", nargs=1, action="store", + help="Build java extension, specify location of swig.exe binary and Java JDK dir separated by comma") + AddOption("--enable-python", dest="lang-python", type="string", nargs=1, action="store", help="Build Python extension, specify location of swig.exe binary") @@ -37,9 +40,6 @@ AddOption("--enable-snappy", dest="snappy", type="string", nargs=1, action="stor AddOption("--enable-tcmalloc", dest="tcmalloc", type="string", nargs=1, action="store", help="Use TCMalloc for memory allocation") -AddOption("--enable-verbose", dest="verbose", action="store_true", default=False, - help="Configure WiredTiger to support the verbose configuration string to wiredtiger_open") - AddOption("--enable-zlib", dest="zlib", type="string", nargs=1, action="store", help="Use zlib compression") @@ -183,9 +183,6 @@ if GetOption("lang-python"): env.Append(LIBPATH=[distutils.sysconfig.PREFIX + r"\libs"]) env.Append(CPPPATH=[distutils.sysconfig.get_python_inc()]) -if GetOption("verbose"): - env.Append(CPPDEFINES = ["HAVE_VERBOSE"]) - # Build WiredTiger.h file # @@ -355,6 +352,61 @@ if GetOption("lang-python"): Default(swiginstall, copySwig) +# Javap SWIG wrapper for WiredTiger +enableJava = GetOption("lang-java") +if enableJava and enableJava.count(",") == 1: + enableJavaPaths = enableJava.split(',') + + swigExe = enableJavaPaths[0] + javaPath = enableJavaPaths[1] + conf.env.Append(CPPPATH=[ javaPath + '/include']) + conf.env.Append(CPPPATH=[ javaPath + '/include/win32']) + + swigJavaFiles = ["lang/java/src/com/wiredtiger/db/AsyncOp.java", + "lang/java/src/com/wiredtiger/db/AsyncOpType.java", + "lang/java/src/com/wiredtiger/db/Connection.java", + "lang/java/src/com/wiredtiger/db/Cursor.java", + "lang/java/src/com/wiredtiger/db/Modify.java", + "lang/java/src/com/wiredtiger/db/SearchStatus.java", + "lang/java/src/com/wiredtiger/db/Session.java", + "lang/java/src/com/wiredtiger/db/WT_ITEM_HOLD.java", + "lang/java/src/com/wiredtiger/db/WT_MODIFY_LIST.java", + "lang/java/src/com/wiredtiger/db/wiredtiger.java", + "lang/java/src/com/wiredtiger/db/wiredtigerConstants.java", + "lang/java/src/com/wiredtiger/db/wiredtigerJNI.java"] + + swigCFile = "wiredtiger_wrap.c" + + swigFiles = env.Command( + swigJavaFiles + [swigCFile], '', + '"' + swigExe + '" -Wall -v -java -nodefaultctor -nodefaultdtor -package com.wiredtiger.db -outdir lang/java/src/com/wiredtiger/db -o wiredtiger_wrap.c lang/java/wiredtiger.i') + env.Depends(swigFiles, wtheader) + objectJavaWrap = env.Object(swigCFile) + env.Depends(objectJavaWrap, swigCFile) + + # + # Dynamically Loaded Library - wiredtiger_java.dll + wtjavadll = env.SharedLibrary( + target="wiredtiger_java", + source=wt_objs + [objectJavaWrap] + ['build_win/wiredtiger.def'], LIBS=wtlibs) + + env.Depends(wtjavadll, [filelistfile, version_file]) + Default(wtjavadll) + + # + # wiredtiger.jar + env['JAVAC'] = '"' + javaPath + '/bin/javac.exe"' + env['JAR'] = '"' + javaPath + '/bin/jar.exe"' + # Build classes + wtClasses = env.Java('lang/java/build', 'lang/java/src/') + env.Depends(wtClasses, swigJavaFiles) + # Pack classes in jar + wtJar = env.Command( 'lang/java/wiredtiger.jar', 'lang/java/build', env['JAR'] + " -cf $TARGET -C $SOURCE .") + env.Depends(wtJar, wtClasses) + Default(wtJar) +else: + print "Error using --enable-java, this option may contain two paths separated by comma, the first is the swig.exe binary and the second is the Java JDK directory. e.g. C:\Python27\python.exe C:\Python27\Scripts\scons.py --enable-java=\"C:\Program Files\swigwin-3.0.12\swig.exe\",\"C:\Program Files\Java\jdk1.8.0_151\"" + # Shim library of functions to emulate POSIX on Windows shim = env.Library("window_shim", ["test/windows/windows_shim.c"]) diff --git a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 index 20d8ed08db5..7d0df5d65ac 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 @@ -253,18 +253,6 @@ AC_ARG_WITH(timestamp-size, AC_MSG_RESULT($with_timestamp_size) AC_DEFINE_UNQUOTED(WT_TIMESTAMP_SIZE, [$with_timestamp_size], [Size of a transaction timestamp in bytes]) -AH_TEMPLATE(HAVE_VERBOSE, [Enable verbose message configuration.]) -AC_MSG_CHECKING(if --enable-verbose option specified) -AC_ARG_ENABLE(verbose, - [AS_HELP_STRING([--enable-verbose], - [Enable verbose message configuration.])], r=$enableval, r=no) -case "$r" in -no) wt_cv_enable_verbose=no;; -*) AC_DEFINE(HAVE_VERBOSE) - wt_cv_enable_verbose=yes;; -esac -AC_MSG_RESULT($wt_cv_enable_verbose) - AC_MSG_CHECKING(if --enable-zlib option specified) AC_ARG_ENABLE(zlib, [AS_HELP_STRING([--enable-zlib], diff --git a/src/third_party/wiredtiger/build_win/wiredtiger_config.h b/src/third_party/wiredtiger/build_win/wiredtiger_config.h index fa8425c4936..55431f59fae 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger_config.h +++ b/src/third_party/wiredtiger/build_win/wiredtiger_config.h @@ -124,9 +124,6 @@ /* Define to 1 if you have the <unistd.h> header file. */ /* #undef HAVE_UNISTD_H */ -/* Enable verbose message configuration. */ -/* #undef HAVE_VERBOSE */ - /* Define to 1 if you have the <x86intrin.h> header file. */ #define HAVE_X86INTRIN_H 1 diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 68a8f4908d8..ac4e80aedcf 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -108,6 +108,18 @@ lsm_config = [ larger than this value. This overrides the \c memory_page_max setting''', min='512K', max='500MB'), + Config('merge_custom', '', r''' + configure the tree to merge into a custom data source''', + type='category', subconfig=[ + Config('prefix', '', r''' + custom data source prefix instead of \c "file"'''), + Config('start_generation', '0', r''' + merge generation at which the custom data source is used + (zero indicates no custom data source)''', + min='0', max='10'), + Config('suffix', '', r''' + custom data source suffix instead of \c ".lsm"'''), + ]), Config('merge_max', '15', r''' the maximum number of chunks to include in a merge operation''', min='2', max='100'), @@ -497,6 +509,19 @@ connection_runtime_config = [ Config('lsm_merge', 'true', r''' merge LSM chunks where possible (deprecated)''', type='boolean', undoc=True), + Config('operation_tracking', '', r''' + enable tracking of performance-critical functions. See + @ref operation_tracking for more information''', + type='category', subconfig=[ + Config('enabled', 'false', r''' + enable operation tracking subsystem''', + type='boolean'), + Config('path', '"."', r''' + the name of a directory into which operation tracking files are + written. The directory must already exist. If the value is not + an absolute path, the path is relative to the database home + (see @ref absolute_path for more information)'''), + ]), Config('shared_cache', '', r''' shared cache configuration options. A database should configure either a cache_size or a shared_cache not both. Enabling a @@ -546,8 +571,7 @@ connection_runtime_config = [ type='list', undoc=True, choices=[ 'checkpoint_slow', 'internal_page_split_race', 'page_split_race']), Config('verbose', '', r''' - enable messages for various events. Only available if WiredTiger - is configured with --enable-verbose. Options are given as a + enable messages for various events. Options are given as a list, such as <code>"verbose=[evictserver,read]"</code>''', type='list', choices=[ 'api', @@ -576,6 +600,7 @@ connection_runtime_config = [ 'salvage', 'shared_cache', 'split', + 'temporary', 'thread_group', 'timestamp', 'transaction', diff --git a/src/third_party/wiredtiger/dist/dist.py b/src/third_party/wiredtiger/dist/dist.py index ec47f509541..e4b76bdbab4 100644 --- a/src/third_party/wiredtiger/dist/dist.py +++ b/src/third_party/wiredtiger/dist/dist.py @@ -22,6 +22,14 @@ def all_c_files(): for line in glob.iglob('../test/*/*.[ci]'): yield line +# all_h_files -- +# Return list of all WiredTiger C include file names. +def all_h_files(): + file_re = re.compile(r'^\w') + for line in glob.iglob('../src/*/*.h'): + yield line + yield "../src/include/wiredtiger.in" + # source_dirs -- # Return a list of the WiredTiger source directory names. def source_dirs(): diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 9755e24f3c7..79590313b89 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -112,6 +112,7 @@ src/meta/meta_ext.c src/meta/meta_table.c src/meta/meta_track.c src/meta/meta_turtle.c +src/optrack/optrack.c src/os_common/filename.c src/os_common/os_abort.c src/os_common/os_alloc.c diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index 28c91486e1a..5d4f3cd2e5c 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -1,226 +1,55 @@ -# Output a C header file using the minimum number of distinct bits to ensure -# flags don't collide. - -import os, re, sys -from dist import compare_srcfile - -flags = { -################################################### -# Internal routine flag declarations -################################################### - 'log_scan' : [ - 'LOGSCAN_FIRST', - 'LOGSCAN_FROM_CKP', - 'LOGSCAN_ONE', - 'LOGSCAN_RECOVER', - ], - 'log_write' : [ - 'LOG_BACKGROUND', - 'LOG_DSYNC', - 'LOG_FLUSH', - 'LOG_FSYNC', - 'LOG_SYNC_ENABLED', - ], - 'page_read' : [ - 'READ_CACHE', - 'READ_IGNORE_CACHE_SIZE', - 'READ_LOOKASIDE', - 'READ_NOTFOUND_OK', - 'READ_NO_EMPTY', - 'READ_NO_GEN', - 'READ_NO_SPLIT', - 'READ_NO_WAIT', - 'READ_PREV', - 'READ_RESTART_OK', - 'READ_SKIP_INTL', - 'READ_TRUNCATE', - 'READ_WONT_NEED', - ], - 'rec_write' : [ - 'REC_CHECKPOINT', - 'REC_EVICT', - 'REC_IN_MEMORY', - 'REC_LOOKASIDE', - 'REC_SCRUB', - 'REC_UPDATE_RESTORE', - 'REC_VISIBILITY_ERR', - 'REC_VISIBLE_ALL', - ], - 'timing_stress_for_test' : [ - 'TIMING_STRESS_CHECKPOINT_SLOW', - 'TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE', - 'TIMING_STRESS_PAGE_SPLIT_RACE', - ], - 'txn_log_checkpoint' : [ - 'TXN_LOG_CKPT_CLEANUP', - 'TXN_LOG_CKPT_PREPARE', - 'TXN_LOG_CKPT_START', - 'TXN_LOG_CKPT_STOP', - 'TXN_LOG_CKPT_SYNC', - ], - 'txn_update_oldest' : [ - 'TXN_OLDEST_STRICT', - 'TXN_OLDEST_WAIT', - ], - 'verbose' : [ - 'VERB_API', - 'VERB_BLOCK', - 'VERB_CHECKPOINT', - 'VERB_CHECKPOINT_PROGRESS', - 'VERB_COMPACT', - 'VERB_EVICT', - 'VERB_EVICTSERVER', - 'VERB_EVICT_STUCK', - 'VERB_FILEOPS', - 'VERB_HANDLEOPS', - 'VERB_LOG', - 'VERB_LOOKASIDE', - 'VERB_LOOKASIDE_ACTIVITY', - 'VERB_LSM', - 'VERB_LSM_MANAGER', - 'VERB_METADATA', - 'VERB_MUTEX', - 'VERB_OVERFLOW', - 'VERB_READ', - 'VERB_REBALANCE', - 'VERB_RECONCILE', - 'VERB_RECOVERY', - 'VERB_RECOVERY_PROGRESS', - 'VERB_SALVAGE', - 'VERB_SHARED_CACHE', - 'VERB_SPLIT', - 'VERB_THREAD_GROUP', - 'VERB_TIMESTAMP', - 'VERB_TRANSACTION', - 'VERB_VERIFY', - 'VERB_VERSION', - 'VERB_WRITE', - ], - -################################################### -# Structure flag declarations -################################################### - 'conn' : [ - 'CONN_CACHE_POOL', - 'CONN_CKPT_SYNC', - 'CONN_CLOSING', - 'CONN_CLOSING_NO_MORE_OPENS', - 'CONN_EVICTION_NO_LOOKASIDE', - 'CONN_EVICTION_RUN', - 'CONN_IN_MEMORY', - 'CONN_LOOKASIDE_OPEN', - 'CONN_LEAK_MEMORY', - 'CONN_LSM_MERGE', - 'CONN_PANIC', - 'CONN_READONLY', - 'CONN_RECOVERING', - 'CONN_SERVER_ASYNC', - 'CONN_SERVER_CHECKPOINT', - 'CONN_SERVER_LOG', - 'CONN_SERVER_LSM', - 'CONN_SERVER_STATISTICS', - 'CONN_SERVER_SWEEP', - 'CONN_WAS_BACKUP', - ], - 'session' : [ - 'SESSION_CAN_WAIT', - 'SESSION_INTERNAL', - 'SESSION_IGNORE_CACHE_SIZE', - 'SESSION_LOCKED_CHECKPOINT', - 'SESSION_LOCKED_HANDLE_LIST_READ', - 'SESSION_LOCKED_HANDLE_LIST_WRITE', - 'SESSION_LOCKED_METADATA', - 'SESSION_LOCKED_PASS', - 'SESSION_LOCKED_SCHEMA', - 'SESSION_LOCKED_SLOT', - 'SESSION_LOCKED_TABLE_READ', - 'SESSION_LOCKED_TABLE_WRITE', - 'SESSION_LOCKED_TURTLE', - 'SESSION_LOGGING_INMEM', - 'SESSION_LOOKASIDE_CURSOR', - 'SESSION_NO_DATA_HANDLES', - 'SESSION_NO_RECONCILE', - 'SESSION_NO_LOGGING', - 'SESSION_NO_SCHEMA_LOCK', - 'SESSION_QUIET_CORRUPT_FILE', - 'SESSION_READ_WONT_NEED', - 'SESSION_SERVER_ASYNC', - ], - 'stat' : [ - 'STAT_CLEAR', - 'STAT_JSON', - 'STAT_ON_CLOSE', - 'STAT_TYPE_ALL', - 'STAT_TYPE_CACHE_WALK', - 'STAT_TYPE_FAST', - 'STAT_TYPE_SIZE', - 'STAT_TYPE_TREE_WALK', - ], -} - -flag_cnt = {} # Dictionary [flag] : [reference count] -flag_name = {} # Dictionary [flag] : [name ...] -name_mask = {} # Dictionary [name] : [used flag mask] - -# Step through the flags dictionary and build our local dictionaries. -for method in flags.items(): - name_mask[method[0]] = 0x0 - for flag in method[1]: - if flag == '__NONE__': - continue - if flag not in flag_cnt: - flag_cnt[flag] = 1 - flag_name[flag] = [] - else: - flag_cnt[flag] += 1 - flag_name[flag].append(method[0]) - -# Create list of possible bit masks. -bits = [2 ** i for i in range(0, 32)] - -# Walk the list of flags in reverse, sorted-by-reference count order. For -# each flag, find a bit that's not currently in use by any method using the -# flag. -flag_bit = {} # Dictionary [flag] : [bit value] -for f in sorted(flag_cnt.items(), key = lambda k_v : (-k_v[1], k_v[0])): - mask = 0xffffffff - for m in flag_name[f[0]]: - mask &= ~name_mask[m] - if mask == 0: - print >>sys.stderr,\ - "flags.py: ran out of flags at " + m + " method", - sys.exit(1) - for b in bits: - if mask & b: - mask = b - break - flag_bit[f[0]] = mask - for m in flag_name[f[0]]: - name_mask[m] |= mask - -# Print out the flag masks in hex. -# Assumes tab stops set to 8 characters. -flag_info = '' -for f in sorted(flag_cnt.items()): - flag_info += "#define\tWT_%s%s%#010x\n" %\ - (f[0],\ - "\t" * max(1, 6 - int((len('WT_') + len(f[0])) / 8)),\ - flag_bit[f[0]]) - -# Update the wiredtiger.in file with the flags information. -tmp_file = '__tmp' -tfile = open(tmp_file, 'w') -skip = 0 -for line in open('../src/include/flags.h', 'r'): - if skip: - if line.count('flags section: END'): - tfile.write('/*\n' + line) - skip = 0 - else: - tfile.write(line) - if line.count('flags section: BEGIN'): - skip = 1 - tfile.write(' */\n') - tfile.write(flag_info) -tfile.close() -compare_srcfile(tmp_file, '../src/include/flags.h') +#!/usr/bin/env python + +import re, sys +from dist import all_c_files, all_h_files, compare_srcfile + +# Automatically build flags values: read through all of the header files, and +# for each group of flags, sort them and give them a unique value. +def flag_declare(name): + tmp_file = '__tmp' + with open(name, 'r') as f: + tfile = open(tmp_file, 'w') + + lcnt = 0 + parsing = False + for line in f: + lcnt = lcnt + 1 + if line.find('AUTOMATIC FLAG VALUE GENERATION START') != -1: + header = line + defines = [] + parsing = True + elif line.find('AUTOMATIC FLAG VALUE GENERATION STOP') != -1: + # We only support 64 bits. + if len(defines) > 64: + print >>sys.stderr, name + ": line " +\ + str(lcnt) + ": exceeds maximum 64 bit flags" + sys.exit(1) + + # Calculate number of hex bytes, create format string + fmt = "0x%%0%dxu" % ((len(defines) + 3) / 4) + + tfile.write(header) + v = 1 + for d in sorted(defines): + tfile.write(re.sub("0x[01248u]*", fmt % v, d)) + v = v * 2 + tfile.write(line) + + parsing = False + elif parsing and line.find('#define') == -1: + print >>sys.stderr, name + ": line " +\ + str(lcnt) + ": unexpected flag line, no #define" + sys.exit(1) + elif parsing: + defines.append(line) + else: + tfile.write(line) + + tfile.close() + compare_srcfile(tmp_file, name) + +# Update function argument declarations. +for name in all_h_files(): + flag_declare(name) +for name in all_c_files(): + flag_declare(name) diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 4be8ceee0e3..5a5af9dd5d7 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -36,6 +36,8 @@ WT_LOG_SLOT_MASK_ON WT_LOG_SLOT_MAXBITS WT_LOG_SLOT_UNBUFFERED_ISSET WT_LOOKASIDE_COMPRESSOR +WT_OPTRACK_BUFSIZE +WT_OPTRACK_MAXRECS WT_PACKED_STRUCT_BEGIN WT_PACKED_STRUCT_END WT_PADDING_CHECK @@ -60,6 +62,10 @@ WT_STAT_INCRV_ATOMIC_BASE WT_STAT_INCRV_BASE WT_STAT_WRITE WT_TIMEDIFF_US +WT_TSCDIFF_NS +WT_TRACK_OP +WT_TRACK_OP_END +WT_TRACK_OP_INIT WT_TRET_ERROR_OK WT_UPDATE_SIZE WT_WITH_LOCK_NOWAIT diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 1a252fdd615..80f6bdca186 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -162,6 +162,7 @@ HFS HHHH HHHHLL HHHLL +HILQr Hendrik HyperLevelDB ID's @@ -268,6 +269,7 @@ Noll Nul OOB OPLOG +OPTRACK OPTYPE OUTBUFF OVFL @@ -746,6 +748,7 @@ fsyncLock fsyncs ftruncate func +funcid fvisibility gcc gdb @@ -775,6 +778,7 @@ hashval havesize hdr highjack +hilq hotbackup hselasky html @@ -986,6 +990,7 @@ notsup notused novalue nowait +nsec nset nsnap nul @@ -1006,6 +1011,7 @@ openfile oplist oplog optimizations +optrack optype ori os @@ -1078,6 +1084,7 @@ rbrace rbracket rcursor rdonly +rdtsc rduppo readlock readonly @@ -1193,6 +1200,7 @@ tV tablename tcbench td +teardown tempdir testutil th @@ -1216,6 +1224,7 @@ trk trk's trunc trylock +tsc tupdate tvalue txn diff --git a/src/third_party/wiredtiger/dist/s_style b/src/third_party/wiredtiger/dist/s_style index 3ae14949fbf..3f4346173e6 100755 --- a/src/third_party/wiredtiger/dist/s_style +++ b/src/third_party/wiredtiger/dist/s_style @@ -61,6 +61,15 @@ else cat $t fi + if ! expr "$f" : 'examples/c/.*' > /dev/null && + ! expr "$f" : 'ext/.*' > /dev/null && + ! expr "$f" : 'src/include/wiredtiger_ext\.h' > /dev/null && + ! expr "$f" : 'src/txn/txn_ext\.c' > /dev/null && + grep WT_TXN_ISO_ $f; then + echo "$f: WT_TXN_ISO_XXX constants only for the extension API" + cat $t + fi + if ! expr "$f" : 'src/include/queue\.h' > /dev/null && egrep 'STAILQ_|SLIST_|\bLIST_' $f ; then echo "$f: use TAILQ for all lists" diff --git a/src/third_party/wiredtiger/examples/c/ex_data_source.c b/src/third_party/wiredtiger/examples/c/ex_data_source.c index 59cfde7313a..f6311adffa6 100644 --- a/src/third_party/wiredtiger/examples/c/ex_data_source.c +++ b/src/third_party/wiredtiger/examples/c/ex_data_source.c @@ -489,6 +489,21 @@ my_salvage(WT_DATA_SOURCE *dsrc, WT_SESSION *session, return (0); } +/*! [WT_DATA_SOURCE size] */ +static int +my_size(WT_DATA_SOURCE *dsrc, WT_SESSION *session, + const char *uri, wt_off_t *size) +/*! [WT_DATA_SOURCE size] */ +{ + /* Unused parameters */ + (void)dsrc; + (void)session; + (void)uri; + (void)size; + + return (0); +} + /*! [WT_DATA_SOURCE truncate] */ static int my_truncate(WT_DATA_SOURCE *dsrc, WT_SESSION *session, @@ -559,6 +574,19 @@ my_terminate(WT_DATA_SOURCE *dsrc, WT_SESSION *session) return (0); } +/*! [WT_DATA_SOURCE lsm_pre_merge] */ +static int +my_lsm_pre_merge(WT_DATA_SOURCE *dsrc, WT_CURSOR *source, WT_CURSOR *dest) +/*! [WT_DATA_SOURCE lsm_pre_merge] */ +{ + /* Unused parameters */ + (void)dsrc; + (void)source; + (void)dest; + + return (0); +} + static const char *home; int @@ -581,11 +609,13 @@ main(int argc, char *argv[]) my_open_cursor, my_rename, my_salvage, + my_size, my_truncate, my_range_truncate, my_verify, my_checkpoint, - my_terminate + my_terminate, + my_lsm_pre_merge }; error_check(conn->add_data_source(conn, "dsrc:", &my_dsrc, NULL)); /*! [WT_DATA_SOURCE register] */ diff --git a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c index 8f857285b2b..9cf8b78b4c0 100644 --- a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c +++ b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c @@ -1026,11 +1026,13 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) kvs_session_open_cursor, /* session.open_cursor */ kvs_session_rename, /* session.rename */ NULL, /* No session.salvage */ + NULL, /* No session.size */ kvs_session_truncate, /* session.truncate */ NULL, /* No range_truncate */ kvs_session_verify, /* session.verify */ NULL, /* session.checkpoint */ - kvs_terminate /* termination */ + kvs_terminate, /* termination */ + NULL /* lsm_pre_merge */ }; DATA_SOURCE *ds; DB_ENV *dbenv; diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 7375a572f39..4a6c00611e6 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "596a3c7c0169cbda0475bfbd4b177fdbf3258058", + "commit": "1a29eac4dc8cf82de437292da546e3f4039268a4", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.8" diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i index e976af6a284..3571b885ec2 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i @@ -511,7 +511,6 @@ COMPARE_NOTFOUND_OK(__wt_cursor::_search_near) %exception wiredtiger_version; %exception diagnostic_build; %exception timestamp_build; -%exception verbose_build; /* WT_ASYNC_OP customization. */ /* First, replace the varargs get / set methods with Python equivalents. */ @@ -1013,18 +1012,9 @@ int diagnostic_build() { int timestamp_build() { return WT_TIMESTAMP_SIZE > 0; } - -int verbose_build() { -#ifdef HAVE_VERBOSE - return 1; -#else - return 0; -#endif -} %} int diagnostic_build(); int timestamp_build(); -int verbose_build(); /* Remove / rename parts of the C API that we don't want in Python. */ %immutable __wt_cursor::session; diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 64d0283a8a3..a7088e31c60 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -62,7 +62,6 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, ci = NULL; -#ifdef HAVE_VERBOSE if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { if (addr != NULL) { WT_ERR(__wt_scr_alloc(session, 0, &tmp)); @@ -72,7 +71,6 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, "%s: load-checkpoint: %s", block->name, addr == NULL ? "[Empty]" : (const char *)tmp->data); } -#endif /* * There's a single checkpoint in the file that can be written, all of @@ -545,7 +543,6 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) !F_ISSET(ckpt, WT_CKPT_DELETE)) continue; -#ifdef HAVE_VERBOSE if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { if (tmp == NULL) WT_ERR(__wt_scr_alloc(session, 0, &tmp)); @@ -555,7 +552,6 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) "%s: delete-checkpoint: %s: %s", block->name, ckpt->name, (const char *)tmp->data); } -#endif /* * Find the checkpoint into which we'll roll this checkpoint's * blocks: it's the next real checkpoint in the list, and it diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 1e510e08596..fa74f1dbb3d 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -8,9 +8,7 @@ #include "wt_internal.h" -#ifdef HAVE_VERBOSE static void __block_dump_avail(WT_SESSION_IMPL *, WT_BLOCK *, bool); -#endif /* * __wt_block_compact_start -- @@ -40,19 +38,15 @@ __wt_block_compact_start(WT_SESSION_IMPL *session, WT_BLOCK *block) int __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) { - WT_UNUSED(session); - /* Restore the original allocation plan. */ __wt_block_configure_first_fit(block, false); -#ifdef HAVE_VERBOSE /* Dump the results of the compaction pass. */ if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) { __wt_spin_lock(session, &block->live_lock); __block_dump_avail(session, block, false); __wt_spin_unlock(session, &block->live_lock); } -#endif return (0); } @@ -80,11 +74,9 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) __wt_spin_lock(session, &block->live_lock); -#ifdef HAVE_VERBOSE /* Dump the current state of the file. */ if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) __block_dump_avail(session, block, true); -#endif /* Sum the available bytes in the initial 80% and 90% of the file. */ avail_eighty = avail_ninety = 0; @@ -186,7 +178,6 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, } __wt_spin_unlock(session, &block->live_lock); -#ifdef HAVE_VERBOSE if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) { ++block->compact_pages_reviewed; if (*skipp) @@ -194,12 +185,10 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, else ++block->compact_pages_written; } -#endif return (0); } -#ifdef HAVE_VERBOSE /* * __block_dump_avail -- * Dump out the avail list so we can see what compaction will look like. @@ -276,4 +265,3 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) (uintmax_t)((v * 100) / (wt_off_t)el->bytes)); } } -#endif diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index c7439104f83..0d8c952721d 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -20,10 +20,8 @@ static int __verify_set_file_size(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *); /* The bit list ignores the first block: convert to/from a frag/offset. */ #define WT_wt_off_TO_FRAG(block, off) \ ((off) / (block)->allocsize - 1) -#ifdef HAVE_VERBOSE #define WT_FRAG_TO_OFF(block, frag) \ (((wt_off_t)((frag) + 1)) * (block)->allocsize) -#endif /* * __wt_block_verify_start -- @@ -385,8 +383,6 @@ __verify_filefrag_add(WT_SESSION_IMPL *session, WT_BLOCK *block, { uint64_t f, frag, frags, i; - WT_UNUSED(type); /* !HAVE_VERBOSE */ - __wt_verbose(session, WT_VERB_VERIFY, "add file block%s%s%s at %" PRIuMAX "-%" PRIuMAX " (%" PRIuMAX ")", type == NULL ? "" : " (", @@ -464,7 +460,6 @@ __verify_filefrag_chk(WT_SESSION_IMPL *session, WT_BLOCK *block) __bit_set(block->fragfile, last); } -#ifdef HAVE_VERBOSE if (!WT_VERBOSE_ISSET(session, WT_VERB_VERIFY)) continue; @@ -472,7 +467,6 @@ __verify_filefrag_chk(WT_SESSION_IMPL *session, WT_BLOCK *block) "file range %" PRIuMAX "-%" PRIuMAX " never verified", (uintmax_t)WT_FRAG_TO_OFF(block, first), (uintmax_t)WT_FRAG_TO_OFF(block, last)); -#endif } if (count == 0) return (0); @@ -557,7 +551,6 @@ __verify_ckptfrag_chk(WT_SESSION_IMPL *session, WT_BLOCK *block) __bit_clear(block->fragckpt, last); } -#ifdef HAVE_VERBOSE if (!WT_VERBOSE_ISSET(session, WT_VERB_VERIFY)) continue; @@ -565,7 +558,6 @@ __verify_ckptfrag_chk(WT_SESSION_IMPL *session, WT_BLOCK *block) "checkpoint range %" PRIuMAX "-%" PRIuMAX " never verified", (uintmax_t)WT_FRAG_TO_OFF(block, first), (uintmax_t)WT_FRAG_TO_OFF(block, last)); -#endif } if (count == 0) diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 3df5920830c..97dd15b963b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -16,17 +16,20 @@ typedef struct __wt_dbg WT_DBG; struct __wt_dbg { WT_SESSION_IMPL *session; /* Enclosing session */ + bool integer_key; /* Integer key formats */ + bool unsigned_key; + /* * When using the standard event handlers, the debugging output has to * do its own message handling because its output isn't line-oriented. */ - FILE *fp; /* Optional file handle */ - WT_ITEM *msg; /* Buffered message */ + FILE *fp; /* Optional file handle */ + WT_ITEM *msg; /* Buffered message */ int (*f)(WT_DBG *, const char *, ...) /* Function to write */ WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))); - WT_ITEM *tmp; /* Temporary space */ + WT_ITEM *tmp; /* Temporary space */ }; static const /* Output separator */ @@ -102,7 +105,7 @@ __debug_bytes(WT_DBG *ds, const void *data_arg, size_t size) /* * __debug_item -- - * Dump a single data/size pair, with an optional tag. + * Dump a single data/size item, with an optional tag. */ static int __debug_item(WT_DBG *ds, const char *tag, const void *data_arg, size_t size) @@ -115,6 +118,34 @@ __debug_item(WT_DBG *ds, const char *tag, const void *data_arg, size_t size) } /* + * __debug_item_key -- + * Dump a single data/size key item, with an optional tag. + */ +static int +__debug_item_key(WT_DBG *ds, const char *tag, const void *data_arg, size_t size) +{ + uint64_t ukey; + int64_t ikey; + const uint8_t *p; + + if (ds->integer_key) { + p = data_arg; + WT_RET(__wt_vunpack_int(&p, 0, &ikey)); + WT_RET(ds->f(ds, "\t%s%s{%" PRId64 "}\n", + tag == NULL ? "" : tag, tag == NULL ? "" : " ", ikey)); + return (0); + } + if (ds->unsigned_key) { + p = data_arg; + WT_RET(__wt_vunpack_uint(&p, 0, &ukey)); + WT_RET(ds->f(ds, "\t%s%s{%" PRIu64 "}\n", + tag == NULL ? "" : tag, tag == NULL ? "" : " ", ukey)); + return (0); + } + return (__debug_item(ds, tag, data_arg, size)); +} + +/* * __dmsg_event -- * Send a debug message to the event handler. */ @@ -193,6 +224,8 @@ __dmsg_file(WT_DBG *ds, const char *fmt, ...) static int __debug_config(WT_SESSION_IMPL *session, WT_DBG *ds, const char *ofile) { + WT_BTREE *btree; + memset(ds, 0, sizeof(WT_DBG)); ds->session = session; @@ -213,6 +246,13 @@ __debug_config(WT_SESSION_IMPL *session, WT_DBG *ds, const char *ofile) ds->f = __dmsg_file; } + btree = S2BT_SAFE(session); + ds->integer_key = btree != NULL && + strchr("hilq", btree->key_format[0]) != NULL && + btree->key_format[1] == '\0'; + ds->unsigned_key = btree != NULL && + strchr("HILQr", btree->key_format[0]) != NULL && + btree->key_format[1] == '\0'; return (0); } @@ -557,8 +597,10 @@ __wt_debug_tree_shape( return (__dmsg_wrapup(ds)); } -#define WT_DEBUG_TREE_LEAF 0x01 /* Debug leaf pages */ -#define WT_DEBUG_TREE_WALK 0x02 /* Descend the tree */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_DEBUG_TREE_LEAF 0x1u /* Debug leaf pages */ +#define WT_DEBUG_TREE_WALK 0x2u /* Descend the tree */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ /* * __wt_debug_tree_all -- @@ -920,7 +962,7 @@ __debug_page_row_int(WT_DBG *ds, WT_PAGE *page, uint32_t flags) WT_INTL_FOREACH_BEGIN(session, page, ref) { __wt_ref_key(page, ref, &p, &len); - WT_RET(__debug_item(ds, "K", p, len)); + WT_RET(__debug_item_key(ds, "K", p, len)); WT_RET(__debug_ref(ds, ref)); } WT_INTL_FOREACH_END; @@ -965,7 +1007,7 @@ __debug_page_row_leaf(WT_DBG *ds, WT_PAGE *page) /* Dump the page's K/V pairs. */ WT_ROW_FOREACH(page, rip, i) { WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); - WT_ERR(__debug_item(ds, "K", key->data, key->size)); + WT_ERR(__debug_item_key(ds, "K", key->data, key->size)); if ((cell = __wt_row_leaf_value_cell(page, rip, NULL)) == NULL) WT_ERR(ds->f(ds, "\tV {}\n")); @@ -1014,7 +1056,7 @@ __debug_row_skip(WT_DBG *ds, WT_INSERT_HEAD *head) WT_INSERT *ins; WT_SKIP_FOREACH(ins, head) { - WT_RET(__debug_item(ds, + WT_RET(__debug_item_key(ds, "insert", WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins))); WT_RET(__debug_update(ds, ins->upd, false)); } diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 4ab88cea01e..304a583c6a7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -292,9 +292,6 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) const char **cfg, *enc_cfg[] = { NULL, NULL }; bool fixed; - WT_UNUSED(maj_version); /* !HAVE_VERBOSE */ - WT_UNUSED(min_version); /* !HAVE_VERBOSE */ - btree = S2BT(session); cfg = btree->dhandle->cfg; conn = S2C(session); diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index e443fbdc94b..f8d0074879c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -173,7 +173,6 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool checkpoint, bool checkpoint_io, bool compressed) { - struct timespec start, stop; WT_BM *bm; WT_BTREE *btree; WT_DECL_ITEM(ctmp); @@ -183,6 +182,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_KEYED_ENCRYPTOR *kencryptor; WT_PAGE_HEADER *dsk; size_t dst_len, len, result_len, size, src_len; + uint64_t time_start, time_stop; uint8_t *dst, *src; int compression_failed; /* Extension API, so not a bool. */ bool data_checksum, encrypted, timer; @@ -190,6 +190,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, btree = S2BT(session); bm = btree->bm; encrypted = false; + time_start = time_stop = 0; /* Checkpoint calls are different than standard calls. */ WT_ASSERT(session, @@ -366,7 +367,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, } timer = !F_ISSET(session, WT_SESSION_INTERNAL); if (timer) - __wt_epoch(session, &start); + time_start = __wt_rdtsc(session); /* Call the block manager to write the block. */ WT_ERR(checkpoint ? @@ -376,10 +377,10 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, /* Update some statistics now that the write is done */ if (timer) { - __wt_epoch(session, &stop); + time_stop = __wt_rdtsc(session); WT_STAT_CONN_INCR(session, cache_write_app_count); WT_STAT_CONN_INCRV(session, cache_write_app_time, - WT_TIMEDIFF_US(stop, start)); + WT_TSCDIFF_US(session, time_stop, time_start)); } WT_STAT_CONN_INCR(session, cache_write); diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 19ff15fb21e..5dfcbf6249c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -314,18 +314,19 @@ __evict_force_check(WT_SESSION_IMPL *session, WT_REF *ref) static int __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) { - struct timespec start, stop; WT_BTREE *btree; WT_DECL_RET; WT_ITEM tmp; WT_PAGE *page; size_t addr_size; + uint64_t time_start, time_stop; uint32_t page_flags, new_state, previous_state; const uint8_t *addr; bool timer; btree = S2BT(session); page = NULL; + time_start = time_stop = 0; /* * Don't pass an allocated buffer to the underlying block read function, @@ -379,13 +380,13 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) */ timer = !F_ISSET(session, WT_SESSION_INTERNAL); if (timer) - __wt_epoch(session, &start); + time_start = __wt_rdtsc(session); WT_ERR(__wt_bt_read(session, &tmp, addr, addr_size)); if (timer) { - __wt_epoch(session, &stop); + time_stop = __wt_rdtsc(session); WT_STAT_CONN_INCR(session, cache_read_app_count); WT_STAT_CONN_INCRV(session, cache_read_app_time, - WT_TIMEDIFF_US(stop, start)); + WT_TSCDIFF_US(session, time_stop, time_start)); } /* @@ -775,7 +776,6 @@ static void __btree_verbose_lookaside_read( WT_SESSION_IMPL *session, uint32_t las_id, uint64_t las_pageid) { -#ifdef HAVE_VERBOSE WT_CACHE *cache; uint64_t ckpt_gen_current, ckpt_gen_last; @@ -808,9 +808,4 @@ __btree_verbose_lookaside_read( las_id, las_pageid); } } -#else - WT_UNUSED(session); - WT_UNUSED(las_id); - WT_UNUSED(las_pageid); -#endif } diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index e2da77348f0..462ee4ec52c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -106,10 +106,12 @@ struct __wt_track { } col; } u; -#define WT_TRACK_CHECK_START 0x01 /* Row: initial key updated */ -#define WT_TRACK_CHECK_STOP 0x02 /* Row: last key updated */ -#define WT_TRACK_MERGE 0x04 /* Page requires merging */ -#define WT_TRACK_OVFL_REFD 0x08 /* Overflow page referenced */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TRACK_CHECK_START 0x1u /* Row: initial key updated */ +#define WT_TRACK_CHECK_STOP 0x2u /* Row: last key updated */ +#define WT_TRACK_MERGE 0x4u /* Page requires merging */ +#define WT_TRACK_OVFL_REFD 0x8u /* Overflow page referenced */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ u_int flags; }; @@ -1893,8 +1895,6 @@ __slvg_row_build_leaf( uint32_t i, skip_start, skip_stop; int cmp; - WT_UNUSED(ss); /* !HAVE_VERBOSE */ - btree = S2BT(session); page = NULL; diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 2338d5be8ed..d1258cef13e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -114,7 +114,6 @@ __sync_dup_walk( static int __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) { - struct timespec end, start; WT_BTREE *btree; WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -122,7 +121,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) WT_REF *prev, *walk; WT_TXN *txn; uint64_t internal_bytes, internal_pages, leaf_bytes, leaf_pages; - uint64_t oldest_id, saved_pinned_id; + uint64_t oldest_id, saved_pinned_id, time_start, time_stop; uint32_t flags; bool timer, tried_eviction; @@ -131,6 +130,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) prev = walk = NULL; txn = &session->txn; tried_eviction = false; + time_start = time_stop = 0; flags = WT_READ_CACHE | WT_READ_NO_GEN; internal_bytes = leaf_bytes = 0; @@ -138,7 +138,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) saved_pinned_id = WT_SESSION_TXN_STATE(session)->pinned_id; timer = WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT); if (timer) - __wt_epoch(session, &start); + time_start = __wt_rdtsc(session); switch (syncop) { case WT_SYNC_WRITE_LEAVES: @@ -330,7 +330,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) } if (timer) { - __wt_epoch(session, &end); + time_stop = __wt_rdtsc(session); __wt_verbose(session, WT_VERB_CHECKPOINT, "__sync_file WT_SYNC_%s wrote: %" PRIu64 " leaf pages (%" PRIu64 "B), %" PRIu64 @@ -338,7 +338,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) syncop == WT_SYNC_WRITE_LEAVES ? "WRITE_LEAVES" : "CHECKPOINT", leaf_pages, leaf_bytes, internal_pages, internal_bytes, - WT_TIMEDIFF_MS(end, start)); + WT_TSCDIFF_MS(session, time_stop, time_start)); } err: /* On error, clear any left-over tree walk. */ diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index e0d90ad836d..cdab377039e 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -333,7 +333,6 @@ __wt_las_cursor_close( static int __las_insert_block_verbose(WT_SESSION_IMPL *session, WT_MULTI *multi) { -#ifdef HAVE_VERBOSE WT_CACHE *cache; WT_CONNECTION_IMPL *conn; #ifdef HAVE_TIMESTAMPS @@ -385,7 +384,7 @@ __las_insert_block_verbose(WT_SESSION_IMPL *session, WT_MULTI *multi) btree_id, multi->page_las.las_pageid, multi->page_las.las_max_txn, hex_timestamp, - multi->page_las.las_skew_newest? "newest" : "oldest", + multi->page_las.las_skew_newest ? "newest" : "oldest", WT_STAT_READ(conn->stats, cache_lookaside_entries), pct_dirty, pct_full); } @@ -393,10 +392,6 @@ __las_insert_block_verbose(WT_SESSION_IMPL *session, WT_MULTI *multi) /* Never skip updating the tracked generation */ if (WT_VERBOSE_ISSET(session, WT_VERB_LOOKASIDE)) cache->las_verb_gen_write = ckpt_gen_current; -#else - WT_UNUSED(session); - WT_UNUSED(multi); -#endif return (0); } @@ -433,7 +428,7 @@ __wt_las_insert_block(WT_SESSION_IMPL *session, WT_CURSOR *cursor, /* Wrap all the updates in a transaction. */ las_session = (WT_SESSION_IMPL *)cursor->session; WT_RET(__wt_txn_begin(las_session, NULL)); - las_session->txn.isolation = WT_TXN_ISO_READ_UNCOMMITTED; + las_session->txn.isolation = WT_ISO_READ_UNCOMMITTED; /* * Make sure there are no leftover entries (e.g., from a handle @@ -533,7 +528,7 @@ __wt_las_insert_block(WT_SESSION_IMPL *session, WT_CURSOR *cursor, if (insert_cnt > 0) { WT_STAT_CONN_INCRV( session, cache_lookaside_entries, insert_cnt); - __wt_atomic_add64( + (void)__wt_atomic_add64( &S2C(session)->cache->las_entry_count, insert_cnt); WT_ERR(__las_insert_block_verbose(session, multi)); } @@ -639,7 +634,7 @@ __wt_las_remove_block(WT_SESSION_IMPL *session, */ if (local_cursor) { WT_ERR(__wt_txn_begin(las_session, NULL)); - las_session->txn.isolation = WT_TXN_ISO_READ_UNCOMMITTED; + las_session->txn.isolation = WT_ISO_READ_UNCOMMITTED; local_txn = true; } @@ -725,12 +720,10 @@ __las_sweep_init(WT_SESSION_IMPL *session) cache->las_sweep_dropmin = UINT32_MAX; cache->las_sweep_dropmax = 0; for (i = 0; i < cache->las_dropped_next; i++) { - cache->las_sweep_dropmin = WT_MIN( - cache->las_sweep_dropmin, - cache->las_dropped[i]); - cache->las_sweep_dropmax = WT_MAX( - cache->las_sweep_dropmax, - cache->las_dropped[i]); + cache->las_sweep_dropmin = + WT_MIN(cache->las_sweep_dropmin, cache->las_dropped[i]); + cache->las_sweep_dropmax = + WT_MAX(cache->las_sweep_dropmax, cache->las_dropped[i]); } /* Initialize the bitmap. */ diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 14234badcfa..da11fa8c98a 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -106,6 +106,13 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK + confchk_wiredtiger_open_operation_tracking_subconfigs[] = { + { "enabled", "boolean", NULL, NULL, NULL, 0 }, + { "path", "string", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + +static const WT_CONFIG_CHECK confchk_wiredtiger_open_shared_cache_subconfigs[] = { { "chunk", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "name", "string", NULL, NULL, NULL, 0 }, @@ -162,6 +169,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2 }, { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, + { "operation_tracking", "category", + NULL, NULL, + confchk_wiredtiger_open_operation_tracking_subconfigs, 2 }, { "shared_cache", "category", NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, @@ -183,8 +193,8 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"thread_group\",\"timestamp\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," + "\"transaction\",\"verify\",\"version\",\"write\"]", NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -277,6 +287,14 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK + confchk_WT_SESSION_create_merge_custom_subconfigs[] = { + { "prefix", "string", NULL, NULL, NULL, 0 }, + { "start_generation", "int", NULL, "min=0,max=10", NULL, 0 }, + { "suffix", "string", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + +static const WT_CONFIG_CHECK confchk_WT_SESSION_create_lsm_subconfigs[] = { { "auto_throttle", "boolean", NULL, NULL, NULL, 0 }, { "bloom", "boolean", NULL, NULL, NULL, 0 }, @@ -287,6 +305,9 @@ static const WT_CONFIG_CHECK { "chunk_count_limit", "int", NULL, NULL, NULL, 0 }, { "chunk_max", "int", NULL, "min=100MB,max=10TB", NULL, 0 }, { "chunk_size", "int", NULL, "min=512K,max=500MB", NULL, 0 }, + { "merge_custom", "category", + NULL, NULL, + confchk_WT_SESSION_create_merge_custom_subconfigs, 3 }, { "merge_max", "int", NULL, "min=2,max=100", NULL, 0 }, { "merge_min", "int", NULL, "max=100", NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } @@ -346,7 +367,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { confchk_WT_SESSION_create_log_subconfigs, 1 }, { "lsm", "category", NULL, NULL, - confchk_WT_SESSION_create_lsm_subconfigs, 11 }, + confchk_WT_SESSION_create_lsm_subconfigs, 12 }, { "memory_page_max", "int", NULL, "min=512B,max=10TB", NULL, 0 }, @@ -680,7 +701,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { confchk_WT_SESSION_create_log_subconfigs, 1 }, { "lsm", "category", NULL, NULL, - confchk_WT_SESSION_create_lsm_subconfigs, 11 }, + confchk_WT_SESSION_create_lsm_subconfigs, 12 }, { "memory_page_max", "int", NULL, "min=512B,max=10TB", NULL, 0 }, @@ -810,6 +831,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "operation_tracking", "category", + NULL, NULL, + confchk_wiredtiger_open_operation_tracking_subconfigs, 2 }, { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, @@ -839,8 +863,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"thread_group\",\"timestamp\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," + "\"transaction\",\"verify\",\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -905,6 +929,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "operation_tracking", "category", + NULL, NULL, + confchk_wiredtiger_open_operation_tracking_subconfigs, 2 }, { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, @@ -934,8 +961,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"thread_group\",\"timestamp\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," + "\"transaction\",\"verify\",\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -997,6 +1024,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "operation_tracking", "category", + NULL, NULL, + confchk_wiredtiger_open_operation_tracking_subconfigs, 2 }, { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, @@ -1024,8 +1054,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"thread_group\",\"timestamp\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," + "\"transaction\",\"verify\",\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -1087,6 +1117,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "operation_tracking", "category", + NULL, NULL, + confchk_wiredtiger_open_operation_tracking_subconfigs, 2 }, { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, @@ -1114,8 +1147,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"thread_group\",\"timestamp\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," + "\"transaction\",\"verify\",\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -1180,11 +1213,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),log=(archive=true,prealloc=true," "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," - "lsm_merge=true,shared_cache=(chunk=10MB,name=,quota=0,reserve=0," - "size=500MB),statistics=none,statistics_log=(json=false," - "on_close=false,sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," + "lsm_merge=true,operation_tracking=(enabled=false,path=\".\")," + "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," + "statistics=none,statistics_log=(json=false,on_close=false," + "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,verbose=", - confchk_WT_CONNECTION_reconfigure, 21 + confchk_WT_CONNECTION_reconfigure, 22 }, { "WT_CONNECTION.rollback_to_stable", "", @@ -1247,7 +1281,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," "bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8," "bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB," - "chunk_size=10MB,merge_max=15,merge_min=0),memory_page_max=5MB," + "chunk_size=10MB,merge_custom=(prefix=,start_generation=0," + "suffix=),merge_max=15,merge_min=0),memory_page_max=5MB," "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," "prefix_compression_min=4,source=,split_deepen_min_child=0," "split_deepen_per_child=0,split_pct=90,type=file,value_format=u", @@ -1387,7 +1422,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "leaf_page_max=32KB,leaf_value_max=0,log=(enabled=true)," "lsm=(auto_throttle=true,bloom=true,bloom_bit_count=16," "bloom_config=,bloom_hash_count=8,bloom_oldest=false," - "chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB,merge_max=15," + "chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB," + "merge_custom=(prefix=,start_generation=0,suffix=),merge_max=15," "merge_min=0),memory_page_max=5MB,old_chunks=," "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," "prefix_compression_min=4,split_deepen_min_child=0," @@ -1414,14 +1450,15 @@ static const WT_CONFIG_ENTRY config_entries[] = { "log=(archive=true,compressor=,enabled=false,file_max=100MB," "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," + "mmap=true,multiprocess=false,operation_tracking=(enabled=false," + "path=\".\"),readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,write_through=", - confchk_wiredtiger_open, 42 + confchk_wiredtiger_open, 43 }, { "wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" @@ -1438,14 +1475,15 @@ static const WT_CONFIG_ENTRY config_entries[] = { "log=(archive=true,compressor=,enabled=false,file_max=100MB," "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," + "mmap=true,multiprocess=false,operation_tracking=(enabled=false," + "path=\".\"),readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 43 + confchk_wiredtiger_open_all, 44 }, { "wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" @@ -1460,13 +1498,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { "log=(archive=true,compressor=,enabled=false,file_max=100MB," "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," + "mmap=true,multiprocess=false,operation_tracking=(enabled=false," + "path=\".\"),readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),verbose=,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 37 + confchk_wiredtiger_open_basecfg, 38 }, { "wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" @@ -1481,13 +1520,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { "log=(archive=true,compressor=,enabled=false,file_max=100MB," "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," + "mmap=true,multiprocess=false,operation_tracking=(enabled=false," + "path=\".\"),readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),verbose=,write_through=", - confchk_wiredtiger_open_usercfg, 36 + confchk_wiredtiger_open_usercfg, 37 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index fd8fd6763db..5727399c301 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1084,6 +1084,8 @@ err: /* WT_TRET(wt_session->close(wt_session, config)); } + WT_TRET(__wt_async_flush(session)); + /* * Disable lookaside eviction: it doesn't help us shut down and can * lead to pages being marked dirty, causing spurious assertions to @@ -1284,6 +1286,35 @@ err: API_END_RET(session, ret); } /* + * __conn_calibrate_ticks -- + * Calibrate a ratio from rdtsc ticks to nanoseconds. + */ +static void +__conn_calibrate_ticks(WT_SESSION_IMPL *session) +{ +#if (defined __i386) || (defined __amd64) + struct timespec start, stop; + uint64_t diff_nsec, diff_tsc, tsc_start, tsc_stop; + volatile uint64_t i; + + __wt_epoch(session, &start); + tsc_start = __wt_rdtsc(session); + /* + * This needs to be CPU intensive and large enough. + */ + for (i = 0; i < WT_MILLION; i++) + ; + tsc_stop = __wt_rdtsc(session); + __wt_epoch(session, &stop); + diff_nsec = WT_TIMEDIFF_NS(stop, start); + diff_tsc = tsc_stop - tsc_start; + S2C(session)->tsc_nsec_ratio = (double)diff_tsc/(double)diff_nsec; +#else + S2C(session)->tsc_nsec_ratio = 1.0; +#endif +} + +/* * __conn_config_append -- * Append an entry to a config stack. */ @@ -1794,7 +1825,7 @@ err: /* /* Simple structure for name and flag configuration searches. */ typedef struct { const char *name; - uint32_t flag; + uint64_t flag; } WT_NAME_FLAG; /* @@ -1831,6 +1862,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "salvage", WT_VERB_SALVAGE }, { "shared_cache", WT_VERB_SHARED_CACHE }, { "split", WT_VERB_SPLIT }, + { "temporary", WT_VERB_TEMPORARY }, { "thread_group", WT_VERB_THREAD_GROUP }, { "timestamp", WT_VERB_TIMESTAMP }, { "transaction", WT_VERB_TRANSACTION }, @@ -1843,7 +1875,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_CONNECTION_IMPL *conn; WT_DECL_RET; const WT_NAME_FLAG *ft; - uint32_t flags; + uint64_t flags; conn = S2C(session); @@ -1852,17 +1884,8 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) flags = 0; for (ft = verbtypes; ft->name != NULL; ft++) { if ((ret = __wt_config_subgets( - session, &cval, ft->name, &sval)) == 0 && sval.val != 0) { -#ifdef HAVE_VERBOSE + session, &cval, ft->name, &sval)) == 0 && sval.val != 0) LF_SET(ft->flag); -#else - WT_RET_MSG(session, EINVAL, - "Verbose option specified when WiredTiger built " - "without verbose support. Add --enable-verbose to " - "configure command and rebuild to include support " - "for verbose messages"); -#endif - } WT_RET_NOTFOUND_OK(ret); } @@ -2002,7 +2025,7 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_CONNECTION_IMPL *conn; WT_DECL_RET; const WT_NAME_FLAG *ft; - uint32_t flags; + uint64_t flags; conn = S2C(session); @@ -2172,6 +2195,44 @@ err: API_END_RET(session, ret); } /* + * __conn_session_size -- + * Return the session count for this run. + */ +static int +__conn_session_size( + WT_SESSION_IMPL *session, const char *cfg[], uint32_t *vp) +{ + WT_CONFIG_ITEM cval; + int64_t v; + + /* + * Start with 20 internal sessions to cover threads the application + * can't configure (for example, checkpoint or statistics log server + * threads). + */ +#define WT_EXTRA_INTERNAL_SESSIONS 20 + v = WT_EXTRA_INTERNAL_SESSIONS; + + /* Then, add in the thread counts applications can configure. */ + WT_RET(__wt_config_gets(session, cfg, "async.threads", &cval)); + v += cval.val; + + WT_RET(__wt_config_gets(session, cfg, "eviction.threads_max", &cval)); + v += cval.val; + + WT_RET(__wt_config_gets( + session, cfg, "lsm_manager.worker_thread_max", &cval)); + v += cval.val; + + WT_RET(__wt_config_gets(session, cfg, "session_max", &cval)); + v += cval.val; + + *vp = (uint32_t)v; + + return (0); +} + +/* * __conn_chk_file_system -- * Check the configured file system. */ @@ -2498,8 +2559,10 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__wt_verbose_config(session, cfg)); WT_ERR(__wt_timing_stress_config(session, cfg)); - WT_ERR(__wt_config_gets(session, cfg, "session_max", &cval)); - conn->session_size = (uint32_t)cval.val + WT_EXTRA_INTERNAL_SESSIONS; + /* Set up operation tracking if configured. */ + WT_ERR(__wt_conn_optrack_setup(session, cfg, false)); + + WT_ERR(__conn_session_size(session, cfg, &conn->session_size)); WT_ERR(__wt_config_gets(session, cfg, "session_scratch_max", &cval)); conn->session_scratch_max = (size_t)cval.val; @@ -2619,6 +2682,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__conn_write_base_config(session, cfg)); /* + * Calibrate the ratio of rdtsc ticks to nanoseconds. + */ + __conn_calibrate_ticks(session); + + /* * Check on the turtle and metadata files, creating them if necessary * (which avoids application threads racing to create the metadata file * later). Once the metadata file exists, get a reference to it in diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 8819bf8b970..c0bada4f827 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -865,12 +865,11 @@ err: WT_PANIC_MSG(session, ret, "log wrlsn server error"); static WT_THREAD_RET __log_server(void *arg) { - struct timespec start, now; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; WT_SESSION_IMPL *session; - uint64_t timediff; + uint64_t time_start, time_stop, timediff; bool did_work, signalled; session = arg; @@ -949,11 +948,11 @@ __log_server(void *arg) } /* Wait until the next event. */ - __wt_epoch(session, &start); + time_start = __wt_rdtsc(session); __wt_cond_auto_wait_signal( session, conn->log_cond, did_work, NULL, &signalled); - __wt_epoch(session, &now); - timediff = WT_TIMEDIFF_MS(now, start); + time_stop = __wt_rdtsc(session); + timediff = WT_TSCDIFF_MS(session, time_stop, time_start); } if (0) { diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index e72fa5c00a4..7b40822f681 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -147,6 +147,13 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) if (conn->lock_fh != NULL) WT_TRET(__wt_close(session, &conn->lock_fh)); + /* Close any optrack files */ + if (session->optrack_fh != NULL) + WT_TRET(__wt_close(session, &session->optrack_fh)); + + /* Close operation tracking */ + WT_TRET(__wt_conn_optrack_teardown(session, false)); + /* Close any file handles left open. */ WT_TRET(__wt_close_connection_close(session)); diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c index 1bde68e1762..2cadc9b78d4 100644 --- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c +++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c @@ -57,6 +57,106 @@ __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg) } /* + * __wt_conn_optrack_setup -- + * Set up operation logging. + */ +int +__wt_conn_optrack_setup(WT_SESSION_IMPL *session, + const char *cfg[], bool reconfig) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + WT_DECL_ITEM(buf); + WT_DECL_RET; + + conn = S2C(session); + + /* Once an operation tracking path has been set it can't be changed. */ + if (!reconfig) { + WT_RET(__wt_config_gets(session, + cfg, "operation_tracking.path", &cval)); + WT_RET(__wt_strndup(session, + cval.str, cval.len, &conn->optrack_path)); + } + + WT_RET(__wt_config_gets(session, + cfg, "operation_tracking.enabled", &cval)); + if (cval.val == 0) { + if (F_ISSET(conn, WT_CONN_OPTRACK)) { + WT_RET(__wt_conn_optrack_teardown(session, reconfig)); + F_CLR(conn, WT_CONN_OPTRACK); + } + return (0); + } else if (F_ISSET(conn, WT_CONN_READONLY)) + /* Operation tracking isn't supported in read-only mode */ + WT_RET_MSG(session, EINVAL, + "Operation tracking is incompatible with read only " + "configuration."); + else if (F_ISSET(conn, WT_CONN_OPTRACK)) + /* Already enabled, nothing else to do */ + return (0); + + /* + * Operation tracking files will include the ID of the creating process + * in their name, so we can distinguish between log files created by + * different WiredTiger processes in the same directory. We cache the + * process id for future use. + */ + conn->optrack_pid = __wt_process_id(); + + /* + * Open the file in the same directory that will hold a map of + * translations between function names and function IDs. If the file + * exists, remove it. + */ + WT_RET(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_filename_construct(session, conn->optrack_path, + "optrack-map", conn->optrack_pid, UINT32_MAX, buf)); + WT_ERR(__wt_open(session, + (const char *)buf->data, WT_FS_OPEN_FILE_TYPE_REGULAR, + WT_FS_OPEN_CREATE, &conn->optrack_map_fh)); + + WT_ERR(__wt_spin_init(session, + &conn->optrack_map_spinlock, "optrack map spinlock")); + + WT_ERR(__wt_malloc(session, WT_OPTRACK_BUFSIZE, + &conn->dummy_session.optrack_buf)); + + /* Set operation tracking on */ + F_SET(conn, WT_CONN_OPTRACK); + +err: __wt_scr_free(session, &buf); + return (ret); +} + +/* + * __wt_conn_optrack_teardown -- + * Clean up connection-wide resources used for operation logging. + */ +int +__wt_conn_optrack_teardown(WT_SESSION_IMPL *session, bool reconfig) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + + conn = S2C(session); + + if (!reconfig) + /* Looks like we are shutting down */ + __wt_free(session, conn->optrack_path); + + if (!F_ISSET(conn, WT_CONN_OPTRACK)) + return (0); + + __wt_spin_destroy(session, &conn->optrack_map_spinlock); + + WT_TRET(__wt_close(session, &conn->optrack_map_fh)); + __wt_free(session, conn->dummy_session.optrack_buf); + + return (ret); +} + +/* * __wt_conn_statistics_config -- * Set statistics configuration. */ @@ -189,6 +289,7 @@ __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_WITH_CHECKPOINT_LOCK(session, ret = __wt_conn_compat_config(session, cfg)); WT_ERR(ret); + WT_ERR(__wt_conn_optrack_setup(session, cfg, true)); WT_ERR(__wt_conn_statistics_config(session, cfg)); WT_ERR(__wt_async_reconfig(session, cfg)); WT_ERR(__wt_cache_config(session, true, cfg)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 9cfa3203aec..6027b1ecdd6 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -96,9 +96,16 @@ __wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) int __wt_cursor_modify_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) { + WT_SESSION_IMPL *session; + WT_UNUSED(entries); WT_UNUSED(nentries); + if (cursor->value_format != NULL && strlen(cursor->value_format) != 0) { + session = (WT_SESSION_IMPL *)cursor->session; + WT_RET_MSG(session, ENOTSUP, + "WT_CURSOR.modify only supported for 'u' value formats"); + } return (__wt_cursor_notsup(cursor)); } diff --git a/src/third_party/wiredtiger/src/docs/build-posix.dox b/src/third_party/wiredtiger/src/docs/build-posix.dox index 3e7f8f37acd..7192a9c2322 100644 --- a/src/third_party/wiredtiger/src/docs/build-posix.dox +++ b/src/third_party/wiredtiger/src/docs/build-posix.dox @@ -142,10 +142,6 @@ Build the WiredTiger <a href="http://www.python.org">Python</a> API. Configure WiredTiger for <a href="http://code.google.com/p/snappy/">snappy</a> compression; see @ref compression for more information. -@par \c --enable-verbose -Configure WiredTiger to support the \c verbose configuration string to -::wiredtiger_open. - @par \c --enable-zlib Configure WiredTiger for <a href="http://www.zlib.net/">zlib</a> compression; see @ref compression for more information. diff --git a/src/third_party/wiredtiger/src/docs/build-windows.dox b/src/third_party/wiredtiger/src/docs/build-windows.dox index 79df6b9712d..dcd16e30ad1 100644 --- a/src/third_party/wiredtiger/src/docs/build-windows.dox +++ b/src/third_party/wiredtiger/src/docs/build-windows.dox @@ -89,6 +89,10 @@ Configure WiredTiger to sleep and wait for a debugger to attach on failure. Configure WiredTiger to perform various run-time diagnostic tests. <b>DO NOT</b> configure this option in production environments. +@par \c --enable-java +Build the WiredTiger <a href="http://www.java.com">Java</a> API, +requires path to swig.exe and path to Java JDK directory, comma separated. + @par \c --enable-python Build the WiredTiger <a href="http://www.python.org">Python</a> API, requires path to swig.exe on Windows. @@ -97,10 +101,6 @@ requires path to swig.exe on Windows. Configure WiredTiger for <a href="http://code.google.com/p/snappy/">snappy</a> compression; see @ref compression for more information. -@par \c --enable-verbose -Configure WiredTiger to support the \c verbose configuration string to -::wiredtiger_open. - @par \c --enable-zlib Configure WiredTiger for <a href="http://www.zlib.net/">zlib</a> compression; see @ref compression for more information. diff --git a/src/third_party/wiredtiger/src/docs/operation_tracking.dox b/src/third_party/wiredtiger/src/docs/operation_tracking.dox new file mode 100644 index 00000000000..b49ec1aa91a --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/operation_tracking.dox @@ -0,0 +1,39 @@ +/*! @m_page{{c,java},operation_tracking,Track function calls} + +When this option is enabled, WiredTiger tracks all API calls as well as certain +functions that are deemed important for performance, such as eviction-related +functions. Tracking is performed by generating a log record when the WiredTiger +library enters and exits a tracked function. A log record contains a function +name and its timestamp. For the purposes of efficiency, the timestamp on most +platforms is obtained directly from the CPU register, so the units of time +correspond to processor clock ticks. + +By default, the log files are produced in the sub-directory +`operation_tracking` of the database home directory. To change the +destination directory, use the `path` sub-option of the +`operation_tracking` option. + +Operation tracking produces two kinds of files: the binary log files +and the map file. The map file is needed to convert the binary logs +into text. The map file contains the mapping between numeric function +ids used in the log files and the human-readable function name. The +map file is called `optrack-map.<pid>`, where `pid` is the id of +the process that generated the map file. Log files have names that +look like `optrack.<pid>.<tid>`, where `tid` is the numeric id of the +WiredTiger session that generated the log file. + +To convert binary log files to text, run the wt_optrack_decode.py script in the +tools directory of the WiredTiger distribution. The script takes as arguments +the list of log files and the name of the map file with the `-m` option. Suppose +you would like to convert the log files generated by a process with `pid +1234`. Then you would run the script as follows: + +``` +python wt_optrack_decode.py -m optrack-map.1234 optrack.1234.* +``` + +Enabling operation tracking may generate performance overhead if the tracked +functions are executed frequently. Please be aware of this consequence and +measure your performance before deciding whether to enable operation tracking. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/programming.dox b/src/third_party/wiredtiger/src/docs/programming.dox index 7274fb8078a..3ddb0c376c5 100644 --- a/src/third_party/wiredtiger/src/docs/programming.dox +++ b/src/third_party/wiredtiger/src/docs/programming.dox @@ -44,6 +44,7 @@ each of which is ordered by one or more columns. - @subpage in_memory - @subpage cursor_join - @subpage cursor_log +- @subpage operation_tracking - @ref transaction_named_snapshots - @subpage rebalance - @subpage shared_cache diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 112e5e815fe..d5347709248 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -44,6 +44,7 @@ Google's HyperDex HyperLevelDB IEC +JDK JavaScript KMS LD @@ -368,6 +369,7 @@ objectsin ol oltp oob +optrack optype os outlier @@ -382,6 +384,7 @@ pget php plantuml png +pid posix pre prepends @@ -462,6 +465,7 @@ substring superset sys sz +tid tRuE tablename tcl diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 343d29d47cf..03f11f29b01 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -230,7 +230,6 @@ __wt_evict_server_wake(WT_SESSION_IMPL *session) conn = S2C(session); cache = conn->cache; -#ifdef HAVE_VERBOSE if (WT_VERBOSE_ISSET(session, WT_VERB_EVICTSERVER)) { uint64_t bytes_inuse, bytes_max; @@ -244,7 +243,6 @@ __wt_evict_server_wake(WT_SESSION_IMPL *session) bytes_inuse <= bytes_max ? "<=" : ">", bytes_max / WT_MEGABYTE); } -#endif __wt_cond_signal(session, cache->evict_cond); } @@ -371,9 +369,7 @@ err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); static int __evict_server(WT_SESSION_IMPL *session, bool *did_work) { -#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) struct timespec now; -#endif WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -421,13 +417,19 @@ __evict_server(WT_SESSION_IMPL *session, bool *did_work) /* Eviction is stuck, check if we have made progress. */ if (*did_work) { -#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) - __wt_epoch(session, &cache->stuck_time); +#if !defined(HAVE_DIAGNOSTIC) + /* Need verbose check only if not in diagnostic build */ + if (WT_VERBOSE_ISSET(session, WT_VERB_EVICT_STUCK)) #endif + __wt_epoch(session, &cache->stuck_time); return (0); } -#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) +#if !defined(HAVE_DIAGNOSTIC) + /* Need verbose check only if not in diagnostic build */ + if (!WT_VERBOSE_ISSET(session, WT_VERB_EVICT_STUCK)) + return (0); +#endif /* * If we're stuck for 5 minutes in diagnostic mode, or the verbose * evict_stuck flag is configured, log the cache and transaction state. @@ -447,11 +449,10 @@ __evict_server(WT_SESSION_IMPL *session, bool *did_work) #if defined(HAVE_DIAGNOSTIC) __wt_err(session, ETIMEDOUT, "Cache stuck for too long, giving up"); - ret = ETIMEDOUT; - WT_TRET(__wt_verbose_dump_txn(session)); - WT_TRET(__wt_verbose_dump_cache(session)); - return (ret); -#elif defined(HAVE_VERBOSE) + WT_RET(__wt_verbose_dump_txn(session)); + WT_RET(__wt_verbose_dump_cache(session)); + return (ETIMEDOUT); +#else if (WT_VERBOSE_ISSET(session, WT_VERB_EVICT_STUCK)) { WT_RET(__wt_verbose_dump_txn(session)); WT_RET(__wt_verbose_dump_cache(session)); @@ -461,7 +462,6 @@ __evict_server(WT_SESSION_IMPL *session, bool *did_work) } #endif } -#endif return (0); } @@ -492,12 +492,14 @@ __wt_evict_create(WT_SESSION_IMPL *session) session_flags, __wt_evict_thread_chk, __wt_evict_thread_run, __wt_evict_thread_stop)); -#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) /* * Ensure the cache stuck timer is initialized when starting eviction. */ - __wt_epoch(session, &conn->cache->stuck_time); +#if !defined(HAVE_DIAGNOSTIC) + /* Need verbose check only if not in diagnostic build */ + if (WT_VERBOSE_ISSET(session, WT_VERB_EVICTSERVER)) #endif + __wt_epoch(session, &conn->cache->stuck_time); /* * Allow queues to be populated now that the eviction threads @@ -651,11 +653,11 @@ __evict_update_work(WT_SESSION_IMPL *session) static int __evict_pass(WT_SESSION_IMPL *session) { - struct timespec now, prev; WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_TXN_GLOBAL *txn_global; uint64_t eviction_progress, oldest_id, prev_oldest_id; + uint64_t time_now, time_prev; u_int loop; conn = S2C(session); @@ -665,13 +667,13 @@ __evict_pass(WT_SESSION_IMPL *session) /* Track whether pages are being evicted and progress is made. */ eviction_progress = cache->eviction_progress; prev_oldest_id = txn_global->oldest_id; - WT_CLEAR(prev); + time_now = time_prev = 0; /* Evict pages from the cache. */ for (loop = 0; cache->pass_intr == 0; loop++) { - __wt_epoch(session, &now); + time_now = __wt_rdtsc(session); if (loop == 0) - prev = now; + time_prev = time_now; __evict_tune_workers(session); /* @@ -739,7 +741,7 @@ __evict_pass(WT_SESSION_IMPL *session) * transactions and writing updates to the lookaside table. */ if (eviction_progress == cache->eviction_progress) { - if (WT_TIMEDIFF_MS(now, prev) >= 20 && + if (WT_TSCDIFF_MS(session, time_now, time_prev) >= 20 && F_ISSET(cache, WT_CACHE_EVICT_CLEAN_HARD | WT_CACHE_EVICT_DIRTY_HARD)) { if (cache->evict_aggressive_score < 100) @@ -749,7 +751,7 @@ __evict_pass(WT_SESSION_IMPL *session) txn_global->current != oldest_id && cache->evict_aggressive_score < 100) ++cache->evict_aggressive_score; - prev = now; + time_prev = time_now; prev_oldest_id = oldest_id; } @@ -1158,7 +1160,9 @@ __evict_lru_pages(WT_SESSION_IMPL *session, bool is_server) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_TRACK_OP_DECL; + WT_TRACK_OP_INIT(session); conn = S2C(session); /* @@ -1175,6 +1179,7 @@ __evict_lru_pages(WT_SESSION_IMPL *session, bool is_server) __wt_cond_wait( session, conn->evict_threads.wait_cond, 10000, NULL); + WT_TRACK_OP_END(session); return (ret == WT_NOTFOUND ? 0 : ret); } @@ -1188,9 +1193,11 @@ __evict_lru_walk(WT_SESSION_IMPL *session) WT_CACHE *cache; WT_DECL_RET; WT_EVICT_QUEUE *queue, *other_queue; + WT_TRACK_OP_DECL; uint64_t read_gen_oldest; uint32_t candidates, entries; + WT_TRACK_OP_INIT(session); cache = S2C(session)->cache; /* Age out the score of how much the queue has been empty recently. */ @@ -1212,12 +1219,14 @@ __evict_lru_walk(WT_SESSION_IMPL *session) */ if (__evict_queue_full(queue) && cache->evict_empty_score < WT_EVICT_SCORE_CUTOFF) - return (0); + goto err; /* Get some more pages to consider for eviction. */ - if ((ret = __evict_walk(cache->walk_session, queue)) == EBUSY) - return (0); /* An interrupt was requested, give up. */ - WT_RET_NOTFOUND_OK(ret); + if ((ret = __evict_walk(cache->walk_session, queue)) == EBUSY) { + ret = 0; + goto err; /* An interrupt was requested, give up. */ + } + WT_ERR_NOTFOUND_OK(ret); /* * If the queue we are filling is empty, pages are being requested @@ -1271,7 +1280,7 @@ __evict_lru_walk(WT_SESSION_IMPL *session) queue->evict_candidates = 0; queue->evict_current = NULL; __wt_spin_unlock(session, &queue->evict_lock); - return (0); + goto err; } /* Decide how many of the candidates we're going to try and evict. */ @@ -1330,7 +1339,8 @@ __evict_lru_walk(WT_SESSION_IMPL *session) */ __wt_cond_signal(session, S2C(session)->evict_threads.wait_cond); - return (0); +err: WT_TRACK_OP_END(session); + return (ret); } /* @@ -1345,9 +1355,12 @@ __evict_walk(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue) WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DECL_RET; + WT_TRACK_OP_DECL; u_int max_entries, retries, slot, start_slot, total_candidates; bool dhandle_locked, incr; + WT_TRACK_OP_INIT(session); + conn = S2C(session); cache = conn->cache; btree = NULL; @@ -1527,9 +1540,10 @@ err: if (dhandle_locked) * let our caller know. */ if (queue->evict_entries == slot && cache->pass_intr == 0) - return (WT_NOTFOUND); + ret = WT_NOTFOUND; queue->evict_entries = slot; + WT_TRACK_OP_END(session); return (ret); } @@ -2242,18 +2256,22 @@ __evict_get_ref( static int __evict_page(WT_SESSION_IMPL *session, bool is_server) { - struct timespec enter, leave; WT_BTREE *btree; WT_CACHE *cache; WT_DECL_RET; WT_REF *ref; + WT_TRACK_OP_DECL; + uint64_t time_start, time_stop; bool app_timer; + WT_TRACK_OP_INIT(session); + WT_RET(__evict_get_ref(session, is_server, &btree, &ref)); WT_ASSERT(session, ref->state == WT_REF_LOCKED); app_timer = false; cache = S2C(session)->cache; + time_start = time_stop = 0; /* * An internal session flags either the server itself or an eviction @@ -2272,7 +2290,7 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) cache->app_evicts++; if (WT_STAT_ENABLED(session)) { app_timer = true; - __wt_epoch(session, &enter); + time_start = __wt_rdtsc(session); } } @@ -2292,10 +2310,12 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) (void)__wt_atomic_subv32(&btree->evict_busy, 1); if (app_timer) { - __wt_epoch(session, &leave); + time_stop = __wt_rdtsc(session); WT_STAT_CONN_INCRV(session, - application_evict_time, WT_TIMEDIFF_US(leave, enter)); + application_evict_time, + WT_TSCDIFF_US(session, time_stop, time_start)); } + WT_TRACK_OP_END(session); return (ret); } @@ -2308,17 +2328,20 @@ int __wt_cache_eviction_worker( WT_SESSION_IMPL *session, bool busy, bool readonly, u_int pct_full) { - struct timespec enter, leave; WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_TRACK_OP_DECL; WT_TXN_GLOBAL *txn_global; WT_TXN_STATE *txn_state; - uint64_t initial_progress, max_progress; + uint64_t initial_progress, max_progress, time_start, time_stop; bool timer; + WT_TRACK_OP_INIT(session); + conn = S2C(session); cache = conn->cache; + time_start = time_stop = 0; txn_global = &conn->txn_global; txn_state = WT_SESSION_TXN_STATE(session); @@ -2326,11 +2349,8 @@ __wt_cache_eviction_worker( * It is not safe to proceed if the eviction server threads aren't * setup yet. */ - if (!conn->evict_server_running) - return (0); - - if (busy && pct_full < 100) - return (0); + if (!conn->evict_server_running || (busy && pct_full < 100)) + goto done; /* Wake the eviction server if we need to do work. */ __wt_evict_server_wake(session); @@ -2339,7 +2359,7 @@ __wt_cache_eviction_worker( timer = WT_STAT_ENABLED(session) && !F_ISSET(session, WT_SESSION_INTERNAL); if (timer) - __wt_epoch(session, &enter); + time_start = __wt_rdtsc(session); for (initial_progress = cache->eviction_progress;; ret = 0) { /* @@ -2406,11 +2426,13 @@ __wt_cache_eviction_worker( } err: if (timer) { - __wt_epoch(session, &leave); + time_stop = __wt_rdtsc(session); WT_STAT_CONN_INCRV(session, - application_cache_time, WT_TIMEDIFF_US(leave, enter)); + application_cache_time, + WT_TSCDIFF_US(session, time_stop, time_start)); } +done: WT_TRACK_OP_END(session); return (ret); /* NOTREACHED */ } @@ -2605,6 +2627,7 @@ __wt_verbose_dump_cache(WT_SESSION_IMPL *session) WT_DECL_RET; uint64_t total_bytes, total_dirty_bytes; u_int pct; + bool needed; conn = S2C(session); total_bytes = total_dirty_bytes = 0; @@ -2615,10 +2638,12 @@ __wt_verbose_dump_cache(WT_SESSION_IMPL *session) WT_RET(__wt_msg(session, "cache full: %s", __wt_cache_full(session) ? "yes" : "no")); - WT_RET(__wt_msg(session, "cache clean check: %s (%u%%)", - __wt_eviction_clean_needed(session, &pct) ? "yes" : "no", pct)); - WT_RET(__wt_msg(session, "cache dirty check: %s (%u%%)", - __wt_eviction_dirty_needed(session, &pct) ? "yes" : "no", pct)); + needed = __wt_eviction_clean_needed(session, &pct); + WT_RET(__wt_msg(session, + "cache clean check: %s (%u%%)", needed ? "yes" : "no", pct)); + needed = __wt_eviction_dirty_needed(session, &pct); + WT_RET(__wt_msg(session, + "cache dirty check: %s (%u%%)", needed ? "yes" : "no", pct)); for (dhandle = NULL;;) { WT_WITH_HANDLE_LIST_READ_LOCK(session, diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 7a84f90eb81..269fbaeaac8 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -52,15 +52,15 @@ __evict_exclusive(WT_SESSION_IMPL *session, WT_REF *ref) int __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) { - struct timespec start, stop; WT_BTREE *btree; WT_DECL_RET; WT_PAGE *page; + uint64_t time_start, time_stop; bool locked, too_big; btree = S2BT(session); page = ref->page; - __wt_epoch(session, &start); + time_start = __wt_rdtsc(session); /* * Take some care with order of operations: if we release the hazard @@ -83,12 +83,12 @@ __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) * we have one of two pairs of stats to increment. */ ret = __wt_evict(session, ref, false); - __wt_epoch(session, &stop); + time_stop = __wt_rdtsc(session); if (ret == 0) { if (too_big) { WT_STAT_CONN_INCR(session, cache_eviction_force); WT_STAT_CONN_INCRV(session, cache_eviction_force_time, - WT_TIMEDIFF_US(stop, start)); + WT_TSCDIFF_US(session, time_stop, time_start)); } else { /* * If the page isn't too big, we are evicting it because @@ -98,12 +98,12 @@ __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) WT_STAT_CONN_INCR(session, cache_eviction_force_delete); WT_STAT_CONN_INCRV(session, cache_eviction_force_delete_time, - WT_TIMEDIFF_US(stop, start)); + WT_TSCDIFF_US(session, time_stop, time_start)); } } else { WT_STAT_CONN_INCR(session, cache_eviction_force_fail); WT_STAT_CONN_INCRV(session, cache_eviction_force_fail_time, - WT_TIMEDIFF_US(stop, start)); + WT_TSCDIFF_US(session, time_stop, time_start)); } (void)__wt_atomic_subv32(&btree->evict_busy, 1); diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 2e3a2fe220f..43b4f733a3d 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -36,10 +36,16 @@ /* Standard entry points to the API: declares/initializes local variables. */ #define API_SESSION_INIT(s, h, n, dh) \ + WT_TRACK_OP_DECL; \ WT_DATA_HANDLE *__olddh = (s)->dhandle; \ const char *__oldname = (s)->name; \ (s)->dhandle = (dh); \ (s)->name = (s)->lastop = #h "." #n; \ + /* \ + * No code before this line, otherwise error handling won't be \ + * correct. \ + */ \ + WT_TRACK_OP_INIT(s); \ WT_SINGLE_THREAD_CHECK_START(s); \ WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ __wt_verbose((s), WT_VERB_API, "%s", "CALL: " #h ":" #n) @@ -57,14 +63,19 @@ #define API_END(s, ret) \ if ((s) != NULL) { \ - WT_SINGLE_THREAD_CHECK_STOP(s); \ - (s)->dhandle = __olddh; \ - (s)->name = __oldname; \ + WT_TRACK_OP_END(s); \ + WT_SINGLE_THREAD_CHECK_STOP(s); \ if (F_ISSET(&(s)->txn, WT_TXN_RUNNING) && \ (ret) != 0 && \ (ret) != WT_NOTFOUND && \ (ret) != WT_DUPLICATE_KEY) \ F_SET(&(s)->txn, WT_TXN_ERROR); \ + /* \ + * No code after this line, otherwise error handling \ + * won't be correct. \ + */ \ + (s)->dhandle = __olddh; \ + (s)->name = __oldname; \ } \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h index 1a445bfac0b..bf99b4babc8 100644 --- a/src/third_party/wiredtiger/src/include/block.h +++ b/src/third_party/wiredtiger/src/include/block.h @@ -341,7 +341,11 @@ struct __wt_block_header { */ uint32_t checksum; /* 04-07: checksum */ -#define WT_BLOCK_DATA_CKSUM 0x01 /* Block data is part of the checksum */ + /* + * No automatic generation: flag values cannot change, they're written + * to disk. + */ +#define WT_BLOCK_DATA_CKSUM 0x1u /* Block data is part of the checksum */ uint8_t flags; /* 08: flags */ /* diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index c5cdfe5850a..d2a7e376273 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -8,6 +8,33 @@ #define WT_RECNO_OOB 0 /* Illegal record number */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_READ_CACHE 0x0001u +#define WT_READ_IGNORE_CACHE_SIZE 0x0002u +#define WT_READ_LOOKASIDE 0x0004u +#define WT_READ_NOTFOUND_OK 0x0008u +#define WT_READ_NO_EMPTY 0x0010u +#define WT_READ_NO_GEN 0x0020u +#define WT_READ_NO_SPLIT 0x0040u +#define WT_READ_NO_WAIT 0x0080u +#define WT_READ_PREV 0x0100u +#define WT_READ_RESTART_OK 0x0200u +#define WT_READ_SKIP_INTL 0x0400u +#define WT_READ_TRUNCATE 0x0800u +#define WT_READ_WONT_NEED 0x1000u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_REC_CHECKPOINT 0x01u +#define WT_REC_EVICT 0x02u +#define WT_REC_IN_MEMORY 0x04u +#define WT_REC_LOOKASIDE 0x08u +#define WT_REC_SCRUB 0x10u +#define WT_REC_UPDATE_RESTORE 0x20u +#define WT_REC_VISIBILITY_ERR 0x40u +#define WT_REC_VISIBLE_ALL 0x80u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + /* * WT_PAGE_HEADER -- * Blocks have a common header, a WT_PAGE_HEADER structure followed by a @@ -41,11 +68,15 @@ struct __wt_page_header { uint8_t type; /* 24: page type */ -#define WT_PAGE_COMPRESSED 0x01 /* Page is compressed on disk */ -#define WT_PAGE_EMPTY_V_ALL 0x02 /* Page has all zero-length values */ -#define WT_PAGE_EMPTY_V_NONE 0x04 /* Page has no zero-length values */ -#define WT_PAGE_ENCRYPTED 0x08 /* Page is encrypted on disk */ -#define WT_PAGE_LAS_UPDATE 0x10 /* Page updates in lookaside store */ + /* + * No automatic generation: flag values cannot change, they're written + * to disk. + */ +#define WT_PAGE_COMPRESSED 0x01u /* Page is compressed on disk */ +#define WT_PAGE_EMPTY_V_ALL 0x02u /* Page has all zero-length values */ +#define WT_PAGE_EMPTY_V_NONE 0x04u /* Page has no zero-length values */ +#define WT_PAGE_ENCRYPTED 0x08u /* Page is encrypted on disk */ +#define WT_PAGE_LAS_UPDATE 0x10u /* Page updates in lookaside store */ uint8_t flags; /* 25: flags */ /* @@ -145,8 +176,10 @@ struct __wt_ovfl_reuse { * skiplist entry; if reconciliation fails for any reason, discard the * newly added skiplist entries, along with their underlying blocks. */ -#define WT_OVFL_REUSE_INUSE 0x01 -#define WT_OVFL_REUSE_JUST_ADDED 0x02 +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_OVFL_REUSE_INUSE 0x1u +#define WT_OVFL_REUSE_JUST_ADDED 0x2u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; /* @@ -604,14 +637,16 @@ struct __wt_page { #define WT_PAGE_ROW_LEAF 7 /* Row-store leaf page */ uint8_t type; /* Page type */ -#define WT_PAGE_BUILD_KEYS 0x01 /* Keys have been built in memory */ -#define WT_PAGE_DISK_ALLOC 0x02 /* Disk image in allocated memory */ -#define WT_PAGE_DISK_MAPPED 0x04 /* Disk image in mapped memory */ -#define WT_PAGE_EVICT_LRU 0x08 /* Page is on the LRU queue */ -#define WT_PAGE_OVERFLOW_KEYS 0x10 /* Page has overflow keys */ -#define WT_PAGE_READ_NO_EVICT 0x20 /* Page read with eviction disabled */ -#define WT_PAGE_SPLIT_INSERT 0x40 /* A leaf page was split for append */ -#define WT_PAGE_UPDATE_IGNORE 0x80 /* Ignore updates on page discard */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_PAGE_BUILD_KEYS 0x01u /* Keys have been built in memory */ +#define WT_PAGE_DISK_ALLOC 0x02u /* Disk image in allocated memory */ +#define WT_PAGE_DISK_MAPPED 0x04u /* Disk image in mapped memory */ +#define WT_PAGE_EVICT_LRU 0x08u /* Page is on the LRU queue */ +#define WT_PAGE_OVERFLOW_KEYS 0x10u /* Page has overflow keys */ +#define WT_PAGE_READ_NO_EVICT 0x20u /* Page read with eviction disabled */ +#define WT_PAGE_SPLIT_INSERT 0x40u /* A leaf page was split for append */ +#define WT_PAGE_UPDATE_IGNORE 0x80u /* Ignore updates on page discard */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags_atomic; /* Atomic flags, use F_*_ATOMIC */ uint8_t unused[2]; /* Unused padding */ diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 8a3273d1b6b..afbe2434e4c 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -97,10 +97,12 @@ struct __wt_btree { uint64_t maxmempage; /* In-memory page max size */ uint64_t splitmempage; /* In-memory split trigger size */ -#define WT_ASSERT_COMMIT_TS_ALWAYS 0x0001 -#define WT_ASSERT_COMMIT_TS_NEVER 0x0002 -#define WT_ASSERT_READ_TS_ALWAYS 0x0004 -#define WT_ASSERT_READ_TS_NEVER 0x0008 +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_ASSERT_COMMIT_TS_ALWAYS 0x1u +#define WT_ASSERT_COMMIT_TS_NEVER 0x2u +#define WT_ASSERT_READ_TS_ALWAYS 0x4u +#define WT_ASSERT_READ_TS_NEVER 0x8u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t assert_flags; /* Debugging assertion information */ void *huffman_key; /* Key huffman encoding */ @@ -192,21 +194,23 @@ struct __wt_btree { } evict_start_type; /* - * Flag values up to 0xff are reserved for WT_DHANDLE_XXX. + * Flag values up to 0xff are reserved for WT_DHANDLE_XXX. We don't + * automatically generate these flag values for that reason, there's + * no way to start at an offset. */ -#define WT_BTREE_ALTER 0x000100 /* Handle is for alter */ -#define WT_BTREE_BULK 0x000200 /* Bulk-load handle */ -#define WT_BTREE_CLOSED 0x000400 /* Handle closed */ -#define WT_BTREE_IGNORE_CACHE 0x000800 /* Cache-resident object */ -#define WT_BTREE_IN_MEMORY 0x001000 /* Cache-resident object */ -#define WT_BTREE_LOOKASIDE 0x002000 /* Look-aside table */ -#define WT_BTREE_NO_CHECKPOINT 0x004000 /* Disable checkpoints */ -#define WT_BTREE_NO_LOGGING 0x008000 /* Disable logging */ -#define WT_BTREE_REBALANCE 0x010000 /* Handle is for rebalance */ -#define WT_BTREE_SALVAGE 0x020000 /* Handle is for salvage */ -#define WT_BTREE_SKIP_CKPT 0x040000 /* Handle skipped checkpoint */ -#define WT_BTREE_UPGRADE 0x080000 /* Handle is for upgrade */ -#define WT_BTREE_VERIFY 0x100000 /* Handle is for verify */ +#define WT_BTREE_ALTER 0x000100u /* Handle is for alter */ +#define WT_BTREE_BULK 0x000200u /* Bulk-load handle */ +#define WT_BTREE_CLOSED 0x000400u /* Handle closed */ +#define WT_BTREE_IGNORE_CACHE 0x000800u /* Cache-resident object */ +#define WT_BTREE_IN_MEMORY 0x001000u /* Cache-resident object */ +#define WT_BTREE_LOOKASIDE 0x002000u /* Look-aside table */ +#define WT_BTREE_NO_CHECKPOINT 0x004000u /* Disable checkpoints */ +#define WT_BTREE_NO_LOGGING 0x008000u /* Disable logging */ +#define WT_BTREE_REBALANCE 0x010000u /* Handle is for rebalance */ +#define WT_BTREE_SALVAGE 0x020000u /* Handle is for salvage */ +#define WT_BTREE_SKIP_CKPT 0x040000u /* Handle skipped checkpoint */ +#define WT_BTREE_UPGRADE 0x080000u /* Handle is for upgrade */ +#define WT_BTREE_VERIFY 0x100000u /* Handle is for verify */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 560cc8eb212..5d51f2cf796 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1469,7 +1469,7 @@ __wt_page_release(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) (LF_ISSET(WT_READ_NO_SPLIT) || (!inmem_split && F_ISSET(session, WT_SESSION_NO_RECONCILE)))) { if (!WT_SESSION_IS_CHECKPOINT(session)) - __wt_page_evict_urgent(session, ref); + (void)__wt_page_evict_urgent(session, ref); } else { WT_RET_BUSY_OK(__wt_page_release_evict(session, ref)); return (0); diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index a2c0e95293c..28e906f373e 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -89,9 +89,7 @@ struct __wt_cache { uint64_t worker_evicts; /* Pages evicted by worker threads */ uint64_t evict_max_page_size; /* Largest page seen at eviction */ -#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) struct timespec stuck_time; /* Stuck time */ -#endif /* * Read information. @@ -233,17 +231,21 @@ struct __wt_cache { /* * Flags. */ -#define WT_CACHE_POOL_MANAGER 0x001 /* The active cache pool manager */ -#define WT_CACHE_POOL_RUN 0x002 /* Cache pool thread running */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CACHE_POOL_MANAGER 0x1u /* The active cache pool manager */ +#define WT_CACHE_POOL_RUN 0x2u /* Cache pool thread running */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t pool_flags; /* Cache pool flags */ -#define WT_CACHE_EVICT_CLEAN 0x001 /* Evict clean pages */ -#define WT_CACHE_EVICT_CLEAN_HARD 0x002 /* Clean % blocking app threads */ -#define WT_CACHE_EVICT_DIRTY 0x004 /* Evict dirty pages */ -#define WT_CACHE_EVICT_DIRTY_HARD 0x008 /* Dirty % blocking app threads */ -#define WT_CACHE_EVICT_LOOKASIDE 0x010 /* Try lookaside eviction */ -#define WT_CACHE_EVICT_SCRUB 0x020 /* Scrub dirty pages */ -#define WT_CACHE_EVICT_URGENT 0x040 /* Pages are in the urgent queue */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CACHE_EVICT_CLEAN 0x01u /* Evict clean pages */ +#define WT_CACHE_EVICT_CLEAN_HARD 0x02u /* Clean % blocking app threads */ +#define WT_CACHE_EVICT_DIRTY 0x04u /* Evict dirty pages */ +#define WT_CACHE_EVICT_DIRTY_HARD 0x08u /* Dirty % blocking app threads */ +#define WT_CACHE_EVICT_LOOKASIDE 0x10u /* Try lookaside eviction */ +#define WT_CACHE_EVICT_SCRUB 0x20u /* Scrub dirty pages */ +#define WT_CACHE_EVICT_URGENT 0x40u /* Pages are in the urgent queue */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ #define WT_CACHE_EVICT_ALL (WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_DIRTY) uint32_t flags; }; @@ -272,6 +274,8 @@ struct __wt_cache_pool { uint8_t pool_managed; /* Cache pool has a manager thread */ -#define WT_CACHE_POOL_ACTIVE 0x01 /* Cache pool is active */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CACHE_POOL_ACTIVE 0x1u /* Cache pool is active */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 9288618c87e..012c50e5f5f 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -104,12 +104,6 @@ struct __wt_named_extractor { }; /* - * Allocate some additional slots for internal sessions so the user cannot - * configure too few sessions for us to run. - */ -#define WT_EXTRA_INTERNAL_SESSIONS 20 - -/* * WT_CONN_CHECK_PANIC -- * Check if we've panicked and return the appropriate error. */ @@ -192,6 +186,12 @@ struct __wt_connection_impl { /* Configuration */ const WT_CONFIG_ENTRY **config_entries; + const char *optrack_path; /* Directory for operation logs */ + WT_FH *optrack_map_fh; /* Name to id translation file. */ + WT_SPINLOCK optrack_map_spinlock; /* Translation file spinlock. */ + uintmax_t optrack_pid; /* Cache the process ID. */ + uint16_t optrack_uid; /* Unique function ID */ + void **foc; /* Free-on-close array */ size_t foc_cnt; /* Array entries */ size_t foc_size; /* Array size */ @@ -312,15 +312,17 @@ struct __wt_connection_impl { const char *stat_stamp; /* Statistics log entry timestamp */ uint64_t stat_usecs; /* Statistics log period */ -#define WT_CONN_LOG_ARCHIVE 0x001 /* Archive is enabled */ -#define WT_CONN_LOG_DOWNGRADED 0x002 /* Running older version */ -#define WT_CONN_LOG_ENABLED 0x004 /* Logging is enabled */ -#define WT_CONN_LOG_EXISTED 0x008 /* Log files found */ -#define WT_CONN_LOG_FORCE_DOWNGRADE 0x010 /* Force downgrade */ -#define WT_CONN_LOG_RECOVER_DIRTY 0x020 /* Recovering unclean */ -#define WT_CONN_LOG_RECOVER_DONE 0x040 /* Recovery completed */ -#define WT_CONN_LOG_RECOVER_ERR 0x080 /* Error if recovery required */ -#define WT_CONN_LOG_ZERO_FILL 0x100 /* Manually zero files */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CONN_LOG_ARCHIVE 0x001u /* Archive is enabled */ +#define WT_CONN_LOG_DOWNGRADED 0x002u /* Running older version */ +#define WT_CONN_LOG_ENABLED 0x004u /* Logging is enabled */ +#define WT_CONN_LOG_EXISTED 0x008u /* Log files found */ +#define WT_CONN_LOG_FORCE_DOWNGRADE 0x010u /* Force downgrade */ +#define WT_CONN_LOG_RECOVER_DIRTY 0x020u /* Recovering unclean */ +#define WT_CONN_LOG_RECOVER_DONE 0x040u /* Recovery completed */ +#define WT_CONN_LOG_RECOVER_ERR 0x080u /* Error if recovery required */ +#define WT_CONN_LOG_ZERO_FILL 0x100u /* Manually zero files */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t log_flags; /* Global logging configuration */ WT_CONDVAR *log_cond; /* Log server wait mutex */ WT_SESSION_IMPL *log_session; /* Log server session */ @@ -391,22 +393,65 @@ struct __wt_connection_impl { wt_off_t data_extend_len; /* file_extend data length */ wt_off_t log_extend_len; /* file_extend log length */ -#define WT_DIRECT_IO_CHECKPOINT 0x01 /* Checkpoints */ -#define WT_DIRECT_IO_DATA 0x02 /* Data files */ -#define WT_DIRECT_IO_LOG 0x04 /* Log files */ - uint32_t direct_io; /* O_DIRECT, FILE_FLAG_NO_BUFFERING */ - - uint32_t write_through; /* FILE_FLAG_WRITE_THROUGH */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_DIRECT_IO_CHECKPOINT 0x1u /* Checkpoints */ +#define WT_DIRECT_IO_DATA 0x2u /* Data files */ +#define WT_DIRECT_IO_LOG 0x4u /* Log files */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint64_t direct_io; /* O_DIRECT, FILE_FLAG_NO_BUFFERING */ + uint64_t write_through; /* FILE_FLAG_WRITE_THROUGH */ bool mmap; /* mmap configuration */ int page_size; /* OS page size for mmap alignment */ - uint32_t verbose; + double tsc_nsec_ratio; /* rdtsc ticks to nanoseconds */ + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_VERB_API 0x000000001u +#define WT_VERB_BLOCK 0x000000002u +#define WT_VERB_CHECKPOINT 0x000000004u +#define WT_VERB_CHECKPOINT_PROGRESS 0x000000008u +#define WT_VERB_COMPACT 0x000000010u +#define WT_VERB_EVICT 0x000000020u +#define WT_VERB_EVICTSERVER 0x000000040u +#define WT_VERB_EVICT_STUCK 0x000000080u +#define WT_VERB_FILEOPS 0x000000100u +#define WT_VERB_HANDLEOPS 0x000000200u +#define WT_VERB_LOG 0x000000400u +#define WT_VERB_LOOKASIDE 0x000000800u +#define WT_VERB_LOOKASIDE_ACTIVITY 0x000001000u +#define WT_VERB_LSM 0x000002000u +#define WT_VERB_LSM_MANAGER 0x000004000u +#define WT_VERB_METADATA 0x000008000u +#define WT_VERB_MUTEX 0x000010000u +#define WT_VERB_OVERFLOW 0x000020000u +#define WT_VERB_READ 0x000040000u +#define WT_VERB_REBALANCE 0x000080000u +#define WT_VERB_RECONCILE 0x000100000u +#define WT_VERB_RECOVERY 0x000200000u +#define WT_VERB_RECOVERY_PROGRESS 0x000400000u +#define WT_VERB_SALVAGE 0x000800000u +#define WT_VERB_SHARED_CACHE 0x001000000u +#define WT_VERB_SPLIT 0x002000000u +#define WT_VERB_TEMPORARY 0x004000000u +#define WT_VERB_THREAD_GROUP 0x008000000u +#define WT_VERB_TIMESTAMP 0x010000000u +#define WT_VERB_TRANSACTION 0x020000000u +#define WT_VERB_VERIFY 0x040000000u +#define WT_VERB_VERSION 0x080000000u +#define WT_VERB_WRITE 0x100000000u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint64_t verbose; /* * Variable with flags for which subsystems the diagnostic stress timing * delays have been requested. */ - uint32_t timing_stress_flags; +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x1u +#define WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE 0x2u +#define WT_TIMING_STRESS_PAGE_SPLIT_RACE 0x4u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint64_t timing_stress_flags; #define WT_STDERR(s) (&S2C(s)->wt_stderr) #define WT_STDOUT(s) (&S2C(s)->wt_stdout) @@ -418,5 +463,28 @@ struct __wt_connection_impl { */ WT_FILE_SYSTEM *file_system; +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CONN_CACHE_POOL 0x000001u +#define WT_CONN_CKPT_SYNC 0x000002u +#define WT_CONN_CLOSING 0x000004u +#define WT_CONN_CLOSING_NO_MORE_OPENS 0x000008u +#define WT_CONN_EVICTION_NO_LOOKASIDE 0x000010u +#define WT_CONN_EVICTION_RUN 0x000020u +#define WT_CONN_IN_MEMORY 0x000040u +#define WT_CONN_LEAK_MEMORY 0x000080u +#define WT_CONN_LOOKASIDE_OPEN 0x000100u +#define WT_CONN_LSM_MERGE 0x000200u +#define WT_CONN_OPTRACK 0x000400u +#define WT_CONN_PANIC 0x000800u +#define WT_CONN_READONLY 0x001000u +#define WT_CONN_RECOVERING 0x002000u +#define WT_CONN_SERVER_ASYNC 0x004000u +#define WT_CONN_SERVER_CHECKPOINT 0x008000u +#define WT_CONN_SERVER_LOG 0x010000u +#define WT_CONN_SERVER_LSM 0x020000u +#define WT_CONN_SERVER_STATISTICS 0x040000u +#define WT_CONN_SERVER_SWEEP 0x080000u +#define WT_CONN_WAS_BACKUP 0x100000u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 8d2f2c80c2a..1d3d44148a3 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -74,7 +74,9 @@ struct __wt_cursor_backup { size_t list_allocated; size_t list_next; -#define WT_CURBACKUP_LOCKER 0x01 /* Hot-backup started */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CURBACKUP_LOCKER 0x1u /* Hot-backup started */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; #define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)(cursor))->maxid) @@ -209,14 +211,15 @@ struct __wt_cursor_btree { uint64_t lastrecno; #endif -#define WT_CBT_ACTIVE 0x01 /* Active in the tree */ -#define WT_CBT_ITERATE_APPEND 0x02 /* Col-store: iterating append list */ -#define WT_CBT_ITERATE_NEXT 0x04 /* Next iteration configuration */ -#define WT_CBT_ITERATE_PREV 0x08 /* Prev iteration configuration */ -#define WT_CBT_NO_TXN 0x10 /* Non-transactional cursor - (e.g. on a checkpoint) */ -#define WT_CBT_SEARCH_SMALLEST 0x20 /* Row-store: small-key insert list */ -#define WT_CBT_VAR_ONPAGE_MATCH 0x40 /* Var-store: on-page recno match */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CBT_ACTIVE 0x01u /* Active in the tree */ +#define WT_CBT_ITERATE_APPEND 0x02u /* Col-store: iterating append list */ +#define WT_CBT_ITERATE_NEXT 0x04u /* Next iteration configuration */ +#define WT_CBT_ITERATE_PREV 0x08u /* Prev iteration configuration */ +#define WT_CBT_NO_TXN 0x10u /* Non-txn cursor (e.g. a checkpoint) */ +#define WT_CBT_SEARCH_SMALLEST 0x20u /* Row-store: small-key insert list */ +#define WT_CBT_VAR_ONPAGE_MATCH 0x40u /* Var-store: on-page recno match */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ #define WT_CBT_POSITION_MASK /* Flags associated with position */ \ (WT_CBT_ITERATE_APPEND | WT_CBT_ITERATE_NEXT | WT_CBT_ITERATE_PREV | \ @@ -336,12 +339,14 @@ struct __wt_cursor_join_endpoint { uint8_t recno_buf[10]; /* holds packed recno */ WT_CURSOR *cursor; -#define WT_CURJOIN_END_LT 0x01 /* include values < cursor */ -#define WT_CURJOIN_END_EQ 0x02 /* include values == cursor */ -#define WT_CURJOIN_END_GT 0x04 /* include values > cursor */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CURJOIN_END_EQ 0x1u /* include values == cursor */ +#define WT_CURJOIN_END_GT 0x2u /* include values > cursor */ +#define WT_CURJOIN_END_LT 0x4u /* include values < cursor */ +#define WT_CURJOIN_END_OWN_CURSOR 0x8u /* must close cursor */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ #define WT_CURJOIN_END_GE (WT_CURJOIN_END_GT | WT_CURJOIN_END_EQ) #define WT_CURJOIN_END_LE (WT_CURJOIN_END_LT | WT_CURJOIN_END_EQ) -#define WT_CURJOIN_END_OWN_CURSOR 0x08 /* must close cursor */ uint8_t flags; /* range for this endpoint */ }; #define WT_CURJOIN_END_RANGE(endp) \ @@ -365,11 +370,12 @@ struct __wt_cursor_join_entry { uint32_t bloom_hash_count; /* hash functions in bloom */ uint64_t count; /* approx number of matches */ -#define WT_CURJOIN_ENTRY_BLOOM 0x01 /* use a bloom filter */ -#define WT_CURJOIN_ENTRY_DISJUNCTION 0x02 /* endpoints are or-ed */ -#define WT_CURJOIN_ENTRY_FALSE_POSITIVES 0x04 /* after bloom filter do not - * filter false positives */ -#define WT_CURJOIN_ENTRY_OWN_BLOOM 0x08 /* this entry owns the bloom */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CURJOIN_ENTRY_BLOOM 0x1u /* use a bloom filter */ +#define WT_CURJOIN_ENTRY_DISJUNCTION 0x2u /* endpoints are or-ed */ +#define WT_CURJOIN_ENTRY_FALSE_POSITIVES 0x4u /* don't filter false pos */ +#define WT_CURJOIN_ENTRY_OWN_BLOOM 0x8u /* this entry owns the bloom */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; WT_CURSOR_JOIN_ENDPOINT *ends; /* reference endpoints */ @@ -392,9 +398,11 @@ struct __wt_cursor_join { u_int entries_next; uint8_t recno_buf[10]; /* holds packed recno */ -#define WT_CURJOIN_DISJUNCTION 0x01 /* Entries are or-ed */ -#define WT_CURJOIN_ERROR 0x02 /* Error in initialization */ -#define WT_CURJOIN_INITIALIZED 0x04 /* Successful initialization */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CURJOIN_DISJUNCTION 0x1u /* Entries are or-ed */ +#define WT_CURJOIN_ERROR 0x2u /* Error in initialization */ +#define WT_CURJOIN_INITIALIZED 0x4u /* Successful initialization */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; @@ -419,7 +427,9 @@ struct __wt_cursor_log { uint32_t rectype; /* Record type */ uint64_t txnid; /* Record txnid */ -#define WT_CURLOG_ARCHIVE_LOCK 0x01 /* Archive lock held */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CURLOG_ARCHIVE_LOCK 0x1u /* Archive lock held */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; @@ -429,9 +439,11 @@ struct __wt_cursor_metadata { WT_CURSOR *file_cursor; /* Queries of regular metadata */ WT_CURSOR *create_cursor; /* Extra cursor for create option */ -#define WT_MDC_CREATEONLY 0x01 -#define WT_MDC_ONMETADATA 0x02 -#define WT_MDC_POSITIONED 0x04 +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_MDC_CREATEONLY 0x1u +#define WT_MDC_ONMETADATA 0x2u +#define WT_MDC_POSITIONED 0x4u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 2844c6e3e73..12a64ed83c3 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -103,12 +103,14 @@ struct __wt_data_handle { WT_DSRC_STATS *stat_array; /* Flags values over 0xff are reserved for WT_BTREE_* */ -#define WT_DHANDLE_DEAD 0x01 /* Dead, awaiting discard */ -#define WT_DHANDLE_DISCARD 0x02 /* Close on release */ -#define WT_DHANDLE_DISCARD_KILL 0x04 /* Mark dead on release */ -#define WT_DHANDLE_EXCLUSIVE 0x08 /* Exclusive access */ -#define WT_DHANDLE_IS_METADATA 0x10 /* Metadata handle */ -#define WT_DHANDLE_LOCK_ONLY 0x20 /* Handle only used as a lock */ -#define WT_DHANDLE_OPEN 0x40 /* Handle is open */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_DHANDLE_DEAD 0x01u /* Dead, awaiting discard */ +#define WT_DHANDLE_DISCARD 0x02u /* Close on release */ +#define WT_DHANDLE_DISCARD_KILL 0x04u /* Mark dead on release */ +#define WT_DHANDLE_EXCLUSIVE 0x08u /* Exclusive access */ +#define WT_DHANDLE_IS_METADATA 0x10u /* Metadata handle */ +#define WT_DHANDLE_LOCK_ONLY 0x20u /* Handle only used as a lock */ +#define WT_DHANDLE_OPEN 0x40u /* Handle is open */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index 3b3c0769c90..af7209f1ad1 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -61,7 +61,6 @@ } while (0) #define WT_RET_BUSY_OK(a) WT_RET_ERROR_OK(a, EBUSY) #define WT_RET_NOTFOUND_OK(a) WT_RET_ERROR_OK(a, WT_NOTFOUND) - /* Set "ret" if not already set. */ #define WT_TRET(a) do { \ int __ret; \ @@ -137,15 +136,7 @@ * there's no portable way to remove the comma before an empty __VA_ARGS__ * value. */ -#ifdef HAVE_VERBOSE #define __wt_verbose(session, flag, fmt, ...) do { \ if (WT_VERBOSE_ISSET(session, flag)) \ __wt_verbose_worker(session, fmt, __VA_ARGS__); \ } while (0) -#else -#define __wt_verbose(session, flag, fmt, ...) do { \ - WT_UNUSED(session); \ - WT_UNUSED(flag); \ - WT_UNUSED(fmt); \ -} while (0) -#endif diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 272544b08f7..b45a0d9e43f 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -286,6 +286,8 @@ extern int __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) WT_ extern int __wt_connection_close(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_optrack_setup(WT_SESSION_IMPL *session, const char *cfg[], bool reconfig) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_optrack_teardown(WT_SESSION_IMPL *session, bool reconfig) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_statistics_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); @@ -464,8 +466,8 @@ extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, extern int __wt_curstat_lsm_init( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_close_all(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_bloom_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_lsm_tree_chunk_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_lsm_tree_set_chunk_size( WT_SESSION_IMPL *session, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_chunk_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, uint32_t generation, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_set_chunk_size( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_setup_chunk( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_setup_bloom( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_create(WT_SESSION_IMPL *session, const char *uri, bool exclusive, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -528,8 +530,13 @@ extern int __wt_meta_track_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_AT extern int __wt_turtle_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_optrack_record_funcid( WT_SESSION_IMPL *session, const char *func, uint16_t *func_idp); +extern int __wt_optrack_open_file(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern size_t __wt_optrack_flush_buffer(WT_SESSION_IMPL *s); +extern uint64_t __wt_optrack_get_expensive_timestamp(WT_SESSION_IMPL *session); extern int __wt_filename(WT_SESSION_IMPL *session, const char *name, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_filename_construct(WT_SESSION_IMPL *session, const char *path, const char *file_prefix, uintmax_t id_1, uint32_t id_2, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); @@ -779,6 +786,7 @@ extern void __wt_thread_group_start_one( WT_SESSION_IMPL *session, WT_THREAD_GRO extern void __wt_thread_group_stop_one(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group); extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_seconds(WT_SESSION_IMPL *session, time_t *timep); +extern uint64_t __wt_tsc_to_nsec(WT_SESSION_IMPL *session, uint64_t tsc_diff); extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); extern void __wt_txn_get_snapshot(WT_SESSION_IMPL *session); extern int __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/extern_posix.h b/src/third_party/wiredtiger/src/include/extern_posix.h index 624cd815dad..38e645a997d 100644 --- a/src/third_party/wiredtiger/src/include/extern_posix.h +++ b/src/third_party/wiredtiger/src/include/extern_posix.h @@ -29,5 +29,6 @@ extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_ extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_thread_id(uintmax_t *id) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_thread_str(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern uintmax_t __wt_process_id(void); extern void __wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp); extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); diff --git a/src/third_party/wiredtiger/src/include/extern_win.h b/src/third_party/wiredtiger/src/include/extern_win.h index ae4195b20a1..4d8d479f0b9 100644 --- a/src/third_party/wiredtiger/src/include/extern_win.h +++ b/src/third_party/wiredtiger/src/include/extern_win.h @@ -27,6 +27,7 @@ extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_ extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_thread_id(uintmax_t *id); extern int __wt_thread_str(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern uintmax_t __wt_process_id(void); extern void __wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp); extern int __wt_to_utf16_string( WT_SESSION_IMPL *session, const char*utf8, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_to_utf8_string( WT_SESSION_IMPL *session, const wchar_t*wide, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h deleted file mode 100644 index b191e8fe01d..00000000000 --- a/src/third_party/wiredtiger/src/include/flags.h +++ /dev/null @@ -1,130 +0,0 @@ -/* - * DO NOT EDIT: automatically built by dist/flags.py. - * flags section: BEGIN - */ -#define WT_CONN_CACHE_POOL 0x00000001 -#define WT_CONN_CKPT_SYNC 0x00000002 -#define WT_CONN_CLOSING 0x00000004 -#define WT_CONN_CLOSING_NO_MORE_OPENS 0x00000008 -#define WT_CONN_EVICTION_NO_LOOKASIDE 0x00000010 -#define WT_CONN_EVICTION_RUN 0x00000020 -#define WT_CONN_IN_MEMORY 0x00000040 -#define WT_CONN_LEAK_MEMORY 0x00000080 -#define WT_CONN_LOOKASIDE_OPEN 0x00000100 -#define WT_CONN_LSM_MERGE 0x00000200 -#define WT_CONN_PANIC 0x00000400 -#define WT_CONN_READONLY 0x00000800 -#define WT_CONN_RECOVERING 0x00001000 -#define WT_CONN_SERVER_ASYNC 0x00002000 -#define WT_CONN_SERVER_CHECKPOINT 0x00004000 -#define WT_CONN_SERVER_LOG 0x00008000 -#define WT_CONN_SERVER_LSM 0x00010000 -#define WT_CONN_SERVER_STATISTICS 0x00020000 -#define WT_CONN_SERVER_SWEEP 0x00040000 -#define WT_CONN_WAS_BACKUP 0x00080000 -#define WT_LOGSCAN_FIRST 0x00000001 -#define WT_LOGSCAN_FROM_CKP 0x00000002 -#define WT_LOGSCAN_ONE 0x00000004 -#define WT_LOGSCAN_RECOVER 0x00000008 -#define WT_LOG_BACKGROUND 0x00000001 -#define WT_LOG_DSYNC 0x00000002 -#define WT_LOG_FLUSH 0x00000004 -#define WT_LOG_FSYNC 0x00000008 -#define WT_LOG_SYNC_ENABLED 0x00000010 -#define WT_READ_CACHE 0x00000001 -#define WT_READ_IGNORE_CACHE_SIZE 0x00000002 -#define WT_READ_LOOKASIDE 0x00000004 -#define WT_READ_NOTFOUND_OK 0x00000008 -#define WT_READ_NO_EMPTY 0x00000010 -#define WT_READ_NO_GEN 0x00000020 -#define WT_READ_NO_SPLIT 0x00000040 -#define WT_READ_NO_WAIT 0x00000080 -#define WT_READ_PREV 0x00000100 -#define WT_READ_RESTART_OK 0x00000200 -#define WT_READ_SKIP_INTL 0x00000400 -#define WT_READ_TRUNCATE 0x00000800 -#define WT_READ_WONT_NEED 0x00001000 -#define WT_REC_CHECKPOINT 0x00000001 -#define WT_REC_EVICT 0x00000002 -#define WT_REC_IN_MEMORY 0x00000004 -#define WT_REC_LOOKASIDE 0x00000008 -#define WT_REC_SCRUB 0x00000010 -#define WT_REC_UPDATE_RESTORE 0x00000020 -#define WT_REC_VISIBILITY_ERR 0x00000040 -#define WT_REC_VISIBLE_ALL 0x00000080 -#define WT_SESSION_CAN_WAIT 0x00000001 -#define WT_SESSION_IGNORE_CACHE_SIZE 0x00000002 -#define WT_SESSION_INTERNAL 0x00000004 -#define WT_SESSION_LOCKED_CHECKPOINT 0x00000008 -#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x00000010 -#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x00000020 -#define WT_SESSION_LOCKED_METADATA 0x00000040 -#define WT_SESSION_LOCKED_PASS 0x00000080 -#define WT_SESSION_LOCKED_SCHEMA 0x00000100 -#define WT_SESSION_LOCKED_SLOT 0x00000200 -#define WT_SESSION_LOCKED_TABLE_READ 0x00000400 -#define WT_SESSION_LOCKED_TABLE_WRITE 0x00000800 -#define WT_SESSION_LOCKED_TURTLE 0x00001000 -#define WT_SESSION_LOGGING_INMEM 0x00002000 -#define WT_SESSION_LOOKASIDE_CURSOR 0x00004000 -#define WT_SESSION_NO_DATA_HANDLES 0x00008000 -#define WT_SESSION_NO_LOGGING 0x00010000 -#define WT_SESSION_NO_RECONCILE 0x00020000 -#define WT_SESSION_NO_SCHEMA_LOCK 0x00040000 -#define WT_SESSION_QUIET_CORRUPT_FILE 0x00080000 -#define WT_SESSION_READ_WONT_NEED 0x00100000 -#define WT_SESSION_SERVER_ASYNC 0x00200000 -#define WT_STAT_CLEAR 0x00000001 -#define WT_STAT_JSON 0x00000002 -#define WT_STAT_ON_CLOSE 0x00000004 -#define WT_STAT_TYPE_ALL 0x00000008 -#define WT_STAT_TYPE_CACHE_WALK 0x00000010 -#define WT_STAT_TYPE_FAST 0x00000020 -#define WT_STAT_TYPE_SIZE 0x00000040 -#define WT_STAT_TYPE_TREE_WALK 0x00000080 -#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x00000001 -#define WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE 0x00000002 -#define WT_TIMING_STRESS_PAGE_SPLIT_RACE 0x00000004 -#define WT_TXN_LOG_CKPT_CLEANUP 0x00000001 -#define WT_TXN_LOG_CKPT_PREPARE 0x00000002 -#define WT_TXN_LOG_CKPT_START 0x00000004 -#define WT_TXN_LOG_CKPT_STOP 0x00000008 -#define WT_TXN_LOG_CKPT_SYNC 0x00000010 -#define WT_TXN_OLDEST_STRICT 0x00000001 -#define WT_TXN_OLDEST_WAIT 0x00000002 -#define WT_VERB_API 0x00000001 -#define WT_VERB_BLOCK 0x00000002 -#define WT_VERB_CHECKPOINT 0x00000004 -#define WT_VERB_CHECKPOINT_PROGRESS 0x00000008 -#define WT_VERB_COMPACT 0x00000010 -#define WT_VERB_EVICT 0x00000020 -#define WT_VERB_EVICTSERVER 0x00000040 -#define WT_VERB_EVICT_STUCK 0x00000080 -#define WT_VERB_FILEOPS 0x00000100 -#define WT_VERB_HANDLEOPS 0x00000200 -#define WT_VERB_LOG 0x00000400 -#define WT_VERB_LOOKASIDE 0x00000800 -#define WT_VERB_LOOKASIDE_ACTIVITY 0x00001000 -#define WT_VERB_LSM 0x00002000 -#define WT_VERB_LSM_MANAGER 0x00004000 -#define WT_VERB_METADATA 0x00008000 -#define WT_VERB_MUTEX 0x00010000 -#define WT_VERB_OVERFLOW 0x00020000 -#define WT_VERB_READ 0x00040000 -#define WT_VERB_REBALANCE 0x00080000 -#define WT_VERB_RECONCILE 0x00100000 -#define WT_VERB_RECOVERY 0x00200000 -#define WT_VERB_RECOVERY_PROGRESS 0x00400000 -#define WT_VERB_SALVAGE 0x00800000 -#define WT_VERB_SHARED_CACHE 0x01000000 -#define WT_VERB_SPLIT 0x02000000 -#define WT_VERB_THREAD_GROUP 0x04000000 -#define WT_VERB_TIMESTAMP 0x08000000 -#define WT_VERB_TRANSACTION 0x10000000 -#define WT_VERB_VERIFY 0x20000000 -#define WT_VERB_VERSION 0x40000000 -#define WT_VERB_WRITE 0x80000000 -/* - * flags section: END - * DO NOT EDIT: automatically built by dist/flags.py. - */ diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index 3ec874ad5b6..e0e7bc599bc 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -6,6 +6,21 @@ * See the file LICENSE for redistribution information. */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LOGSCAN_FIRST 0x1u +#define WT_LOGSCAN_FROM_CKP 0x2u +#define WT_LOGSCAN_ONE 0x4u +#define WT_LOGSCAN_RECOVER 0x8u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LOG_BACKGROUND 0x01u +#define WT_LOG_DSYNC 0x02u +#define WT_LOG_FLUSH 0x04u +#define WT_LOG_FSYNC 0x08u +#define WT_LOG_SYNC_ENABLED 0x10u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + /* * WT_LSN -- * A log sequence number, representing a position in the transaction log. @@ -78,6 +93,17 @@ union __wt_lsn { ((size) - offsetof(WT_LOG_RECORD, record)) /* + * We allocate the buffer size, but trigger a slot switch when we cross + * the maximum size of half the buffer. If a record is more than the buffer + * maximum then we trigger a slot switch and write that record unbuffered. + * We use a larger buffer to provide overflow space so that we can switch + * once we cross the threshold. + */ +#define WT_LOG_SLOT_BUF_SIZE (256 * 1024) /* Must be power of 2 */ +#define WT_LOG_SLOT_BUF_MAX ((uint32_t)log->slot_buf_size / 2) +#define WT_LOG_SLOT_UNBUFFERED (WT_LOG_SLOT_BUF_SIZE << 1) + +/* * Possible values for the consolidation array slot states: * * WT_LOG_SLOT_CLOSE - slot is in use but closed to new joins. @@ -92,6 +118,12 @@ union __wt_lsn { * a few special states, reserve the top few bits for state. That makes * the maximum size less than 32 bits for both joined and released. */ +/* + * XXX + * The log slot bits are signed and should be rewritten as unsigned. For now, + * give the logging subsystem its own flags macro. + */ +#define FLD_LOG_SLOT_ISSET(field, mask) (((field) & (uint64_t)(mask)) != 0) /* * The high bit is reserved for the special states. If the high bit is @@ -101,17 +133,6 @@ union __wt_lsn { #define WT_LOG_SLOT_WRITTEN (-2) /* Slot data written, not processed */ /* - * We allocate the buffer size, but trigger a slot switch when we cross - * the maximum size of half the buffer. If a record is more than the buffer - * maximum then we trigger a slot switch and write that record unbuffered. - * We use a larger buffer to provide overflow space so that we can switch - * once we cross the threshold. - */ -#define WT_LOG_SLOT_BUF_SIZE (256 * 1024) /* Must be power of 2 */ -#define WT_LOG_SLOT_BUF_MAX ((uint32_t)log->slot_buf_size / 2) -#define WT_LOG_SLOT_UNBUFFERED (WT_LOG_SLOT_BUF_SIZE << 1) - -/* * If new slot states are added, adjust WT_LOG_SLOT_BITS and * WT_LOG_SLOT_MASK_OFF accordingly for how much of the top 32 * bits we are using. More slot states here will reduce the maximum @@ -155,8 +176,8 @@ union __wt_lsn { /* Slot is in use, but closed to new joins */ #define WT_LOG_SLOT_CLOSED(state) \ (WT_LOG_SLOT_ACTIVE(state) && \ - (FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ - !FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_RESERVED))) + (FLD_LOG_SLOT_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ + !FLD_LOG_SLOT_ISSET((uint64_t)(state), WT_LOG_SLOT_RESERVED))) /* Slot is in use, all data copied into buffer */ #define WT_LOG_SLOT_INPROGRESS(state) \ (WT_LOG_SLOT_RELEASED(state) != WT_LOG_SLOT_JOINED(state)) @@ -167,7 +188,7 @@ union __wt_lsn { #define WT_LOG_SLOT_OPEN(state) \ (WT_LOG_SLOT_ACTIVE(state) && \ !WT_LOG_SLOT_UNBUFFERED_ISSET(state) && \ - !FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ + !FLD_LOG_SLOT_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ WT_LOG_SLOT_JOINED(state) < WT_LOG_SLOT_BUF_MAX) struct __wt_logslot { @@ -183,11 +204,13 @@ struct __wt_logslot { WT_FH *slot_fh; /* File handle for this group */ WT_ITEM slot_buf; /* Buffer for grouped writes */ -#define WT_SLOT_CLOSEFH 0x01 /* Close old fh on release */ -#define WT_SLOT_FLUSH 0x02 /* Wait for write */ -#define WT_SLOT_SYNC 0x04 /* Needs sync on release */ -#define WT_SLOT_SYNC_DIR 0x08 /* Directory sync on release */ - uint32_t flags; /* Flags */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_SLOT_CLOSEFH 0x1u /* Close old fh on release */ +#define WT_SLOT_FLUSH 0x2u /* Wait for write */ +#define WT_SLOT_SYNC 0x4u /* Needs sync on release */ +#define WT_SLOT_SYNC_DIR 0x8u /* Directory sync on release */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t flags; WT_CACHE_LINE_PAD_END }; @@ -203,10 +226,13 @@ struct __wt_myslot { WT_LOGSLOT *slot; /* Slot I'm using */ wt_off_t end_offset; /* My end offset in buffer */ wt_off_t offset; /* Slot buffer offset */ -#define WT_MYSLOT_CLOSE 0x01 /* This thread is closing the slot */ -#define WT_MYSLOT_NEEDS_RELEASE 0x02 /* This thread is releasing the slot */ -#define WT_MYSLOT_UNBUFFERED 0x04 /* Write directly */ - uint32_t flags; /* Flags */ + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_MYSLOT_CLOSE 0x1u /* This thread is closing the slot */ +#define WT_MYSLOT_NEEDS_RELEASE 0x2u /* This thread is releasing the slot */ +#define WT_MYSLOT_UNBUFFERED 0x4u /* Write directly */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t flags; }; #define WT_LOG_END_HEADER log->allocsize @@ -276,9 +302,11 @@ struct __wt_log { uint64_t write_calls; /* Calls to log_write */ #endif -#define WT_LOG_FORCE_NEWFILE 0x01 /* Force switch to new log file */ -#define WT_LOG_OPENED 0x02 /* Log subsystem successfully open */ -#define WT_LOG_TRUNCATE_NOTSUP 0x04 /* File system truncate not supported */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LOG_FORCE_NEWFILE 0x1u /* Force switch to new log file */ +#define WT_LOG_OPENED 0x2u /* Log subsystem successfully open */ +#define WT_LOG_TRUNCATE_NOTSUP 0x4u /* File system truncate not supported */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; @@ -286,8 +314,12 @@ struct __wt_log_record { uint32_t len; /* 00-03: Record length including hdr */ uint32_t checksum; /* 04-07: Checksum of the record */ -#define WT_LOG_RECORD_COMPRESSED 0x01 /* Compressed except hdr */ -#define WT_LOG_RECORD_ENCRYPTED 0x02 /* Encrypted except hdr */ + /* + * No automatic generation: flag values cannot change, they're written + * to disk. + */ +#define WT_LOG_RECORD_COMPRESSED 0x01u /* Compressed except hdr */ +#define WT_LOG_RECORD_ENCRYPTED 0x02u /* Encrypted except hdr */ uint16_t flags; /* 08-09: Flags */ uint8_t unused[2]; /* 10-11: Padding */ uint32_t mem_len; /* 12-15: Uncompressed len if needed */ @@ -317,7 +349,7 @@ __wt_log_record_byteswap(WT_LOG_RECORD *record) * The log file's description. */ struct __wt_log_desc { -#define WT_LOG_MAGIC 0x101064 +#define WT_LOG_MAGIC 0x101064u uint32_t log_magic; /* 00-03: Magic number */ #define WT_LOG_VERSION 2 uint16_t version; /* 04-05: Log version */ @@ -357,7 +389,9 @@ __wt_log_desc_byteswap(WT_LOG_DESC *desc) /* * Flags for __wt_txn_op_printlog. */ -#define WT_TXN_PRINTLOG_HEX 0x0001 /* Add hex output */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TXN_PRINTLOG_HEX 0x1u /* Add hex output */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ /* * WT_LOG_REC_DESC -- diff --git a/src/third_party/wiredtiger/src/include/lsm.h b/src/third_party/wiredtiger/src/include/lsm.h index 0e026d90da5..2ab16364b09 100644 --- a/src/third_party/wiredtiger/src/include/lsm.h +++ b/src/third_party/wiredtiger/src/include/lsm.h @@ -66,16 +66,17 @@ struct __wt_cursor_lsm { u_int update_count; /* Updates performed. */ -#define WT_CLSM_ACTIVE 0x001 /* Incremented the session count */ -#define WT_CLSM_BULK 0x002 /* Open for snapshot isolation */ -#define WT_CLSM_ITERATE_NEXT 0x004 /* Forward iteration */ -#define WT_CLSM_ITERATE_PREV 0x008 /* Backward iteration */ -#define WT_CLSM_MERGE 0x010 /* Merge cursor, don't update */ -#define WT_CLSM_MINOR_MERGE 0x020 /* Minor merge, include tombstones */ -#define WT_CLSM_MULTIPLE 0x040 /* Multiple cursors have values for the - current key */ -#define WT_CLSM_OPEN_READ 0x080 /* Open for reads */ -#define WT_CLSM_OPEN_SNAPSHOT 0x100 /* Open for snapshot isolation */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CLSM_ACTIVE 0x001u /* Incremented the session count */ +#define WT_CLSM_BULK 0x002u /* Open for snapshot isolation */ +#define WT_CLSM_ITERATE_NEXT 0x004u /* Forward iteration */ +#define WT_CLSM_ITERATE_PREV 0x008u /* Backward iteration */ +#define WT_CLSM_MERGE 0x010u /* Merge cursor, don't update */ +#define WT_CLSM_MINOR_MERGE 0x020u /* Minor merge, include tombstones */ +#define WT_CLSM_MULTIPLE 0x040u /* Multiple cursors have values */ +#define WT_CLSM_OPEN_READ 0x080u /* Open for reads */ +#define WT_CLSM_OPEN_SNAPSHOT 0x100u /* Open for snapshot isolation */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; @@ -112,11 +113,13 @@ struct __wt_lsm_chunk { int8_t evicted; /* 1/0: in-memory chunk was evicted */ uint8_t flushing; /* 1/0: chunk flush in progress */ -#define WT_LSM_CHUNK_BLOOM 0x01 -#define WT_LSM_CHUNK_HAS_TIMESTAMP 0x02 -#define WT_LSM_CHUNK_MERGING 0x04 -#define WT_LSM_CHUNK_ONDISK 0x08 -#define WT_LSM_CHUNK_STABLE 0x10 +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LSM_CHUNK_BLOOM 0x01u +#define WT_LSM_CHUNK_HAS_TIMESTAMP 0x02u +#define WT_LSM_CHUNK_MERGING 0x04u +#define WT_LSM_CHUNK_ONDISK 0x08u +#define WT_LSM_CHUNK_STABLE 0x10u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; @@ -125,11 +128,13 @@ struct __wt_lsm_chunk { * type of work they will execute, and by work units to define which action * is required. */ -#define WT_LSM_WORK_BLOOM 0x01 /* Create a bloom filter */ -#define WT_LSM_WORK_DROP 0x02 /* Drop unused chunks */ -#define WT_LSM_WORK_FLUSH 0x04 /* Flush a chunk to disk */ -#define WT_LSM_WORK_MERGE 0x08 /* Look for a tree merge */ -#define WT_LSM_WORK_SWITCH 0x10 /* Switch to new in-memory chunk */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LSM_WORK_BLOOM 0x01u /* Create a bloom filter */ +#define WT_LSM_WORK_DROP 0x02u /* Drop unused chunks */ +#define WT_LSM_WORK_FLUSH 0x04u /* Flush a chunk to disk */ +#define WT_LSM_WORK_MERGE 0x08u /* Look for a tree merge */ +#define WT_LSM_WORK_SWITCH 0x10u /* Switch to new in-memory chunk */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ /* * WT_LSM_WORK_UNIT -- @@ -138,7 +143,9 @@ struct __wt_lsm_chunk { struct __wt_lsm_work_unit { TAILQ_ENTRY(__wt_lsm_work_unit) q; /* Worker unit queue */ uint32_t type; /* Type of operation */ -#define WT_LSM_WORK_FORCE 0x0001 /* Force operation */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LSM_WORK_FORCE 0x1u /* Force operation */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; /* Flags for operation */ WT_LSM_TREE *lsm_tree; }; @@ -172,7 +179,9 @@ struct __wt_lsm_manager { #define WT_LSM_MIN_WORKERS 3 WT_LSM_WORKER_ARGS lsm_worker_cookies[WT_LSM_MAX_WORKERS]; -#define WT_LSM_MANAGER_SHUTDOWN 0x01 /* Manager has shut down */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LSM_MANAGER_SHUTDOWN 0x1u /* Manager has shut down */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; @@ -192,6 +201,11 @@ struct __wt_lsm_tree { const char *key_format, *value_format; const char *bloom_config, *file_config; + uint32_t custom_generation; /* Level at which a custom data source + should be used for merges. */ + const char *custom_prefix; /* Prefix for custom data source */ + const char *custom_suffix; /* Suffix for custom data source */ + WT_COLLATOR *collator; const char *collator_name; int collator_owned; @@ -226,9 +240,11 @@ struct __wt_lsm_tree { uint64_t chunk_max; /* Maximum chunk a merge creates */ u_int merge_min, merge_max; -#define WT_LSM_BLOOM_MERGED 0x00000001 -#define WT_LSM_BLOOM_OFF 0x00000002 -#define WT_LSM_BLOOM_OLDEST 0x00000004 +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LSM_BLOOM_MERGED 0x1u +#define WT_LSM_BLOOM_OFF 0x2u +#define WT_LSM_BLOOM_OLDEST 0x4u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t bloom; /* Bloom creation policy */ WT_LSM_CHUNK **chunk; /* Array of active LSM chunks */ @@ -274,10 +290,12 @@ struct __wt_lsm_tree { * flags here are not protected for concurrent access, don't put * anything here that is susceptible to races. */ -#define WT_LSM_TREE_COMPACTING 0x01 /* Tree being compacted */ -#define WT_LSM_TREE_MERGES 0x02 /* Tree should run merges */ -#define WT_LSM_TREE_OPEN 0x04 /* The tree is open */ -#define WT_LSM_TREE_THROTTLE 0x08 /* Throttle updates */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LSM_TREE_COMPACTING 0x1u /* Tree being compacted */ +#define WT_LSM_TREE_MERGES 0x2u /* Tree should run merges */ +#define WT_LSM_TREE_OPEN 0x4u /* The tree is open */ +#define WT_LSM_TREE_THROTTLE 0x8u /* Throttle updates */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index 2dd77157caa..2e8ffdded75 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -73,9 +73,11 @@ struct __wt_ckpt { void *bpriv; /* Block manager private */ -#define WT_CKPT_ADD 0x01 /* Checkpoint to be added */ -#define WT_CKPT_DELETE 0x02 /* Checkpoint to be deleted */ -#define WT_CKPT_FAKE 0x04 /* Checkpoint is a fake */ -#define WT_CKPT_UPDATE 0x08 /* Checkpoint requires update */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CKPT_ADD 0x1u /* Checkpoint to be added */ +#define WT_CKPT_DELETE 0x2u /* Checkpoint to be deleted */ +#define WT_CKPT_FAKE 0x4u /* Checkpoint is a fake */ +#define WT_CKPT_UPDATE 0x8u /* Checkpoint requires update */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 2435d37ee20..356ef73dec7 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -132,11 +132,10 @@ * hex constant might be a negative integer), and to ensure the hex constant is * the correct size before applying the bitwise not operator. */ -#define FLD_CLR(field, mask) ((void)((field) &= ~(uint32_t)(mask))) -#define FLD_MASK(field, mask) ((field) & (uint32_t)(mask)) +#define FLD_CLR(field, mask) ((void)((field) &= ~(mask))) +#define FLD_MASK(field, mask) ((field) & (mask)) #define FLD_ISSET(field, mask) (FLD_MASK(field, mask) != 0) -#define FLD64_ISSET(field, mask) (((field) & (uint64_t)(mask)) != 0) -#define FLD_SET(field, mask) ((void)((field) |= (uint32_t)(mask))) +#define FLD_SET(field, mask) ((void)((field) |= (mask))) #define F_CLR(p, mask) FLD_CLR((p)->flags, mask) #define F_ISSET(p, mask) FLD_ISSET((p)->flags, mask) @@ -195,12 +194,8 @@ } while (0) /* Verbose messages. */ -#ifdef HAVE_VERBOSE #define WT_VERBOSE_ISSET(session, f) \ (FLD_ISSET(S2C(session)->verbose, f)) -#else -#define WT_VERBOSE_ISSET(session, f) 0 -#endif #define WT_CLEAR(s) \ memset(&(s), 0, sizeof(s)) diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index bedd3121037..f40f2ac0fa9 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -30,6 +30,31 @@ __wt_hex(int c) } /* + * __wt_rdtsc -- + * Get a timestamp from CPU registers. + */ +static inline uint64_t +__wt_rdtsc(WT_SESSION_IMPL *session) { +#if (defined __i386) + uint64_t x; + + WT_UNUSED(session); + + __asm__ volatile ("rdtsc" : "=A" (x)); + return (x); +#elif (defined __amd64) + uint64_t a, d; + + WT_UNUSED(session); + + __asm__ volatile ("rdtsc" : "=a" (a), "=d" (d)); + return ((d << 32) | a); +#else + return (__wt_optrack_get_expensive_timestamp(session)); +#endif +} + +/* * __wt_strdup -- * ANSI strdup function. */ diff --git a/src/third_party/wiredtiger/src/include/mutex.i b/src/third_party/wiredtiger/src/include/mutex.i index 871ccf63be8..1d9fea9efb4 100644 --- a/src/third_party/wiredtiger/src/include/mutex.i +++ b/src/third_party/wiredtiger/src/include/mutex.i @@ -293,21 +293,23 @@ __wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) static inline void __wt_spin_lock_track(WT_SESSION_IMPL *session, WT_SPINLOCK *t) { - struct timespec enter, leave; + uint64_t time_start, time_stop; int64_t **stats; if (t->stat_count_off != -1 && WT_STAT_ENABLED(session)) { - __wt_epoch(session, &enter); + time_start = __wt_rdtsc(session); __wt_spin_lock(session, t); - __wt_epoch(session, &leave); + time_stop = __wt_rdtsc(session); stats = (int64_t **)S2C(session)->stats; stats[session->stat_bucket][t->stat_count_off]++; if (F_ISSET(session, WT_SESSION_INTERNAL)) stats[session->stat_bucket][t->stat_int_usecs_off] += - (int64_t)WT_TIMEDIFF_US(leave, enter); + (int64_t)WT_TSCDIFF_US( + session, time_stop, time_start); else stats[session->stat_bucket][t->stat_app_usecs_off] += - (int64_t)WT_TIMEDIFF_US(leave, enter); + (int64_t)WT_TSCDIFF_US( + session, time_stop, time_start); } else __wt_spin_lock(session, t); } diff --git a/src/third_party/wiredtiger/src/include/optrack.h b/src/third_party/wiredtiger/src/include/optrack.h new file mode 100644 index 00000000000..6c3d57deea6 --- /dev/null +++ b/src/third_party/wiredtiger/src/include/optrack.h @@ -0,0 +1,86 @@ +/*- + * Copyright (c) 2014-2017 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#define WT_OPTRACK_MAXRECS 16384 +#define WT_OPTRACK_BUFSIZE WT_OPTRACK_MAXRECS * sizeof(WT_OPTRACK_RECORD) +#define WT_OPTRACK_VERSION 1 + +/* + * WT_OPTRACK_HEADER -- + * A header in the operation tracking log file. The internal session + * identifier is a boolean: 1 if the session is internal, 0 otherwise. + */ +struct __wt_optrack_header { + uint32_t optrack_version; + uint32_t optrack_session_internal; +}; + +/* + * WT_OPTRACK_RECORD -- + * A structure for logging function entry and exit events. + * + * We pad the record so that the size of the entire record is 16 bytes. If we + * don't do this, the compiler will pad it for us, because we keep records in + * the record buffer array and each new record must be aligned on the 8-byte + * boundary, since its first element is an 8-byte timestamp. Instead of letting + * the compiler insert the padding silently, we pad explicitly, so that whoever + * writes the binary decoder can refer to this struct to find out the record + * size. + * + * The operation id included in this structure is a unique address of a function + * in the binary. As we log operations, we keep track of the correspondence + * between function addresses and their names. When the log file is decoded, + * operations identifiers are replaced with function names. Therefore, the + * present design assumes that the user will be inserting the tracking macros + * on function boundaries: when we enter into the function and when we exit + * from it. + */ +struct __wt_optrack_record { + uint64_t op_timestamp; /* timestamp */ + uint16_t op_id; /* function ID */ + uint16_t op_type; /* start/stop */ + uint8_t padding[4]; +}; + +#define WT_TRACK_OP(s, optype) do { \ + WT_OPTRACK_RECORD *__tr; \ + __tr = &((s)->optrack_buf[ \ + (s)->optrackbuf_ptr % WT_OPTRACK_MAXRECS]); \ + __tr->op_timestamp = __wt_rdtsc(s); \ + __tr->op_id = __func_id; \ + __tr->op_type = optype; \ + \ + if (++(s)->optrackbuf_ptr == WT_OPTRACK_MAXRECS) { \ + (s)->optrack_offset += __wt_optrack_flush_buffer(s); \ + (s)->optrackbuf_ptr = 0; \ + } \ +} while (0) + +/* + * We do not synchronize access to optrack buffer pointer under the assumption + * that there is no more than one thread using a given session. This assumption + * does not always hold. When it does not, we might have a race. In this case, + * we may lose a few log records. We prefer to risk losing a few log records + * occasionally in order not to synchronize this code, which is intended to be + * very lightweight. + * Exclude the default session (ID 0) because it can be used by multiple + * threads and it is also used in error paths during failed open calls. + */ +#define WT_TRACK_OP_DECL \ + static uint16_t __func_id = 0 +#define WT_TRACK_OP_INIT(s) \ + if (F_ISSET(S2C(s), WT_CONN_OPTRACK) && (s)->id != 0) { \ + if (__func_id == 0) \ + __wt_optrack_record_funcid( \ + s, __func__, &__func_id); \ + WT_TRACK_OP(s, 0); \ + } + +#define WT_TRACK_OP_END(s) \ + if (F_ISSET(S2C(s), WT_CONN_OPTRACK) && (s)->id != 0) \ + WT_TRACK_OP(s, 1); diff --git a/src/third_party/wiredtiger/src/include/os.h b/src/third_party/wiredtiger/src/include/os.h index ec1860d19a6..9f5f35344ba 100644 --- a/src/third_party/wiredtiger/src/include/os.h +++ b/src/third_party/wiredtiger/src/include/os.h @@ -65,6 +65,15 @@ #define WT_TIMEDIFF_SEC(end, begin) \ (WT_TIMEDIFF_NS((end), (begin)) / WT_BILLION) +#define WT_TSCDIFF_NS(s, end, begin) \ + (__wt_tsc_to_nsec((s), (uint64_t)(end) - (uint64_t)(begin))) +#define WT_TSCDIFF_US(s, end, begin) \ + (WT_TSCDIFF_NS((s), (end), (begin)) / WT_THOUSAND) +#define WT_TSCDIFF_MS(s, end, begin) \ + (WT_TSCDIFF_NS((s), (end), (begin)) / WT_MILLION) +#define WT_TSCDIFF_SEC(s, end, begin) \ + (WT_TSCDIFF_NS((s), (end), (begin)) / WT_BILLION) + #define WT_TIMECMP(t1, t2) \ ((t1).tv_sec < (t2).tv_sec ? -1 : \ (t1).tv_sec == (t2).tv_sec ? \ @@ -155,9 +164,11 @@ struct __wt_fstream { wt_off_t size; /* File size */ WT_ITEM buf; /* Data */ -#define WT_STREAM_APPEND 0x01 /* Open a stream for append */ -#define WT_STREAM_READ 0x02 /* Open a stream for read */ -#define WT_STREAM_WRITE 0x04 /* Open a stream for write */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_STREAM_APPEND 0x1u /* Open a stream for append */ +#define WT_STREAM_READ 0x2u /* Open a stream for read */ +#define WT_STREAM_WRITE 0x4u /* Open a stream for write */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; int (*close)(WT_SESSION_IMPL *, WT_FSTREAM *); diff --git a/src/third_party/wiredtiger/src/include/schema.h b/src/third_party/wiredtiger/src/include/schema.h index 80513f1174b..746e83ee358 100644 --- a/src/third_party/wiredtiger/src/include/schema.h +++ b/src/third_party/wiredtiger/src/include/schema.h @@ -40,7 +40,10 @@ struct __wt_index { const char *idxkey_format; /* Index key format (hides primary) */ const char *exkey_format; /* Key format for custom extractors */ -#define WT_INDEX_IMMUTABLE 0x01 + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_INDEX_IMMUTABLE 0x1u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; /* Index configuration flags */ }; diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 23cf136d0aa..269ca281ed2 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -148,6 +148,30 @@ struct __wt_session_impl { /* Sessions have an associated statistics bucket based on its ID. */ u_int stat_bucket; /* Statistics bucket offset */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_SESSION_CAN_WAIT 0x000001u +#define WT_SESSION_IGNORE_CACHE_SIZE 0x000002u +#define WT_SESSION_INTERNAL 0x000004u +#define WT_SESSION_LOCKED_CHECKPOINT 0x000008u +#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x000010u +#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x000020u +#define WT_SESSION_LOCKED_METADATA 0x000040u +#define WT_SESSION_LOCKED_PASS 0x000080u +#define WT_SESSION_LOCKED_SCHEMA 0x000100u +#define WT_SESSION_LOCKED_SLOT 0x000200u +#define WT_SESSION_LOCKED_TABLE_READ 0x000400u +#define WT_SESSION_LOCKED_TABLE_WRITE 0x000800u +#define WT_SESSION_LOCKED_TURTLE 0x001000u +#define WT_SESSION_LOGGING_INMEM 0x002000u +#define WT_SESSION_LOOKASIDE_CURSOR 0x004000u +#define WT_SESSION_NO_DATA_HANDLES 0x008000u +#define WT_SESSION_NO_LOGGING 0x010000u +#define WT_SESSION_NO_RECONCILE 0x020000u +#define WT_SESSION_NO_SCHEMA_LOCK 0x040000u +#define WT_SESSION_QUIET_CORRUPT_FILE 0x080000u +#define WT_SESSION_READ_WONT_NEED 0x100000u +#define WT_SESSION_SERVER_ASYNC 0x200000u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; /* @@ -213,4 +237,12 @@ struct __wt_session_impl { uint32_t hazard_inuse; /* Hazard pointer array slots in-use */ uint32_t nhazard; /* Count of active hazard pointers */ WT_HAZARD *hazard; /* Hazard pointer array */ + + /* + * Operation tracking. + */ + WT_OPTRACK_RECORD *optrack_buf; + u_int optrackbuf_ptr; + uint64_t optrack_offset; + WT_FH *optrack_fh; }; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 438333858b9..b27ac3df3b4 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -84,6 +84,17 @@ #define WT_STATS_FIELD_TO_OFFSET(stats, fld) \ (int)(&(stats)[0]->fld - (int64_t *)(stats)[0]) +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_STAT_CLEAR 0x01u +#define WT_STAT_JSON 0x02u +#define WT_STAT_ON_CLOSE 0x04u +#define WT_STAT_TYPE_ALL 0x08u +#define WT_STAT_TYPE_CACHE_WALK 0x10u +#define WT_STAT_TYPE_FAST 0x20u +#define WT_STAT_TYPE_SIZE 0x40u +#define WT_STAT_TYPE_TREE_WALK 0x80u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + /* * Sum the values from all structures in the array. */ diff --git a/src/third_party/wiredtiger/src/include/thread_group.h b/src/third_party/wiredtiger/src/include/thread_group.h index 97eda6ab674..feab58f13d5 100644 --- a/src/third_party/wiredtiger/src/include/thread_group.h +++ b/src/third_party/wiredtiger/src/include/thread_group.h @@ -21,11 +21,13 @@ struct __wt_thread { * WT_THREAD and thread-group function flags, merged because * WT_THREAD_PANIC_FAIL appears in both groups. */ -#define WT_THREAD_ACTIVE 0x01 /* thread is active or paused */ -#define WT_THREAD_CAN_WAIT 0x02 /* WT_SESSION_CAN_WAIT */ -#define WT_THREAD_LOOKASIDE 0x04 /* open lookaside cursor */ -#define WT_THREAD_PANIC_FAIL 0x08 /* panic if the thread fails */ -#define WT_THREAD_RUN 0x10 /* thread is running */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_THREAD_ACTIVE 0x01u /* thread is active or paused */ +#define WT_THREAD_CAN_WAIT 0x02u /* WT_SESSION_CAN_WAIT */ +#define WT_THREAD_LOOKASIDE 0x04u /* open lookaside cursor */ +#define WT_THREAD_PANIC_FAIL 0x08u /* panic if the thread fails */ +#define WT_THREAD_RUN 0x10u /* thread is running */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; /* diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 6b78c78a5cd..ba5897e98f6 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -10,6 +10,19 @@ #define WT_TXN_FIRST 1 /* First transaction to run. */ #define WT_TXN_ABORTED UINT64_MAX /* Update rolled back, ignore. */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TXN_LOG_CKPT_CLEANUP 0x01u +#define WT_TXN_LOG_CKPT_PREPARE 0x02u +#define WT_TXN_LOG_CKPT_START 0x04u +#define WT_TXN_LOG_CKPT_STOP 0x08u +#define WT_TXN_LOG_CKPT_SYNC 0x10u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TXN_OLDEST_STRICT 0x1u +#define WT_TXN_OLDEST_WAIT 0x2u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + /* * Transaction ID comparison dealing with edge cases. * @@ -246,19 +259,21 @@ struct __wt_txn { WT_ITEM *ckpt_snapshot; bool full_ckpt; -#define WT_TXN_AUTOCOMMIT 0x00001 -#define WT_TXN_ERROR 0x00002 -#define WT_TXN_HAS_ID 0x00004 -#define WT_TXN_HAS_SNAPSHOT 0x00008 -#define WT_TXN_HAS_TS_COMMIT 0x00010 -#define WT_TXN_HAS_TS_READ 0x00020 -#define WT_TXN_NAMED_SNAPSHOT 0x00040 -#define WT_TXN_PUBLIC_TS_COMMIT 0x00080 -#define WT_TXN_PUBLIC_TS_READ 0x00100 -#define WT_TXN_READONLY 0x00200 -#define WT_TXN_RUNNING 0x00400 -#define WT_TXN_SYNC_SET 0x00800 -#define WT_TXN_TS_COMMIT_ALWAYS 0x01000 -#define WT_TXN_TS_COMMIT_NEVER 0x02000 +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TXN_AUTOCOMMIT 0x0001u +#define WT_TXN_ERROR 0x0002u +#define WT_TXN_HAS_ID 0x0004u +#define WT_TXN_HAS_SNAPSHOT 0x0008u +#define WT_TXN_HAS_TS_COMMIT 0x0010u +#define WT_TXN_HAS_TS_READ 0x0020u +#define WT_TXN_NAMED_SNAPSHOT 0x0040u +#define WT_TXN_PUBLIC_TS_COMMIT 0x0080u +#define WT_TXN_PUBLIC_TS_READ 0x0100u +#define WT_TXN_READONLY 0x0200u +#define WT_TXN_RUNNING 0x0400u +#define WT_TXN_SYNC_SET 0x0800u +#define WT_TXN_TS_COMMIT_ALWAYS 0x1000u +#define WT_TXN_TS_COMMIT_NEVER 0x2000u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index d4c1fa956c3..69ef8af0650 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -125,9 +125,11 @@ struct __wt_item { /*! Managed memory size (internal use). */ size_t memsize; -#define WT_ITEM_ALIGNED 0x00000001 -#define WT_ITEM_INUSE 0x00000002 /*! Object flags (internal use). */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_ITEM_ALIGNED 0x1u +#define WT_ITEM_INUSE 0x2u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; #endif }; @@ -673,22 +675,24 @@ struct __wt_cursor { */ const char *internal_uri; -#define WT_CURSTD_APPEND 0x00001 -#define WT_CURSTD_BULK 0x00002 -#define WT_CURSTD_DUMP_HEX 0x00004 -#define WT_CURSTD_DUMP_JSON 0x00008 -#define WT_CURSTD_DUMP_PRINT 0x00010 -#define WT_CURSTD_JOINED 0x00020 -#define WT_CURSTD_KEY_EXT 0x00040 /* Key points out of the tree. */ -#define WT_CURSTD_KEY_INT 0x00080 /* Key points into the tree. */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CURSTD_APPEND 0x0001u +#define WT_CURSTD_BULK 0x0002u +#define WT_CURSTD_DUMP_HEX 0x0004u +#define WT_CURSTD_DUMP_JSON 0x0008u +#define WT_CURSTD_DUMP_PRINT 0x0010u +#define WT_CURSTD_JOINED 0x0020u +#define WT_CURSTD_KEY_EXT 0x0040u /* Key points out of the tree. */ +#define WT_CURSTD_KEY_INT 0x0080u /* Key points into the tree. */ +#define WT_CURSTD_META_INUSE 0x0100u +#define WT_CURSTD_OPEN 0x0200u +#define WT_CURSTD_OVERWRITE 0x0400u +#define WT_CURSTD_RAW 0x0800u +#define WT_CURSTD_RAW_SEARCH 0x1000u +#define WT_CURSTD_VALUE_EXT 0x2000u /* Value points out of the tree. */ +#define WT_CURSTD_VALUE_INT 0x4000u /* Value points into the tree. */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ #define WT_CURSTD_KEY_SET (WT_CURSTD_KEY_EXT | WT_CURSTD_KEY_INT) -#define WT_CURSTD_META_INUSE 0x00100 -#define WT_CURSTD_OPEN 0x00200 -#define WT_CURSTD_OVERWRITE 0x00400 -#define WT_CURSTD_RAW 0x00800 -#define WT_CURSTD_RAW_SEARCH 0x01000 -#define WT_CURSTD_VALUE_EXT 0x02000 /* Value points out of the tree. */ -#define WT_CURSTD_VALUE_INT 0x04000 /* Value points into the tree. */ #define WT_CURSTD_VALUE_SET (WT_CURSTD_VALUE_EXT | WT_CURSTD_VALUE_INT) uint32_t flags; #endif @@ -1320,6 +1324,20 @@ struct __wt_session { * for chunks to be temporarily larger than this value. This overrides * the \c memory_page_max setting., an integer between 512K and 500MB; * default \c 10MB.} + * @config{ merge_custom = (, + * configure the tree to merge into a custom data source., a set of + * related configuration options defined below.} + * @config{ prefix, + * custom data source prefix instead of \c "file"., a string; default + * empty.} + * @config{ start + * _generation, merge generation at which the custom data source is used + * (zero indicates no custom data source)., an integer between 0 and 10; + * default \c 0.} + * @config{ suffix, + * custom data source suffix instead of \c ".lsm"., a string; default + * empty.} + * @config{ ),,} * @config{ merge_max, the * maximum number of chunks to include in a merge operation., an integer * between 2 and 100; default \c 15.} @@ -2137,6 +2155,17 @@ struct __wt_connection { * thread uses a session handle from the configured session_max., an * integer between 3 and 20; default \c 4.} * @config{ ),,} + * @config{operation_tracking = (, enable tracking of + * performance-critical functions. See @ref operation_tracking for more + * information., a set of related configuration options defined below.} + * @config{ enabled, enable operation tracking + * subsystem., a boolean flag; default \c false.} + * @config{ path, the name of a directory into + * which operation tracking files are written. The directory must + * already exist. If the value is not an absolute path\, the path is + * relative to the database home (see @ref absolute_path for more + * information)., a string; default \c ".".} + * @config{ ),,} * @config{shared_cache = (, shared cache configuration options. A * database should configure either a cache_size or a shared_cache not * both. Enabling a shared cache uses a session from the configured @@ -2196,17 +2225,17 @@ struct __wt_connection { * write of the log records; setting this value above 0 configures * statistics logging., an integer between 0 and 100000; default \c 0.} * @config{ ),,} - * @config{verbose, enable messages for various events. Only available - * if WiredTiger is configured with --enable-verbose. Options are given - * as a list\, such as <code>"verbose=[evictserver\,read]"</code>., a - * list\, with values chosen from the following options: \c "api"\, \c - * "block"\, \c "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, - * \c "evict"\, \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c - * "handleops"\, \c "log"\, \c "lookaside"\, \c "lookaside_activity"\, - * \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c - * "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c - * "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c - * "shared_cache"\, \c "split"\, \c "thread_group"\, \c "timestamp"\, \c + * @config{verbose, enable messages for various events. Options are + * given as a list\, such as + * <code>"verbose=[evictserver\,read]"</code>., a list\, with values + * chosen from the following options: \c "api"\, \c "block"\, \c + * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c "evict"\, + * \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, + * \c "log"\, \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c + * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c + * "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, \c + * "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, + * \c "temporary"\, \c "thread_group"\, \c "timestamp"\, \c * "transaction"\, \c "verify"\, \c "version"\, \c "write"; default * empty.} * @configend @@ -2759,6 +2788,16 @@ struct __wt_connection { * start an RPC server for primary processes and use RPC for secondary * processes). <b>Not yet supported in WiredTiger</b>., a boolean flag; default * \c false.} + * @config{operation_tracking = (, enable tracking of performance-critical + * functions. See @ref operation_tracking for more information., a set of + * related configuration options defined below.} + * @config{ enabled, enable operation tracking + * subsystem., a boolean flag; default \c false.} + * @config{ path, the name of a directory into which + * operation tracking files are written. The directory must already exist. If + * the value is not an absolute path\, the path is relative to the database home + * (see @ref absolute_path for more information)., a string; default \c ".".} + * @config{ ),,} * @config{readonly, open connection in read-only mode. The database must * exist. All methods that may modify a database are disabled. See @ref * readonly for more information., a boolean flag; default \c false.} @@ -2839,8 +2878,7 @@ struct __wt_connection { * WIREDTIGER_HOME environment variables even if the process is running with * special privileges. See @ref home for more information., a boolean flag; * default \c false.} - * @config{verbose, enable messages for various events. Only available if - * WiredTiger is configured with --enable-verbose. Options are given as a + * @config{verbose, enable messages for various events. Options are given as a * list\, such as <code>"verbose=[evictserver\,read]"</code>., a list\, with * values chosen from the following options: \c "api"\, \c "block"\, \c * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c "evict"\, \c @@ -2848,8 +2886,9 @@ struct __wt_connection { * \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c * "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c * "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c - * "shared_cache"\, \c "split"\, \c "thread_group"\, \c "timestamp"\, \c - * "transaction"\, \c "verify"\, \c "version"\, \c "write"; default empty.} + * "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c + * "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; + * default empty.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to * files. Ignored on non-Windows systems. Options are given as a list\, such * as <code>"write_through=[data]"</code>. Configuring \c write_through requires @@ -3864,6 +3903,14 @@ struct __wt_data_source { const char *uri, WT_CONFIG_ARG *config); /*! + * Callback to get the size of an object. + * + * @snippet ex_data_source.c WT_DATA_SOURCE size + */ + int (*size)(WT_DATA_SOURCE *dsrc, WT_SESSION *session, + const char *uri, wt_off_t *size); + + /*! * Callback to truncate an object. * * @snippet ex_data_source.c WT_DATA_SOURCE truncate @@ -3904,6 +3951,17 @@ struct __wt_data_source { * @snippet ex_data_source.c WT_DATA_SOURCE terminate */ int (*terminate)(WT_DATA_SOURCE *dsrc, WT_SESSION *session); + + /*! + * If non-NULL, a callback performed before an LSM merge. + * + * @param[in] source a cursor configured with the data being merged + * @param[in] dest a cursor on the new object being filled by the merge + * + * @snippet ex_data_source.c WT_DATA_SOURCE lsm_pre_merge + */ + int (*lsm_pre_merge)( + WT_DATA_SOURCE *dsrc, WT_CURSOR *source, WT_CURSOR *dest); }; /*! @@ -4118,31 +4176,45 @@ typedef enum { WT_FS_OPEN_FILE_TYPE_REGULAR /*!< open a regular file */ } WT_FS_OPEN_FILE_TYPE; +#ifdef DOXYGEN /*! WT_FILE_SYSTEM::open_file flags: random access pattern */ -#define WT_FS_OPEN_ACCESS_RAND 0x001 +#define WT_FS_OPEN_ACCESS_RAND 0x0 /*! WT_FILE_SYSTEM::open_file flags: sequential access pattern */ -#define WT_FS_OPEN_ACCESS_SEQ 0x002 +#define WT_FS_OPEN_ACCESS_SEQ 0x0 /*! WT_FILE_SYSTEM::open_file flags: create if does not exist */ -#define WT_FS_OPEN_CREATE 0x004 +#define WT_FS_OPEN_CREATE 0x0 /*! WT_FILE_SYSTEM::open_file flags: direct I/O requested */ -#define WT_FS_OPEN_DIRECTIO 0x008 +#define WT_FS_OPEN_DIRECTIO 0x0 /*! WT_FILE_SYSTEM::open_file flags: file creation must be durable */ -#define WT_FS_OPEN_DURABLE 0x010 +#define WT_FS_OPEN_DURABLE 0x0 /*! * WT_FILE_SYSTEM::open_file flags: return EBUSY if exclusive use not available */ -#define WT_FS_OPEN_EXCLUSIVE 0x020 -#ifndef DOXYGEN -#define WT_FS_OPEN_FIXED 0x040 /* Path not home relative (internal) */ -#endif +#define WT_FS_OPEN_EXCLUSIVE 0x0 /*! WT_FILE_SYSTEM::open_file flags: open is read-only */ -#define WT_FS_OPEN_READONLY 0x080 +#define WT_FS_OPEN_READONLY 0x0 /*! * WT_FILE_SYSTEM::remove or WT_FILE_SYSTEM::rename flags: the remove or rename * operation must be durable */ -#define WT_FS_DURABLE 0x001 +#define WT_FS_DURABLE 0x0 +#else +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_FS_OPEN_ACCESS_RAND 0x01u +#define WT_FS_OPEN_ACCESS_SEQ 0x02u +#define WT_FS_OPEN_CREATE 0x04u +#define WT_FS_OPEN_DIRECTIO 0x08u +#define WT_FS_OPEN_DURABLE 0x10u +#define WT_FS_OPEN_EXCLUSIVE 0x20u +#define WT_FS_OPEN_FIXED 0x40u /* Path not home relative (internal) */ +#define WT_FS_OPEN_READONLY 0x80u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_FS_DURABLE 0x1u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ +#endif /*! * The interface implemented by applications to provide a custom file system diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index b25ed08e30f..9fcf1401a4c 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -246,6 +246,10 @@ struct __wt_named_extractor; typedef struct __wt_named_extractor WT_NAMED_EXTRACTOR; struct __wt_named_snapshot; typedef struct __wt_named_snapshot WT_NAMED_SNAPSHOT; +struct __wt_optrack_header; + typedef struct __wt_optrack_header WT_OPTRACK_HEADER; +struct __wt_optrack_record; + typedef struct __wt_optrack_record WT_OPTRACK_RECORD; struct __wt_ovfl_reuse; typedef struct __wt_ovfl_reuse WT_OVFL_REUSE; struct __wt_ovfl_track; @@ -352,10 +356,10 @@ union __wt_rand_state; #include "cursor.h" #include "dlh.h" #include "error.h" -#include "flags.h" #include "log.h" #include "lsm.h" #include "meta.h" +#include "optrack.h" #include "os.h" #include "schema.h" #include "thread_group.h" @@ -374,6 +378,7 @@ union __wt_rand_state; #include "ctype.i" /* required by packing.i */ #include "intpack.i" /* required by cell.i, packing.i */ +#include "misc.i" /* required by mutex.i */ #include "buf.i" /* required by cell.i */ #include "cache.i" /* required by txn.i */ @@ -387,7 +392,6 @@ union __wt_rand_state; #include "column.i" #include "cursor.i" #include "log.i" -#include "misc.i" #include "os_fhandle.i" #include "os_fs.i" #include "os_fstream.i" diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 17c77f532bb..a04ab84845f 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -16,8 +16,9 @@ static int __log_write_internal( #define WT_LOG_COMPRESS_SKIP (offsetof(WT_LOG_RECORD, record)) #define WT_LOG_ENCRYPT_SKIP (offsetof(WT_LOG_RECORD, record)) -/* Flags to __log_openfile */ -#define WT_LOG_OPEN_CREATE_OK 0x01 +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_LOG_OPEN_CREATE_OK 0x1u /* Flag to __log_openfile() */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ /* * __wt_log_printf -- @@ -258,14 +259,14 @@ __wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn) int __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) { - struct timespec fsync_start, fsync_stop; WT_DECL_RET; WT_FH *log_fh; WT_LOG *log; - uint64_t fsync_duration_usecs; + uint64_t fsync_duration_usecs, time_start, time_stop; log = S2C(session)->log; log_fh = NULL; + time_start = time_stop = 0; /* * We need to wait for the previous log file to get written @@ -289,10 +290,11 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) "log_force_sync: sync directory %s to LSN %" PRIu32 "/%" PRIu32, log->log_dir_fh->name, min_lsn->l.file, min_lsn->l.offset); - __wt_epoch(session, &fsync_start); + time_start = __wt_rdtsc(session); WT_ERR(__wt_fsync(session, log->log_dir_fh, true)); - __wt_epoch(session, &fsync_stop); - fsync_duration_usecs = WT_TIMEDIFF_US(fsync_stop, fsync_start); + time_stop = __wt_rdtsc(session); + fsync_duration_usecs = WT_TSCDIFF_US(session, + time_stop, time_start); log->sync_dir_lsn = *min_lsn; WT_STAT_CONN_INCR(session, log_sync_dir); WT_STAT_CONN_INCRV(session, @@ -312,10 +314,11 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) __wt_verbose(session, WT_VERB_LOG, "log_force_sync: sync %s to LSN %" PRIu32 "/%" PRIu32, log_fh->name, min_lsn->l.file, min_lsn->l.offset); - __wt_epoch(session, &fsync_start); + time_start = __wt_rdtsc(session); WT_ERR(__wt_fsync(session, log_fh, true)); - __wt_epoch(session, &fsync_stop); - fsync_duration_usecs = WT_TIMEDIFF_US(fsync_stop, fsync_start); + time_stop = __wt_rdtsc(session); + fsync_duration_usecs = WT_TSCDIFF_US(session, + time_stop, time_start); log->sync_lsn = *min_lsn; WT_STAT_CONN_INCR(session, log_sync); WT_STAT_CONN_INCRV(session, @@ -484,18 +487,8 @@ static int __log_filename(WT_SESSION_IMPL *session, uint32_t id, const char *file_prefix, WT_ITEM *buf) { - const char *log_path; - - log_path = S2C(session)->log_path; - - if (log_path != NULL && log_path[0] != '\0') - WT_RET(__wt_buf_fmt(session, buf, "%s/%s.%010" PRIu32, - log_path, file_prefix, id)); - else - WT_RET(__wt_buf_fmt(session, buf, "%s.%010" PRIu32, - file_prefix, id)); - - return (0); + return (__wt_filename_construct(session, + S2C(session)->log_path, file_prefix, UINTMAX_MAX, id, buf)); } /* @@ -1012,6 +1005,7 @@ err: __wt_scr_free(session, &buf); static int __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) { + WT_CONNECTION_IMPL *conn; WT_DECL_ITEM(from_path); WT_DECL_ITEM(to_path); WT_DECL_RET; @@ -1023,7 +1017,8 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) /* * If there are no pre-allocated files, return WT_NOTFOUND. */ - log = S2C(session)->log; + conn = S2C(session); + log = conn->log; logfiles = NULL; WT_ERR(__log_get_files(session, WT_LOG_PREPNAME, &logfiles, &logcount)); if (logcount == 0) @@ -1137,7 +1132,14 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) * If we need to create the log file, do so now. */ if (create_log) { - log->prep_missed++; + /* + * Increment the missed pre-allocated file counter only + * if a hot backup is not in progress. We are deliberately + * not using pre-allocated log files during backup + * (see comment above). + */ + if (!conn->hot_backup) + log->prep_missed++; WT_RET(__wt_log_allocfile( session, log->fileid, WT_LOG_FILENAME)); } @@ -1709,18 +1711,18 @@ err: __wt_free(session, buf); int __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) { - struct timespec fsync_start, fsync_stop; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; WT_LSN sync_lsn; - uint64_t fsync_duration_usecs; + uint64_t fsync_duration_usecs, time_start, time_stop; int64_t release_buffered, release_bytes; bool locked; conn = S2C(session); log = conn->log; locked = false; + time_start = time_stop = 0; if (freep != NULL) *freep = 1; release_buffered = WT_LOG_SLOT_RELEASED_BUFFERED(slot->slot_state); @@ -1825,11 +1827,11 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) "/%" PRIu32, log->log_dir_fh->name, sync_lsn.l.file, sync_lsn.l.offset); - __wt_epoch(session, &fsync_start); + time_start = __wt_rdtsc(session); WT_ERR(__wt_fsync(session, log->log_dir_fh, true)); - __wt_epoch(session, &fsync_stop); - fsync_duration_usecs = - WT_TIMEDIFF_US(fsync_stop, fsync_start); + time_stop = __wt_rdtsc(session); + fsync_duration_usecs = WT_TSCDIFF_US(session, + time_stop, time_start); log->sync_dir_lsn = sync_lsn; WT_STAT_CONN_INCR(session, log_sync_dir); WT_STAT_CONN_INCRV(session, @@ -1847,11 +1849,11 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) log->log_fh->name, sync_lsn.l.file, sync_lsn.l.offset); WT_STAT_CONN_INCR(session, log_sync); - __wt_epoch(session, &fsync_start); + time_start = __wt_rdtsc(session); WT_ERR(__wt_fsync(session, log->log_fh, true)); - __wt_epoch(session, &fsync_stop); - fsync_duration_usecs = - WT_TIMEDIFF_US(fsync_stop, fsync_start); + time_stop = __wt_rdtsc(session); + fsync_duration_usecs = WT_TSCDIFF_US(session, + time_stop, time_start); WT_STAT_CONN_INCRV(session, log_sync_duration, fsync_duration_usecs); log->sync_lsn = sync_lsn; diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index 61dfb82083d..74ce16d0e21 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -100,7 +100,7 @@ __log_slot_close( WT_LOG *log; int64_t end_offset, new_state, old_state; #ifdef HAVE_DIAGNOSTIC - struct timespec begin, now; + uint64_t time_start, time_stop; int count; #endif @@ -133,7 +133,8 @@ retry: /* * If someone completely processed this slot, we're done. */ - if (FLD64_ISSET((uint64_t)slot->slot_state, WT_LOG_SLOT_RESERVED)) { + if (FLD_LOG_SLOT_ISSET( + (uint64_t)slot->slot_state, WT_LOG_SLOT_RESERVED)) { WT_STAT_CONN_INCR(session, log_slot_close_race); return (WT_NOTFOUND); } @@ -160,7 +161,7 @@ retry: */ #ifdef HAVE_DIAGNOSTIC count = 0; - __wt_epoch(session, &begin); + time_start = __wt_rdtsc(session); #endif if (WT_LOG_SLOT_UNBUFFERED_ISSET(old_state)) { while (slot->slot_unbuffered == 0) { @@ -169,8 +170,9 @@ retry: #ifdef HAVE_DIAGNOSTIC ++count; if (count > WT_MILLION) { - __wt_epoch(session, &now); - if (WT_TIMEDIFF_SEC(now, begin) > 10) { + time_stop = __wt_rdtsc(session); + if (WT_TSCDIFF_SEC( + session, time_stop, time_start) > 10) { __wt_errx(session, "SLOT_CLOSE: Slot %" PRIu32 " Timeout unbuffered, state 0x%" PRIx64 " unbuffered %" PRId64, @@ -211,7 +213,7 @@ __log_slot_new(WT_SESSION_IMPL *session) WT_LOGSLOT *slot; int32_t i, pool_i; #ifdef HAVE_DIAGNOSTIC - struct timespec begin, now; + uint64_t time_start, time_stop; int count; #endif @@ -229,7 +231,7 @@ __log_slot_new(WT_SESSION_IMPL *session) #ifdef HAVE_DIAGNOSTIC count = 0; - __wt_epoch(session, &begin); + time_start = __wt_rdtsc(session); #endif /* * Keep trying until we can find a free slot. @@ -269,8 +271,9 @@ __log_slot_new(WT_SESSION_IMPL *session) #ifdef HAVE_DIAGNOSTIC ++count; if (count > WT_MILLION) { - __wt_epoch(session, &now); - if (WT_TIMEDIFF_SEC(now, begin) > 10) { + time_stop = __wt_rdtsc(session); + if (WT_TSCDIFF_SEC( + session, time_stop, time_start) > 10) { __wt_errx(session, "SLOT_NEW: Timeout free slot"); __log_slot_dump(session); @@ -482,7 +485,7 @@ __wt_log_slot_destroy(WT_SESSION_IMPL *session) */ for (i = 0; i < WT_SLOT_POOL; i++) { slot = &log->slot_pool[i]; - if (!FLD64_ISSET( + if (!FLD_LOG_SLOT_ISSET( (uint64_t)slot->slot_state, WT_LOG_SLOT_RESERVED)) { rel = WT_LOG_SLOT_RELEASED_BUFFERED(slot->slot_state); if (rel != 0) @@ -503,17 +506,17 @@ void __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot) { - struct timespec start, stop; WT_CONNECTION_IMPL *conn; WT_LOG *log; WT_LOGSLOT *slot; - uint64_t usecs; + uint64_t time_start, time_stop, usecs; int64_t flag_state, new_state, old_state, released; int32_t join_offset, new_join, wait_cnt; bool closed, diag_yield, raced, slept, unbuffered, yielded; conn = S2C(session); log = conn->log; + time_start = time_stop = 0; WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_SLOT)); WT_ASSERT(session, mysize != 0); @@ -575,7 +578,7 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, ++wait_cnt; } if (!yielded) - __wt_epoch(session, &start); + time_start = __wt_rdtsc(session); yielded = true; /* * The slot is no longer open or we lost the race to @@ -596,8 +599,8 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, WT_STAT_CONN_INCR(session, log_slot_immediate); else { WT_STAT_CONN_INCR(session, log_slot_yield); - __wt_epoch(session, &stop); - usecs = WT_TIMEDIFF_US(stop, start); + time_stop = __wt_rdtsc(session); + usecs = WT_TSCDIFF_US(session, time_stop, time_start); WT_STAT_CONN_INCRV(session, log_slot_yield_duration, usecs); if (closed) WT_STAT_CONN_INCR(session, log_slot_yield_close); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index e3b2b50ad28..04c6d5c00c4 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -557,13 +557,19 @@ retry: if (F_ISSET(clsm, WT_CLSM_MERGE)) { if (strcmp(cursor->uri, chunk->uri) != 0) break; - /* Make sure the checkpoint config matches. */ - checkpoint = ((WT_CURSOR_BTREE *)cursor)-> - btree->dhandle->checkpoint; - if (checkpoint == NULL && - F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && - !chunk->empty) - break; + /* + * Make sure the checkpoint config matches when not + * using a custom data source. + */ + if (lsm_tree->custom_generation == 0 || + chunk->generation < lsm_tree->custom_generation) { + checkpoint = ((WT_CURSOR_BTREE *)cursor)-> + btree->dhandle->checkpoint; + if (checkpoint == NULL && + F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && + !chunk->empty) + break; + } /* Make sure the Bloom config matches. */ if (clsm->chunks[ngood]->bloom == NULL && @@ -721,13 +727,19 @@ err: WT_ASSERT( session, strcmp(cursor->uri, chunk->uri) == 0); - /* Make sure the checkpoint config matches. */ - checkpoint = ((WT_CURSOR_BTREE *)cursor)-> - btree->dhandle->checkpoint; - WT_ASSERT(session, - (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && - !chunk->empty) ? - checkpoint != NULL : checkpoint == NULL); + /* + * Make sure the checkpoint config matches when not + * using a custom data source. + */ + if (lsm_tree->custom_generation == 0 || + chunk->generation < lsm_tree->custom_generation) { + checkpoint = ((WT_CURSOR_BTREE *)cursor)-> + btree->dhandle->checkpoint; + WT_ASSERT(session, + (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && + !chunk->empty) ? + checkpoint != NULL : checkpoint == NULL); + } /* Make sure the Bloom config matches. */ WT_ASSERT(session, diff --git a/src/third_party/wiredtiger/src/lsm/lsm_merge.c b/src/third_party/wiredtiger/src/lsm/lsm_merge.c index 7a20686fb97..2a02792d392 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_merge.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_merge.c @@ -137,6 +137,25 @@ __lsm_merge_aggressive_update(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) } /* + * __lsm_merge_clear -- + * Clear merge flag on chunks that was set during __lsm_merge_span. + */ +static void +__lsm_merge_clear(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, + u_int start, u_int nchunks) +{ + WT_LSM_CHUNK *chunk; + u_int i; + + for (i = 0; i < nchunks; i++) { + chunk = lsm_tree->chunk[start + i]; + WT_ASSERT(session, + F_ISSET(chunk, WT_LSM_CHUNK_MERGING)); + F_CLR(chunk, WT_LSM_CHUNK_MERGING); + } +} + +/* * __lsm_merge_span -- * Figure out the best span of chunks to merge. Return an error if * there is no need to do any merges. Called with the LSM tree @@ -146,12 +165,14 @@ static int __lsm_merge_span(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id, u_int *start, u_int *end, uint64_t *records) { - WT_LSM_CHUNK *chunk, *previous, *youngest; + WT_LSM_CHUNK *chunk, *youngest; uint64_t chunk_size, record_count; - uint32_t aggressive, max_gap, max_gen, max_level; - u_int end_chunk, i, merge_max, merge_min, nchunks, start_chunk; + uint32_t aggressive, max_gap, max_level; + u_int end_chunk, merge_max, merge_min, nchunks, start_chunk; u_int oldest_gen, youngest_gen; - +#ifdef HAVE_DIAGNOSTIC + u_int i; +#endif /* Clear the return parameters */ *start = *end = 0; *records = 0; @@ -235,6 +256,14 @@ retry_find: break; /* + * If we have enough chunks for a merge and the next chunk is + * in too high a generation, stop. + */ + if (nchunks >= merge_min && + chunk->generation > youngest_gen + max_gap) + break; + + /* * If the size of the chunks selected so far exceeds the * configured maximum chunk size, stop. Keep going if we can * slide the window further into the tree: we don't want to @@ -255,21 +284,10 @@ retry_find: if (oldest_gen - youngest_gen > max_gap) break; - /* - * If we have enough chunks for a merge and the next chunk is - * in too high a generation, stop. - */ - if (nchunks >= merge_min) { - previous = lsm_tree->chunk[start_chunk]; - max_gen = youngest->generation + max_gap; - if (previous->generation <= max_gen && - chunk->generation > max_gen) - break; - } - F_SET(chunk, WT_LSM_CHUNK_MERGING); record_count += chunk->count; --start_chunk; + ++nchunks; /* * If the merge would be too big, or we have a full window @@ -280,12 +298,15 @@ retry_find: (nchunks == merge_max && start_chunk > 0 && chunk->generation == lsm_tree->chunk[start_chunk - 1]->generation)) { - WT_ASSERT(session, - F_ISSET(youngest, WT_LSM_CHUNK_MERGING)); - F_CLR(youngest, WT_LSM_CHUNK_MERGING); - record_count -= youngest->count; - chunk_size -= youngest->size; + /* + * Try again with smaller range. Unfortunately all the + * intermediate state will be reset. Since there's no + * easy way to restore youngest_gen and oldest_gen. + */ + __lsm_merge_clear( + session, lsm_tree, start_chunk, nchunks); --end_chunk; + goto retry_find; } else if (nchunks == merge_max) /* We've found the best full merge we can */ break; @@ -308,12 +329,7 @@ retry_find: * Don't do merges that are too small or across too many generations. */ if (nchunks < merge_min || oldest_gen - youngest_gen > max_gap) { - for (i = 0; i < nchunks; i++) { - chunk = lsm_tree->chunk[start_chunk + i]; - WT_ASSERT(session, - F_ISSET(chunk, WT_LSM_CHUNK_MERGING)); - F_CLR(chunk, WT_LSM_CHUNK_MERGING); - } + __lsm_merge_clear(session, lsm_tree, start_chunk, nchunks); /* * If we didn't find a merge with appropriate gaps, try again * with a smaller range. @@ -417,6 +433,7 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) WT_ERR(__wt_calloc_one(session, &chunk)); created_chunk = true; chunk->id = dest_id; + chunk->generation = generation; if (FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_MERGED) && (FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST) || @@ -453,6 +470,20 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) cfg[2] = NULL; WT_ERR(__wt_open_cursor(session, chunk->uri, NULL, cfg, &dest)); + if (lsm_tree->custom_generation != 0 && + chunk->generation >= lsm_tree->custom_generation) { + WT_DATA_SOURCE *dsrc = + __wt_schema_get_source(session, chunk->uri); + + if (dsrc != NULL && dsrc->lsm_pre_merge != NULL) { + /* Call the callback. */ + WT_ERR(dsrc->lsm_pre_merge(dsrc, src, dest)); + + /* Make sure the source is ready to start the scan. */ + WT_ERR(src->reset(src)); + } + } + #define LSM_MERGE_CHECK_INTERVAL WT_THOUSAND for (insert_count = 0; (ret = src->next(src)) == 0; insert_count++) { if (insert_count % LSM_MERGE_CHECK_INTERVAL == 0) { @@ -541,7 +572,7 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) in_sync = false; WT_ERR_NOTFOUND_OK(ret); - WT_ERR(__wt_lsm_tree_set_chunk_size(session, chunk)); + WT_ERR(__wt_lsm_tree_set_chunk_size(session, lsm_tree, chunk)); __wt_lsm_tree_writelock(session, lsm_tree); locked = true; @@ -567,7 +598,6 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) if (create_bloom) F_SET(chunk, WT_LSM_CHUNK_BLOOM); chunk->count = insert_count; - chunk->generation = generation; F_SET(chunk, WT_LSM_CHUNK_ONDISK); /* diff --git a/src/third_party/wiredtiger/src/lsm/lsm_meta.c b/src/third_party/wiredtiger/src/lsm/lsm_meta.c index 3a4960aaec1..7cb82f94ad4 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_meta.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_meta.c @@ -104,7 +104,8 @@ __lsm_meta_read_v0( lsm_tree->chunk[nchunks++] = chunk; chunk->id = (uint32_t)lv.val; WT_RET(__wt_lsm_tree_chunk_name(session, - lsm_tree, chunk->id, &chunk->uri)); + lsm_tree, chunk->id, + chunk->generation, &chunk->uri)); F_SET(chunk, WT_LSM_CHUNK_ONDISK | WT_LSM_CHUNK_STABLE); @@ -200,6 +201,21 @@ __lsm_meta_read_v1( cv.str, cv.len, &lsm_tree->collator_name)); } + WT_ERR(__wt_config_getones( + session, lsmconf, "lsm.merge_custom.start_generation", &cv)); + lsm_tree->custom_generation = (uint32_t)cv.val; + if (lsm_tree->custom_generation != 0) { + WT_ERR(__wt_config_getones( + session, lsmconf, "lsm.merge_custom.prefix", &cv)); + WT_ERR(__wt_strndup(session, + cv.str, cv.len, &lsm_tree->custom_prefix)); + + WT_ERR(__wt_config_getones( + session, lsmconf, "lsm.merge_custom.suffix", &cv)); + WT_ERR(__wt_strndup(session, + cv.str, cv.len, &lsm_tree->custom_suffix)); + } + WT_ERR(__wt_config_getones(session, lsmconf, "lsm.auto_throttle", &cv)); if (cv.val) F_SET(lsm_tree, WT_LSM_TREE_THROTTLE); @@ -267,15 +283,18 @@ __lsm_meta_read_v1( __wt_config_subinit(session, &lparser, &cv); for (nchunks = 0; (ret = __wt_config_next(&lparser, &lk, &lv)) == 0; ) { - if (WT_STRING_MATCH("id", lk.str, lk.len)) { + if (WT_STRING_MATCH("generation", lk.str, lk.len)) { WT_ERR(__wt_realloc_def(session, &lsm_tree->chunk_alloc, nchunks + 1, &lsm_tree->chunk)); WT_ERR(__wt_calloc_one(session, &chunk)); lsm_tree->chunk[nchunks++] = chunk; + chunk->generation = (uint32_t)lv.val; + continue; + } else if (WT_STRING_MATCH("id", lk.str, lk.len)) { chunk->id = (uint32_t)lv.val; - WT_ERR(__wt_lsm_tree_chunk_name(session, - lsm_tree, chunk->id, &chunk->uri)); + WT_ERR(__wt_lsm_tree_chunk_name(session, lsm_tree, + chunk->id, chunk->generation, &chunk->uri)); F_SET(chunk, WT_LSM_CHUNK_ONDISK | WT_LSM_CHUNK_STABLE); @@ -290,9 +309,6 @@ __lsm_meta_read_v1( } else if (WT_STRING_MATCH("count", lk.str, lk.len)) { chunk->count = (uint64_t)lv.val; continue; - } else if (WT_STRING_MATCH("generation", lk.str, lk.len)) { - chunk->generation = (uint32_t)lv.val; - continue; } } WT_ERR_NOTFOUND_OK(ret); @@ -473,7 +489,14 @@ __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, chunk = lsm_tree->chunk[i]; if (i > 0) WT_ERR(__wt_buf_catfmt(session, buf, ",")); - WT_ERR(__wt_buf_catfmt(session, buf, "id=%" PRIu32, chunk->id)); + /* + * Note that we need the generation before the ID for custom + * data sources, or the wrong URI will be generated. + */ + WT_ERR(__wt_buf_catfmt( + session, buf, "generation=%" PRIu32, chunk->generation)); + WT_ERR(__wt_buf_catfmt( + session, buf, ",id=%" PRIu32, chunk->id)); if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) WT_ERR(__wt_buf_catfmt(session, buf, ",bloom")); if (chunk->size != 0) @@ -482,8 +505,6 @@ __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, if (chunk->count != 0) WT_ERR(__wt_buf_catfmt( session, buf, ",count=%" PRIu64, chunk->count)); - WT_ERR(__wt_buf_catfmt( - session, buf, ",generation=%" PRIu32, chunk->generation)); } WT_ERR(__wt_buf_catfmt(session, buf, "]")); WT_ERR(__wt_buf_catfmt(session, buf, ",old_chunks=[")); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index c5b63df099f..cef5e51e214 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -28,6 +28,8 @@ __lsm_tree_discard_state(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) __wt_free(session, lsm_tree->key_format); __wt_free(session, lsm_tree->value_format); __wt_free(session, lsm_tree->collator_name); + __wt_free(session, lsm_tree->custom_prefix); + __wt_free(session, lsm_tree->custom_suffix); __wt_free(session, lsm_tree->bloom_config); __wt_free(session, lsm_tree->file_config); @@ -208,14 +210,22 @@ err: __wt_scr_free(session, &tmp); */ int __wt_lsm_tree_chunk_name(WT_SESSION_IMPL *session, - WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) + WT_LSM_TREE *lsm_tree, uint32_t id, uint32_t generation, const char **retp) { WT_DECL_ITEM(tmp); WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 0, &tmp)); - WT_ERR(__wt_buf_fmt( - session, tmp, "file:%s-%06" PRIu32 ".lsm", lsm_tree->filename, id)); + + if (lsm_tree->custom_generation != 0 && + generation >= lsm_tree->custom_generation) + WT_ERR(__wt_buf_fmt(session, tmp, "%s:%s-%06" PRIu32 "%s", + lsm_tree->custom_prefix, lsm_tree->filename, id, + lsm_tree->custom_suffix)); + else + WT_ERR(__wt_buf_fmt(session, tmp, "file:%s-%06" PRIu32 ".lsm", + lsm_tree->filename, id)); + WT_ERR(__wt_strndup(session, tmp->data, tmp->size, retp)); err: __wt_scr_free(session, &tmp); @@ -229,16 +239,32 @@ err: __wt_scr_free(session, &tmp); */ int __wt_lsm_tree_set_chunk_size( - WT_SESSION_IMPL *session, WT_LSM_CHUNK *chunk) + WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) { + WT_DATA_SOURCE *dsrc; wt_off_t size; const char *filename; - filename = chunk->uri; - if (!WT_PREFIX_SKIP(filename, "file:")) - WT_RET_MSG(session, EINVAL, - "Expected a 'file:' URI: %s", chunk->uri); - WT_RET(__wt_fs_size(session, filename, &size)); + size = 0; + if (lsm_tree->custom_generation != 0 && + chunk->generation >= lsm_tree->custom_generation) { + dsrc = __wt_schema_get_source(session, chunk->uri); + /* + * We can only retrieve a size if the data source exposes the + * information. + */ + if (dsrc != NULL && dsrc->size != NULL) { + /* Call the callback. */ + WT_RET(dsrc->size( + dsrc, (WT_SESSION*)session, chunk->uri, &size)); + } + } else { + filename = chunk->uri; + if (!WT_PREFIX_SKIP(filename, "file:")) + WT_RET_MSG(session, EINVAL, + "Expected a 'file:' URI: %s", chunk->uri); + WT_RET(__wt_fs_size(session, filename, &size)); + } chunk->size = (uint64_t)size; @@ -257,10 +283,13 @@ __lsm_tree_cleanup_old(WT_SESSION_IMPL *session, const char *uri) WT_DECL_RET; const char *cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_drop), "force", NULL }; - bool exists; + bool exists, is_file; - WT_RET(__wt_fs_exist(session, uri + strlen("file:"), &exists)); - if (exists) + exists = false; + is_file = WT_PREFIX_MATCH(uri, "file:"); + if (is_file) + WT_RET(__wt_fs_exist(session, uri + strlen("file:"), &exists)); + if (!is_file || exists) WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_drop(session, uri, cfg)); return (ret); @@ -280,7 +309,7 @@ __wt_lsm_tree_setup_chunk( WT_RET(__wt_spin_init(session, &chunk->timestamp_spinlock, "LSM chunk timestamp")); WT_RET(__wt_lsm_tree_chunk_name( - session, lsm_tree, chunk->id, &chunk->uri)); + session, lsm_tree, chunk->id, chunk->generation, &chunk->uri)); /* * If the underlying file exists, drop the chunk first - there may be @@ -959,8 +988,8 @@ __wt_lsm_tree_rename(WT_SESSION_IMPL *session, old = chunk->uri; chunk->uri = NULL; - WT_ERR(__wt_lsm_tree_chunk_name( - session, lsm_tree, chunk->id, &chunk->uri)); + WT_ERR(__wt_lsm_tree_chunk_name(session, lsm_tree, + chunk->id, chunk->generation, &chunk->uri)); WT_ERR(__wt_schema_rename(session, old, chunk->uri, cfg)); __wt_free(session, old); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index 76827f7888c..697c2986652 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -409,7 +409,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_ERR_MSG(session, ret, "LSM checkpoint"); /* Now the file is written, get the chunk size. */ - WT_ERR(__wt_lsm_tree_set_chunk_size(session, chunk)); + WT_ERR(__wt_lsm_tree_set_chunk_size(session, lsm_tree, chunk)); ++lsm_tree->chunks_flushed; diff --git a/src/third_party/wiredtiger/src/optrack/optrack.c b/src/third_party/wiredtiger/src/optrack/optrack.c new file mode 100644 index 00000000000..72789c4a154 --- /dev/null +++ b/src/third_party/wiredtiger/src/optrack/optrack.c @@ -0,0 +1,123 @@ +/*- + * Copyright (c) 2014-2017 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_optrack_record_funcid -- + * Record optrack function id + */ +void +__wt_optrack_record_funcid( + WT_SESSION_IMPL *session, const char *func, uint16_t *func_idp) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_ITEM(tmp); + WT_DECL_RET; + wt_off_t fsize; + + conn = S2C(session); + + WT_ERR(__wt_scr_alloc(session, strlen(func) + 32, &tmp)); + + __wt_spin_lock(session, &conn->optrack_map_spinlock); + if (*func_idp == 0) { + *func_idp = ++conn->optrack_uid; + + WT_ERR(__wt_buf_fmt( + session, tmp, "%" PRIu16 " %s\n", *func_idp, func)); + WT_ERR(__wt_filesize(session, conn->optrack_map_fh, &fsize)); + WT_ERR(__wt_write(session, + conn->optrack_map_fh, fsize, tmp->size, tmp->data)); + } + + if (0) { +err: WT_PANIC_MSG(session, ret, "%s", __func__); + } + + __wt_spin_unlock(session, &conn->optrack_map_spinlock); + __wt_scr_free(session, &tmp); +} + +/* + * __wt_optrack_open_file -- + * Open the per-session operation-tracking file. + */ +int +__wt_optrack_open_file(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_ITEM(buf); + WT_DECL_RET; + WT_OPTRACK_HEADER optrack_header = {WT_OPTRACK_VERSION, 0}; + + conn = S2C(session); + + if (!F_ISSET(conn, WT_CONN_OPTRACK)) + return (WT_ERROR); + + WT_RET(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_filename_construct(session, conn->optrack_path, + "optrack", conn->optrack_pid, session->id, buf)); + WT_ERR(__wt_open(session, + (const char *)buf->data, WT_FS_OPEN_FILE_TYPE_REGULAR, + WT_FS_OPEN_CREATE, &session->optrack_fh)); + + /* Write the header into the operation-tracking file. */ + if (F_ISSET(session, WT_SESSION_INTERNAL)) + optrack_header.optrack_session_internal = 1; + + WT_ERR(session->optrack_fh->handle->fh_write( + session->optrack_fh->handle, (WT_SESSION *)session, + 0, sizeof(WT_OPTRACK_HEADER), &optrack_header)); + + session->optrack_offset = sizeof(WT_OPTRACK_HEADER); + + if (0) { +err: WT_TRET(__wt_close(session, &session->optrack_fh)); + } + __wt_scr_free(session, &buf); + + return (ret); +} + +/* + * __wt_optrack_flush_buffer -- + * Flush optrack buffer. Returns the number of bytes flushed to the file. + */ +size_t +__wt_optrack_flush_buffer(WT_SESSION_IMPL *s) +{ + WT_DECL_RET; + + if (s->optrack_fh == NULL) + if (__wt_optrack_open_file(s)) + return (0); + + ret = s->optrack_fh->handle->fh_write(s->optrack_fh->handle, + (WT_SESSION *)s, (wt_off_t)s->optrack_offset, + s->optrackbuf_ptr * sizeof(WT_OPTRACK_RECORD), s->optrack_buf); + if (ret == 0) + return (s->optrackbuf_ptr * sizeof(WT_OPTRACK_RECORD)); + else + return (0); +} + +/* + * __wt_optrack_get_expensive_timestamp -- + * Obtain a timestamp via a system call on platforms where obtaining it + * directly from the hardware register is not supported. + */ +uint64_t +__wt_optrack_get_expensive_timestamp(WT_SESSION_IMPL *session) +{ + struct timespec tsp; + + __wt_epoch_raw(session, &tsp); + return ((uint64_t)(tsp.tv_sec * WT_BILLION + tsp.tv_nsec)); +} diff --git a/src/third_party/wiredtiger/src/os_common/filename.c b/src/third_party/wiredtiger/src/os_common/filename.c index 16825410dc3..3430f15fec8 100644 --- a/src/third_party/wiredtiger/src/os_common/filename.c +++ b/src/third_party/wiredtiger/src/os_common/filename.c @@ -54,6 +54,27 @@ err: __wt_free(session, buf); } /* + * __wt_filename_construct -- + * Given unique identifiers, return a WT_ITEM of a generated file name of + * the given prefix type. Any identifier that is 0 will be skipped. + */ +int +__wt_filename_construct(WT_SESSION_IMPL *session, const char *path, + const char *file_prefix, uintmax_t id_1, uint32_t id_2, WT_ITEM *buf) +{ + if (path != NULL && path[0] != '\0') + WT_RET(__wt_buf_catfmt( + session, buf, "%s%s", path, __wt_path_separator())); + WT_RET(__wt_buf_catfmt(session, buf, "%s", file_prefix)); + if (id_1 != UINTMAX_MAX) + WT_RET(__wt_buf_catfmt(session, buf, ".%010" PRIuMAX, id_1)); + if (id_2 != UINT32_MAX) + WT_RET(__wt_buf_catfmt(session, buf, ".%010" PRIu32, id_2)); + + return (0); +} + +/* * __wt_remove_if_exists -- * Remove a file if it exists. */ diff --git a/src/third_party/wiredtiger/src/os_common/os_fhandle.c b/src/third_party/wiredtiger/src/os_common/os_fhandle.c index a353353216b..3b9e4fa26dc 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fhandle.c +++ b/src/third_party/wiredtiger/src/os_common/os_fhandle.c @@ -135,9 +135,8 @@ static inline int __open_verbose( WT_SESSION_IMPL *session, const char *name, int file_type, u_int flags) { -#ifdef HAVE_VERBOSE - WT_DECL_RET; WT_DECL_ITEM(tmp); + WT_DECL_RET; const char *file_type_tag, *sep; if (!WT_VERBOSE_ISSET(session, WT_VERB_FILEOPS)) @@ -193,13 +192,6 @@ __open_verbose( err: __wt_scr_free(session, &tmp); return (ret); -#else - WT_UNUSED(session); - WT_UNUSED(name); - WT_UNUSED(file_type); - WT_UNUSED(flags); - return (0); -#endif } /* @@ -219,6 +211,8 @@ __wt_open(WT_SESSION_IMPL *session, WT_ASSERT(session, file_type != 0); /* A file type is required. */ + *fhp = NULL; + conn = S2C(session); file_system = conn->file_system; fh = NULL; diff --git a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c index 533d2a0ab08..c8819de599d 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c @@ -57,8 +57,11 @@ __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, { struct timespec ts; WT_DECL_RET; + WT_TRACK_OP_DECL; bool locked; + WT_TRACK_OP_INIT(session); + locked = false; /* Fast path if already signalled. */ @@ -135,8 +138,10 @@ err: (void)__wt_atomic_subi32(&cond->waiters, 1); if (locked) WT_TRET(pthread_mutex_unlock(&cond->mtx)); - if (ret == 0) + if (ret == 0) { + WT_TRACK_OP_END(session); return; + } WT_PANIC_MSG(session, ret, "pthread_cond_wait: %s", cond->name); } diff --git a/src/third_party/wiredtiger/src/os_posix/os_thread.c b/src/third_party/wiredtiger/src/os_posix/os_thread.c index dc4d49ad493..4e93cdcb1af 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_thread.c +++ b/src/third_party/wiredtiger/src/os_posix/os_thread.c @@ -110,3 +110,13 @@ __wt_thread_str(char *buf, size_t buflen) "%" PRIuMAX ":%p", (uintmax_t)getpid(), (void *)self)); #endif } + +/* + * __wt_process_id -- + * Return the process ID assigned by the operating system. + */ +uintmax_t +__wt_process_id(void) +{ + return ((uintmax_t)getpid()); +} diff --git a/src/third_party/wiredtiger/src/os_win/os_map.c b/src/third_party/wiredtiger/src/os_win/os_map.c index 25835d2615d..c0aa6dac28f 100644 --- a/src/third_party/wiredtiger/src/os_win/os_map.c +++ b/src/third_party/wiredtiger/src/os_win/os_map.c @@ -82,8 +82,6 @@ __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; - WT_UNUSED(length); /* !HAVE_VERBOSE */ - win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; diff --git a/src/third_party/wiredtiger/src/os_win/os_thread.c b/src/third_party/wiredtiger/src/os_win/os_thread.c index 1d549cf4712..8761bf1e617 100644 --- a/src/third_party/wiredtiger/src/os_win/os_thread.c +++ b/src/third_party/wiredtiger/src/os_win/os_thread.c @@ -96,3 +96,13 @@ __wt_thread_str(char *buf, size_t buflen) "%" PRIu64 ":%" PRIu64, (uint64_t)GetCurrentProcessId(), (uint64_t)GetCurrentThreadId)); } + +/* + * __wt_process_id -- + * Return the process ID assigned by the operating system. + */ +uintmax_t +__wt_process_id(void) +{ + return (uintmax_t)GetCurrentProcessId(); +} diff --git a/src/third_party/wiredtiger/src/reconcile/rec_track.c b/src/third_party/wiredtiger/src/reconcile/rec_track.c index d0a6df2a7ec..8842da50f00 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_track.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_track.c @@ -36,9 +36,6 @@ __ovfl_discard_verbose( WT_CELL_UNPACK *unpack, _unpack; WT_DECL_ITEM(tmp); - WT_UNUSED(page); /* !HAVE_VERBOSE */ - WT_UNUSED(tag); /* !HAVE_VERBOSE */ - WT_RET(__wt_scr_alloc(session, 512, &tmp)); unpack = &_unpack; @@ -172,10 +169,6 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, { WT_DECL_ITEM(tmp); - WT_UNUSED(page); /* !HAVE_VERBOSE */ - WT_UNUSED(reuse); /* !HAVE_VERBOSE */ - WT_UNUSED(tag); /* !HAVE_VERBOSE */ - WT_RET(__wt_scr_alloc(session, 64, &tmp)); __wt_verbose(session, WT_VERB_OVERFLOW, diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 233e0ec61f6..d686f87a637 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -5475,8 +5475,7 @@ __rec_row_leaf(WT_SESSION_IMPL *session, goto leaf_insert; case WT_UPDATE_MODIFIED: cbt->slot = WT_ROW_SLOT(page, rip); - WT_ERR(__wt_value_return_upd( - session, cbt, upd, + WT_ERR(__wt_value_return_upd(session, cbt, upd, F_ISSET(r, WT_REC_VISIBLE_ALL))); WT_ERR(__rec_cell_build_val(session, r, cbt->iface.value.data, @@ -5697,8 +5696,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) */ cbt->slot = UINT32_MAX; WT_RET(__wt_value_return_upd( - session, cbt, upd, - F_ISSET(r, WT_REC_VISIBLE_ALL))); + session, cbt, upd, F_ISSET(r, WT_REC_VISIBLE_ALL))); WT_RET(__rec_cell_build_val(session, r, cbt->iface.value.data, cbt->iface.value.size, (uint64_t)0)); diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index 291778068f4..d924fe30d51 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -396,18 +396,28 @@ __create_index(WT_SESSION_IMPL *session, "should be <table name>:<index name>: %s", name); /* - * Note: it would be better to get the table exclusive here, while + * Note: it would be better to keep the table exclusive here, while * changing its indexes. We don't because some operation we perform * below reacquire the table handle (such as opening a cursor on the - * table in order to fill the index). If we get the handle exclusive + * table in order to fill the index). If we keep the handle exclusive * here, those operations wanting ordinary access will conflict, - * leading to errors. + * leading to errors. At the same time, we don't want to allow + * table cursors that have already been fully opened to remain open + * across this call. * - * Instead, we rely on the global table lock to protect the set of - * available indexes. + * Temporarily getting the table exclusively serves the purpose + * of ensuring that cursors on the table that are already open + * must at least be closed before this call proceeds. */ tlen = (size_t)(idxname++ - tablename); if ((ret = __wt_schema_get_table( + session, tablename, tlen, true, WT_DHANDLE_EXCLUSIVE, &table)) != 0) + WT_RET_MSG(session, ret, + "Can't create an index for table: %.*s", + (int)tlen, tablename); + WT_RET(__wt_schema_release_table(session, table)); + + if ((ret = __wt_schema_get_table( session, tablename, tlen, true, 0, &table)) != 0) WT_RET_MSG(session, ret, "Can't create an index for a non-existent table: %.*s", diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index 4ee34f68a8c..8501de8b805 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -124,7 +124,14 @@ __drop_table( * being reopened while it is being dropped. One issue is that the * WT_WITHOUT_LOCKS macro can drop and reacquire the global table lock, * avoiding deadlocks while waiting for LSM operation to quiesce. + * + * Temporarily getting the table exclusively serves the purpose + * of ensuring that cursors on the table that are already open + * must at least be closed before this call proceeds. */ + WT_ERR(__wt_schema_get_table_uri(session, uri, true, + WT_DHANDLE_EXCLUSIVE, &table)); + WT_ERR(__wt_schema_release_table(session, table)); WT_ERR(__wt_schema_get_table_uri(session, uri, true, 0, &table)); /* Drop the column groups. */ diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index d81735234a0..71665d9996f 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -191,6 +191,24 @@ __session_close(WT_SESSION *wt_session, const char *config) /* Free transaction information. */ __wt_txn_destroy(session); + /* + * Close the file where we tracked long operations. Do this before + * releasing resources, as we do scratch buffer management when we flush + * optrack buffers to disk + */ + if (F_ISSET(conn, WT_CONN_OPTRACK)) { + if (session->optrackbuf_ptr > 0) { + WT_IGNORE_RET((int)__wt_optrack_flush_buffer(session)); + WT_IGNORE_RET(__wt_close(session, + &session->optrack_fh)); + /* Indicate that the file is closed */ + session->optrack_fh = NULL; + } + + /* Free the operation tracking buffer */ + __wt_free(session, session->optrack_buf); + } + /* Release common session resources. */ WT_TRET(__wt_session_release_resources(session)); @@ -1552,17 +1570,18 @@ __transaction_sync_run_chk(WT_SESSION_IMPL *session) static int __session_transaction_sync(WT_SESSION *wt_session, const char *config) { - struct timespec now, start; WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; WT_SESSION_IMPL *session; uint64_t remaining_usec, timeout_ms, waited_ms; + uint64_t time_start, time_stop; session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, transaction_sync, config, cfg); WT_STAT_CONN_INCR(session, txn_sync); + time_start = time_stop = 0; conn = S2C(session); WT_ERR(__wt_txn_context_check(session, false)); @@ -1600,7 +1619,7 @@ __session_transaction_sync(WT_SESSION *wt_session, const char *config) if (timeout_ms == 0) WT_ERR(ETIMEDOUT); - __wt_epoch(session, &start); + time_start = __wt_rdtsc(session); /* * Keep checking the LSNs until we find it is stable or we reach * our timeout, or there's some other reason to quit. @@ -1610,8 +1629,8 @@ __session_transaction_sync(WT_SESSION *wt_session, const char *config) WT_ERR(ETIMEDOUT); __wt_cond_signal(session, conn->log_file_cond); - __wt_epoch(session, &now); - waited_ms = WT_TIMEDIFF_MS(now, start); + time_stop = __wt_rdtsc(session); + waited_ms = WT_TSCDIFF_MS(session, time_stop, time_start); if (waited_ms < timeout_ms) { remaining_usec = (timeout_ms - waited_ms) * WT_THOUSAND; __wt_cond_wait(session, log->log_sync_cond, @@ -1847,9 +1866,9 @@ __open_session(WT_CONNECTION_IMPL *conn, break; if (i == conn->session_size) WT_ERR_MSG(session, WT_ERROR, - "out of sessions, only configured to support %" PRIu32 - " sessions (including %d additional internal sessions)", - conn->session_size, WT_EXTRA_INTERNAL_SESSIONS); + "out of sessions, configured for %" PRIu32 " (including " + "internal sessions)", + conn->session_size); /* * If the active session count is increasing, update it. We don't worry @@ -1904,6 +1923,12 @@ __open_session(WT_CONNECTION_IMPL *conn, /* Cache the offset of this session's statistics bucket. */ session_ret->stat_bucket = WT_STATS_SLOT_ID(session); + /* Allocate the buffer for operation tracking */ + if (F_ISSET(conn, WT_CONN_OPTRACK)) { + WT_ERR(__wt_malloc( + session, WT_OPTRACK_BUFSIZE, &session_ret->optrack_buf)); + session_ret->optrackbuf_ptr = 0; + } /* * Configuration: currently, the configuration for open_session is the * same as session.reconfigure, so use that function. diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c index 1e7a8a1b4df..dc9ca4932c5 100644 --- a/src/third_party/wiredtiger/src/support/mtx_rw.c +++ b/src/third_party/wiredtiger/src/support/mtx_rw.c @@ -165,8 +165,8 @@ __read_blocked(WT_SESSION_IMPL *session) void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - struct timespec enter, leave; WT_RWLOCK new, old; + uint64_t time_start, time_stop; int64_t **stats; int16_t writers_active; uint8_t ticket; @@ -176,6 +176,7 @@ __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_STAT_CONN_INCR(session, rwlock_read); stats = (int64_t **)S2C(session)->stats; set_stats = (l->stat_read_count_off != -1 && WT_STAT_ENABLED(session)); + time_start = time_stop = 0; if (set_stats) stats[session->stat_bucket][l->stat_read_count_off]++; @@ -236,7 +237,7 @@ stall: __wt_cond_wait(session, } if (set_stats) - __wt_epoch(session, &enter); + time_start = __wt_rdtsc(session); /* Wait for our group to start. */ for (pause_cnt = 0; ticket != l->u.s.current; pause_cnt++) { if (pause_cnt < 1000) @@ -251,13 +252,15 @@ stall: __wt_cond_wait(session, } } if (set_stats) { - __wt_epoch(session, &leave); + time_stop = __wt_rdtsc(session); if (F_ISSET(session, WT_SESSION_INTERNAL)) stats[session->stat_bucket][l->stat_int_usecs_off] += - (int64_t)WT_TIMEDIFF_US(leave, enter); + (int64_t)WT_TSCDIFF_US( + session, time_stop, time_start); else stats[session->stat_bucket][l->stat_app_usecs_off] += - (int64_t)WT_TIMEDIFF_US(leave, enter); + (int64_t)WT_TSCDIFF_US( + session, time_stop, time_start); } /* @@ -362,8 +365,8 @@ __write_blocked(WT_SESSION_IMPL *session) void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - struct timespec enter, leave; WT_RWLOCK new, old; + uint64_t time_start, time_stop; int64_t **stats; uint8_t ticket; int pause_cnt; @@ -372,6 +375,7 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_STAT_CONN_INCR(session, rwlock_write); stats = (int64_t **)S2C(session)->stats; set_stats = (l->stat_write_count_off != -1 && WT_STAT_ENABLED(session)); + time_start = time_stop = 0; if (set_stats) stats[session->stat_bucket][l->stat_write_count_off]++; @@ -405,7 +409,7 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) * we have the lock. */ if (set_stats) - __wt_epoch(session, &enter); + time_start = __wt_rdtsc(session); for (pause_cnt = 0, old.u.v = l->u.v; ticket != old.u.s.current || old.u.s.readers_active != 0; pause_cnt++, old.u.v = l->u.v) { @@ -421,13 +425,15 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) } } if (set_stats) { - __wt_epoch(session, &leave); + time_stop = __wt_rdtsc(session); if (F_ISSET(session, WT_SESSION_INTERNAL)) stats[session->stat_bucket][l->stat_int_usecs_off] += - (int64_t)WT_TIMEDIFF_US(leave, enter); + (int64_t)WT_TSCDIFF_US( + session, time_stop, time_start); else stats[session->stat_bucket][l->stat_app_usecs_off] += - (int64_t)WT_TIMEDIFF_US(leave, enter); + (int64_t)WT_TSCDIFF_US( + session, time_stop, time_start); } /* diff --git a/src/third_party/wiredtiger/src/support/time.c b/src/third_party/wiredtiger/src/support/time.c index 240a77591a3..e3444aea75a 100644 --- a/src/third_party/wiredtiger/src/support/time.c +++ b/src/third_party/wiredtiger/src/support/time.c @@ -71,3 +71,13 @@ __wt_seconds(WT_SESSION_IMPL *session, time_t *timep) *timep = t.tv_sec; } + +/* + * __wt_tsc_to_nsec -- + * Convert from rdtsc ticks to nanoseconds. + */ +uint64_t +__wt_tsc_to_nsec(WT_SESSION_IMPL *session, uint64_t tsc_diff) +{ + return ((uint64_t)((double)tsc_diff / S2C(session)->tsc_nsec_ratio)); +} diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index fb61b037723..cfb1dfc3458 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -367,7 +367,6 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) if (WT_TXNID_LT(txn_global->last_running, last_running)) { txn_global->last_running = last_running; -#ifdef HAVE_VERBOSE /* Output a verbose message about long-running transactions, * but only when some progress is being made. */ if (WT_VERBOSE_ISSET(session, WT_VERB_TRANSACTION) && @@ -380,7 +379,6 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) oldest_session->lastop, oldest_session->txn.snap_min); } -#endif } done: __wt_writeunlock(session, &txn_global->rwlock); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index c82187daf85..d5c2951c8f8 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -369,25 +369,26 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) static void __checkpoint_reduce_dirty_cache(WT_SESSION_IMPL *session) { - struct timespec last, start, stop; WT_CACHE *cache; WT_CONNECTION_IMPL *conn; double current_dirty, delta, scrub_min; uint64_t bytes_written_last, bytes_written_start, bytes_written_total; uint64_t cache_size, max_write; uint64_t current_us, stepdown_us, total_ms, work_us; + uint64_t time_last, time_start, time_stop; bool progress; conn = S2C(session); cache = conn->cache; + time_last = time_start = time_stop = 0; /* Give up if scrubbing is disabled. */ if (cache->eviction_checkpoint_target == 0 || cache->eviction_checkpoint_target >= cache->eviction_dirty_trigger) return; - __wt_epoch(session, &start); - last = start; + time_start = __wt_rdtsc(session); + time_last = time_start; bytes_written_last = 0; bytes_written_start = cache->bytes_written; cache_size = conn->cache_size; @@ -448,8 +449,8 @@ __checkpoint_reduce_dirty_cache(WT_SESSION_IMPL *session) break; __wt_sleep(0, stepdown_us / 10); - __wt_epoch(session, &stop); - current_us = WT_TIMEDIFF_US(stop, last); + time_stop = __wt_rdtsc(session); + current_us = WT_TSCDIFF_US(session, time_stop, time_last); bytes_written_total = cache->bytes_written - bytes_written_start; @@ -503,11 +504,11 @@ __checkpoint_reduce_dirty_cache(WT_SESSION_IMPL *session) WT_MAX(cache->eviction_dirty_target, current_dirty - delta); WT_STAT_CONN_SET(session, txn_checkpoint_scrub_target, cache->eviction_scrub_limit); - __wt_epoch(session, &last); + time_last = __wt_rdtsc(session); } - __wt_epoch(session, &stop); - total_ms = WT_TIMEDIFF_MS(stop, start); + time_stop = __wt_rdtsc(session); + total_ms = WT_TSCDIFF_MS(session, time_stop, time_start); WT_STAT_CONN_SET(session, txn_checkpoint_scrub_time, total_ms); } @@ -575,7 +576,6 @@ __checkpoint_stats(WT_SESSION_IMPL *session) static void __checkpoint_verbose_track(WT_SESSION_IMPL *session, const char *msg) { -#ifdef HAVE_VERBOSE struct timespec stop; WT_CONNECTION_IMPL *conn; uint64_t msec; @@ -593,10 +593,6 @@ __checkpoint_verbose_track(WT_SESSION_IMPL *session, const char *msg) ": Full database checkpoint %s", msec, __wt_gen(session, WT_GEN_CHECKPOINT), msg); -#else - WT_UNUSED(session); - WT_UNUSED(msg); -#endif } /* @@ -745,14 +741,13 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, const char *cfg[]) static int __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) { - struct timespec fsync_start, fsync_stop; WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_TXN *txn; WT_TXN_GLOBAL *txn_global; WT_TXN_ISOLATION saved_isolation; - uint64_t fsync_duration_usecs, generation; + uint64_t fsync_duration_usecs, generation, time_start, time_stop; u_int i; bool failed, full, idle, logging, tracking; void *saved_meta_next; @@ -763,6 +758,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) txn_global = &conn->txn_global; saved_isolation = session->isolation; full = idle = logging = tracking = false; + time_start = time_stop = 0; /* * Do a pass over the configuration arguments and figure out what kind @@ -887,10 +883,10 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * Checkpoints have to hit disk (it would be reasonable to configure for * lazy checkpoints, but we don't support them yet). */ - __wt_epoch(session, &fsync_start); + time_start = __wt_rdtsc(session); WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); - __wt_epoch(session, &fsync_stop); - fsync_duration_usecs = WT_TIMEDIFF_US(fsync_stop, fsync_start); + time_stop = __wt_rdtsc(session); + fsync_duration_usecs = WT_TSCDIFF_US(session, time_stop, time_start); WT_STAT_CONN_INCR(session, txn_checkpoint_fsync_post); WT_STAT_CONN_SET(session, txn_checkpoint_fsync_post_duration, fsync_duration_usecs); diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 6735946fbb3..99d52f5a4b4 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -65,25 +65,20 @@ __wt_timestamp_to_hex_string( /* * __wt_verbose_timestamp -- - * Output a verbose message along with the specified timestamp + * Output a verbose message along with the specified timestamp. */ void __wt_verbose_timestamp(WT_SESSION_IMPL *session, const wt_timestamp_t *ts, const char *msg) { -#ifdef HAVE_VERBOSE char timestamp_buf[2 * WT_TIMESTAMP_SIZE + 1]; - if (__wt_timestamp_to_hex_string(session, timestamp_buf, ts) != 0) + if (!WT_VERBOSE_ISSET(session, WT_VERB_TIMESTAMP) || + (__wt_timestamp_to_hex_string(session, timestamp_buf, ts) != 0)) return; __wt_verbose(session, WT_VERB_TIMESTAMP, "Timestamp %s : %s", timestamp_buf, msg); -#else - WT_UNUSED(session); - WT_UNUSED(ts); - WT_UNUSED(msg); -#endif } /* diff --git a/src/third_party/wiredtiger/src/utilities/util_load.h b/src/third_party/wiredtiger/src/utilities/util_load.h index 53fce665ddc..390b9523214 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.h +++ b/src/third_party/wiredtiger/src/utilities/util_load.h @@ -22,7 +22,9 @@ int config_reorder(WT_SESSION *, char **); int config_update(WT_SESSION *, char **); /* Flags for util_load_json */ -#define LOAD_JSON_APPEND 0x0001 /* append (ignore record number keys) */ -#define LOAD_JSON_NO_OVERWRITE 0x0002 /* don't overwrite existing data */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define LOAD_JSON_APPEND 0x1u /* append (ignore record number keys) */ +#define LOAD_JSON_NO_OVERWRITE 0x2u /* don't overwrite existing data */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ int util_load_json(WT_SESSION *, const char *, uint32_t); diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 79b232b532a..530f7393374 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -476,7 +476,7 @@ print_missing(REPORT *r, const char *fname, const char *msg) ". Then keys %" PRIu64 "-%" PRIu64 " exist." " Key range %" PRIu64 "-%" PRIu64 "\n", fname, msg, - r->exist_key - r->first_miss - 1, + (r->exist_key - r->first_miss) - 1, r->first_miss, r->exist_key - 1, r->exist_key, r->last_key, r->first_key, r->last_key); diff --git a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c index 58e2a0bc113..d7d408f6ea4 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c @@ -149,7 +149,6 @@ static const char * const list[] = { ",statistics_log=(wait=37)", ",statistics_log=(wait=0)", -#ifdef HAVE_VERBOSE ",verbose=(\"api\")", ",verbose=(\"block\")", ",verbose=(\"checkpoint\")", @@ -176,7 +175,6 @@ static const char * const list[] = { ",verbose=(\"version\")", ",verbose=(\"write\")", ",verbose=()" -#endif }; static int diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c index fcbbdcabd73..eecb8a10f06 100644 --- a/src/third_party/wiredtiger/test/fops/t.c +++ b/src/third_party/wiredtiger/test/fops/t.c @@ -160,7 +160,8 @@ wt_startup(char *config_open) testutil_make_work_dir(home); testutil_check(__wt_snprintf(config_buf, sizeof(config_buf), - "create,error_prefix=\"%s\",cache_size=5MB%s%s", + "create,error_prefix=\"%s\",cache_size=5MB%s%s," + "operation_tracking=(enabled=false)", progname, config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 7ac65147462..e6c31550cc9 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -35,19 +35,19 @@ typedef struct { const char *desc; /* Configuration description */ /* Value is a boolean, yes if roll of 1-to-100 is <= CONFIG->min. */ -#define C_BOOL 0x001 +#define C_BOOL 0x01u /* Not a simple randomization, handle outside the main loop. */ -#define C_IGNORE 0x002 +#define C_IGNORE 0x02u /* Value was set from command-line or file, ignore for all runs. */ -#define C_PERM 0x004 +#define C_PERM 0x04u /* Value isn't random for this run, ignore just for this run. */ -#define C_TEMP 0x008 +#define C_TEMP 0x08u /* Value is a string. */ -#define C_STRING 0x020 +#define C_STRING 0x20u u_int flags; uint32_t min; /* Minimum value */ diff --git a/src/third_party/wiredtiger/test/mciproject.yml b/src/third_party/wiredtiger/test/mciproject.yml index 16e103e5366..271f392ae7c 100644 --- a/src/third_party/wiredtiger/test/mciproject.yml +++ b/src/third_party/wiredtiger/test/mciproject.yml @@ -60,10 +60,10 @@ tasks: set -o errexit set -o verbose if [ "Windows_NT" = "$OS" ]; then - scons.bat --enable-python=c:\\swigwin-3.0.2\\swig.exe --enable-diagnostic --enable-verbose ${smp_command|} + scons.bat --enable-python=c:\\swigwin-3.0.2\\swig.exe --enable-diagnostic ${smp_command|} else ./build_posix/reconf - ${configure_env_vars|} ./configure --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-verbose + ${configure_env_vars|} ./configure --enable-diagnostic --enable-python --enable-zlib --enable-strict ${make_command|make} ${smp_command|} 2>&1 ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 ${make_command|make} VERBOSE=1 check 2>&1 fi @@ -113,7 +113,7 @@ tasks: set -o errexit set -o verbose - scons.bat ${smp_command|} "CFLAGS=/Gv /wd4090 /wd4996 /we4047 /we4024 /TC /we4100 /w4133" wiredtiger.dll libwiredtiger.lib + scons.bat ${smp_command|} "CFLAGS=/Gv /wd4090 /wd4996 /we4047 /we4024 /TC /we4100 /we4133" wiredtiger.dll libwiredtiger.lib - name: fops depends_on: diff --git a/src/third_party/wiredtiger/test/readonly/readonly.c b/src/third_party/wiredtiger/test/readonly/readonly.c index e01b5b85fbd..40de5fff1a4 100644 --- a/src/third_party/wiredtiger/test/readonly/readonly.c +++ b/src/third_party/wiredtiger/test/readonly/readonly.c @@ -44,9 +44,9 @@ static const char * const uri = "table:main"; #define ENV_CONFIG \ "create,log=(file_max=10M,archive=false,enabled)," \ - "transaction_sync=(enabled,method=none)" -#define ENV_CONFIG_RD "readonly=true" -#define ENV_CONFIG_WR "readonly=false" + "operation_tracking=(enabled=false),transaction_sync=(enabled,method=none)" +#define ENV_CONFIG_RD "operation_tracking=(enabled=false),readonly=true" +#define ENV_CONFIG_WR "operation_tracking=(enabled=false),readonly=false" #define MAX_VAL 4096 #define MAX_KV 10000 diff --git a/src/third_party/wiredtiger/test/suite/test_backup02.py b/src/third_party/wiredtiger/test/suite/test_backup02.py index 7d8f653feae..c0410b41708 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup02.py +++ b/src/third_party/wiredtiger/test/suite/test_backup02.py @@ -32,7 +32,7 @@ from wtthread import backup_thread, checkpoint_thread, op_thread from wtscenario import make_scenarios # test_backup02.py -# Run background checkpoints and backsups repeatedly while doing inserts +# Run background checkpoints and backups repeatedly while doing inserts # in another thread class test_backup02(wttest.WiredTigerTestCase): scenarios = make_scenarios([ diff --git a/src/third_party/wiredtiger/test/suite/test_bug018.py b/src/third_party/wiredtiger/test/suite/test_bug018.py index 7d20ebcaacb..b904b688586 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug018.py +++ b/src/third_party/wiredtiger/test/suite/test_bug018.py @@ -65,7 +65,7 @@ class test_bug018(wttest.WiredTigerTestCase): # table out from underneath WiredTiger. We do this right before # closing the connection so that the write error happens during close # when writing out the final data. Allow table 1 to succeed and force - # an erorr writing out table 2. + # an error writing out table 2. # # This is Linux-specific code to figure out the file descriptor. for f in os.listdir('/proc/self/fd'): diff --git a/src/third_party/wiredtiger/test/suite/test_collator.py b/src/third_party/wiredtiger/test/suite/test_collator.py index 320c4e7d7b4..c614ea10c83 100644 --- a/src/third_party/wiredtiger/test/suite/test_collator.py +++ b/src/third_party/wiredtiger/test/suite/test_collator.py @@ -108,6 +108,7 @@ class test_collator(wttest.WiredTigerTestCase): key = c.get_key() self.assertEqual(value, expect) i += 1 + cursor.close() self.assertEqual(self.nentries, i) for i in range(0, self.nindices): c = icursor[i] diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index eb1eb641191..bea370280d1 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -30,7 +30,7 @@ # Test that compact reduces the file size. # -import wiredtiger, wttest +import time, wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios @@ -146,7 +146,12 @@ class test_compact02(wttest.WiredTigerTestCase): self.session.checkpoint() # 5. Call compact. - self.session.compact(self.uri, None) + # Compact can collide with eviction, if that happens we retry. + for i in range(1, 5): + if not self.raisesBusy( + lambda: self.session.compact(self.uri, None)): + break + time.sleep(2) # 6. Get stats on compacted table. sz = self.getSize() diff --git a/src/third_party/wiredtiger/test/suite/test_config05.py b/src/third_party/wiredtiger/test/suite/test_config05.py index 5960f01dc8e..0004851b905 100644 --- a/src/third_party/wiredtiger/test/suite/test_config05.py +++ b/src/third_party/wiredtiger/test/suite/test_config05.py @@ -90,7 +90,7 @@ class test_config05(wttest.WiredTigerTestCase): self.conn = self.wiredtiger_open('.', 'create,session_max=1') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: [self.conn.open_session(None) for i in range(100)], - '/configured to support/') + '/out of sessions/') def test_exclusive_create(self): self.conn = self.wiredtiger_open('.', 'create,exclusive') diff --git a/src/third_party/wiredtiger/test/suite/test_index03.py b/src/third_party/wiredtiger/test/suite/test_index03.py new file mode 100644 index 00000000000..1052cc291d6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_index03.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_index03.py +# Make sure cursors cannot stay open while a new index is created. +class test_index03(wttest.WiredTigerTestCase): + + def key(self, i): + return str('%015d' % i) + + def value(self, i): + return tuple([str(i+1), str(i+2), str(i+3), str(i+4)]) + + def test_index_create(self): + uri = 'table:test_index03' + index1_uri = 'index:test_index03:indx1' + index2_uri = 'index:test_index03:indx2' + config = ',key_format=S,value_format=SSSS' + + session = self.session + session.create(uri, 'columns=(key,col1,col2,col3,col4)' + config) + session.create(index1_uri, 'columns=(col1)' + config) + c1 = session.open_cursor(uri, None) + + # Having cursors open across index creates is not currently allowed. + with self.expectedStderrPattern("Can't create an index for table"): + self.assertRaises(wiredtiger.WiredTigerError, + lambda: session.create(index2_uri, 'columns=(col2)' + config)) + c1.close() + + session.create(index2_uri, 'columns=(col2)' + config) + c1 = session.open_cursor(uri, None) + # Having cursors open across drops is not currently allowed. + # On the drop side, we need to begin using the cursor + for i in xrange(100, 200): + c1[self.key(i)] = self.value(i) + + self.assertRaises(wiredtiger.WiredTigerError, + lambda: session.drop(index2_uri)) + c1.close() + session.drop(index2_uri) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_las.py b/src/third_party/wiredtiger/test/suite/test_las.py index 07938c6d80b..d07476382b9 100644 --- a/src/third_party/wiredtiger/test/suite/test_las.py +++ b/src/third_party/wiredtiger/test/suite/test_las.py @@ -41,7 +41,7 @@ class test_las(wttest.WiredTigerTestCase): return 'cache_size=50MB' def large_updates(self, session, uri, value, ds, nrows, timestamp=False): - # Insert a large number of records, we'll hang if the lookaside table + # Update a large number of records, we'll hang if the lookaside table # isn't doing its thing. cursor = session.open_cursor(uri) for i in range(1, 10000): @@ -54,6 +54,23 @@ class test_las(wttest.WiredTigerTestCase): session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) cursor.close() + def large_modifies(self, session, uri, offset, ds, nrows, timestamp=False): + # Modify a large number of records, we'll hang if the lookaside table + # isn't doing its thing. + cursor = session.open_cursor(uri) + for i in range(1, 10000): + if timestamp == True: + session.begin_transaction() + cursor.set_key(ds.key(nrows + i)) + mods = [] + mod = wiredtiger.Modify('A', offset, 1) + mods.append(mod) + + self.assertEqual(cursor.modify(mods), 0) + if timestamp == True: + session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) + cursor.close() + def durable_check(self, check_value, uri, ds, nrows): # Checkpoint and backup so as to simulate recovery self.session.checkpoint() @@ -66,8 +83,9 @@ class test_las(wttest.WiredTigerTestCase): # Skip the initial rows, which were not updated for i in range(0, nrows+1): self.assertEquals(cursor.next(), 0) - #print "Check value : " + str(check_value) - #print "value : " + str(cursor.get_value()) + if (check_value != cursor.get_value()): + print "Check value : " + str(check_value) + print "value : " + str(cursor.get_value()) self.assertTrue(check_value == cursor.get_value()) cursor.close() session.close() @@ -77,7 +95,7 @@ class test_las(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_las" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format="S") + ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') ds.populate() bigvalue = "aaaaa" * 100 @@ -112,16 +130,30 @@ class test_las(wttest.WiredTigerTestCase): session2.close() # Scenario: 3 + # Check to see LAS working with modify operations + bigvalue3 = "ccccc" * 100 + bigvalue3 = 'AA' + bigvalue3[2:] + session2 = self.conn.open_session() + session2.begin_transaction('isolation=snapshot') + # Apply two modify operations - replacing the first two items with 'A' + self.large_modifies(self.session, uri, 0, ds, nrows) + self.large_modifies(self.session, uri, 1, ds, nrows) + # Check to see the value after recovery + self.durable_check(bigvalue3, uri, ds, nrows) + session2.rollback_transaction() + session2.close() + + # Scenario: 4 # Check to see LAS working with old timestamp - bigvalue3 = "ddddd" * 100 + bigvalue4 = "ddddd" * 100 self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) - self.large_updates(self.session, uri, bigvalue3, ds, nrows, timestamp=True) + self.large_updates(self.session, uri, bigvalue4, ds, nrows, timestamp=True) # Check to see data can be see only till the stable_timestamp - self.durable_check(bigvalue2, uri, ds, nrows) + self.durable_check(bigvalue3, uri, ds, nrows) self.conn.set_timestamp('stable_timestamp=' + timestamp_str(i + 1)) # Check to see latest data can be seen - self.durable_check(bigvalue3, uri, ds, nrows) + self.durable_check(bigvalue4, uri, ds, nrows) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_pack.py b/src/third_party/wiredtiger/test/suite/test_pack.py index a24ef4fdfe1..6e5614f1a01 100644 --- a/src/third_party/wiredtiger/test/suite/test_pack.py +++ b/src/third_party/wiredtiger/test/suite/test_pack.py @@ -74,6 +74,8 @@ class test_pack(wttest.WiredTigerTestCase): forw_idx.set_key(*v) self.assertEquals(forw_idx.search(), 0) self.assertEquals(forw_idx.get_value(), 1234) + forw.close() + forw_idx.close() def test_packing(self): self.check('iii', 0, 101, -99) diff --git a/src/third_party/wiredtiger/test/suite/test_readonly01.py b/src/third_party/wiredtiger/test/suite/test_readonly01.py index ee5f78294f4..cd769bc4e88 100644 --- a/src/third_party/wiredtiger/test/suite/test_readonly01.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly01.py @@ -44,8 +44,8 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): # We want a list of directory writable or readonly. # basecfg_list = [ - ('basecfg', dict(basecfg='config_base=true,')), - ('no_basecfg', dict(basecfg='config_base=false,')), + ('basecfg', dict(basecfg='config_base=true,operation_tracking=(enabled=false),')), + ('no_basecfg', dict(basecfg='config_base=false,operation_tracking=(enabled=false),')), ] dir_list = [ ('write', dict(dirchmod=False)), diff --git a/src/third_party/wiredtiger/test/suite/test_readonly02.py b/src/third_party/wiredtiger/test/suite/test_readonly02.py index 3d3de8186d9..5fceb9e2ea8 100644 --- a/src/third_party/wiredtiger/test/suite/test_readonly02.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly02.py @@ -42,11 +42,16 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): entries = 10 conn_params = \ - 'create,statistics=(fast),log=(enabled,file_max=100K,zero_fill=true),' + 'create,statistics=(fast),' + \ + 'log=(enabled,file_max=100K,zero_fill=true),' + \ + 'operation_tracking=(enabled=false),' conn_params_rd = \ - 'create,readonly=true,statistics=(fast),log=(enabled,zero_fill=false),' + 'create,readonly=true,statistics=(fast),' + \ + 'log=(enabled,zero_fill=false),' + \ + 'operation_tracking=(enabled=false),' conn_params_rdcfg = \ - 'create,readonly=true,statistics=(fast),log=(enabled),' + 'create,readonly=true,statistics=(fast),log=(enabled),' + \ + 'operation_tracking=(enabled=false),' # # Run to make sure incompatible configuration options return an error. diff --git a/src/third_party/wiredtiger/test/suite/test_readonly03.py b/src/third_party/wiredtiger/test/suite/test_readonly03.py index 474e23981a2..e3e92edd8fc 100644 --- a/src/third_party/wiredtiger/test/suite/test_readonly03.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly03.py @@ -40,8 +40,8 @@ class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): uri2 = 'table:test_readonly03_2' create = True - conn_params = 'create,log=(enabled),' - conn_params_rd = 'readonly=true' + conn_params = 'create,log=(enabled),operation_tracking=(enabled=false),' + conn_params_rd = 'readonly=true,operation_tracking=(enabled=false),' session_ops = [ 'alter', 'create', 'compact', 'drop', 'log_flush', 'log_printf', 'rebalance', 'rename', 'salvage', 'truncate', 'upgrade', ] diff --git a/src/third_party/wiredtiger/test/suite/test_schema05.py b/src/third_party/wiredtiger/test/suite/test_schema05.py index f3a75447ee4..98044cdca09 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema05.py +++ b/src/third_party/wiredtiger/test/suite/test_schema05.py @@ -122,6 +122,7 @@ class test_schema05(wttest.WiredTigerTestCase): value = c.get_value() self.assertEqual(value, expect) i += 1 + cursor.close() self.assertEqual(self.nentries, i) for i in range(0, self.nindices): icursor[i].close() diff --git a/src/third_party/wiredtiger/test/suite/test_sweep01.py b/src/third_party/wiredtiger/test/suite/test_sweep01.py index 4d11942dc54..ac6150262ad 100644 --- a/src/third_party/wiredtiger/test/suite/test_sweep01.py +++ b/src/third_party/wiredtiger/test/suite/test_sweep01.py @@ -44,7 +44,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): numkv = 1000 conn_config = 'file_manager=(close_handle_minimum=0,' + \ 'close_idle_time=6,close_scan_interval=2),' + \ - 'statistics=(fast),' + 'statistics=(fast),operation_tracking=(enabled=false),' types = [ ('row', dict(tabletype='row', diff --git a/src/third_party/wiredtiger/test/suite/test_txn06.py b/src/third_party/wiredtiger/test/suite/test_txn06.py index 520c25f9b86..21d5d886a6a 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn06.py +++ b/src/third_party/wiredtiger/test/suite/test_txn06.py @@ -40,11 +40,6 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): source_uri = 'table:' + tablename + "_src" nrows = 100000 - def conn_config(self): - if not wiredtiger.verbose_build(): - self.skipTest('requires a verbose build') - return '' - def test_long_running(self): # Populate a table SimpleDataSet(self, self.source_uri, self.nrows).populate() diff --git a/src/third_party/wiredtiger/test/suite/test_txn18.py b/src/third_party/wiredtiger/test/suite/test_txn18.py index ec3cc7bae00..e139ebe60b5 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn18.py +++ b/src/third_party/wiredtiger/test/suite/test_txn18.py @@ -93,7 +93,7 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): lambda:self.wiredtiger_open(errdir, self.conn_recerror), msg) # If recover=error is run on the directory and returns an error, - # make sure when we subsequenty open with recover=on it properly + # make sure when we subsequently open with recover=on it properly # recovers all the data. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.wiredtiger_open(newdir, self.conn_recerror), msg) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index c654370718c..f6d81d88b86 100644 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -490,6 +490,39 @@ class WiredTigerTestCase(unittest.TestCase): with self.expectedStderr(message): self.assertRaises(exceptionType, expr) + def assertRaisesException(self, exceptionType, expr, + exceptionString=None, optional=False): + """ + Like TestCase.assertRaises(), with some additional options. + If the exceptionString argument is used, the exception's string + must match it. If optional is set, then no assertion occurs + if the exception doesn't occur. + Returns true if the assertion is raised. + """ + raised = False + try: + expr() + except BaseException, err: + if not isinstance(err, exceptionType): + self.fail('Exception of incorrect type raised, got type: ' + \ + str(type(err))) + if exceptionString != None and exceptionString != str(err): + self.fail('Exception with incorrect string raised, got: "' + \ + str(err) + '"') + raised = True + if not raised and not optional: + self.fail('no assertion raised') + return raised + + def raisesBusy(self, expr): + """ + Execute the expression, returning true if a 'Resource busy' + exception is raised, returning false if no exception is raised. + Any other exception raises a test suite failure. + """ + return self.assertRaisesException(wiredtiger.WiredTigerError, \ + expr, exceptionString='Resource busy', optional=True) + def assertTimestampsEqual(self, ts1, ts2): """ TestCase.assertEqual() for timestamps diff --git a/src/third_party/wiredtiger/tools/wt_optrack_decode.py b/src/third_party/wiredtiger/tools/wt_optrack_decode.py new file mode 100755 index 00000000000..1e063d1887c --- /dev/null +++ b/src/third_party/wiredtiger/tools/wt_optrack_decode.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import argparse +import colorsys +from multiprocessing import Process +import multiprocessing +import os +import os.path +import struct +import sys +import subprocess +import time +import traceback + +# +# This log version must be the same as that defined in ../src/include/optrack.h +# +currentLogVersion = 1; + +class color: + PURPLE = '\033[95m' + CYAN = '\033[96m' + DARKCYAN = '\033[36m' + BLUE = '\033[94m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + END = '\033[0m' + +functionMap = {}; + +def buildTranslationMap(mapFileName): + + mapFile = None; + + if not os.path.exists(mapFileName): + return False; + + try: + mapFile = open(mapFileName, "r"); + except: + print(color.BOLD + color.RED); + print("Could not open " + mapFileName + " for reading"); + print(color.END); + return; + + # Read lines from the map file and build an in-memory map + # of translations. Each line has a function ID followed by space and + # followed by the function name. + # + lines = mapFile.readlines(); # a map file is usually small + + for line in lines: + + words = line.split(" "); + if (len(words) < 2): + continue; + + try: + funcID = int(words[0]); + except: + continue; + + funcName = words[1].strip(); + + functionMap[funcID] = funcName; + + return True; + +def funcIDtoName(funcID): + + if (functionMap.has_key(funcID)): + return functionMap[funcID]; + else: + return "NULL"; + +# +# The format of the record is written down in src/include/optrack.h +# file in the WiredTiger source tree. The current implementation assumes +# a record of three fields. The first field is the 8-byte timestamp. +# The second field is the 2-byte function ID. The third field is the +# 2-byte operation type: '0' for function entry, '1' for function exit. +# The record size would be padded to 16 bytes in the C implementation by +# the compiler, because we keep an array of records, and each new record +# has to be 8-byte aligned, since the first field has the size 8 bytes. +# So we explicitly pad the track record structure in the implementation +# to make it clear what the record size is. +# +def parseOneRecord(file): + + bytesRead = ""; + record = (); + RECORD_SIZE = 16; + + try: + bytesRead = file.read(RECORD_SIZE); + except: + return None; + + if (len(bytesRead) < RECORD_SIZE): + return None; + + record = struct.unpack('Qhhxxxx', bytesRead); + + return record; + +# +# HEADER_SIZE must be the same as the size of WT_OPTRACK_HEADER +# structure defined in ../src/include/optrack.h +# +def validateHeader(file): + + global currentLogVersion; + + bytesRead = ""; + HEADER_SIZE = 8; + + try: + bytesRead = file.read(HEADER_SIZE); + except: + return False, -1; + + if (len(bytesRead) < HEADER_SIZE): + return False, -1; + + version, threadType = struct.unpack('II', bytesRead); + + if (version == currentLogVersion): + return True, threadType; + else: + return False, -1; + +def getStringFromThreadType(threadType): + + if (threadType == 0): + return "external"; + elif (threadType == 1): + return "internal"; + else: + return unknown; + + +def parseFile(fileName): + + done = False; + file = None; + threadType = 0; + threadTypeString = None; + outputFile = None; + outputFileName = ""; + totalRecords = 0; + validVersion = False; + + print(color.BOLD + "Processing file " + fileName + color.END); + + # Open the log file for reading + try: + file = open(fileName, "r"); + except: + print(color.BOLD + color.RED + + "Could not open " + fileName + " for reading" + color.END); + return; + + # Read and validate log header + validVersion, threadType = validateHeader(file); + if (not validVersion): + return; + + threadTypeString = getStringFromThreadType(threadType); + + # Open the text file for writing + try: + outputFileName = fileName + "-" + threadTypeString + ".txt"; + outputFile = open(outputFileName, "w"); + except: + print(color.BOLD + color.RED + + "Could not open file " + outputfileName + ".txt for writing." + + color.END); + return; + + print(color.BOLD + color.PURPLE + + "Writing to output file " + outputFileName + "." + color.END); + + while (not done): + record = parseOneRecord(file); + + if ((record is None) or len(record) < 3): + done = True; + else: + try: + time = record[0]; + funcName = funcIDtoName(record[1]); + opType = record[2]; + + outputFile.write(str(opType) + " " + funcName + " " + str(time) + + "\n"); + totalRecords += 1; + except: + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback.print_exception(exc_type, exc_value, exc_traceback); + print(color.BOLD + color.RED); + print("Could not write record " + str(record) + + " to file " + fileName + ".txt."); + print(color.END); + done = True; + + print("Wrote " + str(totalRecords) + " records to " + outputFileName + "."); + file.close(); + outputFile.close(); + +def waitOnOneProcess(runningProcesses): + + success = False; + for fname, p in runningProcesses.items(): + if (not p.is_alive()): + del runningProcesses[fname]; + success = True; + + # If we have not found a terminated process, sleep for a while + if (not success): + time.sleep(5); + +def main(): + + runnableProcesses = {}; + returnValues = {}; + spawnedProcesses = {}; + successfullyProcessedFiles = []; + targetParallelism = multiprocessing.cpu_count(); + terminatedProcesses = {}; + + parser = argparse.ArgumentParser(description= + 'Convert WiredTiger operation \ + tracking logs from binary to \ + text format.'); + + parser.add_argument('files', type=str, nargs='*', + help='optrack log files to process'); + + parser.add_argument('-j', dest='jobParallelism', type=int, + default='0'); + + parser.add_argument('-m', '--mapfile', dest='mapFileName', type=str, + default='optrack-map'); + + args = parser.parse_args(); + + print("Running with the following parameters:"); + for key, value in vars(args).items(): + print ("\t" + key + ": " + str(value)); + + # Parse the map of function ID to name translations. + if (buildTranslationMap(args.mapFileName) is False): + print("Failed to locate or parse the map file " + + args.mapFileName); + print("Cannot proceed."); + return; + + # Determine the target job parallelism + if (args.jobParallelism > 0): + targetParallelism = args.jobParallelism; + if (targetParallelism == 0): + targetParallelism = len(args.files); + print(color.BLUE + color.BOLD + + "Will process " + str(targetParallelism) + " files in parallel." + + color.END); + + # Prepare the processes that will parse files, one per file + if (len(args.files) > 0): + for fname in args.files: + p = Process(target=parseFile, args=(fname,)); + runnableProcesses[fname] = p; + + # Spawn these processes, not exceeding the desired parallelism + while (len(runnableProcesses) > 0): + while (len(spawnedProcesses) < targetParallelism + and len(runnableProcesses) > 0): + + fname, p = runnableProcesses.popitem(); + p.start(); + spawnedProcesses[fname] = p; + + # Find at least one terminated process + waitOnOneProcess(spawnedProcesses); + + # Wait for all processes to terminate + while (len(spawnedProcesses) > 0): + waitOnOneProcess(spawnedProcesses); + +if __name__ == '__main__': + main() |