Import wiredtiger: 1a29eac4dc8cf82de437292da546e3f4039268a4 from branch mongodb-3.8

ref: 596a3c7c01..1a29eac4dc
for: 3.7.1

WT-3079       Make sure eviction visits all trees
WT-3133       Detect or track long latency operations
WT-3295       Allow LSM to merge into custom data sources
WT-3587       Remove HAVE_VERBOSE conditional compilation
WT-3654       Fix warning in Windows build on evergreen
WT-3716       Restore the WT_VERB_TEMPORARY verbose flag.
WT-3720       flags macros cast flags to unsigned values, hiding warnings.
WT-3732       Handle adding WT indices while cursors on the table are open
WT-3734       Fix undefined behavior in verbose output
WT-3738       Review internal session allocation accounting
WT-3753       Building on Windows --enable-java
WT-3772       Hot backup causes uncontrolled growth of WiredTigerPreplog files
WT-3774       Enhance Python lookaside testing to cover cursor modify
WT-3776       Cursor remove operation unpins page too early
WT-3780       Improve error messages on invalid WT_CURSOR::modify usage
WT-3783       Fix transaction isolation to use the correct enum
WT-3786       Transactions with timestamps should read their writes
WT-3787       test_compact02 failed as compaction halted due to eviction pressure
WT-3790       Switch statistics to rdtsc from epoch calls
WT-3793       WiredTiger page debug dump functions should unpack integer keys
WT-3794       Coverity 1383547 and lint
WT-3795       lint cleanups for the op-tracking software, reduce record write size.

7 files changed, 115 insertions, 228 deletions

diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index 68a8f4908d8..ac4e80aedcf 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -108,6 +108,18 @@ lsm_config = [
             larger than this value.  This overrides the \c memory_page_max
             setting''',
             min='512K', max='500MB'),
+        Config('merge_custom', '', r'''
+            configure the tree to merge into a custom data source''',
+            type='category', subconfig=[
+            Config('prefix', '', r'''
+                custom data source prefix instead of \c "file"'''),
+            Config('start_generation', '0', r'''
+                merge generation at which the custom data source is used
+                (zero indicates no custom data source)''',
+                min='0', max='10'),
+            Config('suffix', '', r'''
+                custom data source suffix instead of \c ".lsm"'''),
+            ]),
         Config('merge_max', '15', r'''
             the maximum number of chunks to include in a merge operation''',
             min='2', max='100'),
@@ -497,6 +509,19 @@ connection_runtime_config = [
     Config('lsm_merge', 'true', r'''
         merge LSM chunks where possible (deprecated)''',
         type='boolean', undoc=True),
+    Config('operation_tracking', '', r'''
+        enable tracking of performance-critical functions. See
+        @ref operation_tracking for more information''',
+        type='category', subconfig=[
+            Config('enabled', 'false', r'''
+                enable operation tracking subsystem''',
+                type='boolean'),
+            Config('path', '"."', r'''
+                the name of a directory into which operation tracking files are
+                written. The directory must already exist. If the value is not
+                an absolute path, the path is relative to the database home
+                (see @ref absolute_path for more information)'''),
+        ]),
     Config('shared_cache', '', r'''
         shared cache configuration options. A database should configure
         either a cache_size or a shared_cache not both. Enabling a
@@ -546,8 +571,7 @@ connection_runtime_config = [
         type='list', undoc=True, choices=[
             'checkpoint_slow', 'internal_page_split_race', 'page_split_race']),
     Config('verbose', '', r'''
-        enable messages for various events. Only available if WiredTiger
-        is configured with --enable-verbose. Options are given as a
+        enable messages for various events. Options are given as a
         list, such as <code>"verbose=[evictserver,read]"</code>''',
         type='list', choices=[
             'api',
@@ -576,6 +600,7 @@ connection_runtime_config = [
             'salvage',
             'shared_cache',
             'split',
+            'temporary',
             'thread_group',
             'timestamp',
             'transaction',
diff --git a/src/third_party/wiredtiger/dist/dist.py b/src/third_party/wiredtiger/dist/dist.py
index ec47f509541..e4b76bdbab4 100644
--- a/src/third_party/wiredtiger/dist/dist.py
+++ b/src/third_party/wiredtiger/dist/dist.py
@@ -22,6 +22,14 @@ def all_c_files():
     for line in glob.iglob('../test/*/*.[ci]'):
         yield line
 
+# all_h_files --
+#       Return list of all WiredTiger C include file names.
+def all_h_files():
+    file_re = re.compile(r'^\w')
+    for line in glob.iglob('../src/*/*.h'):
+        yield line
+    yield "../src/include/wiredtiger.in"
+
 # source_dirs --
 #    Return a list of the WiredTiger source directory names.
 def source_dirs():
diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist
index 9755e24f3c7..79590313b89 100644
--- a/src/third_party/wiredtiger/dist/filelist
+++ b/src/third_party/wiredtiger/dist/filelist
@@ -112,6 +112,7 @@ src/meta/meta_ext.c
 src/meta/meta_table.c
 src/meta/meta_track.c
 src/meta/meta_turtle.c
+src/optrack/optrack.c
 src/os_common/filename.c
 src/os_common/os_abort.c
 src/os_common/os_alloc.c
diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py
index 28c91486e1a..5d4f3cd2e5c 100644
--- a/src/third_party/wiredtiger/dist/flags.py
+++ b/src/third_party/wiredtiger/dist/flags.py
@@ -1,226 +1,55 @@
-# Output a C header file using the minimum number of distinct bits to ensure
-# flags don't collide.
-
-import os, re, sys
-from dist import compare_srcfile
-
-flags = {
-###################################################
-# Internal routine flag declarations
-###################################################
-    'log_scan' : [
-        'LOGSCAN_FIRST',
-        'LOGSCAN_FROM_CKP',
-        'LOGSCAN_ONE',
-        'LOGSCAN_RECOVER',
-    ],
-    'log_write' : [
-        'LOG_BACKGROUND',
-        'LOG_DSYNC',
-        'LOG_FLUSH',
-        'LOG_FSYNC',
-        'LOG_SYNC_ENABLED',
-    ],
-    'page_read' : [
-        'READ_CACHE',
-        'READ_IGNORE_CACHE_SIZE',
-        'READ_LOOKASIDE',
-        'READ_NOTFOUND_OK',
-        'READ_NO_EMPTY',
-        'READ_NO_GEN',
-        'READ_NO_SPLIT',
-        'READ_NO_WAIT',
-        'READ_PREV',
-        'READ_RESTART_OK',
-        'READ_SKIP_INTL',
-        'READ_TRUNCATE',
-        'READ_WONT_NEED',
-    ],
-    'rec_write' : [
-        'REC_CHECKPOINT',
-        'REC_EVICT',
-        'REC_IN_MEMORY',
-        'REC_LOOKASIDE',
-        'REC_SCRUB',
-        'REC_UPDATE_RESTORE',
-        'REC_VISIBILITY_ERR',
-        'REC_VISIBLE_ALL',
-    ],
-    'timing_stress_for_test' : [
-        'TIMING_STRESS_CHECKPOINT_SLOW',
-        'TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE',
-        'TIMING_STRESS_PAGE_SPLIT_RACE',
-    ],
-    'txn_log_checkpoint' : [
-        'TXN_LOG_CKPT_CLEANUP',
-        'TXN_LOG_CKPT_PREPARE',
-        'TXN_LOG_CKPT_START',
-        'TXN_LOG_CKPT_STOP',
-        'TXN_LOG_CKPT_SYNC',
-    ],
-    'txn_update_oldest' : [
-        'TXN_OLDEST_STRICT',
-        'TXN_OLDEST_WAIT',
-    ],
-    'verbose' : [
-        'VERB_API',
-        'VERB_BLOCK',
-        'VERB_CHECKPOINT',
-        'VERB_CHECKPOINT_PROGRESS',
-        'VERB_COMPACT',
-        'VERB_EVICT',
-        'VERB_EVICTSERVER',
-        'VERB_EVICT_STUCK',
-        'VERB_FILEOPS',
-        'VERB_HANDLEOPS',
-        'VERB_LOG',
-        'VERB_LOOKASIDE',
-        'VERB_LOOKASIDE_ACTIVITY',
-        'VERB_LSM',
-        'VERB_LSM_MANAGER',
-        'VERB_METADATA',
-        'VERB_MUTEX',
-        'VERB_OVERFLOW',
-        'VERB_READ',
-        'VERB_REBALANCE',
-        'VERB_RECONCILE',
-        'VERB_RECOVERY',
-        'VERB_RECOVERY_PROGRESS',
-        'VERB_SALVAGE',
-        'VERB_SHARED_CACHE',
-        'VERB_SPLIT',
-        'VERB_THREAD_GROUP',
-        'VERB_TIMESTAMP',
-        'VERB_TRANSACTION',
-        'VERB_VERIFY',
-        'VERB_VERSION',
-        'VERB_WRITE',
-    ],
-
-###################################################
-# Structure flag declarations
-###################################################
-    'conn' : [
-        'CONN_CACHE_POOL',
-        'CONN_CKPT_SYNC',
-        'CONN_CLOSING',
-        'CONN_CLOSING_NO_MORE_OPENS',
-        'CONN_EVICTION_NO_LOOKASIDE',
-        'CONN_EVICTION_RUN',
-        'CONN_IN_MEMORY',
-        'CONN_LOOKASIDE_OPEN',
-        'CONN_LEAK_MEMORY',
-        'CONN_LSM_MERGE',
-        'CONN_PANIC',
-        'CONN_READONLY',
-        'CONN_RECOVERING',
-        'CONN_SERVER_ASYNC',
-        'CONN_SERVER_CHECKPOINT',
-        'CONN_SERVER_LOG',
-        'CONN_SERVER_LSM',
-        'CONN_SERVER_STATISTICS',
-        'CONN_SERVER_SWEEP',
-        'CONN_WAS_BACKUP',
-    ],
-    'session' : [
-        'SESSION_CAN_WAIT',
-        'SESSION_INTERNAL',
-        'SESSION_IGNORE_CACHE_SIZE',
-        'SESSION_LOCKED_CHECKPOINT',
-        'SESSION_LOCKED_HANDLE_LIST_READ',
-        'SESSION_LOCKED_HANDLE_LIST_WRITE',
-        'SESSION_LOCKED_METADATA',
-        'SESSION_LOCKED_PASS',
-        'SESSION_LOCKED_SCHEMA',
-        'SESSION_LOCKED_SLOT',
-        'SESSION_LOCKED_TABLE_READ',
-        'SESSION_LOCKED_TABLE_WRITE',
-        'SESSION_LOCKED_TURTLE',
-        'SESSION_LOGGING_INMEM',
-        'SESSION_LOOKASIDE_CURSOR',
-        'SESSION_NO_DATA_HANDLES',
-        'SESSION_NO_RECONCILE',
-        'SESSION_NO_LOGGING',
-        'SESSION_NO_SCHEMA_LOCK',
-        'SESSION_QUIET_CORRUPT_FILE',
-        'SESSION_READ_WONT_NEED',
-        'SESSION_SERVER_ASYNC',
-    ],
-    'stat' : [
-        'STAT_CLEAR',
-        'STAT_JSON',
-        'STAT_ON_CLOSE',
-        'STAT_TYPE_ALL',
-        'STAT_TYPE_CACHE_WALK',
-        'STAT_TYPE_FAST',
-        'STAT_TYPE_SIZE',
-        'STAT_TYPE_TREE_WALK',
-    ],
-}
-
-flag_cnt = {}    # Dictionary [flag] : [reference count]
-flag_name = {}   # Dictionary [flag] : [name ...]
-name_mask = {}   # Dictionary [name] : [used flag mask]
-
-# Step through the flags dictionary and build our local dictionaries.
-for method in flags.items():
-    name_mask[method[0]] = 0x0
-    for flag in method[1]:
-        if flag == '__NONE__':
-            continue
-        if flag not in flag_cnt:
-            flag_cnt[flag] = 1
-            flag_name[flag] = []
-        else:
-            flag_cnt[flag] += 1
-        flag_name[flag].append(method[0])
-
-# Create list of possible bit masks.
-bits = [2 ** i for i in range(0, 32)]
-
-# Walk the list of flags in reverse, sorted-by-reference count order.  For
-# each flag, find a bit that's not currently in use by any method using the
-# flag.
-flag_bit = {}    # Dictionary [flag] : [bit value]
-for f in sorted(flag_cnt.items(), key = lambda k_v : (-k_v[1], k_v[0])):
-    mask = 0xffffffff
-    for m in flag_name[f[0]]:
-        mask &= ~name_mask[m]
-    if mask == 0:
-        print >>sys.stderr,\
-            "flags.py: ran out of flags at " + m + " method",
-        sys.exit(1)
-    for b in bits:
-        if mask & b:
-            mask = b
-            break
-    flag_bit[f[0]] = mask
-    for m in flag_name[f[0]]:
-        name_mask[m] |= mask
-
-# Print out the flag masks in hex.
-#    Assumes tab stops set to 8 characters.
-flag_info = ''
-for f in sorted(flag_cnt.items()):
-    flag_info += "#define\tWT_%s%s%#010x\n" %\
-        (f[0],\
-        "\t" * max(1, 6 - int((len('WT_') + len(f[0])) / 8)),\
-        flag_bit[f[0]])
-
-# Update the wiredtiger.in file with the flags information.
-tmp_file = '__tmp'
-tfile = open(tmp_file, 'w')
-skip = 0
-for line in open('../src/include/flags.h', 'r'):
-    if skip:
-        if line.count('flags section: END'):
-            tfile.write('/*\n' + line)
-            skip = 0
-    else:
-        tfile.write(line)
-    if line.count('flags section: BEGIN'):
-        skip = 1
-        tfile.write(' */\n')
-        tfile.write(flag_info)
-tfile.close()
-compare_srcfile(tmp_file, '../src/include/flags.h')
+#!/usr/bin/env python
+
+import re, sys
+from dist import all_c_files, all_h_files, compare_srcfile
+
+# Automatically build flags values: read through all of the header files, and
+# for each group of flags, sort them and give them a unique value.
+def flag_declare(name):
+    tmp_file = '__tmp'
+    with open(name, 'r') as f:
+        tfile = open(tmp_file, 'w')
+
+        lcnt = 0
+        parsing = False
+        for line in f:
+            lcnt = lcnt + 1
+            if line.find('AUTOMATIC FLAG VALUE GENERATION START') != -1:
+                header = line
+                defines = []
+                parsing = True
+            elif line.find('AUTOMATIC FLAG VALUE GENERATION STOP') != -1:
+                # We only support 64 bits.
+                if len(defines) > 64:
+                    print >>sys.stderr, name + ": line " +\
+                        str(lcnt) + ": exceeds maximum 64 bit flags"
+                    sys.exit(1)
+
+                # Calculate number of hex bytes, create format string
+                fmt = "0x%%0%dxu" % ((len(defines) + 3) / 4)
+
+                tfile.write(header)
+                v = 1
+                for d in sorted(defines):
+                    tfile.write(re.sub("0x[01248u]*", fmt % v, d))
+                    v = v * 2
+                tfile.write(line)
+
+                parsing = False
+            elif parsing and line.find('#define') == -1:
+                print >>sys.stderr, name + ": line " +\
+                    str(lcnt) + ": unexpected flag line, no #define"
+                sys.exit(1)
+            elif parsing:
+                defines.append(line)
+            else:
+                tfile.write(line)
+
+        tfile.close()
+        compare_srcfile(tmp_file, name)
+
+# Update function argument declarations.
+for name in all_h_files():
+    flag_declare(name)
+for name in all_c_files():
+    flag_declare(name)
diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list
index 4be8ceee0e3..5a5af9dd5d7 100644
--- a/src/third_party/wiredtiger/dist/s_define.list
+++ b/src/third_party/wiredtiger/dist/s_define.list
@@ -36,6 +36,8 @@ WT_LOG_SLOT_MASK_ON
 WT_LOG_SLOT_MAXBITS
 WT_LOG_SLOT_UNBUFFERED_ISSET
 WT_LOOKASIDE_COMPRESSOR
+WT_OPTRACK_BUFSIZE
+WT_OPTRACK_MAXRECS
 WT_PACKED_STRUCT_BEGIN
 WT_PACKED_STRUCT_END
 WT_PADDING_CHECK
@@ -60,6 +62,10 @@ WT_STAT_INCRV_ATOMIC_BASE
 WT_STAT_INCRV_BASE
 WT_STAT_WRITE
 WT_TIMEDIFF_US
+WT_TSCDIFF_NS
+WT_TRACK_OP
+WT_TRACK_OP_END
+WT_TRACK_OP_INIT
 WT_TRET_ERROR_OK
 WT_UPDATE_SIZE
 WT_WITH_LOCK_NOWAIT
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok
index 1a252fdd615..80f6bdca186 100644
--- a/src/third_party/wiredtiger/dist/s_string.ok
+++ b/src/third_party/wiredtiger/dist/s_string.ok
@@ -162,6 +162,7 @@ HFS
 HHHH
 HHHHLL
 HHHLL
+HILQr
 Hendrik
 HyperLevelDB
 ID's
@@ -268,6 +269,7 @@ Noll
 Nul
 OOB
 OPLOG
+OPTRACK
 OPTYPE
 OUTBUFF
 OVFL
@@ -746,6 +748,7 @@ fsyncLock
 fsyncs
 ftruncate
 func
+funcid
 fvisibility
 gcc
 gdb
@@ -775,6 +778,7 @@ hashval
 havesize
 hdr
 highjack
+hilq
 hotbackup
 hselasky
 html
@@ -986,6 +990,7 @@ notsup
 notused
 novalue
 nowait
+nsec
 nset
 nsnap
 nul
@@ -1006,6 +1011,7 @@ openfile
 oplist
 oplog
 optimizations
+optrack
 optype
 ori
 os
@@ -1078,6 +1084,7 @@ rbrace
 rbracket
 rcursor
 rdonly
+rdtsc
 rduppo
 readlock
 readonly
@@ -1193,6 +1200,7 @@ tV
 tablename
 tcbench
 td
+teardown
 tempdir
 testutil
 th
@@ -1216,6 +1224,7 @@ trk
 trk's
 trunc
 trylock
+tsc
 tupdate
 tvalue
 txn
diff --git a/src/third_party/wiredtiger/dist/s_style b/src/third_party/wiredtiger/dist/s_style
index 3ae14949fbf..3f4346173e6 100755
--- a/src/third_party/wiredtiger/dist/s_style
+++ b/src/third_party/wiredtiger/dist/s_style
@@ -61,6 +61,15 @@ else
 		cat $t
 	fi
 
+	if ! expr "$f" : 'examples/c/.*' > /dev/null &&
+	   ! expr "$f" : 'ext/.*' > /dev/null &&
+	   ! expr "$f" : 'src/include/wiredtiger_ext\.h' > /dev/null &&
+	   ! expr "$f" : 'src/txn/txn_ext\.c' > /dev/null &&
+	   grep WT_TXN_ISO_ $f; then
+		echo "$f: WT_TXN_ISO_XXX constants only for the extension API"
+		cat $t
+	fi
+
 	if ! expr "$f" : 'src/include/queue\.h' > /dev/null &&
 	    egrep 'STAILQ_|SLIST_|\bLIST_' $f ; then
 		echo "$f: use TAILQ for all lists"