Merge branch 'develop' into wt-2360-join-disjunction

12 files changed, 126 insertions, 75 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index 624c519b101..71ef30fa787 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -569,6 +569,7 @@ connection_runtime_config = [
             'evict',
             'evictserver',
             'fileops',
+            'handleops',
             'log',
             'lsm',
             'lsm_manager',
diff --git a/dist/filelist b/dist/filelist
index 350e0c50087..af9b2319b75 100644
--- a/dist/filelist
+++ b/dist/filelist
@@ -101,34 +101,31 @@ src/meta/meta_ext.c
 src/meta/meta_table.c
 src/meta/meta_track.c
 src/meta/meta_turtle.c
-src/os_posix/os_abort.c
-src/os_posix/os_alloc.c
+src/os_common/filename.c
+src/os_common/os_abort.c
+src/os_common/os_alloc.c
+src/os_common/os_fhandle.c
+src/os_common/os_fs_init.c
+src/os_common/os_fs_inmemory.c
+src/os_common/os_fs_stdio.c
+src/os_common/os_getline.c
+src/os_common/os_getopt.c
+src/os_common/os_init.c
+src/os_common/os_strtouq.c
 src/os_posix/os_dir.c
 src/os_posix/os_dlopen.c
 src/os_posix/os_errno.c
-src/os_posix/os_exist.c
 src/os_posix/os_fallocate.c
-src/os_posix/os_filesize.c
-src/os_posix/os_flock.c
-src/os_posix/os_fsync.c
-src/os_posix/os_ftruncate.c
+src/os_posix/os_fs.c
 src/os_posix/os_getenv.c
-src/os_posix/os_getline.c
-src/os_posix/os_getopt.c
 src/os_posix/os_map.c
 src/os_posix/os_mtx_cond.c
-src/os_posix/os_mtx_rw.c
 src/os_posix/os_once.c
-src/os_posix/os_open.c
 src/os_posix/os_pagesize.c
 src/os_posix/os_path.c
 src/os_posix/os_priv.c
-src/os_posix/os_remove.c
-src/os_posix/os_rename.c
-src/os_posix/os_rw.c
+src/os_posix/os_setvbuf.c
 src/os_posix/os_sleep.c
-src/os_posix/os_stdio.c
-src/os_posix/os_strtouq.c
 src/os_posix/os_thread.c
 src/os_posix/os_time.c
 src/os_posix/os_yield.c
@@ -156,13 +153,13 @@ src/support/cksum.c
 src/support/cond_auto.c
 src/support/crypto.c
 src/support/err.c
-src/support/filename.c
 src/support/global.c
 src/support/hash_city.c
 src/support/hash_fnv.c
 src/support/hazard.c
 src/support/hex.c
 src/support/huffman.c
+src/support/mtx_rw.c
 src/support/pow.c
 src/support/power8/crc32.S
 src/support/power8/crc32_wrapper.c
diff --git a/dist/flags.py b/dist/flags.py
index f500e3b1ae1..8f7827ad160 100644
--- a/dist/flags.py
+++ b/dist/flags.py
@@ -13,7 +13,7 @@ flags = {
         'FILE_TYPE_DATA',
         'FILE_TYPE_DIRECTORY',
         'FILE_TYPE_LOG',
-        'FILE_TYPE_TURTLE',
+        'FILE_TYPE_REGULAR',
     ],
     'log_scan' : [
         'LOGSCAN_FIRST',
@@ -65,6 +65,7 @@ flags = {
         'VERB_EVICT',
         'VERB_EVICTSERVER',
         'VERB_FILEOPS',
+        'VERB_HANDLEOPS',
         'VERB_LOG',
         'VERB_LSM',
         'VERB_LSM_MANAGER',
diff --git a/dist/log.py b/dist/log.py
index 6d35bf2e718..9201b20054b 100644
--- a/dist/log.py
+++ b/dist/log.py
@@ -89,7 +89,7 @@ def printf_line(f, optype, i, ishex):
         ifbegin = 'if (LF_ISSET(WT_TXN_PRINTLOG_HEX)) {' + nl_indent
         if postcomma == '':
             precomma = ',\\n'
-    body = '%s%s(__wt_fprintf(out,' % (
+    body = '%s%s(__wt_fprintf(session, WT_STDOUT(session),' % (
         printf_setup(f, ishex, nl_indent),
         'WT_ERR' if has_escape(optype.fields) else 'WT_RET') + \
         '%s    "%s        \\"%s\\": \\"%s\\"%s",%s));' % (
@@ -292,16 +292,16 @@ __wt_logop_%(name)s_unpack(
     last_field = optype.fields[-1]
     tfile.write('''
 int
-__wt_logop_%(name)s_print(
-    WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end,
-    FILE *out, uint32_t flags)
+__wt_logop_%(name)s_print(WT_SESSION_IMPL *session,
+    const uint8_t **pp, const uint8_t *end, uint32_t flags)
 {
 %(arg_ret)s\t%(arg_decls)s
 
 \t%(arg_unused)s%(arg_init)sWT_RET(__wt_logop_%(name)s_unpack(
 \t    session, pp, end%(arg_addrs)s));
 
-\tWT_RET(__wt_fprintf(out, " \\"optype\\": \\"%(name)s\\",\\n"));
+\tWT_RET(__wt_fprintf(session, WT_STDOUT(session),
+\t    " \\"optype\\": \\"%(name)s\\",\\n"));
 \t%(print_args)s
 %(arg_fini)s
 }
@@ -324,9 +324,8 @@ __wt_logop_%(name)s_print(
 # Emit the printlog entry point
 tfile.write('''
 int
-__wt_txn_op_printlog(
-    WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end,
-    FILE *out, uint32_t flags)
+__wt_txn_op_printlog(WT_SESSION_IMPL *session,
+    const uint8_t **pp, const uint8_t *end, uint32_t flags)
 {
 \tuint32_t optype, opsize;
 
@@ -342,8 +341,7 @@ for optype in log_data.optypes:
 
     tfile.write('''
 \tcase %(macro)s:
-\t\tWT_RET(%(print_func)s(session, pp, end, out,
-\t\t    flags));
+\t\tWT_RET(%(print_func)s(session, pp, end, flags));
 \t\tbreak;
 ''' % {
     'macro' : optype.macro_name(),
diff --git a/dist/s_define.list b/dist/s_define.list
index e3f0dc7f181..c9777c86675 100644
--- a/dist/s_define.list
+++ b/dist/s_define.list
@@ -16,6 +16,7 @@ WIN32_LEAN_AND_MEAN
 WT_ATOMIC_CAS
 WT_ATOMIC_FUNC
 WT_BLOCK_DESC_SIZE
+WT_BLOCK_HEADER_SIZE
 WT_CACHE_LINE_ALIGNMENT
 WT_COMPILER_TYPE_ALIGN
 WT_CONN_CHECK_PANIC
diff --git a/dist/s_funcs.list b/dist/s_funcs.list
index 8d32eecdfb7..c0d9f2e688f 100644
--- a/dist/s_funcs.list
+++ b/dist/s_funcs.list
@@ -22,7 +22,6 @@ __wt_debug_set_verbose
 __wt_debug_tree
 __wt_debug_tree_all
 __wt_debug_tree_shape
-__wt_fsync
 __wt_lex_compare
 __wt_lex_compare_skip
 __wt_log_scan
@@ -31,6 +30,7 @@ __wt_nlpo2_round
 __wt_print_huffman_code
 __wt_stat_join_aggregate
 __wt_stat_join_clear_all
+__wt_stream_set_no_buffer
 __wt_try_readlock
 wiredtiger_config_parser_open
 wiredtiger_config_validate
diff --git a/dist/s_prototypes b/dist/s_prototypes
index 603c0f5633d..4ceb69f4c77 100755
--- a/dist/s_prototypes
+++ b/dist/s_prototypes
@@ -4,13 +4,10 @@
 t=__wt.$$
 trap 'rm -f $t; exit 0' 0 1 2 3 13 15
 
-(
-cat <<EOF
-/* DO NOT EDIT: automatically built by dist/s_prototypes. */
-
-EOF
-
-for i in `sed -e '/^[a-z]/!d' filelist`; do
+# proto --
+#	extract public functions.
+proto()
+{
 	sed -n \
 	    -e '/^__wt_[a-z]/!{' \
 		-e h \
@@ -32,9 +29,34 @@ for i in `sed -e '/^[a-z]/!d' filelist`; do
 	    -e 's/  */ /g' \
 	    -e 's/^/extern /' \
 	    -e 's/WT_GCC_FUNC_/WT_GCC_FUNC_DECL_/' \
-	    -e 's/$/;/p' \
-	    < ../$i
-done) > $t
+	    -e 's/$/;/p' < $1
+}
+
+(
+cat <<EOF
+/* DO NOT EDIT: automatically built by dist/s_prototypes. */
+
+EOF
+
+# First, get prototypes for everything but the OS directories.
+# Second, get prototypes for the OS directories.
+# The reason for this is because the OS directories repeat names (that is, there
+# are common names in both os_posix and os_win), and so we sort the prototypes
+# to avoid repeating them in the output (which some compilers won't tolerate).
+# We'd sort everything and discard duplicates, but we can't sort when function
+# signatures are on multiple lines, that is, #ifdef'd function signatures. Since
+# the OS directories are the only places with repeated names, and they have no
+# #ifdef'd signatures, we do it this way.
+l=`sed -e '/^[a-z]/!d' -e '/src\/os/d' filelist`
+for i in $l; do
+	proto ../$i
+done
+l=`echo ../src\/os*/*.c`
+
+for i in $l; do
+	proto $i
+done | tee xxx | env LC_ALL=C sort -u
+) > $t
 
 f=../src/include/extern.h
 cmp $t $f > /dev/null 2>&1 ||
diff --git a/dist/s_stat b/dist/s_stat
index 44c22ab56bb..3938b8e65eb 100755
--- a/dist/s_stat
+++ b/dist/s_stat
@@ -11,7 +11,7 @@ l=`sed \
     -e 's,#.*,,' \
     -e '/^$/d' \
     -e 's,^,../,' filelist`
-l="$l `echo ../src/include/*.i`"
+l="$l `echo ../src/include/*.i ../src/include/os.h`"
 
 (
 # Get the list of statistics fields.
diff --git a/dist/s_string.ok b/dist/s_string.ok
index 79f662aa851..eed034abb47 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -67,7 +67,9 @@ CloseHandle
 Comparator
 Config
 Coverity
+CreateFileA
 CreateFileMapping
+CreateFileMappingA
 Crummey
 CustomersPhone
 DECL
@@ -76,11 +78,13 @@ DESC
 DHANDLE
 DNE
 DOI
+DONTNEED
 DUPLICATEV
 DbCursor
 DbEnv
 Decrement
 Decrypt
+DeleteFileA
 EAGAIN
 EBUSY
 EEXIST
@@ -187,6 +191,7 @@ LoadLoad
 LockFile
 Lookaside
 Lookup
+MADV
 MALLOC
 MEM
 MEMALIGN
@@ -249,6 +254,7 @@ Prepend
 Qsort
 RCS
 RDNOLOCK
+RDONLY
 RECNO
 REF's
 REFs
@@ -261,6 +267,7 @@ RNG
 RPC
 RUNDIR
 Radu
+ReadFile
 Readonly
 Rebalance
 RedHat
@@ -318,6 +325,7 @@ UTF
 UltraSparc
 Unbuffered
 UnixLib
+UnlockFile
 Unmap
 UnmapViewOfFile
 Unmarshall
@@ -333,9 +341,11 @@ Vixie
 Vo
 VxWorks
 WAL
+WILLNEED
 WIREDTIGER
 WRLSN
 WRNOLOCK
+WaitForSingleObject
 WakeAllConditionVariable
 Wconditional
 WeakHashLen
@@ -353,6 +363,7 @@ WiredTigerPreplog
 WiredTigerTmplog
 WiredTigerTxn
 WithSeeds
+WriteFile
 Wuninitialized
 Wunused
 XP
@@ -455,6 +466,7 @@ ckpt
 ckptfrag
 ckptlist
 cksum
+cloexec
 clsm
 cmd
 cmp
@@ -596,12 +608,17 @@ fallocate
 fblocks
 fclose
 fcntl
+fd
+fdatasync
+fdopen
 ffc
 fflush
 ffs
+fgetc
 fgetln
 fh
 filefrag
+filehandle
 fileid
 filename
 filenames
@@ -632,7 +649,9 @@ func
 gcc
 gdb
 ge
+getc
 getenv
+getlasterror
 getline
 getone
 getones
@@ -648,6 +667,7 @@ gostring
 gostruct
 goutf
 gt
+handleops
 hashval
 havesize
 hdr
@@ -664,6 +684,7 @@ icount
 idx
 ifdef's
 ikey
+im
 impl
 incase
 incr
@@ -680,6 +701,7 @@ initsize
 initval
 inline
 inmem
+inmemory
 insertK
 insertV
 inserters
@@ -687,6 +709,7 @@ instantiation
 intl
 intnum
 intpack
+intptr
 intrin
 inuse
 io
@@ -839,6 +862,7 @@ optimizations
 optype
 ori
 os
+osfhandle
 ovfl
 ownp
 packv
@@ -864,8 +888,10 @@ postsize
 powerpc
 pragmas
 pre
+pread
 prealloc
 preload
+preloaded
 prepend
 prepended
 prepending
@@ -884,6 +910,7 @@ pushms
 putK
 putV
 pv
+pwrite
 py
 qdown
 qrrSS
@@ -942,6 +969,7 @@ sessionp
 setkv
 setstr
 setv
+setvbuf
 sfence
 sii
 sizeof
diff --git a/dist/s_style b/dist/s_style
index 78fb7a6eb03..a163eb83b25 100755
--- a/dist/s_style
+++ b/dist/s_style
@@ -60,11 +60,12 @@ else
 		echo "$f: use TAILQ for all lists"
 	fi
 
-	if ! expr "$f" : 'src/os_posix/.*' > /dev/null &&
+	if ! expr "$f" : 'src/os_common/.*' > /dev/null &&
+	   ! expr "$f" : 'src/os_posix/.*' > /dev/null &&
 	   ! expr "$f" : 'src/os_win/.*' > /dev/null &&
 	   ! expr "$f" : 'src/include/extern.h' > /dev/null &&
 	   ! expr "$f" : 'src/include/os.h' > /dev/null &&
-	   grep '__wt_errno' $f > $t; then
+	    grep '__wt_errno' $f > $t; then
 		echo "$f: upper-level code should not call __wt_errno"
 		cat $t
 	fi
@@ -83,6 +84,12 @@ else
 		cat $t
 	}
 
+	if ! expr "$f" : 'src/.*/os_setvbuf.c' > /dev/null &&
+	    egrep -w 'setvbuf' $f > $t; then
+		echo "$f: setvbuf call, use WiredTiger library replacements"
+		cat $t
+	fi
+
 	# Alignment directive before "struct".
 	egrep 'WT_COMPILER_TYPE_ALIGN.*struct' $f > $t
 	test -s $t && {
diff --git a/dist/s_win b/dist/s_win
index 0b7d5184037..562e89f94c6 100755
--- a/dist/s_win
+++ b/dist/s_win
@@ -43,40 +43,33 @@ win_filelist()
 {
 	f='../build_win/filelist.win'
 
-	# Process the files for which there's a Windows-specific version, then
-	# append Windows-only files and discard POSIX-only files.
-	(sed \
-	    -e 's;os_posix/os_dir.c;os_win/os_dir.c;' \
-	    -e 's;os_posix/os_dlopen.c;os_win/os_dlopen.c;' \
-	    -e 's;os_posix/os_dlopen.c;os_win/os_dlopen.c;' \
-	    -e 's;os_posix/os_dlopen.c;os_win/os_dlopen.c;' \
-	    -e 's;os_posix/os_errno.c;os_win/os_errno.c;' \
-	    -e 's;os_posix/os_exist.c;os_win/os_exist.c;' \
-	    -e 's;os_posix/os_fallocate.c;os_win/os_fallocate.c;' \
-	    -e 's;os_posix/os_filesize.c;os_win/os_filesize.c;' \
-	    -e 's;os_posix/os_flock.c;os_win/os_flock.c;' \
-	    -e 's;os_posix/os_fsync.c;os_win/os_fsync.c;' \
-	    -e 's;os_posix/os_ftruncate.c;os_win/os_ftruncate.c;' \
-	    -e 's;os_posix/os_getenv.c;os_win/os_getenv.c;' \
-	    -e 's;os_posix/os_map.c;os_win/os_map.c;' \
-	    -e 's;os_posix/os_mtx_cond.c;os_win/os_mtx_cond.c;' \
-	    -e 's;os_posix/os_once.c;os_win/os_once.c;' \
-	    -e 's;os_posix/os_open.c;os_win/os_open.c;' \
-	    -e 's;os_posix/os_pagesize.c;os_win/os_pagesize.c;' \
-	    -e 's;os_posix/os_path.c;os_win/os_path.c;' \
-	    -e 's;os_posix/os_priv.c;os_win/os_priv.c;' \
-	    -e 's;os_posix/os_remove.c;os_win/os_remove.c;' \
-	    -e 's;os_posix/os_rename.c;os_win/os_rename.c;' \
-	    -e 's;os_posix/os_rw.c;os_win/os_rw.c;' \
-	    -e 's;os_posix/os_sleep.c;os_win/os_sleep.c;' \
-	    -e 's;os_posix/os_thread.c;os_win/os_thread.c;' \
-	    -e 's;os_posix/os_time.c;os_win/os_time.c;' \
-	    -e 's;os_posix/os_yield.c;os_win/os_yield.c;' \
+	# Discard POSIX-only and PPC-only files, add in Windows-only files.
+	(
+	sed \
+	    -e '/\/os_posix\//d' \
 	    -e '/src\/support\/power8\/crc32.S/d' \
 	    -e '/src\/support\/power8\/crc32_wrapper.c/d'
-	 echo 'src/os_win/os_snprintf.c'
-	 echo 'src/os_win/os_vsnprintf.c') < filelist | sort > $t
-	 cmp $t $f > /dev/null 2>&1 ||
+
+	echo 'src/os_win/os_dir.c'
+	echo 'src/os_win/os_dlopen.c'
+	echo 'src/os_win/os_errno.c'
+	echo 'src/os_win/os_fs.c'
+	echo 'src/os_win/os_getenv.c'
+	echo 'src/os_win/os_map.c'
+	echo 'src/os_win/os_mtx_cond.c'
+	echo 'src/os_win/os_once.c'
+	echo 'src/os_win/os_pagesize.c'
+	echo 'src/os_win/os_path.c'
+	echo 'src/os_win/os_priv.c'
+	echo 'src/os_win/os_setvbuf.c'
+	echo 'src/os_win/os_sleep.c'
+	echo 'src/os_win/os_snprintf.c'
+	echo 'src/os_win/os_thread.c'
+	echo 'src/os_win/os_time.c'
+	echo 'src/os_win/os_vsnprintf.c'
+	echo 'src/os_win/os_yield.c') < filelist | sort > $t
+
+	cmp $t $f > /dev/null 2>&1 ||
 	     (echo "Building $f" && rm -f $f && cp $t $f)
 }
 
diff --git a/dist/stat_data.py b/dist/stat_data.py
index bd951e64999..3a8ddbfa7d1 100644
--- a/dist/stat_data.py
+++ b/dist/stat_data.py
@@ -170,13 +170,16 @@ connection_stats = [
     CacheStat('cache_eviction_force', 'pages evicted because they exceeded the in-memory maximum'),
     CacheStat('cache_eviction_force_delete', 'pages evicted because they had chains of deleted items'),
     CacheStat('cache_eviction_force_fail', 'failed eviction of pages that exceeded the in-memory maximum'),
+    CacheStat('cache_eviction_get_ref', 'eviction calls to get a page'),
+    CacheStat('cache_eviction_get_ref_empty', 'eviction calls to get a page found queue empty'),
+    CacheStat('cache_eviction_get_ref_empty2', 'eviction calls to get a page found queue empty after locking'),
     CacheStat('cache_eviction_hazard', 'hazard pointer blocked page eviction'),
     CacheStat('cache_eviction_internal', 'internal pages evicted'),
     CacheStat('cache_eviction_maximum_page_size', 'maximum page size at eviction', 'no_clear,no_scale,size'),
     CacheStat('cache_eviction_queue_empty', 'eviction server candidate queue empty when topping up'),
     CacheStat('cache_eviction_queue_not_empty', 'eviction server candidate queue not empty when topping up'),
     CacheStat('cache_eviction_server_evicting', 'eviction server evicting pages'),
-    CacheStat('cache_eviction_server_not_evicting', 'eviction server populating queue, but not evicting pages'),
+    CacheStat('cache_eviction_server_toobig', 'eviction server skipped very large page'),
     CacheStat('cache_eviction_slow', 'eviction server unable to reach eviction goal'),
     CacheStat('cache_eviction_split_internal', 'internal pages split during eviction'),
     CacheStat('cache_eviction_split_leaf', 'leaf pages split during eviction'),