Import wiredtiger: 1a29eac4dc8cf82de437292da546e3f4039268a4 from branch mongodb-3.8

ref: 596a3c7c01..1a29eac4dc
for: 3.7.1

WT-3079       Make sure eviction visits all trees
WT-3133       Detect or track long latency operations
WT-3295       Allow LSM to merge into custom data sources
WT-3587       Remove HAVE_VERBOSE conditional compilation
WT-3654       Fix warning in Windows build on evergreen
WT-3716       Restore the WT_VERB_TEMPORARY verbose flag.
WT-3720       flags macros cast flags to unsigned values, hiding warnings.
WT-3732       Handle adding WT indices while cursors on the table are open
WT-3734       Fix undefined behavior in verbose output
WT-3738       Review internal session allocation accounting
WT-3753       Building on Windows --enable-java
WT-3772       Hot backup causes uncontrolled growth of WiredTigerPreplog files
WT-3774       Enhance Python lookaside testing to cover cursor modify
WT-3776       Cursor remove operation unpins page too early
WT-3780       Improve error messages on invalid WT_CURSOR::modify usage
WT-3783       Fix transaction isolation to use the correct enum
WT-3786       Transactions with timestamps should read their writes
WT-3787       test_compact02 failed as compaction halted due to eviction pressure
WT-3790       Switch statistics to rdtsc from epoch calls
WT-3793       WiredTiger page debug dump functions should unpack integer keys
WT-3794       Coverity 1383547 and lint
WT-3795       lint cleanups for the op-tracking software, reduce record write size.

4 files changed, 195 insertions, 20 deletions

diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index fd8fd6763db..5727399c301 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -1084,6 +1084,8 @@ err:	/*
 			WT_TRET(wt_session->close(wt_session, config));
 		}
 
+	WT_TRET(__wt_async_flush(session));
+
 	/*
 	 * Disable lookaside eviction: it doesn't help us shut down and can
 	 * lead to pages being marked dirty, causing spurious assertions to
@@ -1284,6 +1286,35 @@ err:	API_END_RET(session, ret);
 }
 
 /*
+ * __conn_calibrate_ticks --
+ *	Calibrate a ratio from rdtsc ticks to nanoseconds.
+ */
+static void
+__conn_calibrate_ticks(WT_SESSION_IMPL *session)
+{
+#if (defined __i386) || (defined __amd64)
+	struct timespec start, stop;
+	uint64_t diff_nsec, diff_tsc, tsc_start, tsc_stop;
+	volatile uint64_t i;
+
+	__wt_epoch(session, &start);
+	tsc_start = __wt_rdtsc(session);
+	/*
+	 * This needs to be CPU intensive and large enough.
+	 */
+	for (i = 0; i < WT_MILLION; i++)
+		;
+	tsc_stop = __wt_rdtsc(session);
+	__wt_epoch(session, &stop);
+	diff_nsec = WT_TIMEDIFF_NS(stop, start);
+	diff_tsc = tsc_stop - tsc_start;
+	S2C(session)->tsc_nsec_ratio = (double)diff_tsc/(double)diff_nsec;
+#else
+	S2C(session)->tsc_nsec_ratio = 1.0;
+#endif
+}
+
+/*
  * __conn_config_append --
  *	Append an entry to a config stack.
  */
@@ -1794,7 +1825,7 @@ err:	/*
 /* Simple structure for name and flag configuration searches. */
 typedef struct {
 	const char *name;
-	uint32_t flag;
+	uint64_t flag;
 } WT_NAME_FLAG;
 
 /*
@@ -1831,6 +1862,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[])
 		{ "salvage",		WT_VERB_SALVAGE },
 		{ "shared_cache",	WT_VERB_SHARED_CACHE },
 		{ "split",		WT_VERB_SPLIT },
+		{ "temporary",		WT_VERB_TEMPORARY },
 		{ "thread_group",	WT_VERB_THREAD_GROUP },
 		{ "timestamp",		WT_VERB_TIMESTAMP },
 		{ "transaction",	WT_VERB_TRANSACTION },
@@ -1843,7 +1875,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[])
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
 	const WT_NAME_FLAG *ft;
-	uint32_t flags;
+	uint64_t flags;
 
 	conn = S2C(session);
 
@@ -1852,17 +1884,8 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[])
 	flags = 0;
 	for (ft = verbtypes; ft->name != NULL; ft++) {
 		if ((ret = __wt_config_subgets(
-		    session, &cval, ft->name, &sval)) == 0 && sval.val != 0) {
-#ifdef HAVE_VERBOSE
+		    session, &cval, ft->name, &sval)) == 0 && sval.val != 0)
 			LF_SET(ft->flag);
-#else
-			WT_RET_MSG(session, EINVAL,
-			    "Verbose option specified when WiredTiger built "
-			    "without verbose support. Add --enable-verbose to "
-			    "configure command and rebuild to include support "
-			    "for verbose messages");
-#endif
-		}
 		WT_RET_NOTFOUND_OK(ret);
 	}
 
@@ -2002,7 +2025,7 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[])
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
 	const WT_NAME_FLAG *ft;
-	uint32_t flags;
+	uint64_t flags;
 
 	conn = S2C(session);
 
@@ -2172,6 +2195,44 @@ err:	API_END_RET(session, ret);
 }
 
 /*
+ * __conn_session_size --
+ *	Return the session count for this run.
+ */
+static int
+__conn_session_size(
+    WT_SESSION_IMPL *session, const char *cfg[], uint32_t *vp)
+{
+	WT_CONFIG_ITEM cval;
+	int64_t v;
+
+	/*
+	 * Start with 20 internal sessions to cover threads the application
+	 * can't configure (for example, checkpoint or statistics log server
+	 * threads).
+	 */
+#define	WT_EXTRA_INTERNAL_SESSIONS	20
+	v = WT_EXTRA_INTERNAL_SESSIONS;
+
+	/* Then, add in the thread counts applications can configure. */
+	WT_RET(__wt_config_gets(session, cfg, "async.threads", &cval));
+	v += cval.val;
+
+	WT_RET(__wt_config_gets(session, cfg, "eviction.threads_max", &cval));
+	v += cval.val;
+
+	WT_RET(__wt_config_gets(
+	    session, cfg, "lsm_manager.worker_thread_max", &cval));
+	v += cval.val;
+
+	WT_RET(__wt_config_gets(session, cfg, "session_max", &cval));
+	v += cval.val;
+
+	*vp = (uint32_t)v;
+
+	return (0);
+}
+
+/*
  * __conn_chk_file_system --
  *	Check the configured file system.
  */
@@ -2498,8 +2559,10 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
 	WT_ERR(__wt_verbose_config(session, cfg));
 	WT_ERR(__wt_timing_stress_config(session, cfg));
 
-	WT_ERR(__wt_config_gets(session, cfg, "session_max", &cval));
-	conn->session_size = (uint32_t)cval.val + WT_EXTRA_INTERNAL_SESSIONS;
+	/* Set up operation tracking if configured. */
+	WT_ERR(__wt_conn_optrack_setup(session, cfg, false));
+
+	WT_ERR(__conn_session_size(session, cfg, &conn->session_size));
 
 	WT_ERR(__wt_config_gets(session, cfg, "session_scratch_max", &cval));
 	conn->session_scratch_max = (size_t)cval.val;
@@ -2619,6 +2682,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
 	WT_ERR(__conn_write_base_config(session, cfg));
 
 	/*
+	 * Calibrate the ratio of rdtsc ticks to nanoseconds.
+	 */
+	__conn_calibrate_ticks(session);
+
+	/*
 	 * Check on the turtle and metadata files, creating them if necessary
 	 * (which avoids application threads racing to create the metadata file
 	 * later).  Once the metadata file exists, get a reference to it in
diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c
index 8819bf8b970..c0bada4f827 100644
--- a/src/third_party/wiredtiger/src/conn/conn_log.c
+++ b/src/third_party/wiredtiger/src/conn/conn_log.c
@@ -865,12 +865,11 @@ err:		WT_PANIC_MSG(session, ret, "log wrlsn server error");
 static WT_THREAD_RET
 __log_server(void *arg)
 {
-	struct timespec start, now;
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
 	WT_LOG *log;
 	WT_SESSION_IMPL *session;
-	uint64_t timediff;
+	uint64_t time_start, time_stop, timediff;
 	bool did_work, signalled;
 
 	session = arg;
@@ -949,11 +948,11 @@ __log_server(void *arg)
 		}
 
 		/* Wait until the next event. */
-		__wt_epoch(session, &start);
+		time_start = __wt_rdtsc(session);
 		__wt_cond_auto_wait_signal(
 		    session, conn->log_cond, did_work, NULL, &signalled);
-		__wt_epoch(session, &now);
-		timediff = WT_TIMEDIFF_MS(now, start);
+		time_stop = __wt_rdtsc(session);
+		timediff = WT_TSCDIFF_MS(session, time_stop, time_start);
 	}
 
 	if (0) {
diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c
index e72fa5c00a4..7b40822f681 100644
--- a/src/third_party/wiredtiger/src/conn/conn_open.c
+++ b/src/third_party/wiredtiger/src/conn/conn_open.c
@@ -147,6 +147,13 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn)
 	if (conn->lock_fh != NULL)
 		WT_TRET(__wt_close(session, &conn->lock_fh));
 
+	/* Close any optrack files */
+	if (session->optrack_fh != NULL)
+		WT_TRET(__wt_close(session, &session->optrack_fh));
+
+	/* Close operation tracking */
+	WT_TRET(__wt_conn_optrack_teardown(session, false));
+
 	/* Close any file handles left open. */
 	WT_TRET(__wt_close_connection_close(session));
 
diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c
index 1bde68e1762..2cadc9b78d4 100644
--- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c
+++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c
@@ -57,6 +57,106 @@ __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg)
 }
 
 /*
+ * __wt_conn_optrack_setup --
+ *     Set up operation logging.
+ */
+int
+__wt_conn_optrack_setup(WT_SESSION_IMPL *session,
+    const char *cfg[], bool reconfig)
+{
+	WT_CONFIG_ITEM cval;
+	WT_CONNECTION_IMPL *conn;
+	WT_DECL_ITEM(buf);
+	WT_DECL_RET;
+
+	conn = S2C(session);
+
+	/* Once an operation tracking path has been set it can't be changed. */
+	if (!reconfig) {
+		WT_RET(__wt_config_gets(session,
+		    cfg, "operation_tracking.path", &cval));
+		WT_RET(__wt_strndup(session,
+		    cval.str, cval.len, &conn->optrack_path));
+	}
+
+	WT_RET(__wt_config_gets(session,
+	    cfg, "operation_tracking.enabled", &cval));
+	if (cval.val == 0) {
+		if (F_ISSET(conn, WT_CONN_OPTRACK)) {
+			WT_RET(__wt_conn_optrack_teardown(session, reconfig));
+			F_CLR(conn, WT_CONN_OPTRACK);
+		}
+		return (0);
+	} else if (F_ISSET(conn, WT_CONN_READONLY))
+		/* Operation tracking isn't supported in read-only mode */
+		WT_RET_MSG(session, EINVAL,
+		    "Operation tracking is incompatible with read only "
+		    "configuration.");
+	else if (F_ISSET(conn, WT_CONN_OPTRACK))
+		/* Already enabled, nothing else to do */
+		return (0);
+
+	/*
+	 * Operation tracking files will include the ID of the creating process
+	 * in their name, so we can distinguish between log files created by
+	 * different WiredTiger processes in the same directory. We cache the
+	 * process id for future use.
+	 */
+	conn->optrack_pid = __wt_process_id();
+
+	/*
+	 * Open the file in the same directory that will hold a map of
+	 * translations between function names and function IDs. If the file
+	 * exists, remove it.
+	 */
+	WT_RET(__wt_scr_alloc(session, 0, &buf));
+	WT_ERR(__wt_filename_construct(session, conn->optrack_path,
+	    "optrack-map", conn->optrack_pid, UINT32_MAX, buf));
+	WT_ERR(__wt_open(session,
+	    (const char *)buf->data, WT_FS_OPEN_FILE_TYPE_REGULAR,
+	    WT_FS_OPEN_CREATE, &conn->optrack_map_fh));
+
+	WT_ERR(__wt_spin_init(session,
+	    &conn->optrack_map_spinlock, "optrack map spinlock"));
+
+	WT_ERR(__wt_malloc(session, WT_OPTRACK_BUFSIZE,
+	    &conn->dummy_session.optrack_buf));
+
+	/* Set operation tracking on */
+	F_SET(conn, WT_CONN_OPTRACK);
+
+err:	__wt_scr_free(session, &buf);
+	return (ret);
+}
+
+/*
+ * __wt_conn_optrack_teardown --
+ *      Clean up connection-wide resources used for operation logging.
+ */
+int
+__wt_conn_optrack_teardown(WT_SESSION_IMPL *session, bool reconfig)
+{
+	WT_CONNECTION_IMPL *conn;
+	WT_DECL_RET;
+
+	conn = S2C(session);
+
+	if (!reconfig)
+		/* Looks like we are shutting down */
+		__wt_free(session, conn->optrack_path);
+
+	if (!F_ISSET(conn, WT_CONN_OPTRACK))
+		return (0);
+
+	__wt_spin_destroy(session, &conn->optrack_map_spinlock);
+
+	WT_TRET(__wt_close(session, &conn->optrack_map_fh));
+	__wt_free(session, conn->dummy_session.optrack_buf);
+
+	return (ret);
+}
+
+/*
  * __wt_conn_statistics_config --
  *	Set statistics configuration.
  */
@@ -189,6 +289,7 @@ __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg)
 	WT_WITH_CHECKPOINT_LOCK(session,
 	    ret = __wt_conn_compat_config(session, cfg));
 	WT_ERR(ret);
+	WT_ERR(__wt_conn_optrack_setup(session, cfg, true));
 	WT_ERR(__wt_conn_statistics_config(session, cfg));
 	WT_ERR(__wt_async_reconfig(session, cfg));
 	WT_ERR(__wt_cache_config(session, true, cfg));