WT-2402 Pad structures to avoid cache line sharing. (#3191)

* WT-2402 Pad structures to avoid cache line sharing.

We previously used compiler alignment to avoid cache line sharing, which
is problematic because we don't allocate memory with the necessary
alignment, so some behavior is undefined.  Instead, this change merely
pads structs to ensure they won't share a cache line when allocated in
an array.

* Add some #if barriers around the new union's

* Change where CACHE_LINE_PAD defines live and rename _START to _BEGIN

* Add ax_check_compile_flag to aclocal

* Lint

* Allocate the TINFO structures separately to minimize false sharing.

* Comment cleanup to make spell happy.

* Pad the read-write locks to avoid false sharing.

* Add a comment as to why we're adding -std=c11 to the compile.

* whitespace

* Add asserts that we're padding the structures we intended to pad.

Minor cleanup, don't use #defines outside of the WiredTiger namespace.

18 files changed, 156 insertions, 54 deletions

diff --git a/build_posix/aclocal/ax_check_compile_flag.m4 b/build_posix/aclocal/ax_check_compile_flag.m4
new file mode 100644
index 00000000000..ca3639715e7
--- /dev/null
+++ b/build_posix/aclocal/ax_check_compile_flag.m4
@@ -0,0 +1,74 @@
+# ===========================================================================
+#   http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT])
+#
+# DESCRIPTION
+#
+#   Check whether the given FLAG works with the current language's compiler
+#   or gives an error.  (Warnings, however, are ignored)
+#
+#   ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on
+#   success/failure.
+#
+#   If EXTRA-FLAGS is defined, it is added to the current language's default
+#   flags (e.g. CFLAGS) when the check is done.  The check is thus made with
+#   the flags: "CFLAGS EXTRA-FLAGS FLAG".  This can for example be used to
+#   force the compiler to issue an error when a bad flag is given.
+#
+#   INPUT gives an alternative input source to AC_COMPILE_IFELSE.
+#
+#   NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this
+#   macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de>
+#   Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com>
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+#   Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#   As a special exception, the respective Autoconf Macro's copyright owner
+#   gives unlimited permission to copy, distribute and modify the configure
+#   scripts that are the output of Autoconf when processing the Macro. You
+#   need not follow the terms of the GNU General Public License when using
+#   or distributing such scripts, even though portions of the text of the
+#   Macro appear in them. The GNU General Public License (GPL) does govern
+#   all other use of the material that constitutes the Autoconf Macro.
+#
+#   This special exception to the GPL applies to versions of the Autoconf
+#   Macro released by the Autoconf Archive. When you make and distribute a
+#   modified version of the Autoconf Macro, you may extend this special
+#   exception to the GPL to apply to your modified version as well.
+
+#serial 4
+
+AC_DEFUN([AX_CHECK_COMPILE_FLAG],
+[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF
+AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl
+AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [
+  ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS
+  _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1"
+  AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])],
+    [AS_VAR_SET(CACHEVAR,[yes])],
+    [AS_VAR_SET(CACHEVAR,[no])])
+  _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags])
+AS_VAR_IF(CACHEVAR,yes,
+  [m4_default([$2], :)],
+  [m4_default([$3], :)])
+AS_VAR_POPDEF([CACHEVAR])dnl
+])dnl AX_CHECK_COMPILE_FLAGS
diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in
index ad00b19a3bb..c6258f38f01 100644
--- a/build_posix/configure.ac.in
+++ b/build_posix/configure.ac.in
@@ -22,6 +22,10 @@ AC_PROG_CC(cc gcc)
 AC_PROG_CXX(c++ g++)
 AM_PROG_AS(as gas)
 
+# WiredTiger uses anonymous unions to pad structures. It's part of C11, but
+# some compilers require a flag to support them.
+AX_CHECK_COMPILE_FLAG([-std=c11], [AM_CFLAGS="$AM_CFLAGS -std=c11"])
+
 # Configure options.
 AM_OPTIONS
 
diff --git a/dist/s_define.list b/dist/s_define.list
index 372a251bf01..53a3df87615 100644
--- a/dist/s_define.list
+++ b/dist/s_define.list
@@ -1,5 +1,4 @@
 # List of WiredTiger #defines that are "unused", but it's OK.
-ALIGN_CHECK
 API_CALL
 API_CALL_NOCONF
 API_SESSION_INIT
@@ -8,17 +7,18 @@ JOINABLE_CURSOR_CALL_CHECK
 LF_MASK
 LLONG_MAX
 LLONG_MIN
-SIZE_CHECK
 TXN_API_CALL
 TXN_API_CALL_NOCONF
 TXN_API_END
 WIN32_LEAN_AND_MEAN
+WT_ALIGN_CHECK
 WT_ATOMIC_CAS
 WT_ATOMIC_FUNC
 WT_BLOCK_DESC_SIZE
 WT_BLOCK_HEADER_SIZE
 WT_CACHE_LINE_ALIGNMENT
-WT_COMPILER_TYPE_ALIGN
+WT_CACHE_LINE_PAD_BEGIN
+WT_CACHE_LINE_PAD_END
 WT_CONN_CHECK_PANIC
 WT_DEADLOCK
 WT_DEBUG_BYTE
@@ -35,10 +35,12 @@ WT_LOG_SLOT_MAXBITS
 WT_LOG_SLOT_UNBUFFERED_ISSET
 WT_PACKED_STRUCT_BEGIN
 WT_PACKED_STRUCT_END
+WT_PADDING_CHECK
 WT_READ_BARRIER
 WT_REF_SIZE
 WT_SESSION_LOCKED_CHECKPOINT
 WT_SESSION_LOCKED_TURTLE
+WT_SIZE_CHECK
 WT_STATS_FIELD_TO_OFFSET
 WT_STATS_SLOT_ID
 WT_STAT_CONN_DECRV
diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c
index 3571cc60115..02182daa7dc 100644
--- a/src/conn/conn_handle.c
+++ b/src/conn/conn_handle.c
@@ -68,7 +68,6 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn)
 	    session, &conn->hot_backup_lock, "hot backup"));
 
 	WT_RET(__wt_calloc_def(session, WT_PAGE_LOCKS, &conn->page_lock));
-	WT_CACHE_LINE_ALIGNMENT_VERIFY(session, conn->page_lock);
 	for (i = 0; i < WT_PAGE_LOCKS; ++i)
 		WT_RET(
 		    __wt_spin_init(session, &conn->page_lock[i], "btree page"));
diff --git a/src/conn/conn_open.c b/src/conn/conn_open.c
index 6454503d6cb..d4ace127bb2 100644
--- a/src/conn/conn_open.c
+++ b/src/conn/conn_open.c
@@ -30,7 +30,6 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[])
 	/* WT_SESSION_IMPL array. */
 	WT_RET(__wt_calloc(session,
 	    conn->session_size, sizeof(WT_SESSION_IMPL), &conn->sessions));
-	WT_CACHE_LINE_ALIGNMENT_VERIFY(session, conn->sessions);
 
 	/*
 	 * Open the default session.  We open this before starting service
diff --git a/src/include/gcc.h b/src/include/gcc.h
index ce6afdd6e9c..22d78fc165a 100644
--- a/src/include/gcc.h
+++ b/src/include/gcc.h
@@ -10,8 +10,6 @@
 #define	WT_SIZET_FMT	"zu"			/* size_t format string */
 
 /* Add GCC-specific attributes to types and function declarations. */
-#define	WT_COMPILER_TYPE_ALIGN(x)	__attribute__((aligned(x)))
-
 #define	WT_PACKED_STRUCT_BEGIN(name)					\
 	struct __attribute__ ((__packed__)) name {
 #define	WT_PACKED_STRUCT_END						\
diff --git a/src/include/hardware.h b/src/include/hardware.h
index 0e52818ae05..2530659db21 100644
--- a/src/include/hardware.h
+++ b/src/include/hardware.h
@@ -55,7 +55,19 @@
 #else
 #define	WT_CACHE_LINE_ALIGNMENT	64
 #endif
-#define	WT_CACHE_LINE_ALIGNMENT_VERIFY(session, a)			\
-	WT_ASSERT(session,						\
-	    WT_PTRDIFF(&(a)[1], &(a)[0]) >= WT_CACHE_LINE_ALIGNMENT &&	\
-	    WT_PTRDIFF(&(a)[1], &(a)[0]) % WT_CACHE_LINE_ALIGNMENT == 0)
+
+/*
+ * Pad a structure so an array of structures get separate cache lines.
+ *
+ * Note that we avoid compiler structure alignment because that requires
+ * allocating aligned blocks of memory, and alignment pollutes any other type
+ * that contains an aligned field.  It is possible that a hot field positioned
+ * before this one will be on the same cache line, but not if it is also
+ * padded.
+ *
+ * This alignment has a small impact on portability as well, as we are using an
+ * anonymous union here which is supported under C11, earlier versions of
+ * the GNU standard, and MSVC versions as early as 2003.
+ */
+#define	WT_CACHE_LINE_PAD_BEGIN union { struct {
+#define	WT_CACHE_LINE_PAD_END   }; char __padding[WT_CACHE_LINE_ALIGNMENT]; };
diff --git a/src/include/lint.h b/src/include/lint.h
index 1b64186cbab..e20a83144ee 100644
--- a/src/include/lint.h
+++ b/src/include/lint.h
@@ -9,8 +9,6 @@
 #define	WT_PTRDIFFT_FMT	"td"			/* ptrdiff_t format string */
 #define	WT_SIZET_FMT	"zu"			/* size_t format string */
 
-#define	WT_COMPILER_TYPE_ALIGN(x)
-
 #define	WT_PACKED_STRUCT_BEGIN(name)					\
 	struct name {
 #define	WT_PACKED_STRUCT_END						\
diff --git a/src/include/log.h b/src/include/log.h
index 6272b63b8a9..3f2cb2ba8e6 100644
--- a/src/include/log.h
+++ b/src/include/log.h
@@ -159,7 +159,8 @@ union __wt_lsn {
     !FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) &&		\
     WT_LOG_SLOT_JOINED(state) < WT_LOG_SLOT_BUF_MAX)
 
-struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_logslot {
+struct __wt_logslot {
+	WT_CACHE_LINE_PAD_BEGIN
 	volatile int64_t slot_state;	/* Slot state */
 	int64_t	 slot_unbuffered;	/* Unbuffered data in this slot */
 	int32_t	 slot_error;		/* Error value */
@@ -176,6 +177,7 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_logslot {
 #define	WT_SLOT_SYNC		0x04		/* Needs sync on release */
 #define	WT_SLOT_SYNC_DIR	0x08		/* Directory sync on release */
 	uint32_t flags;			/* Flags */
+	WT_CACHE_LINE_PAD_END
 };
 
 #define	WT_SLOT_INIT_FLAGS	0
diff --git a/src/include/msvc.h b/src/include/msvc.h
index d5be5bd8c60..6c5c8b67647 100644
--- a/src/include/msvc.h
+++ b/src/include/msvc.h
@@ -19,8 +19,6 @@
 /*
  * Add MSVC-specific attributes and pragmas to types and function declarations.
  */
-#define	WT_COMPILER_TYPE_ALIGN(x)	__declspec(align(x))
-
 #define	WT_PACKED_STRUCT_BEGIN(name)					\
 	__pragma(pack(push,1))						\
 	struct name {
diff --git a/src/include/mutex.h b/src/include/mutex.h
index 84f015d6b67..6b81b1a6265 100644
--- a/src/include/mutex.h
+++ b/src/include/mutex.h
@@ -53,9 +53,11 @@ typedef union {				/* Read/write lock */
  * WiredTiger uses read/write locks for shared/exclusive access to resources.
  */
 struct __wt_rwlock {
+	WT_CACHE_LINE_PAD_BEGIN
 	const char *name;		/* Lock name for debugging */
 
 	wt_rwlock_t rwlock;		/* Read/write lock */
+	WT_CACHE_LINE_PAD_END
 };
 
 /*
@@ -72,7 +74,8 @@ struct __wt_rwlock {
 
 #if SPINLOCK_TYPE == SPINLOCK_GCC
 
-struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock {
+struct __wt_spinlock {
+	WT_CACHE_LINE_PAD_BEGIN
 	volatile int lock;
 
 	/*
@@ -84,13 +87,15 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock {
 	int16_t stat_count_off;		/* acquisitions offset */
 	int16_t stat_app_usecs_off;	/* waiting application threads offset */
 	int16_t stat_int_usecs_off;	/* waiting server threads offset */
+	WT_CACHE_LINE_PAD_END
 };
 
 #elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX ||\
 	SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE ||\
 	SPINLOCK_TYPE == SPINLOCK_MSVC
 
-struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock {
+struct __wt_spinlock {
+	WT_CACHE_LINE_PAD_BEGIN
 	wt_mutex_t lock;
 
 	const char *name;		/* Mutex name */
@@ -106,6 +111,7 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock {
 	int16_t stat_int_usecs_off;	/* waiting server threads offset */
 
 	int8_t initialized;		/* Lock initialized, for cleanup */
+	WT_CACHE_LINE_PAD_END
 };
 
 #else
diff --git a/src/include/session.h b/src/include/session.h
index 7e855a3db25..7dd523aea26 100644
--- a/src/include/session.h
+++ b/src/include/session.h
@@ -41,7 +41,7 @@ struct __wt_hazard {
  * WT_SESSION_IMPL --
  *	Implementation of WT_SESSION.
  */
-struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl {
+struct __wt_session_impl {
 	WT_SESSION iface;
 
 	void	*lang_private;		/* Language specific private storage */
diff --git a/src/include/txn.h b/src/include/txn.h
index afe5a1965c4..12fc2a0a5b7 100644
--- a/src/include/txn.h
+++ b/src/include/txn.h
@@ -67,10 +67,12 @@ struct __wt_named_snapshot {
 	uint32_t snapshot_count;
 };
 
-struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_txn_state {
+struct __wt_txn_state {
+	WT_CACHE_LINE_PAD_BEGIN
 	volatile uint64_t id;
 	volatile uint64_t pinned_id;
 	volatile uint64_t metadata_pinned;
+	WT_CACHE_LINE_PAD_END
 };
 
 struct __wt_txn_global {
diff --git a/src/include/verify_build.h b/src/include/verify_build.h
index 477b9b7c134..8abc192892e 100644
--- a/src/include/verify_build.h
+++ b/src/include/verify_build.h
@@ -6,9 +6,6 @@
  * See the file LICENSE for redistribution information.
  */
 
-#undef ALIGN_CHECK
-#undef SIZE_CHECK
-
 /*
  * NOTE: If you see a compile failure in this file, your compiler is laying out
  * structs in memory in a way WiredTiger does not expect.  Please refer to the
@@ -36,12 +33,12 @@
  */
 #define	WT_STATIC_ASSERT(cond)	(void)sizeof(char[1 - 2 * !(cond)])
 
-#define	SIZE_CHECK(type, e)	do {					\
+#define	WT_SIZE_CHECK(type, e)	do {					\
 	char __check_##type[1 - 2 * !(sizeof(type) == (e))];		\
 	(void)__check_##type;						\
 } while (0)
 
-#define	ALIGN_CHECK(type, a)						\
+#define	WT_ALIGN_CHECK(type, a)						\
 	WT_STATIC_ASSERT(WT_ALIGN(sizeof(type), (a)) == sizeof(type))
 
 /*
@@ -53,8 +50,18 @@ static inline void
 __wt_verify_build(void)
 {
 	/* Check specific structures weren't padded. */
-	SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE);
-	SIZE_CHECK(WT_REF, WT_REF_SIZE);
+	WT_SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE);
+	WT_SIZE_CHECK(WT_REF, WT_REF_SIZE);
+
+	/* Check specific structures were padded. */
+#define	WT_PADDING_CHECK(s)						\
+	WT_STATIC_ASSERT(						\
+	    sizeof(s) > WT_CACHE_LINE_ALIGNMENT ||			\
+	    sizeof(s) % WT_CACHE_LINE_ALIGNMENT == 0)
+	WT_PADDING_CHECK(WT_LOGSLOT);
+	WT_PADDING_CHECK(WT_RWLOCK);
+	WT_PADDING_CHECK(WT_SPINLOCK);
+	WT_PADDING_CHECK(WT_TXN_STATE);
 
 	/*
 	 * The btree code encodes key/value pairs in size_t's, and requires at
@@ -71,6 +78,3 @@ __wt_verify_build(void)
 	 */
 	WT_STATIC_ASSERT(sizeof(wt_off_t) == 8);
 }
-
-#undef ALIGN_CHECK
-#undef SIZE_CHECK
diff --git a/src/log/log_slot.c b/src/log/log_slot.c
index 3ad6814a5d5..a29a34e5652 100644
--- a/src/log/log_slot.c
+++ b/src/log/log_slot.c
@@ -283,7 +283,6 @@ __wt_log_slot_init(WT_SESSION_IMPL *session)
 
 	conn = S2C(session);
 	log = conn->log;
-	WT_CACHE_LINE_ALIGNMENT_VERIFY(session, log->slot_pool);
 	for (i = 0; i < WT_SLOT_POOL; i++)
 		log->slot_pool[i].slot_state = WT_LOG_SLOT_FREE;
 
diff --git a/src/txn/txn.c b/src/txn/txn.c
index a70551cdeb2..26a0ed679e2 100644
--- a/src/txn/txn.c
+++ b/src/txn/txn.c
@@ -777,7 +777,6 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[])
 
 	WT_RET(__wt_calloc_def(
 	    session, conn->session_size, &txn_global->states));
-	WT_CACHE_LINE_ALIGNMENT_VERIFY(session, txn_global->states);
 
 	for (i = 0, s = txn_global->states; i < conn->session_size; i++, s++)
 		s->id = s->metadata_pinned = s->pinned_id = WT_TXN_NONE;
diff --git a/test/format/format.h b/test/format/format.h
index 820bc020c9b..530156fe661 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -242,7 +242,7 @@ typedef struct {
 } GLOBAL;
 extern GLOBAL g;
 
-typedef struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) {
+typedef struct {
 	WT_RAND_STATE rnd;			/* thread RNG state */
 
 	uint64_t search;			/* operations */
diff --git a/test/format/ops.c b/test/format/ops.c
index 283e2912daa..b50ce3c2e2f 100644
--- a/test/format/ops.c
+++ b/test/format/ops.c
@@ -50,7 +50,7 @@ static void  print_item(const char *, WT_ITEM *);
 void
 wts_ops(int lastrun)
 {
-	TINFO *tinfo, total;
+	TINFO **tinfo_list, *tinfo, total;
 	WT_CONNECTION *conn;
 	WT_SESSION *session;
 	pthread_t backup_tid, compact_tid, lrt_tid;
@@ -102,13 +102,16 @@ wts_ops(int lastrun)
 		    "=============== thread ops start ===============");
 	}
 
-	/* Create thread structure; start the worker threads. */
-	tinfo = dcalloc((size_t)g.c_threads, sizeof(*tinfo));
+	/*
+	 * Create the per-thread structures and start the worker threads.
+	 * Allocate the thread structures separately to minimize false sharing.
+	 */
+	tinfo_list = dcalloc((size_t)g.c_threads, sizeof(TINFO *));
 	for (i = 0; i < g.c_threads; ++i) {
-		tinfo[i].id = (int)i + 1;
-		tinfo[i].state = TINFO_RUNNING;
-		testutil_check(
-		    pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i]));
+		tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO));
+		tinfo->id = (int)i + 1;
+		tinfo->state = TINFO_RUNNING;
+		testutil_check(pthread_create(&tinfo->tid, NULL, ops, tinfo));
 	}
 
 	/*
@@ -128,21 +131,22 @@ wts_ops(int lastrun)
 		/* Clear out the totals each pass. */
 		memset(&total, 0, sizeof(total));
 		for (i = 0, running = 0; i < g.c_threads; ++i) {
-			total.commit += tinfo[i].commit;
-			total.deadlock += tinfo[i].deadlock;
-			total.insert += tinfo[i].insert;
-			total.remove += tinfo[i].remove;
-			total.rollback += tinfo[i].rollback;
-			total.search += tinfo[i].search;
-			total.update += tinfo[i].update;
-
-			switch (tinfo[i].state) {
+			tinfo = tinfo_list[i];
+			total.commit += tinfo->commit;
+			total.deadlock += tinfo->deadlock;
+			total.insert += tinfo->insert;
+			total.remove += tinfo->remove;
+			total.rollback += tinfo->rollback;
+			total.search += tinfo->search;
+			total.update += tinfo->update;
+
+			switch (tinfo->state) {
 			case TINFO_RUNNING:
 				running = 1;
 				break;
 			case TINFO_COMPLETE:
-				tinfo[i].state = TINFO_JOINED;
-				(void)pthread_join(tinfo[i].tid, NULL);
+				tinfo->state = TINFO_JOINED;
+				(void)pthread_join(tinfo->tid, NULL);
 				break;
 			case TINFO_JOINED:
 				break;
@@ -154,7 +158,7 @@ wts_ops(int lastrun)
 			 */
 			if (fourths == 0 ||
 			    (thread_ops != -1 &&
-			    tinfo[i].ops >= (uint64_t)thread_ops)) {
+			    tinfo->ops >= (uint64_t)thread_ops)) {
 				/*
 				 * On the last execution, optionally drop core
 				 * for recovery testing.
@@ -163,7 +167,7 @@ wts_ops(int lastrun)
 					static char *core = NULL;
 					*core = 0;
 				}
-				tinfo[i].quit = 1;
+				tinfo->quit = 1;
 			}
 		}
 		track("ops", 0ULL, &total);
@@ -173,7 +177,9 @@ wts_ops(int lastrun)
 		if (fourths != -1)
 			--fourths;
 	}
-	free(tinfo);
+	for (i = 0; i < g.c_threads; ++i)
+		free(tinfo_list[i]);
+	free(tinfo_list);
 
 	/* Wait for the backup, compaction, long-running reader threads. */
 	g.workers_finished = 1;