WT-2309 Add option to cause delays in internal page split code to aid testing (#3531)

6 files changed, 47 insertions, 13 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index 726c79590e5..9fd043d5b25 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -526,9 +526,9 @@ connection_runtime_config = [
         This option is intended for use with internal stress
         testing of WiredTiger. Options are given as a list, such as
         <code>"timing_stress_for_test=[checkpoint_slow,
-            page_split_race]"</code>''',
+            internal_page_split_race, page_split_race]"</code>''',
         type='list', undoc=True, choices=[
-            'checkpoint_slow', 'page_split_race']),
+            'checkpoint_slow', 'internal_page_split_race', 'page_split_race']),
     Config('verbose', '', r'''
         enable messages for various events. Only available if WiredTiger
         is configured with --enable-verbose. Options are given as a
diff --git a/dist/flags.py b/dist/flags.py
index 1c3dd67aba6..48952768c18 100644
--- a/dist/flags.py
+++ b/dist/flags.py
@@ -47,6 +47,7 @@ flags = {
     ],
     'timing_stress_for_test' : [
         'TIMING_STRESS_CHECKPOINT_SLOW',
+        'TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE',
         'TIMING_STRESS_PAGE_SPLIT_RACE',
     ],
     'txn_log_checkpoint' : [
diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c
index c0c3ed5c9ff..2c0e247ce06 100644
--- a/src/btree/bt_split.c
+++ b/src/btree/bt_split.c
@@ -37,16 +37,16 @@ typedef enum {
  *  the race conditions in page split.
  */
 static void
-__page_split_timing_stress(WT_SESSION_IMPL *session)
+__page_split_timing_stress(WT_SESSION_IMPL *session,
+    uint32_t flag, uint64_t micro_seconds)
 {
 	WT_CONNECTION_IMPL *conn;
 
 	conn = S2C(session);
 
 	/* We only want to sleep when page split race flag is set. */
-	if (FLD_ISSET(conn->timing_stress_flags,
-	    WT_TIMING_STRESS_PAGE_SPLIT_RACE))
-		__wt_sleep(0, WT_THOUSAND);
+	if (FLD_ISSET(conn->timing_stress_flags, flag))
+		__wt_sleep(0, micro_seconds);
 
 }
 
@@ -532,6 +532,10 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root)
 	__split_ref_prepare(session, alloc_index,
 	    __wt_session_gen(session, WT_GEN_SPLIT), false);
 
+	/* Encourage a race */
+	__page_split_timing_stress(session,
+	    WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE, 100 * WT_THOUSAND);
+
 	/*
 	 * Confirm the root page's index hasn't moved, then update it, which
 	 * makes the split visible to threads descending the tree.
@@ -540,6 +544,10 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root)
 	WT_INTL_INDEX_SET(root, alloc_index);
 	alloc_index = NULL;
 
+	/* Encourage a race */
+	__page_split_timing_stress(session,
+	    WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE, 100 * WT_THOUSAND);
+
 	WT_LEAVE_PAGE_INDEX(session);
 
 	/*
@@ -720,6 +728,10 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
 	/* Start making real changes to the tree, errors are fatal. */
 	complete = WT_ERR_PANIC;
 
+	/* Encourage a race */
+	__page_split_timing_stress(session,
+	    WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE, 100 * WT_THOUSAND);
+
 	/*
 	 * Confirm the parent page's index hasn't moved then update it, which
 	 * makes the split visible to threads descending the tree.
@@ -728,6 +740,10 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
 	WT_INTL_INDEX_SET(parent, alloc_index);
 	alloc_index = NULL;
 
+	/* Encourage a race */
+	__page_split_timing_stress(session,
+	    WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE, 100 * WT_THOUSAND);
+
 	/*
 	 * Get a generation for this split, mark the page.  This must be after
 	 * the new index is swapped into place in order to know that no readers
@@ -1061,6 +1077,10 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page)
 	__split_ref_prepare(session, alloc_index,
 	    __wt_session_gen(session, WT_GEN_SPLIT), true);
 
+	/* Encourage a race */
+	__page_split_timing_stress(session,
+	    WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE, 100 * WT_THOUSAND);
+
 	/* Split into the parent. */
 	if ((ret = __split_parent(session, page_ref, alloc_index->index,
 	    alloc_index->entries, parent_incr, false, false)) == 0) {
@@ -1075,6 +1095,10 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page)
 	WT_LEAVE_PAGE_INDEX(session);
 	WT_ERR(ret);
 
+	/* Encourage a race */
+	__page_split_timing_stress(session,
+	    WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE, 100 * WT_THOUSAND);
+
 	/*
 	 * Get a generation for this split, mark the parent page.  This must be
 	 * after the new index is swapped into place in order to know that no
@@ -1186,7 +1210,8 @@ __split_internal_lock_worker(WT_SESSION_IMPL *session,
 	for (;;) {
 		parent = ref->home;
 
-		__page_split_timing_stress(session);
+		__page_split_timing_stress(session,
+		    WT_TIMING_STRESS_PAGE_SPLIT_RACE, WT_THOUSAND);
 		/*
 		 * The page will be marked dirty, and we can only lock a page
 		 * with a modify structure.
diff --git a/src/config/config_def.c b/src/config/config_def.c
index 9106385bf9e..2fca9dcf69f 100644
--- a/src/config/config_def.c
+++ b/src/config/config_def.c
@@ -162,7 +162,8 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = {
 	    NULL, NULL,
 	    confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"page_split_race\"]",
+	    NULL, "choices=[\"checkpoint_slow\",\"internal_page_split_race\""
+	    ",\"page_split_race\"]",
 	    NULL, 0 },
 	{ "verbose", "list",
 	    NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\","
@@ -785,7 +786,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_statistics_log_subconfigs, 6 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"page_split_race\"]",
+	    NULL, "choices=[\"checkpoint_slow\",\"internal_page_split_race\""
+	    ",\"page_split_race\"]",
 	    NULL, 0 },
 	{ "transaction_sync", "category",
 	    NULL, NULL,
@@ -879,7 +881,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_statistics_log_subconfigs, 6 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"page_split_race\"]",
+	    NULL, "choices=[\"checkpoint_slow\",\"internal_page_split_race\""
+	    ",\"page_split_race\"]",
 	    NULL, 0 },
 	{ "transaction_sync", "category",
 	    NULL, NULL,
@@ -970,7 +973,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_statistics_log_subconfigs, 6 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"page_split_race\"]",
+	    NULL, "choices=[\"checkpoint_slow\",\"internal_page_split_race\""
+	    ",\"page_split_race\"]",
 	    NULL, 0 },
 	{ "transaction_sync", "category",
 	    NULL, NULL,
@@ -1059,7 +1063,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_statistics_log_subconfigs, 6 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"page_split_race\"]",
+	    NULL, "choices=[\"checkpoint_slow\",\"internal_page_split_race\""
+	    ",\"page_split_race\"]",
 	    NULL, 0 },
 	{ "transaction_sync", "category",
 	    NULL, NULL,
diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c
index ccb1abff6ab..ded0e39b218 100644
--- a/src/conn/conn_api.c
+++ b/src/conn/conn_api.c
@@ -1943,6 +1943,8 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[])
 {
 	static const WT_NAME_FLAG stress_types[] = {
 		{ "checkpoint_slow",	WT_TIMING_STRESS_CHECKPOINT_SLOW },
+		{ "internal_page_split_race",
+		    WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE },
 		{ "page_split_race",	WT_TIMING_STRESS_PAGE_SPLIT_RACE },
 		{ NULL, 0 }
 	};
diff --git a/src/include/flags.h b/src/include/flags.h
index 2f4d1c2ed20..4f7b59c7849 100644
--- a/src/include/flags.h
+++ b/src/include/flags.h
@@ -79,7 +79,8 @@
 #define	WT_STAT_TYPE_SIZE				0x00000040
 #define	WT_STAT_TYPE_TREE_WALK				0x00000080
 #define	WT_TIMING_STRESS_CHECKPOINT_SLOW		0x00000001
-#define	WT_TIMING_STRESS_PAGE_SPLIT_RACE		0x00000002
+#define	WT_TIMING_STRESS_INTERNAL_PAGE_SPLIT_RACE	0x00000002
+#define	WT_TIMING_STRESS_PAGE_SPLIT_RACE		0x00000004
 #define	WT_TXN_LOG_CKPT_CLEANUP				0x00000001
 #define	WT_TXN_LOG_CKPT_PREPARE				0x00000002
 #define	WT_TXN_LOG_CKPT_START				0x00000004