WT-2615 Enabling checkpoints in test/format leads to reduced concurrency (#2720)

Don't queue threads for checkpoint operations, just skip the scheduled
checkpoint operation if a checkpoint is already in progress.

Don't wait for backups to complete if scheduled to do a named checkpoint,
do an unnamed checkpoint instead.

Fix a bug: we were scheduling named checkpoints 80% of the time instead
of the documented 20%; change that number to 5%, named checkpoints aren't
worth testing that heavily.

Don't bother avoiding scheduling two named checkpoints in a row, it's
not worth the effort.

4 files changed, 31 insertions, 14 deletions

diff --git a/test/format/backup.c b/test/format/backup.c
index cc4435ef688..69fdf771de9 100644
--- a/test/format/backup.c
+++ b/test/format/backup.c
@@ -118,7 +118,12 @@ backup(void *arg)
 			sleep(1);
 		}
 
-		/* Lock out named checkpoints */
+		/*
+		 * We can't drop named checkpoints while there's a backup in
+		 * progress, serialize backups with named checkpoints. Wait
+		 * for the checkpoint to complete, otherwise backups might be
+		 * starved out.
+		 */
 		testutil_check(pthread_rwlock_wrlock(&g.backup_lock));
 		if (g.workers_finished) {
 			testutil_check(pthread_rwlock_unlock(&g.backup_lock));
diff --git a/test/format/format.h b/test/format/format.h
index 5660b7a8423..beaabe7e83c 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -121,7 +121,8 @@ typedef struct {
 	int replay;				/* Replaying a run. */
 	int workers_finished;			/* Operations completed */
 
-	pthread_rwlock_t backup_lock;		/* Hot backup running */
+	pthread_rwlock_t backup_lock;		/* Backup running */
+	pthread_rwlock_t checkpoint_lock;	/* Checkpoint running */
 
 	WT_RAND_STATE rnd;			/* Global RNG state */
 
diff --git a/test/format/ops.c b/test/format/ops.c
index a336b3c6d50..9275d7f3856 100644
--- a/test/format/ops.c
+++ b/test/format/ops.c
@@ -504,14 +504,25 @@ ops(void *arg)
 		/* Checkpoint the database. */
 		if (tinfo->ops == ckpt_op && g.c_checkpoints) {
 			/*
-			 * LSM and data-sources don't support named checkpoints,
+			 * Checkpoints are single-threaded inside WiredTiger,
+			 * skip our checkpoint if another thread is already
+			 * doing one.
+			 */
+			ret = pthread_rwlock_trywrlock(&g.checkpoint_lock);
+			if (ret == EBUSY)
+				goto skip_checkpoint;
+			testutil_check(ret);
+
+			/*
+			 * LSM and data-sources don't support named checkpoints
 			 * and we can't drop a named checkpoint while there's a
-			 * cursor open on it, otherwise 20% of the time name the
-			 * checkpoint.
+			 * backup in progress, otherwise name the checkpoint 5%
+			 * of the time.
 			 */
-			if (DATASOURCE("helium") || DATASOURCE("kvsbdb") ||
-			    DATASOURCE("lsm") ||
-			    readonly || mmrand(&tinfo->rnd, 1, 5) == 1)
+			if (mmrand(&tinfo->rnd, 1, 20) != 1 ||
+			    DATASOURCE("helium") ||
+			    DATASOURCE("kvsbdb") || DATASOURCE("lsm") ||
+			    pthread_rwlock_trywrlock(&g.backup_lock) == EBUSY)
 				ckpt_config = NULL;
 			else {
 				(void)snprintf(ckpt_name, sizeof(ckpt_name),
@@ -519,11 +530,6 @@ ops(void *arg)
 				ckpt_config = ckpt_name;
 			}
 
-			/* Named checkpoints lock out backups */
-			if (ckpt_config != NULL)
-				testutil_check(
-				    pthread_rwlock_wrlock(&g.backup_lock));
-
 			testutil_checkfmt(
 			    session->checkpoint(session, ckpt_config),
 			    "%s", ckpt_config == NULL ? "" : ckpt_config);
@@ -531,6 +537,8 @@ ops(void *arg)
 			if (ckpt_config != NULL)
 				testutil_check(
 				    pthread_rwlock_unlock(&g.backup_lock));
+			testutil_check(
+			    pthread_rwlock_unlock(&g.checkpoint_lock));
 
 			/* Rephrase the checkpoint name for cursor open. */
 			if (ckpt_config == NULL)
@@ -541,7 +549,7 @@ ops(void *arg)
 				    "checkpoint=thread-%d", tinfo->id);
 			ckpt_available = true;
 
-			/* Pick the next checkpoint operation. */
+skip_checkpoint:	/* Pick the next checkpoint operation. */
 			ckpt_op += mmrand(&tinfo->rnd, 5000, 20000);
 		}
 
diff --git a/test/format/t.c b/test/format/t.c
index b4f0fffda54..085163befe2 100644
--- a/test/format/t.c
+++ b/test/format/t.c
@@ -181,6 +181,7 @@ main(int argc, char *argv[])
 	 */
 	testutil_check(pthread_rwlock_init(&g.append_lock, NULL));
 	testutil_check(pthread_rwlock_init(&g.backup_lock, NULL));
+	testutil_check(pthread_rwlock_init(&g.checkpoint_lock, NULL));
 	testutil_check(pthread_rwlock_init(&g.death_lock, NULL));
 
 	printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid());
@@ -275,6 +276,8 @@ main(int argc, char *argv[])
 
 	testutil_check(pthread_rwlock_destroy(&g.append_lock));
 	testutil_check(pthread_rwlock_destroy(&g.backup_lock));
+	testutil_check(pthread_rwlock_destroy(&g.checkpoint_lock));
+	testutil_check(pthread_rwlock_destroy(&g.death_lock));
 
 	config_clear();