Import wiredtiger: 465b66827f105c184d317979d1b7c437c5ba6391 from branch mongodb-master

ref: b86eb4c244..465b66827f
for: 5.1.0

WT-7958       Include recovery in test/checkpoint

5 files changed, 139 insertions, 42 deletions

diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c
index a2445225e2e..2c32277c27a 100644
--- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c
+++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c
@@ -33,7 +33,6 @@ static WT_THREAD_RET clock_thread(void *);
 static int compare_cursors(WT_CURSOR *, const char *, WT_CURSOR *, const char *);
 static int diagnose_key_error(WT_CURSOR *, int, WT_CURSOR *, int);
 static int real_checkpointer(void);
-static int verify_consistency(WT_SESSION *, char *);
 
 /*
  * set_stable --
@@ -44,7 +43,11 @@ set_stable(void)
 {
     char buf[128];
 
-    testutil_check(__wt_snprintf(buf, sizeof(buf), "stable_timestamp=%x", g.ts_stable));
+    if (g.race_timetamps)
+        testutil_check(__wt_snprintf(
+          buf, sizeof(buf), "stable_timestamp=%x,oldest_timestamp=%x", g.ts_stable, g.ts_stable));
+    else
+        testutil_check(__wt_snprintf(buf, sizeof(buf), "stable_timestamp=%x", g.ts_stable));
     testutil_check(g.conn->set_timestamp(g.conn, buf));
 }
 
@@ -97,7 +100,14 @@ clock_thread(void *arg)
 
     while (g.running) {
         __wt_writelock(session, &g.clock_lock);
-        ++g.ts_stable;
+        if (g.prepare)
+            /*
+             * Leave a gap between timestamps so prepared insert followed by remove don't overlap
+             * with stable timestamp.
+             */
+            g.ts_stable += 5;
+        else
+            ++g.ts_stable;
         set_stable();
         if (g.ts_stable % 997 == 0) {
             /*
@@ -229,7 +239,7 @@ done:
  *     Open a cursor on each table at the last checkpoint and walk through the tables in parallel.
  *     The key/values should match across all tables.
  */
-static int
+int
 verify_consistency(WT_SESSION *session, char *stable_timestamp)
 {
     WT_CURSOR **cursors;
@@ -245,8 +255,8 @@ verify_consistency(WT_SESSION *session, char *stable_timestamp)
         return (log_print_err("verify_consistency", ENOMEM, 1));
 
     if (stable_timestamp != NULL) {
-        testutil_check(__wt_snprintf(
-          cfg_buf, sizeof(cfg_buf), "isolation=snapshot,read_timestamp=%s", stable_timestamp));
+        testutil_check(__wt_snprintf(cfg_buf, sizeof(cfg_buf),
+          "isolation=snapshot,read_timestamp=%s,roundup_timestamps=read", stable_timestamp));
     } else {
         testutil_check(__wt_snprintf(cfg_buf, sizeof(cfg_buf), "isolation=snapshot"));
     }
@@ -267,13 +277,19 @@ verify_consistency(WT_SESSION *session, char *stable_timestamp)
     }
 
     while (ret == 0) {
-        ret = cursors[0]->next(cursors[0]);
+        while ((ret = cursors[0]->next(cursors[0])) != 0) {
+            if (ret == WT_NOTFOUND)
+                break;
+            if (ret != WT_PREPARE_CONFLICT) {
+                (void)log_print_err("cursor->next", ret, 1);
+                goto err;
+            }
+            __wt_yield();
+        }
+
         if (ret == 0)
             ++key_count;
-        else if (ret != WT_NOTFOUND) {
-            (void)log_print_err("cursor->next", ret, 1);
-            goto err;
-        }
+
         /*
          * Check to see that all remaining cursors have the same key/value pair.
          */
@@ -283,10 +299,14 @@ verify_consistency(WT_SESSION *session, char *stable_timestamp)
              */
             if (g.cookies[i].type == LSM)
                 continue;
-            t_ret = cursors[i]->next(cursors[i]);
-            if (t_ret != 0 && t_ret != WT_NOTFOUND) {
-                (void)log_print_err("cursor->next", t_ret, 1);
-                goto err;
+            while ((t_ret = cursors[i]->next(cursors[i])) != 0) {
+                if (t_ret == WT_NOTFOUND)
+                    break;
+                if (t_ret != WT_PREPARE_CONFLICT) {
+                    (void)log_print_err("cursor->next", t_ret, 1);
+                    goto err;
+                }
+                __wt_yield();
             }
 
             if (ret == WT_NOTFOUND && t_ret == WT_NOTFOUND)
diff --git a/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh b/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh
new file mode 100755
index 00000000000..2ec1df3e69d
--- /dev/null
+++ b/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -x
+
+home=${1:-WT_TEST}
+backup=$home.backup
+recovery=$home.recovery
+
+#./t -t r -W 3 -D -X -n 100000 -k 100000 -C cache_size=100MB -h $home > $home.out 2>&1 &
+./t -t r -W 3 -D -n 100000 -k 100000 -C cache_size=100MB -h $home > $home.out 2>&1 &
+pid=$!
+
+trap "kill -9 $pid" 0 1 2 3 13 15
+
+# Wait for the test to start running
+while ! grep -q "Finished a checkpoint" $home.out ; do
+	sleep 1
+done
+
+while kill -STOP $pid ; do
+	rm -rf $backup $recovery ; mkdir $backup ; mkdir $recovery
+	# Make sure all threads are stopped before copying files
+	sleep 1
+	cp $home/* $backup
+	kill -CONT $pid
+	cp $backup/* $recovery
+	./t -t r -D -v -h $recovery || exit 1
+done
+
+exit 0
diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
index 1d4c99a2b03..0ead6293038 100644
--- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
+++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
@@ -45,9 +45,10 @@ int
 main(int argc, char *argv[])
 {
     table_type ttype;
-    int ch, cnt, ret, runs;
+    int ch, cnt, i, ret, runs;
     char *working_dir;
     const char *config_open;
+    bool verify_only;
 
     (void)testutil_set_progname(argv);
 
@@ -64,8 +65,9 @@ main(int argc, char *argv[])
     g.nworkers = 1;
     g.sweep_stress = g.use_timestamps = false;
     runs = 1;
+    verify_only = false;
 
-    while ((ch = __wt_getopt(progname, argc, argv, "C:c:Dh:k:l:n:pr:sT:t:W:x")) != EOF)
+    while ((ch = __wt_getopt(progname, argc, argv, "C:c:Dh:k:l:n:pr:sT:t:vW:xX")) != EOF)
         switch (ch) {
         case 'c':
             g.checkpoint_name = __wt_optarg;
@@ -121,12 +123,18 @@ main(int argc, char *argv[])
         case 'T':
             g.ntables = atoi(__wt_optarg);
             break;
+        case 'v':
+            verify_only = true;
+            break;
         case 'W':
             g.nworkers = atoi(__wt_optarg);
             break;
         case 'x':
             g.use_timestamps = true;
             break;
+        case 'X':
+            g.use_timestamps = g.race_timetamps = true;
+            break;
         default:
             return (usage());
         }
@@ -145,7 +153,7 @@ main(int argc, char *argv[])
 
     printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid());
     for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) {
-        cleanup(cnt == 1); /* Clean up previous runs */
+        cleanup(cnt == 1 && !verify_only); /* Clean up previous runs */
 
         printf("    %d: %d workers, %d tables\n", cnt, g.nworkers, g.ntables);
 
@@ -155,6 +163,16 @@ main(int argc, char *argv[])
             break;
         }
 
+        for (i = 0; i < g.ntables; ++i) {
+            g.cookies[i].id = i;
+            if (ttype == MIX)
+                g.cookies[i].type = (table_type)((i % MAX_TABLE_TYPE) + 1);
+            else
+                g.cookies[i].type = ttype;
+            testutil_check(__wt_snprintf(
+              g.cookies[i].uri, sizeof(g.cookies[i].uri), "%s%04d", URI_BASE, g.cookies[i].id));
+        }
+
         g.running = 1;
 
         if ((ret = wt_connect(config_open)) != 0) {
@@ -162,8 +180,20 @@ main(int argc, char *argv[])
             break;
         }
 
+        if (verify_only) {
+            WT_SESSION *session;
+
+            if ((ret = g.conn->open_session(g.conn, NULL, NULL, &session)) != 0) {
+                (void)log_print_err("conn.open_session", ret, 1);
+                break;
+            }
+
+            verify_consistency(session, NULL);
+            goto run_complete;
+        }
+
         start_checkpoints();
-        if ((ret = start_workers(ttype)) != 0) {
+        if ((ret = start_workers()) != 0) {
             (void)log_print_err("Start workers failed", ret, 1);
             break;
         }
@@ -171,6 +201,7 @@ main(int argc, char *argv[])
         g.running = 0;
         end_checkpoints();
 
+run_complete:
         free(g.cookies);
         g.cookies = NULL;
         if ((ret = wt_shutdown()) != 0) {
@@ -187,7 +218,7 @@ main(int argc, char *argv[])
     return (g.status);
 }
 
-#define DEBUG_MODE_CFG ",debug_mode=(eviction=true,table_logging=true)"
+#define DEBUG_MODE_CFG ",debug_mode=(eviction=true,table_logging=true),verbose=(recovery)"
 /*
  * wt_connect --
  *     Configure the WiredTiger connection.
@@ -352,7 +383,9 @@ usage(void)
       "\t-r set number of runs (0 for continuous)\n"
       "\t-T specify a table configuration\n"
       "\t-t set a file type ( col | mix | row | lsm )\n"
+      "\t-v verify only\n"
       "\t-W set number of worker threads\n"
-      "\t-x use timestamps\n");
+      "\t-x use timestamps\n"
+      "\t-X race timestamp updates with checkpoints\n");
     return (EXIT_FAILURE);
 }
diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h
index 7950fc8bb2e..b092751a5fc 100644
--- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h
+++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h
@@ -68,6 +68,7 @@ typedef struct {
     u_int ts_oldest;               /* Current oldest timestamp */
     u_int ts_stable;               /* Current stable timestamp */
     bool use_timestamps;           /* Use txn timestamps */
+    bool race_timetamps;           /* Async update to oldest timestamp */
     bool prepare;                  /* Use prepare transactions */
     COOKIE *cookies;               /* Per-thread info */
     WT_RWLOCK clock_lock;          /* Clock synchronization */
@@ -79,5 +80,6 @@ extern GLOBAL g;
 void end_checkpoints(void);
 int log_print_err(const char *, int, int);
 void start_checkpoints(void);
-int start_workers(table_type);
+int start_workers(void);
 const char *type_to_string(table_type);
+int verify_consistency(WT_SESSION *, char *);
diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c
index 05b9a83b75b..87184dd8a7f 100644
--- a/src/third_party/wiredtiger/test/checkpoint/workers.c
+++ b/src/third_party/wiredtiger/test/checkpoint/workers.c
@@ -67,7 +67,7 @@ create_table(WT_SESSION *session, COOKIE *cookie)
  *     wait for them to finish.
  */
 int
-start_workers(table_type type)
+start_workers(void)
 {
     struct timeval start, stop;
     WT_SESSION *session;
@@ -85,16 +85,9 @@ start_workers(table_type type)
         (void)log_print_err("conn.open_session", ret, 1);
         goto err;
     }
-    /* Setup the cookies */
-    for (i = 0; i < g.ntables; ++i) {
-        g.cookies[i].id = i;
-        if (type == MIX)
-            g.cookies[i].type = (table_type)((i % MAX_TABLE_TYPE) + 1);
-        else
-            g.cookies[i].type = type;
-        testutil_check(__wt_snprintf(
-          g.cookies[i].uri, sizeof(g.cookies[i].uri), "%s%04d", URI_BASE, g.cookies[i].id));
 
+    /* Create tables */
+    for (i = 0; i < g.ntables; ++i) {
         /* Should probably be atomic to avoid races. */
         if ((ret = create_table(session, &g.cookies[i])) != 0)
             goto err;
@@ -280,29 +273,48 @@ real_worker(void)
                         } else
                             testutil_check(__wt_snprintf(
                               buf, sizeof(buf), "commit_timestamp=%x", g.ts_stable + 1));
-                        if ((ret = session->commit_transaction(session, buf)) != 0) {
-                            __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock);
-                            (void)log_print_err("real_worker:commit_transaction", ret, 1);
-                            goto err;
+
+                        // Commit majority of times
+                        if (next_rnd % 49 != 0) {
+                            if ((ret = session->commit_transaction(session, buf)) != 0) {
+                                __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock);
+                                (void)log_print_err("real_worker:commit_transaction", ret, 1);
+                                goto err;
+                            }
+                        } else {
+                            if ((ret = session->rollback_transaction(session, NULL)) != 0) {
+                                __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock);
+                                (void)log_print_err("real_worker:rollback_transaction", ret, 1);
+                                goto err;
+                            }
                         }
                         __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock);
                         start_txn = true;
-                        /* Occasionally reopen cursors after committing. */
+                        /* Occasionally reopen cursors after transaction finish. */
                         if (next_rnd % 13 == 0) {
                             reopen_cursors = true;
                         }
                     }
                 } else {
-                    if ((ret = session->commit_transaction(session, NULL)) != 0) {
-                        (void)log_print_err("real_worker:commit_transaction", ret, 1);
-                        goto err;
+                    // Commit majority of times
+                    if (next_rnd % 49 != 0) {
+                        if ((ret = session->commit_transaction(session, NULL)) != 0) {
+                            __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock);
+                            (void)log_print_err("real_worker:commit_transaction", ret, 1);
+                            goto err;
+                        }
+                    } else {
+                        if ((ret = session->rollback_transaction(session, NULL)) != 0) {
+                            __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock);
+                            (void)log_print_err("real_worker:rollback_transaction", ret, 1);
+                            goto err;
+                        }
                     }
                     start_txn = true;
                 }
-            } else if (next_rnd % 15 == 0) {
+            } else if (next_rnd % 15 == 0)
                 /* Occasionally reopen cursors during a running transaction. */
                 reopen_cursors = true;
-            }
         } else {
             if ((ret = session->rollback_transaction(session, NULL)) != 0) {
                 (void)log_print_err("real_worker:rollback_transaction", ret, 1);