Import wiredtiger: 0fbad5023a23823b1dd8f63e2a9b582f3723cd64 from branch mongodb-master

ref: c7ee7a966f..0fbad5023a
for: 6.3.0-rc0

WT-9904 test tiered: develop microbenchmarks for SS-PUSH

10 files changed, 395 insertions, 17 deletions

diff --git a/src/third_party/wiredtiger/bench/tiered/push_pull.c b/src/third_party/wiredtiger/bench/tiered/push_pull.c
new file mode 100644
index 00000000000..59c267c18c5
--- /dev/null
+++ b/src/third_party/wiredtiger/bench/tiered/push_pull.c
@@ -0,0 +1,308 @@
+/*-
+ * Public Domain 2014-present MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "test_util.h"
+
+/*
+ * This test is to calculate benchmarks for tiered storage:
+ * - This test populates tables of different sizes, say 100K, 1MB, 10MB,
+ * 100MB and checkpoints with/without flush call and calculates time taken for
+ * populate and checkpoint.
+ */
+
+#define HOME_BUF_SIZE 512
+#define MAX_RUN 10
+#define MAX_TIERED_FILES 10
+#define NUM_RECORDS 500
+
+/* Constants and variables declaration. */
+static const char conn_config[] =
+  "create,cache_size=2GB,statistics=(all),statistics_log=(json,on_close,wait=1)";
+static const char table_config_row[] = "leaf_page_max=64KB,key_format=i,value_format=S";
+static char data_str[200] = "";
+
+static TEST_OPTS *opts, _opts;
+
+/* Forward declarations. */
+
+static void compute_wt_file_size(const char *, const char *, int64_t *);
+static void compute_tiered_file_size(const char *, const char *, int64_t *);
+static void get_file_size(const char *, int64_t *);
+static void run_test_clean(const char *, uint64_t, bool);
+static void run_test(const char *, uint64_t, bool, int);
+static void populate(WT_SESSION *, uint64_t);
+
+static double avg_time_array[MAX_RUN];
+static double avg_throughput_array[MAX_RUN];
+static int64_t avg_filesize_array[MAX_RUN];
+
+/*
+ * main --
+ *     Methods implementation.
+ */
+int
+main(int argc, char *argv[])
+{
+    bool flush;
+    int i;
+    opts = &_opts;
+    memset(opts, 0, sizeof(*opts));
+    testutil_check(testutil_parse_opts(argc, argv, opts));
+
+    flush = false;
+    printf("The below benchmarks are average of %d runs\n", MAX_RUN);
+    for (i = 0; i < 2; ++i) {
+
+        printf("############################################\n");
+        printf(
+          "            Flush call %s\n", (opts->tiered_storage && flush) ? "enabled" : "disabled");
+        printf("############################################\n");
+
+        /*
+         * Run test with 100K file size. Row store case.
+         */
+        run_test_clean("100KB", NUM_RECORDS, flush);
+
+        /*
+         * Run test with 1Mb file size. Row store case.
+         */
+        run_test_clean("1MB", NUM_RECORDS * 10, flush);
+
+        /*
+         * Run test with 10 Mb file size. Row store case.
+         */
+        run_test_clean("10MB", NUM_RECORDS * 100, flush);
+
+        /*
+         * Run test with 100 Mb file size. Row store case.
+         */
+        run_test_clean("100MB", NUM_RECORDS * 1000, flush);
+        flush = true;
+    }
+
+    testutil_cleanup(opts);
+
+    return (EXIT_SUCCESS);
+}
+
+/*
+ * difftime_msecs --
+ *     Return the time in msecs.
+ */
+static double
+difftime_msecs(struct timeval t0, struct timeval t1)
+{
+    return (t1.tv_sec - t0.tv_sec) * (double)WT_THOUSAND +
+      (t1.tv_usec - t0.tv_usec) / (double)WT_THOUSAND;
+}
+
+/*
+ * difftime_sec --
+ *     Return the time in seconds.
+ */
+static double
+difftime_sec(struct timeval t0, struct timeval t1)
+{
+    return difftime_msecs(t0, t1) / (double)WT_THOUSAND;
+}
+
+/*
+ * run_test_clean --
+ *     This function runs the test for configured number of times to compute the average time taken.
+ */
+static void
+run_test_clean(const char *suffix, uint64_t num_records, bool flush)
+{
+    char home_full[HOME_BUF_SIZE];
+    double avg_time, avg_throughput;
+    int64_t avg_file_size;
+    int counter;
+
+    avg_file_size = 0;
+    avg_time = avg_throughput = 0;
+
+    for (counter = 0; counter < MAX_RUN; ++counter) {
+        testutil_check(__wt_snprintf(
+          home_full, HOME_BUF_SIZE, "%s_%s_%d_%d", opts->home, suffix, flush, counter));
+        run_test(home_full, num_records, flush, counter);
+
+        /* Cleanup */
+        if (!opts->preserve)
+            testutil_clean_work_dir(home_full);
+    }
+
+    for (counter = 0; counter < MAX_RUN; ++counter) {
+        avg_time += avg_time_array[counter];
+        avg_throughput += avg_throughput_array[counter];
+        avg_file_size += avg_filesize_array[counter];
+    }
+
+    printf("Bytes Written- %" PRIi64
+           " (~%s), Time took- %.3f seconds, Throughput- %.3f MB/second\n",
+      avg_file_size / MAX_RUN, suffix, avg_time / MAX_RUN, avg_throughput / MAX_RUN);
+}
+
+/*
+ * run_test --
+ *     This function runs the actual test and checkpoints with/without flush call based on the
+ *     parameter.
+ */
+static void
+run_test(const char *home, uint64_t num_records, bool flush, int counter)
+{
+    struct timeval start, end;
+
+    char buf[1024];
+    double diff_sec;
+    int64_t file_size;
+
+    WT_CONNECTION *conn;
+    WT_SESSION *session;
+
+    testutil_make_work_dir(home);
+    if (opts->tiered_storage) {
+        testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/bucket", home));
+        testutil_make_work_dir(buf);
+    }
+
+    testutil_wiredtiger_open(opts, home, conn_config, NULL, &conn, false);
+    testutil_check(conn->open_session(conn, NULL, NULL, &session));
+
+    /* Create and populate table. Checkpoint the data after that. */
+    testutil_check(session->create(session, opts->uri, table_config_row));
+
+    testutil_check(__wt_snprintf(buf, sizeof(buf), flush ? "flush_tier=(enabled,force=true)" : ""));
+
+    gettimeofday(&start, 0);
+    populate(session, num_records);
+
+    testutil_check(session->checkpoint(session, buf));
+    gettimeofday(&end, 0);
+
+    diff_sec = difftime_sec(start, end);
+
+    /* Sleep to guarantee the tables are created to read the size. */
+    sleep(1);
+
+    get_file_size(home, &file_size);
+    testutil_assert(diff_sec > 0);
+
+    avg_time_array[counter] = diff_sec;
+    avg_throughput_array[counter] = ((file_size / diff_sec) / WT_MEGABYTE);
+    avg_filesize_array[counter] = file_size;
+}
+
+/*
+ * populate --
+ *     Populate the table.
+ */
+static void
+populate(WT_SESSION *session, uint64_t num_records)
+{
+    WT_CURSOR *cursor;
+    WT_RAND_STATE rnd;
+    uint64_t i, str_len;
+
+    __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd);
+
+    str_len = sizeof(data_str) / sizeof(data_str[0]);
+    for (i = 0; i < str_len - 1; i++)
+        data_str[i] = 'a' + (uint32_t)__wt_random(&rnd) % 26;
+
+    data_str[str_len - 1] = '\0';
+
+    testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &cursor));
+    for (i = 0; i < num_records; i++) {
+        cursor->set_key(cursor, i + 1);
+        cursor->set_value(cursor, data_str);
+        testutil_check(cursor->insert(cursor));
+    }
+
+    testutil_check(cursor->close(cursor));
+    cursor = NULL;
+}
+
+/*
+ * compute_tiered_file_size --
+ *     Iterate over all the tiered files and compute file size..
+ */
+static void
+compute_tiered_file_size(const char *home, const char *tablename, int64_t *file_size)
+{
+    char stat_path[512];
+    int index;
+    struct stat stats;
+
+    *file_size = 0;
+    for (index = 1; index < MAX_TIERED_FILES; ++index) {
+        testutil_check(__wt_snprintf(
+          stat_path, sizeof(stat_path), "%s/%s-%10.10d.wtobj", home, tablename, index));
+
+        /* Return if the stat fails that means the file does not exist. */
+        if (stat(stat_path, &stats) == 0)
+            *file_size += stats.st_size;
+        else
+            return;
+    }
+}
+
+/*
+ * compute_wt_file_size --
+ *     Compute wt file size.
+ */
+static void
+compute_wt_file_size(const char *home, const char *tablename, int64_t *file_size)
+{
+    char stat_path[512];
+    struct stat stats;
+
+    *file_size = 0;
+    testutil_check(__wt_snprintf(stat_path, sizeof(stat_path), "%s/%s.wt", home, tablename));
+    if (stat(stat_path, &stats) == 0)
+        *file_size = stats.st_size;
+    else
+        testutil_die(ENOENT, "%s does not exist", stat_path);
+}
+
+/*
+ * get_file_size --
+ *     Retrieve the file size of the table.
+ */
+static void
+get_file_size(const char *home, int64_t *file_size)
+{
+    const char *tablename;
+
+    tablename = strchr(opts->uri, ':');
+    testutil_assert(tablename != NULL);
+    tablename++;
+
+    if (opts->tiered_storage)
+        compute_tiered_file_size(home, tablename, file_size);
+    else
+        compute_wt_file_size(home, tablename, file_size);
+}
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok
index 31a99a4c86e..b83ab99ddec 100644
--- a/src/third_party/wiredtiger/dist/s_string.ok
+++ b/src/third_party/wiredtiger/dist/s_string.ok
@@ -1463,6 +1463,7 @@ strerror
 strftime
 strget
 strlen
+strncat
 strncpy
 strndup
 strnlen
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index efce2182957..70fa701e8f8 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "c7ee7a966f253f3d66606a91c55b3dd4bd2e251b"
+    "commit": "0fbad5023a23823b1dd8f63e2a9b582f3723cd64"
 }
diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c
index 090a3190bcb..85d21873179 100644
--- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c
+++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c
@@ -830,7 +830,7 @@ run_workload(uint32_t nth)
         strcpy(envconf, ENV_CONFIG_TXNSYNC);
 
     /* Open WiredTiger without recovery. */
-    testutil_wiredtiger_open(opts, envconf, &event_handler, &conn, false);
+    testutil_wiredtiger_open(opts, NULL, envconf, &event_handler, &conn, false);
 
     testutil_check(conn->open_session(conn, NULL, NULL, &session));
     /*
@@ -1103,7 +1103,7 @@ main(int argc, char *argv[])
     /*
      * Open the connection which forces recovery to be run.
      */
-    testutil_wiredtiger_open(opts, buf, &event_handler, &conn, true);
+    testutil_wiredtiger_open(opts, NULL, buf, &event_handler, &conn, true);
 
     testutil_check(conn->open_session(conn, NULL, NULL, &session));
     /*
diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c
index c171807fd12..cf032fee1b2 100644
--- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c
+++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c
@@ -624,7 +624,7 @@ run_workload(void)
     if (!opts->compat && !opts->inmem)
         strcat(envconf, ENV_CONFIG_ADD_EVICT_DIRTY);
 
-    testutil_wiredtiger_open(opts, envconf, NULL, &conn, false);
+    testutil_wiredtiger_open(opts, NULL, envconf, NULL, &conn, false);
     testutil_check(conn->open_session(conn, NULL, NULL, &session));
 
     /*
@@ -751,7 +751,7 @@ main(int argc, char *argv[])
     uint64_t commit_fp, durable_fp, stable_val;
     uint32_t i, timeout;
     int ch, status, ret;
-    char buf[512], fname[64], kname[64], statname[1024];
+    char buf[512], fname[64], kname[64], statname[1024], bucket[512];
     char ts_string[WT_TS_HEX_STRING_SIZE];
     bool fatal, rand_th, rand_time, verify_only;
 
@@ -825,6 +825,11 @@ main(int argc, char *argv[])
     if (!verify_only) {
         testutil_make_work_dir(home);
 
+        if (opts->tiered_storage) {
+            testutil_check(__wt_snprintf(bucket, sizeof(bucket), "%s/bucket", home));
+            testutil_make_work_dir(bucket);
+        }
+
         __wt_random_init_seed(NULL, &rnd);
         if (rand_time) {
             timeout = __wt_random(&rnd) % MAX_TIME;
@@ -900,7 +905,7 @@ main(int argc, char *argv[])
     /*
      * Open the connection which forces recovery to be run.
      */
-    testutil_wiredtiger_open(opts, buf, &my_event, &conn, true);
+    testutil_wiredtiger_open(opts, NULL, buf, &my_event, &conn, true);
 
     printf("Connection open and recovery complete. Verify content\n");
     /* Sleep to guarantee the statistics thread has enough time to run. */
diff --git a/src/third_party/wiredtiger/test/csuite/wt9937_parse_opts/main.c b/src/third_party/wiredtiger/test/csuite/wt9937_parse_opts/main.c
index 49926ee36d1..50a7adf283e 100644
--- a/src/third_party/wiredtiger/test/csuite/wt9937_parse_opts/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt9937_parse_opts/main.c
@@ -110,6 +110,7 @@ static TEST_DRIVER driver[] = {
   {{"parse_opts", "-bbuilddir", "-T21", NULL}, {"builddir", NULL, 0, 0, false, false, 21},
     {NULL, 0, 0, 0}},
 
+  {{"parse_opts", "-T21", NULL}, {NULL, NULL, 0, 0, false, false, 21}, {NULL, 0, 0, 0}},
   /*
    * If -PT is used, the tiered_storage source is set to dir_store, even if -Po is not used. Also
    * when -PT is used, random seeds are initialized to some non-zero value.
@@ -302,9 +303,11 @@ verify_expect(
     } while (0)
 
     /*
-     * opts->home is always set, even without -h on the command line, so don't check it here.
+     * opts->home is always set, even without -h on the command line, so don't check it here. If
+     * tiered_storage is set then build_dir is deduced from the test program.
      */
-    VERIFY_STR(opts, expect, build_dir);
+    if (opts->tiered_storage != true)
+        VERIFY_STR(opts, expect, build_dir);
     VERIFY_STR(opts, expect, tiered_storage_source);
     VERIFY_INT(opts, expect, table_type);
     VERIFY_RANDOM_INT(opts, expect, data_seed);
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index ed4c346dd0b..75ebc903817 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -2473,6 +2473,19 @@ tasks:
               ${test_env_vars|} ./test_fops
             fi
 
+  - name: bench-tiered-push-pull
+    depends_on:
+    - name: compile
+    commands:
+      - func: "fetch artifacts"
+      - command: shell.exec
+        params:
+          working_dir: "wiredtiger/cmake_build/bench/tiered"
+          script: |
+            set -o errexit
+            set -o verbose
+            ${test_env_vars|} ./test_push_pull
+
   - name: compatibility-test-for-newer-releases
     commands:
       - func: "get project"
@@ -4686,6 +4699,7 @@ buildvariants:
     - name: ".data-validation-stress-test"
     - name: unittest-test
     - name: tiered-storage-extensions-test
+    - name: bench-tiered-push-pull
     - name: compile-nonstandalone
     - name: make-check-nonstandalone
     - name: unit-test-nonstandalone
diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c
index 530ca18c11a..04be1ed5bc7 100644
--- a/src/third_party/wiredtiger/test/utility/misc.c
+++ b/src/third_party/wiredtiger/test/utility/misc.c
@@ -139,6 +139,45 @@ testutil_clean_work_dir(const char *dir)
 }
 
 /*
+ * testutil_deduce_build_dir --
+ *     Deduce the build directory.
+ */
+void
+testutil_deduce_build_dir(TEST_OPTS *opts)
+{
+    struct stat stats;
+
+    char path[512], pwd[512], stat_path[512];
+    char *token;
+    int index;
+
+    if (getcwd(pwd, sizeof(pwd)) == NULL)
+        testutil_die(ENOENT, "No such directory");
+
+    /* This condition is when the full path name is used for argv0. */
+    if (opts->argv0[0] == '/')
+        testutil_check(__wt_snprintf(path, sizeof(path), "%s", opts->argv0));
+    else
+        testutil_check(__wt_snprintf(path, sizeof(path), "%s/%s", pwd, opts->argv0));
+
+    token = strrchr(path, '/');
+    while (strlen(path) > 0) {
+        testutil_assert(token != NULL);
+        index = (int)(token - path);
+        path[index] = '\0';
+
+        testutil_check(__wt_snprintf(stat_path, sizeof(stat_path), "%s/wt", path));
+
+        if (stat(stat_path, &stats) == 0) {
+            opts->build_dir = dstrdup(path);
+            return;
+        }
+        token = strrchr(path, '/');
+    }
+    return;
+}
+
+/*
  * testutil_build_dir --
  *     Get the build directory.
  */
@@ -363,16 +402,20 @@ testutil_print_command_line(int argc, char *const *argv)
  *     Call wiredtiger_open with the tiered storage configuration if enabled.
  */
 void
-testutil_wiredtiger_open(TEST_OPTS *opts, const char *config, WT_EVENT_HANDLER *event_handler,
-  WT_CONNECTION **connectionp, bool rerun)
+testutil_wiredtiger_open(TEST_OPTS *opts, const char *home, const char *config,
+  WT_EVENT_HANDLER *event_handler, WT_CONNECTION **connectionp, bool rerun)
 {
-    char buf[1024];
+    char buf[1024], tiered_ext_cfg[512];
+
+    if (opts->tiered_storage)
+        testutil_check(__wt_snprintf(tiered_ext_cfg, sizeof(tiered_ext_cfg),
+          TESTUTIL_ENV_CONFIG_TIERED_EXT TESTUTIL_ENV_CONFIG_TIERED, opts->build_dir));
 
     testutil_check(__wt_snprintf(buf, sizeof(buf), "%s%s%s%s", config,
       (rerun ? TESTUTIL_ENV_CONFIG_REC : ""), (opts->compat ? TESTUTIL_ENV_CONFIG_COMPAT : ""),
-      (opts->tiered_storage ? TESTUTIL_ENV_CONFIG_TIERED_EXT TESTUTIL_ENV_CONFIG_TIERED : "")));
-    printf("wiredtiger_open configuration: %s\n", buf);
-    testutil_check(wiredtiger_open(NULL, event_handler, buf, connectionp));
+      (opts->tiered_storage ? tiered_ext_cfg : "")));
+    // printf("wiredtiger_open configuration: %s\n", buf);
+    testutil_check(wiredtiger_open(home, event_handler, buf, connectionp));
 }
 
 #ifndef _WIN32
diff --git a/src/third_party/wiredtiger/test/utility/parse_opts.c b/src/third_party/wiredtiger/test/utility/parse_opts.c
index f94fab187ff..17d31b80a85 100644
--- a/src/third_party/wiredtiger/test/utility/parse_opts.c
+++ b/src/third_party/wiredtiger/test/utility/parse_opts.c
@@ -211,6 +211,9 @@ testutil_parse_end_opt(TEST_OPTS *opts)
         /* Initialize the state for the random number generators. */
         parse_init_random(&opts->data_rnd, &opts->data_seed);
         parse_init_random(&opts->extra_rnd, &opts->extra_seed);
+
+        /* Deduce the build directory. */
+        testutil_deduce_build_dir(opts);
     }
 }
 
diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h
index d7f4a9d8fdd..4703b9a478c 100644
--- a/src/third_party/wiredtiger/test/utility/test_util.h
+++ b/src/third_party/wiredtiger/test/utility/test_util.h
@@ -52,8 +52,8 @@
 #define TESTUTIL_ENV_CONFIG_TIERED \
     ",tiered_storage=(bucket=./"   \
     "bucket,bucket_prefix=pfx-,local_retention=2,name=dir_store)"
-#define TESTUTIL_ENV_CONFIG_TIERED_EXT                        \
-    ",extensions=(../../../../ext/storage_sources/dir_store/" \
+#define TESTUTIL_ENV_CONFIG_TIERED_EXT               \
+    ",extensions=(%s/ext/storage_sources/dir_store/" \
     "libwiredtiger_dir_store.so=(early_load=true))"
 #define TESTUTIL_ENV_CONFIG_REC \
     ",log=(recover=on,remove=false),statistics=(all),statistics_log=(json,on_close,wait=1)"
@@ -397,6 +397,7 @@ void testutil_copy_data(const char *);
 void testutil_copy_file(WT_SESSION *, const char *);
 void testutil_copy_if_exists(WT_SESSION *, const char *);
 void testutil_create_backup_directory(const char *);
+void testutil_deduce_build_dir(TEST_OPTS *opts);
 int testutil_general_event_handler(
   WT_EVENT_HANDLER *, WT_CONNECTION *, WT_SESSION *, WT_EVENT_TYPE, void *);
 void testutil_make_work_dir(const char *);
@@ -411,7 +412,7 @@ void testutil_progress(TEST_OPTS *, const char *);
 void testutil_sleep_wait(uint32_t, pid_t);
 #endif
 void testutil_wiredtiger_open(
-  TEST_OPTS *, const char *, WT_EVENT_HANDLER *, WT_CONNECTION **, bool);
+  TEST_OPTS *, const char *, const char *, WT_EVENT_HANDLER *, WT_CONNECTION **, bool);
 void testutil_tiered_begin(TEST_OPTS *);
 void testutil_tiered_flush_complete(TEST_OPTS *, WT_SESSION *, void *);
 void testutil_tiered_sleep(TEST_OPTS *, WT_SESSION *, uint32_t, bool *);