Import wiredtiger: 1ed75dd0937da05deeafa874b30f2dfe505f22be from branch mongodb-master

ref: 1daa7d05a7..1ed75dd093
for: 7.1.0-rc0

WT-10786 Use unique object prefixes in C tiered storage tests

9 files changed, 198 insertions, 29 deletions

diff --git a/src/third_party/wiredtiger/bench/tiered/push_pull.c b/src/third_party/wiredtiger/bench/tiered/push_pull.c
index 88db1ce814e..3c8a95dd542 100644
--- a/src/third_party/wiredtiger/bench/tiered/push_pull.c
+++ b/src/third_party/wiredtiger/bench/tiered/push_pull.c
@@ -286,7 +286,7 @@ recover_validate(const char *home, uint32_t num_records, uint64_t file_size, uin
 {
     struct timeval start, end;
 
-    char buf[1024];
+    char buf[1024], pwd[1024];
     double diff_sec;
     size_t val_1_size, val_2_size;
     uint64_t key, i, v;
@@ -297,7 +297,10 @@ recover_validate(const char *home, uint32_t num_records, uint64_t file_size, uin
     WT_SESSION *session;
 
     /* Copy the data to a separate folder for debugging purpose. */
-    testutil_copy_data(home);
+    testutil_assert(getcwd(pwd, sizeof(pwd)) != NULL);
+    testutil_check(chdir(home));
+    testutil_copy_data(home); /* This function assumes we are inside the home directory. */
+    testutil_check(chdir(pwd));
 
     key = 0;
     buf[0] = '\0';
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok
index 29521488fa3..c5c703d6a87 100644
--- a/src/third_party/wiredtiger/dist/s_string.ok
+++ b/src/third_party/wiredtiger/dist/s_string.ok
@@ -894,6 +894,7 @@ libmemkind
 libs
 libsodium
 libwiredtiger
+lifecycle
 linux
 llll
 llu
diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp
index 14a09f6b63f..5a341dc76bd 100644
--- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp
+++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp
@@ -117,7 +117,8 @@ AwsManager AwsManager::aws_instance;
 static int S3GetDirectory(
   const S3Storage &, const std::string &, const std::string &, bool, std::string &);
 static bool S3CacheExists(WT_FILE_SYSTEM *, const std::string &);
-static std::string S3Path(const std::string &, const std::string &);
+static std::string S3CachePath(const std::string &, const std::string &);
+static std::string S3SourcePath(const std::string &, const std::string &);
 static int S3FileExists(WT_FILE_SYSTEM *, WT_SESSION *, const char *, bool *);
 static int S3CustomizeFileSystem(
   WT_STORAGE_SOURCE *, WT_SESSION *, const char *, const char *, const char *, WT_FILE_SYSTEM **);
@@ -143,9 +144,23 @@ static int S3FileSize(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *);
 static int S3FileLock(WT_FILE_HANDLE *, WT_SESSION *, bool);
 static int S3ObjectSize(WT_FILE_SYSTEM *, WT_SESSION *, const char *, wt_off_t *);
 
-// Construct a pathname from the directory and the object name.
+// Construct a pathname from the cache directory and the object name. This takes care of translating
+// illegal characters in the object name (e.g., '/') into legal characters.
 static std::string
-S3Path(const std::string &dir, const std::string &name)
+S3CachePath(const std::string &dir, const std::string &name)
+{
+    int i;
+    std::string cacheFileName = name;
+    for (i = 0; i < cacheFileName.length(); i++)
+        if (cacheFileName[i] == '/')
+            cacheFileName[i] = '-';
+
+    return (dir + "/" + cacheFileName);
+}
+
+// Construct a pathname from the source directory and the base file name.
+static std::string
+S3SourcePath(const std::string &dir, const std::string &name)
 {
     // Skip over "./" and variations (".//", ".///./././//") at the beginning of the name.
     int i = 0;
@@ -190,7 +205,7 @@ S3FileExists(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name,
 static bool
 S3CacheExists(WT_FILE_SYSTEM *fileSystem, const std::string &name)
 {
-    const std::string path = S3Path(((S3FileSystem *)fileSystem)->cacheDir, name);
+    const std::string path = S3CachePath(((S3FileSystem *)fileSystem)->cacheDir, name);
     return (LocalFileExists(path));
 }
 
@@ -307,7 +322,7 @@ S3FileOpen(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name,
     }
 
     // Make a copy from S3 if the file is not in the cache.
-    const std::string cachePath = S3Path(fs->cacheDir, name);
+    const std::string cachePath = S3CachePath(fs->cacheDir, name);
     if (!LocalFileExists(cachePath)) {
         s3->statistics.getObjectCount++;
         if ((ret = fs->connection->GetObject(name, cachePath)) != 0) {
@@ -783,7 +798,7 @@ S3Flush(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, WT_FILE_SYSTEM *f
     FS2S3(fileSystem)->statistics.putObjectCount++;
 
     // Confirm that the file exists on the native filesystem.
-    std::string srcPath = S3Path(fs->homeDir, source);
+    std::string srcPath = S3SourcePath(fs->homeDir, source);
     bool nativeExist = false;
     int ret = wtFileSystem->fs_exist(wtFileSystem, session, srcPath.c_str(), &nativeExist);
     if (ret != 0) {
@@ -816,10 +831,11 @@ S3FlushFinish(WT_STORAGE_SOURCE *storage, WT_SESSION *session, WT_FILE_SYSTEM *f
     S3Storage *s3 = (S3Storage *)storage;
     S3FileSystem *fs = (S3FileSystem *)fileSystem;
     // Constructing the pathname for source and cache from file system and local.
-    std::string srcPath = S3Path(fs->homeDir, source);
-    std::string destPath = S3Path(fs->cacheDir, object);
+    std::string srcPath = S3SourcePath(fs->homeDir, source);
+    std::string destPath = S3CachePath(fs->cacheDir, object);
 
-    // Converting S3 object name to cache directory strcture to link the cache file with local file.
+    // Converting S3 object name to cache directory structure to link the cache file with local
+    // file.
     std::filesystem::create_directories(std::filesystem::path(destPath).parent_path());
 
     // Linking file with the local file.
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 7c613437694..f9b8604b26b 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "1daa7d05a761f3968fbcefa46f2d71803c3cc17d"
+    "commit": "1ed75dd0937da05deeafa874b30f2dfe505f22be"
 }
diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c
index 6485e4c79a8..038b4d669d7 100644
--- a/src/third_party/wiredtiger/test/format/wts.c
+++ b/src/third_party/wiredtiger/test/format/wts.c
@@ -292,7 +292,7 @@ configure_tiered_storage(const char *home, char **p, size_t max, char *ext_cfg,
     opts.absolute_bucket_dir = true;
 
     testutil_tiered_storage_configuration(
-      &opts, tiered_cfg, sizeof(tiered_cfg), ext_cfg, ext_cfg_size);
+      &opts, home, tiered_cfg, sizeof(tiered_cfg), ext_cfg, ext_cfg_size);
     CONFIG_APPEND(*p, ",%s", tiered_cfg);
 }
 
diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c
index 74f7a845179..e3431da6023 100644
--- a/src/third_party/wiredtiger/test/utility/misc.c
+++ b/src/third_party/wiredtiger/test/utility/misc.c
@@ -592,7 +592,7 @@ testutil_wiredtiger_open(TEST_OPTS *opts, const char *home, const char *config,
 
     opts->local_retention = benchmarkrun ? 0 : 2;
     testutil_tiered_storage_configuration(
-      opts, tiered_cfg, sizeof(tiered_cfg), tiered_ext_cfg, sizeof(tiered_ext_cfg));
+      opts, home, tiered_cfg, sizeof(tiered_cfg), tiered_ext_cfg, sizeof(tiered_ext_cfg));
 
     testutil_check(__wt_snprintf(buf, sizeof(buf), "%s%s%s%s,extensions=[%s]",
       config == NULL ? "" : config, (rerun ? TESTUTIL_ENV_CONFIG_REC : ""),
diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h
index a3e3bb99432..0faa50ddc59 100644
--- a/src/third_party/wiredtiger/test/utility/test_util.h
+++ b/src/third_party/wiredtiger/test/utility/test_util.h
@@ -65,10 +65,10 @@
 #define DIR_STORE "dir_store"
 #define S3_STORE "s3_store"
 
-#define TESTUTIL_ENV_CONFIG_TIERED                 \
-    ",tiered_storage=(bucket=%s"                   \
-    ",bucket_prefix=pfx-,local_retention=%" PRIu32 \
-    ",name=%s"                                     \
+#define TESTUTIL_ENV_CONFIG_TIERED               \
+    ",tiered_storage=(bucket=%s"                 \
+    ",bucket_prefix=%s,local_retention=%" PRIu32 \
+    ",name=%s"                                   \
     ",auth_token=%s)"
 #define TESTUTIL_ENV_CONFIG_TIERED_EXT                                         \
     "\"%s/ext/storage_sources/%s/libwiredtiger_%s.so\"=("                      \
@@ -462,6 +462,7 @@ int testutil_parse_single_opt(TEST_OPTS *, int);
 int testutil_parse_opts(int, char *const *, TEST_OPTS *);
 void testutil_print_command_line(int argc, char *const *argv);
 void testutil_progress(TEST_OPTS *, const char *);
+uint32_t testutil_random(WT_RAND_STATE *);
 void testutil_random_init(WT_RAND_STATE *, uint64_t *, uint32_t);
 void testutil_random_from_random(WT_RAND_STATE *, WT_RAND_STATE *);
 void testutil_random_from_seed(WT_RAND_STATE *, uint64_t);
@@ -474,7 +475,8 @@ void testutil_wiredtiger_open(
 void testutil_tiered_begin(TEST_OPTS *);
 void testutil_tiered_flush_complete(TEST_OPTS *, WT_SESSION *, void *);
 void testutil_tiered_sleep(TEST_OPTS *, WT_SESSION *, uint64_t, bool *);
-void testutil_tiered_storage_configuration(TEST_OPTS *, char *, size_t, char *, size_t);
+void testutil_tiered_storage_configuration(
+  TEST_OPTS *, const char *, char *, size_t, char *, size_t);
 uint64_t testutil_time_us(WT_SESSION *);
 void testutil_verify_src_backup(WT_CONNECTION *, const char *, const char *, char *);
 void testutil_work_dir_from_path(char *, size_t, const char *);
diff --git a/src/third_party/wiredtiger/test/utility/tiered.c b/src/third_party/wiredtiger/test/utility/tiered.c
index d1bbd00ae50..94741d40ddf 100644
--- a/src/third_party/wiredtiger/test/utility/tiered.c
+++ b/src/third_party/wiredtiger/test/utility/tiered.c
@@ -27,6 +27,8 @@
  */
 #include "test_util.h"
 
+#define TIERED_STORAGE_CONFIG_FILE "wt-test-tiered-config.txt"
+
 /*
  * testutil_tiered_begin --
  *     Begin processing for a test program that supports tiered storage.
@@ -103,28 +105,145 @@ testutil_tiered_flush_complete(TEST_OPTS *opts, WT_SESSION *session, void *arg)
 }
 
 /*
+ * tiered_storage_read_config --
+ *     Read configuration from a file, if exists.
+ */
+static bool
+tiered_storage_read_config(const char *home, char *s3_prefix, size_t s3_prefix_size)
+{
+    FILE *f;
+    char config_path[512], str[512];
+    char *s, *value;
+
+    testutil_assert(s3_prefix_size > 0);
+    s3_prefix[0] = '\0';
+
+    testutil_check(__wt_snprintf(config_path, sizeof(config_path), "%s/%s",
+      home == NULL ? "." : home, TIERED_STORAGE_CONFIG_FILE));
+    f = fopen(config_path, "r");
+    if (f == NULL) {
+        testutil_assert_errno(errno == ENOENT);
+        return (false);
+    }
+
+    /*
+     * For now, we only support specifying prefixes in the file, but this can be easily expanded to
+     * include more information, such as the bucket name.
+     */
+    while (fgets(str, sizeof(str), f) != NULL) {
+        if (str[0] == '\0' || str[0] == '#')
+            continue;
+        s = str + strlen(str) - 1;
+        if (*s == '\n')
+            *s = '\0';
+        if (str[0] == '\0')
+            continue;
+
+        value = strchr(str, '=');
+        testutil_assertfmt(value != NULL, "Unexpected format of %s", config_path);
+        *(value++) = '\0';
+
+        if (strcmp(str, "prefix") == 0) {
+            testutil_check(__wt_snprintf(s3_prefix, s3_prefix_size, "%s", value));
+            continue;
+        }
+
+        testutil_die(EINVAL, "Unsupported key in the tiered storage config: %s", str);
+    };
+
+    /* Check that everything is specified. */
+    testutil_assert(s3_prefix[0] != '\0');
+
+    testutil_assert_errno(fclose(f) == 0);
+    return (true);
+}
+
+/*
+ * tiered_storage_write_config --
+ *     Write configuration to a file.
+ */
+static void
+tiered_storage_write_config(const char *home, const char *s3_prefix)
+{
+    FILE *f;
+    char config_path[512];
+
+    testutil_check(__wt_snprintf(config_path, sizeof(config_path), "%s/%s",
+      home == NULL ? "." : home, TIERED_STORAGE_CONFIG_FILE));
+    f = fopen(config_path, "w");
+    testutil_assert_errno(f != NULL);
+
+    testutil_assert_errno(fprintf(f, "# Tiered storage configuration written by testutil\n") >= 0);
+    testutil_assert_errno(fprintf(f, "prefix=%s\n", s3_prefix) >= 0);
+
+    testutil_assert_errno(fclose(f) == 0);
+}
+
+/*
+ * tiered_storage_generate_prefix --
+ *     Generate a unique prefix for objects when creating a new database; reuse the prefix when
+ *     opening an existing database.
+ */
+static void
+tiered_storage_generate_prefix(char *out, size_t size)
+{
+    struct tm time_parsed;
+    size_t n;
+#ifdef _WIN32
+    __time64_t time_now;
+#else
+    time_t time_now;
+#endif
+    char time_str[100];
+
+    /*
+     * Generates a unique prefix to be used with the object keys, e.g.:
+     * "s3test/test/2022-31-01-16-34-10/623843294--".
+     *
+     * Objects with the prefix pattern "s3test/" are deleted after a certain period of time
+     * according to the lifecycle rule on the S3 bucket. Should you wish to make any changes to the
+     * prefix pattern or lifecycle of the object, please speak to the release manager.
+     */
+#ifdef _WIN32
+    time_now = _time64(NULL);
+    testutil_check(_localtime64_s(&time_parsed, &time_now));
+#else
+    time_now = time(NULL);
+    (void)localtime_r(&time_now, &time_parsed);
+#endif
+    n = strftime(time_str, sizeof(time_str), "%F-%H-%M-%S", &time_parsed);
+    testutil_assert(n > 0);
+    testutil_check(
+      __wt_snprintf(out, size, "s3test/test/%s/%" PRIu32 "--", time_str, testutil_random(NULL)));
+}
+
+/*
  * testutil_tiered_storage_configuration --
  *     Set up tiered storage configuration.
  */
 void
-testutil_tiered_storage_configuration(
-  TEST_OPTS *opts, char *tiered_cfg, size_t tiered_cfg_size, char *ext_cfg, size_t ext_cfg_size)
+testutil_tiered_storage_configuration(TEST_OPTS *opts, const char *home, char *tiered_cfg,
+  size_t tiered_cfg_size, char *ext_cfg, size_t ext_cfg_size)
 {
     char auth_token[256];
-    char cwd[256], dir[256];
+    char cwd[256], dir[256], s3_prefix[128];
     const char *s3_access_key, *s3_secret_key, *s3_bucket_name;
+    bool is_dir_store;
 
     s3_bucket_name = NULL;
     auth_token[0] = '\0';
 
     if (opts->tiered_storage) {
-        if (!testutil_is_dir_store(opts)) {
+        is_dir_store = testutil_is_dir_store(opts);
+        if (!is_dir_store) {
             s3_access_key = getenv("aws_sdk_s3_ext_access_key");
             s3_secret_key = getenv("aws_sdk_s3_ext_secret_key");
             s3_bucket_name = getenv("WT_S3_EXT_BUCKET");
 
             if (s3_access_key == NULL || s3_secret_key == NULL)
                 testutil_die(EINVAL, "AWS S3 access key or secret key is not set");
+            testutil_check(
+              __wt_snprintf(auth_token, sizeof(auth_token), "%s;%s", s3_access_key, s3_secret_key));
 
             /*
              * By default the S3 bucket name is S3_DEFAULT_BUCKET_NAME, but it can be overridden
@@ -133,14 +252,24 @@ testutil_tiered_storage_configuration(
             if (s3_bucket_name == NULL)
                 s3_bucket_name = S3_DEFAULT_BUCKET_NAME;
 
-            testutil_check(
-              __wt_snprintf(auth_token, sizeof(auth_token), "%s;%s", s3_access_key, s3_secret_key));
+            /*
+             * Read configuration that we might have saved before to a file, which is what we need
+             * to do when opening an existing database (e.g., for tests that crash, recover, and
+             * verify), so that we use the same object prefix.
+             */
+            if (!tiered_storage_read_config(home, s3_prefix, sizeof(s3_prefix))) {
+                /* Generate a random prefix for the new database. */
+                tiered_storage_generate_prefix(s3_prefix, sizeof(s3_prefix));
+
+                /* Remember it for the next time. */
+                tiered_storage_write_config(home, s3_prefix);
+            }
         }
         testutil_check(__wt_snprintf(ext_cfg, ext_cfg_size, TESTUTIL_ENV_CONFIG_TIERED_EXT,
           opts->build_dir, opts->tiered_storage_source, opts->tiered_storage_source, opts->delay_ms,
           opts->error_ms, opts->force_delay, opts->force_error));
 
-        if (testutil_is_dir_store(opts)) {
+        if (is_dir_store) {
             if (opts->absolute_bucket_dir) {
                 if (opts->home[0] == '/')
                     testutil_check(
@@ -155,9 +284,9 @@ testutil_tiered_storage_configuration(
                 testutil_check(__wt_snprintf(dir, sizeof(dir), "%s", DIR_STORE_BUCKET_NAME));
         }
         testutil_check(__wt_snprintf(tiered_cfg, tiered_cfg_size, TESTUTIL_ENV_CONFIG_TIERED,
-          testutil_is_dir_store(opts) ? dir : s3_bucket_name, opts->local_retention,
-          opts->tiered_storage_source, auth_token));
-        if (testutil_is_dir_store(opts) && opts->make_bucket_dir) {
+          is_dir_store ? dir : s3_bucket_name, is_dir_store ? "pfx-" : s3_prefix,
+          opts->local_retention, opts->tiered_storage_source, auth_token));
+        if (is_dir_store && opts->make_bucket_dir) {
             testutil_check(
               __wt_snprintf(dir, sizeof(dir), "%s/%s", opts->home, DIR_STORE_BUCKET_NAME));
             testutil_check(mkdir(dir, 0777));
diff --git a/src/third_party/wiredtiger/test/utility/util_random.c b/src/third_party/wiredtiger/test/utility/util_random.c
index fdfc5c873de..50cb4fb2dc0 100644
--- a/src/third_party/wiredtiger/test/utility/util_random.c
+++ b/src/third_party/wiredtiger/test/utility/util_random.c
@@ -28,6 +28,24 @@
 #include "test_util.h"
 
 /*
+ * testutil_random --
+ *     Return a random number, without the need to specify a randomness state (i.e., it can be
+ *     NULL). If the randomness state is not provided, generate a completely random number.
+ */
+uint32_t
+testutil_random(WT_RAND_STATE *rnd)
+{
+    WT_RAND_STATE rnd_inner;
+
+    if (rnd != NULL) {
+        return __wt_random(rnd);
+    } else {
+        __wt_random_init_seed(NULL, &rnd_inner); /* The session argument is not required. */
+        return __wt_random(&rnd_inner);
+    }
+}
+
+/*
  * testutil_random_from_random --
  *     Seed a destination random number generator from a source random number generator. The source
  *     generator's state is advanced.