From 5b82c98a40721e2668f209de38b00c90d346fb9e Mon Sep 17 00:00:00 2001 From: Will Korteland Date: Thu, 17 Mar 2022 09:57:44 +0000 Subject: Import wiredtiger: fadba1cd56e27dc8ccc97dcccd1883e1ecdaf6ce from branch mongodb-master ref: f1c6393e01..fadba1cd56 for: 6.0.0 WT-8845 Rename directory based storage source extension to be dir_store --- src/third_party/wiredtiger/dist/s_void | 10 +- .../ext/storage_sources/dir_store/dir_store.c | 1344 ++++++++++++++++++++ .../ext/storage_sources/local_store/local_store.c | 1331 ------------------- src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/src/docs/custom-storage-sources.dox | 4 +- src/third_party/wiredtiger/src/docs/spell.ok | 1 + .../wiredtiger/test/csuite/tiered_abort/main.c | 4 +- .../wiredtiger/test/suite/helper_tiered.py | 6 +- .../wiredtiger/test/suite/hook_tiered.py | 2 +- .../wiredtiger/test/suite/test_tiered02.py | 16 +- .../wiredtiger/test/suite/test_tiered03.py | 12 +- .../wiredtiger/test/suite/test_tiered04.py | 12 +- .../wiredtiger/test/suite/test_tiered05.py | 10 +- .../wiredtiger/test/suite/test_tiered06.py | 34 +- .../wiredtiger/test/suite/test_tiered07.py | 10 +- .../wiredtiger/test/suite/test_tiered08.py | 10 +- .../wiredtiger/test/suite/test_tiered09.py | 18 +- .../wiredtiger/test/suite/test_tiered10.py | 14 +- .../wiredtiger/test/suite/test_tiered11.py | 10 +- .../wiredtiger/test/suite/test_tiered12.py | 14 +- .../wiredtiger/test/suite/test_tiered13.py | 12 +- .../wiredtiger/test/suite/test_tiered14.py | 10 +- 22 files changed, 1450 insertions(+), 1436 deletions(-) create mode 100644 src/third_party/wiredtiger/ext/storage_sources/dir_store/dir_store.c delete mode 100644 src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c mode change 100755 => 100644 src/third_party/wiredtiger/test/suite/helper_tiered.py mode change 100755 => 100644 src/third_party/wiredtiger/test/suite/hook_tiered.py mode change 100755 => 100644 src/third_party/wiredtiger/test/suite/test_tiered04.py mode change 100755 => 100644 src/third_party/wiredtiger/test/suite/test_tiered05.py mode change 100755 => 100644 src/third_party/wiredtiger/test/suite/test_tiered07.py mode change 100755 => 100644 src/third_party/wiredtiger/test/suite/test_tiered08.py mode change 100755 => 100644 src/third_party/wiredtiger/test/suite/test_tiered11.py mode change 100755 => 100644 src/third_party/wiredtiger/test/suite/test_tiered13.py mode change 100755 => 100644 src/third_party/wiredtiger/test/suite/test_tiered14.py diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index 6d19030ba0d..0090048d346 100755 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -113,11 +113,11 @@ func_ok() -e '/int index_compare_primary$/d' \ -e '/int index_compare_u$/d' \ -e '/int index_extractor_u$/d' \ - -e '/int local_directory_list_free$/d' \ - -e '/int local_err$/d' \ - -e '/int local_file_lock$/d' \ - -e '/int local_file_sync$/d' \ - -e '/int local_fs_terminate$/d' \ + -e '/int dir_store_directory_list_free$/d' \ + -e '/int dir_store_err$/d' \ + -e '/int dir_store_file_lock$/d' \ + -e '/int dir_store_file_sync$/d' \ + -e '/int dir_store_fs_terminate$/d' \ -e '/int log_print_err$/d' \ -e '/int lz4_error$/d' \ -e '/int lz4_pre_size$/d' \ diff --git a/src/third_party/wiredtiger/ext/storage_sources/dir_store/dir_store.c b/src/third_party/wiredtiger/ext/storage_sources/dir_store/dir_store.c new file mode 100644 index 00000000000..586ed3a7fcb --- /dev/null +++ b/src/third_party/wiredtiger/ext/storage_sources/dir_store/dir_store.c @@ -0,0 +1,1344 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "queue.h" + +/* + * This storage source implementation is used for demonstration and testing. All objects are stored + * as local files in a designated directory. + */ + +#ifdef __GNUC__ +#if __GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ > 0) +/* + * !!! + * GCC with -Wformat-truncation complains about calls to snprintf in this file. + * There's nothing wrong, this makes the warning go away. + */ +#pragma GCC diagnostic ignored "-Wformat-truncation" +#endif +#endif + +/* Directory storage source structure. */ +typedef struct { + WT_STORAGE_SOURCE storage_source; /* Must come first */ + + WT_EXTENSION_API *wt_api; /* Extension API */ + + /* + * Locks are used to protect the file handle queue and flush queue. + */ + pthread_rwlock_t file_handle_lock; + + /* + * Keep the number of references to this storage source. + */ + uint32_t reference_count; + + /* + * Configuration values are set at startup. + */ + uint32_t delay_ms; /* Average length of delay when simulated */ + uint32_t force_delay; /* Force a simulated network delay every N operations */ + uint32_t force_error; /* Force a simulated network error every N operations */ + uint32_t verbose; /* Verbose level */ + + /* + * Statistics are collected but not yet exposed. + */ + uint64_t fh_ops; /* Non-read/write operations in file handles */ + uint64_t object_writes; /* (What would be) writes to the cloud */ + uint64_t object_reads; /* (What would be) reads to the cloud */ + uint64_t op_count; /* Number of operations done on dir_store */ + uint64_t read_ops; + uint64_t write_ops; + + /* Queue of file handles */ + TAILQ_HEAD(dir_store_file_handle_qh, dir_store_file_handle) fileq; + +} DIR_STORE; + +typedef struct { + /* Must come first - this is the interface for the file system we are implementing. */ + WT_FILE_SYSTEM file_system; + DIR_STORE *dir_store; + + /* This is WiredTiger's file system, it is used in implementing the dir_store file system. */ + WT_FILE_SYSTEM *wt_fs; + + char *auth_token; /* Identifier for key management system */ + char *bucket_dir; /* Directory that stands in for cloud storage bucket */ + char *cache_dir; /* Directory for cached objects */ + const char *home_dir; /* Owned by the connection */ +} DIR_STORE_FILE_SYSTEM; + +typedef struct dir_store_file_handle { + WT_FILE_HANDLE iface; /* Must come first */ + + DIR_STORE *dir_store; /* Enclosing storage source */ + WT_FILE_HANDLE *fh; /* File handle */ + + TAILQ_ENTRY(dir_store_file_handle) q; /* Queue of handles */ +} DIR_STORE_FILE_HANDLE; + +/* + * Forward function declarations for internal functions + */ +static int dir_store_bucket_path(WT_FILE_SYSTEM *, const char *, char **); +static int dir_store_cache_path(WT_FILE_SYSTEM *, const char *, char **); +static int dir_store_home_path(WT_FILE_SYSTEM *, const char *, char **); +static int dir_store_configure(DIR_STORE *, WT_CONFIG_ARG *); +static int dir_store_configure_int(DIR_STORE *, WT_CONFIG_ARG *, const char *, uint32_t *); +static int dir_store_delay(DIR_STORE *); +static int dir_store_err(DIR_STORE *, WT_SESSION *, int, const char *, ...); +static int dir_store_file_copy( + DIR_STORE *, WT_SESSION *, const char *, const char *, WT_FS_OPEN_FILE_TYPE); +static int dir_store_get_directory(const char *, const char *, ssize_t len, bool, char **); +static int dir_store_path(WT_FILE_SYSTEM *, const char *, const char *, char **); +static int dir_store_stat( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, bool, struct stat *); + +/* + * Forward function declarations for storage source API implementation + */ +static int dir_store_add_reference(WT_STORAGE_SOURCE *); +static int dir_store_customize_file_system( + WT_STORAGE_SOURCE *, WT_SESSION *, const char *, const char *, const char *, WT_FILE_SYSTEM **); +static int dir_store_flush( + WT_STORAGE_SOURCE *, WT_SESSION *, WT_FILE_SYSTEM *, const char *, const char *, const char *); +static int dir_store_flush_finish( + WT_STORAGE_SOURCE *, WT_SESSION *, WT_FILE_SYSTEM *, const char *, const char *, const char *); +static int dir_store_terminate(WT_STORAGE_SOURCE *, WT_SESSION *); + +/* + * Forward function declarations for file system API implementation + */ +static int dir_store_directory_list( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, char ***, uint32_t *); +static int dir_store_directory_list_add(DIR_STORE *, char ***, const char *, uint32_t, uint32_t *); +static int dir_store_directory_list_internal( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, uint32_t, char ***, uint32_t *); +static int dir_store_directory_list_single( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, char ***, uint32_t *); +static int dir_store_directory_list_free(WT_FILE_SYSTEM *, WT_SESSION *, char **, uint32_t); +static int dir_store_exist(WT_FILE_SYSTEM *, WT_SESSION *, const char *, bool *); +static int dir_store_fs_terminate(WT_FILE_SYSTEM *, WT_SESSION *); +static int dir_store_open(WT_FILE_SYSTEM *, WT_SESSION *, const char *, + WT_FS_OPEN_FILE_TYPE file_type, uint32_t, WT_FILE_HANDLE **); +static int dir_store_remove(WT_FILE_SYSTEM *, WT_SESSION *, const char *, uint32_t); +static int dir_store_rename(WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, uint32_t); +static int dir_store_size(WT_FILE_SYSTEM *, WT_SESSION *, const char *, wt_off_t *); + +/* + * Forward function declarations for file handle API implementation + */ +static int dir_store_file_close(WT_FILE_HANDLE *, WT_SESSION *); +static int dir_store_file_close_internal(DIR_STORE *, WT_SESSION *, DIR_STORE_FILE_HANDLE *); +static int dir_store_file_lock(WT_FILE_HANDLE *, WT_SESSION *, bool); +static int dir_store_file_read(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, void *); +static int dir_store_file_size(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); +static int dir_store_file_sync(WT_FILE_HANDLE *, WT_SESSION *); +static int dir_store_file_write(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, const void *); + +#define FS2DS(fs) (((DIR_STORE_FILE_SYSTEM *)(fs))->dir_store) +#define SHOW_STRING(s) (((s) == NULL) ? "" : (s)) +#define VERBOSE_LS(dir_store, ...) \ + do { \ + if ((dir_store)->verbose > 0) \ + fprintf(stderr, __VA_ARGS__); \ + } while (0); + +/* + * dir_store_configure + * Parse the configuration for the keys we care about. + */ +static int +dir_store_configure(DIR_STORE *dir_store, WT_CONFIG_ARG *config) +{ + int ret; + + if ((ret = dir_store_configure_int(dir_store, config, "delay_ms", &dir_store->delay_ms)) != 0) + return (ret); + if ((ret = dir_store_configure_int( + dir_store, config, "force_delay", &dir_store->force_delay)) != 0) + return (ret); + if ((ret = dir_store_configure_int( + dir_store, config, "force_error", &dir_store->force_error)) != 0) + return (ret); + if ((ret = dir_store_configure_int(dir_store, config, "verbose", &dir_store->verbose)) != 0) + return (ret); + + return (0); +} + +/* + * dir_store_configure_int + * Look for a particular configuration key, and return its integer value. + */ +static int +dir_store_configure_int( + DIR_STORE *dir_store, WT_CONFIG_ARG *config, const char *key, uint32_t *valuep) +{ + WT_CONFIG_ITEM v; + int ret; + + ret = 0; + + if ((ret = dir_store->wt_api->config_get(dir_store->wt_api, NULL, config, key, &v)) == 0) { + if (v.len == 0 || v.type != WT_CONFIG_ITEM_NUM) + ret = + dir_store_err(dir_store, NULL, EINVAL, "force_error config arg: integer required"); + else + *valuep = (uint32_t)v.val; + } else if (ret == WT_NOTFOUND) + ret = 0; + else + ret = dir_store_err(dir_store, NULL, EINVAL, "WT_API->config_get"); + + return (ret); +} + +/* + * dir_store_delay -- + * Add any artificial delay or simulated network error during an object transfer. + */ +static int +dir_store_delay(DIR_STORE *dir_store) +{ + struct timeval tv; + int ret; + + ret = 0; + if (dir_store->force_delay != 0 && + (dir_store->object_reads + dir_store->object_writes) % dir_store->force_delay == 0) { + VERBOSE_LS(dir_store, + "Artificial delay %" PRIu32 " milliseconds after %" PRIu64 " object reads, %" PRIu64 + " object writes\n", + dir_store->delay_ms, dir_store->object_reads, dir_store->object_writes); + /* + * tv_usec has type suseconds_t, which is signed (hence the s), but ->delay_ms is unsigned. + * In both gcc8 and gcc10 with -Wsign-conversion enabled (as we do) this causes a spurious + * warning about the implicit conversion possibly changing the value. Hence the explicit + * cast. (both struct timeval and suseconds_t are POSIX) + */ + tv.tv_sec = dir_store->delay_ms / 1000; + tv.tv_usec = (suseconds_t)(dir_store->delay_ms % 1000) * 1000; + (void)select(0, NULL, NULL, NULL, &tv); + } + if (dir_store->force_error != 0 && + (dir_store->object_reads + dir_store->object_writes) % dir_store->force_error == 0) { + VERBOSE_LS(dir_store, + "Artificial error returned after %" PRIu64 " object reads, %" PRIu64 " object writes\n", + dir_store->object_reads, dir_store->object_writes); + ret = ENETUNREACH; + } + + return (ret); +} + +/* + * dir_store_err -- + * Print errors from the interface. Returns "ret", the third argument. + */ +static int +dir_store_err(DIR_STORE *dir_store, WT_SESSION *session, int ret, const char *format, ...) +{ + va_list ap; + WT_EXTENSION_API *wt_api; + char buf[1000]; + + va_start(ap, format); + wt_api = dir_store->wt_api; + if (vsnprintf(buf, sizeof(buf), format, ap) >= (int)sizeof(buf)) + wt_api->err_printf(wt_api, session, "dir_store: error overflow"); + wt_api->err_printf( + wt_api, session, "dir_store: %s: %s", wt_api->strerror(wt_api, session, ret), buf); + va_end(ap); + + return (ret); +} + +/* + * dir_store_get_directory -- + * Return a copy of a directory name after verifying that it is a directory. + */ +static int +dir_store_get_directory(const char *home, const char *s, ssize_t len, bool create, char **copy) +{ + struct stat sb; + size_t buflen; + int ret; + char *dirname; + + *copy = NULL; + + if (len == -1) + len = (ssize_t)strlen(s); + + /* For relative pathnames, the path is considered to be relative to the home directory. */ + if (*s == '/') + dirname = strndup(s, (size_t)len + 1); /* Room for null */ + else { + buflen = (size_t)len + strlen(home) + 2; /* Room for slash, null */ + if ((dirname = malloc(buflen)) != NULL) + if (snprintf(dirname, buflen, "%s/%.*s", home, (int)len, s) >= (int)buflen) + return (EINVAL); + } + if (dirname == NULL) + return (ENOMEM); + + ret = stat(dirname, &sb); + if (ret != 0 && errno == ENOENT && create) { + (void)mkdir(dirname, 0777); + ret = stat(dirname, &sb); + } + if (ret != 0) + ret = errno; + else if ((sb.st_mode & S_IFMT) != S_IFDIR) + ret = EINVAL; + if (ret != 0) + free(dirname); + else + *copy = dirname; + return (ret); +} + +/* + * dir_store_bucket_path -- + * Construct the bucket pathname from the file system and dir_store name. + */ +static int +dir_store_bucket_path(WT_FILE_SYSTEM *file_system, const char *name, char **pathp) +{ + return ( + dir_store_path(file_system, ((DIR_STORE_FILE_SYSTEM *)file_system)->bucket_dir, name, pathp)); +} + +/* + * dir_store_cache_path -- + * Construct the cache pathname from the file system and dir_store name. + */ +static int +dir_store_cache_path(WT_FILE_SYSTEM *file_system, const char *name, char **pathp) +{ + return ( + dir_store_path(file_system, ((DIR_STORE_FILE_SYSTEM *)file_system)->cache_dir, name, pathp)); +} + +/* + * dir_store_home_path -- + * Construct the source pathname from the file system and dir_store name. + */ +static int +dir_store_home_path(WT_FILE_SYSTEM *file_system, const char *name, char **pathp) +{ + return ( + dir_store_path(file_system, ((DIR_STORE_FILE_SYSTEM *)file_system)->home_dir, name, pathp)); +} + +/* + * dir_store_path -- + * Construct a pathname from the file system and dir_store name. + */ +static int +dir_store_path(WT_FILE_SYSTEM *file_system, const char *dir, const char *name, char **pathp) +{ + size_t len; + int ret; + char *p; + + ret = 0; + + /* Skip over "./" and variations (".//", ".///./././//") at the beginning of the name. */ + while (*name == '.') { + if (name[1] != '/') + break; + name += 2; + while (*name == '/') + name++; + } + len = strlen(dir) + strlen(name) + 2; + if ((p = malloc(len)) == NULL) + return (dir_store_err(FS2DS(file_system), NULL, ENOMEM, "dir_store_path")); + if (snprintf(p, len, "%s/%s", dir, name) >= (int)len) + return (dir_store_err(FS2DS(file_system), NULL, EINVAL, "overflow sprintf")); + *pathp = p; + return (ret); +} + +/* + * dir_store_stat -- + * Perform the stat system call for a name in the file system. + */ +static int +dir_store_stat(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, + const char *caller, bool must_exist, struct stat *statp) +{ + int ret; + char *path; + + path = NULL; + + /* + * We check to see if the file exists in the cache first, and if not the bucket directory. This + * maps what a real cloud implementation would do. This will allow us to instrument this code to + * try out and measure caching implementations. + */ + if ((ret = dir_store_cache_path(file_system, name, &path)) != 0) + goto err; + + ret = stat(path, statp); + if (ret != 0 && errno == ENOENT) { + /* It's not in the cache, try the bucket directory. */ + free(path); + if ((ret = dir_store_bucket_path(file_system, name, &path)) != 0) + goto err; + ret = stat(path, statp); + } + if (ret != 0) { + /* + * If the file must exist, report the error no matter what. + */ + if (must_exist || errno != ENOENT) + ret = dir_store_err(FS2DS(file_system), session, errno, "%s: %s stat", path, caller); + else + ret = errno; + } +err: + free(path); + return (ret); +} + +/* + * dir_store_add_reference -- + * Add a reference to the storage source so we can reference count to know when to really + * terminate. + */ +static int +dir_store_add_reference(WT_STORAGE_SOURCE *storage_source) +{ + DIR_STORE *dir_store; + + dir_store = (DIR_STORE *)storage_source; + + /* + * Missing reference or overflow? + */ + if (dir_store->reference_count == 0 || dir_store->reference_count + 1 == 0) + return (EINVAL); + ++dir_store->reference_count; + return (0); +} + +/* + * dir_store_customize_file_system -- + * Return a customized file system to access the dir_store storage source objects. + */ +static int +dir_store_customize_file_system(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + const char *bucket_name, const char *auth_token, const char *config, + WT_FILE_SYSTEM **file_systemp) +{ + DIR_STORE *dir_store; + DIR_STORE_FILE_SYSTEM *fs; + WT_CONFIG_ITEM cachedir; + WT_FILE_SYSTEM *wt_fs; + int ret; + const char *p; + char buf[1024]; + + dir_store = (DIR_STORE *)storage_source; + + fs = NULL; + ret = 0; + + /* Parse configuration string. */ + if ((ret = dir_store->wt_api->config_get_string( + dir_store->wt_api, session, config, "cache_directory", &cachedir)) != 0) { + if (ret == WT_NOTFOUND) { + ret = 0; + cachedir.len = 0; + } else { + ret = dir_store_err(dir_store, session, ret, "customize_file_system: config parsing"); + goto err; + } + } + + if ((ret = dir_store->wt_api->file_system_get(dir_store->wt_api, session, &wt_fs)) != 0) { + ret = dir_store_err( + dir_store, session, ret, "dir_store_file_system: cannot get WiredTiger file system"); + goto err; + } + if ((fs = calloc(1, sizeof(DIR_STORE_FILE_SYSTEM))) == NULL) { + ret = dir_store_err(dir_store, session, ENOMEM, "dir_store_file_system"); + goto err; + } + fs->dir_store = dir_store; + fs->wt_fs = wt_fs; + + if ((fs->auth_token = strdup(auth_token)) == NULL) { + ret = dir_store_err(dir_store, session, ENOMEM, "dir_store_file_system.auth_token"); + goto err; + } + + /* + * The home directory owned by the connection will not change, and will be valid memory, for as + * long as the connection is open. That is longer than this file system will be open, so we can + * use the string without copying. + */ + fs->home_dir = session->connection->get_home(session->connection); + + /* + * Get the bucket directory and the cache directory. + */ + if ((ret = dir_store_get_directory(fs->home_dir, bucket_name, -1, false, &fs->bucket_dir)) != + 0) { + ret = dir_store_err(dir_store, session, ret, "%s: bucket directory", bucket_name); + goto err; + } + + /* + * The default cache directory is named "cache-", where name is the last component of the + * bucket name's path. We'll create it if it doesn't exist. + */ + if (cachedir.len == 0) { + if ((p = strrchr(bucket_name, '/')) != NULL) + p++; + else + p = bucket_name; + if (snprintf(buf, sizeof(buf), "cache-%s", p) >= (int)sizeof(buf)) { + ret = dir_store_err(dir_store, session, EINVAL, "overflow snprintf"); + goto err; + } + cachedir.str = buf; + cachedir.len = strlen(buf); + } + if ((ret = dir_store_get_directory( + fs->home_dir, cachedir.str, (ssize_t)cachedir.len, true, &fs->cache_dir)) != 0) { + ret = dir_store_err( + dir_store, session, ret, "%*s: cache directory", (int)cachedir.len, cachedir.str); + goto err; + } + fs->file_system.fs_directory_list = dir_store_directory_list; + fs->file_system.fs_directory_list_single = dir_store_directory_list_single; + fs->file_system.fs_directory_list_free = dir_store_directory_list_free; + fs->file_system.fs_exist = dir_store_exist; + fs->file_system.fs_open_file = dir_store_open; + fs->file_system.fs_remove = dir_store_remove; + fs->file_system.fs_rename = dir_store_rename; + fs->file_system.fs_size = dir_store_size; + fs->file_system.terminate = dir_store_fs_terminate; + +err: + if (ret == 0) + *file_systemp = &fs->file_system; + else if (fs != NULL) { + free(fs->auth_token); + free(fs->bucket_dir); + free(fs->cache_dir); + free(fs); + } + return (ret); +} + +/* + * dir_store_exist -- + * Return if the file exists. + */ +static int +dir_store_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, bool *existp) +{ + struct stat sb; + DIR_STORE *dir_store; + int ret; + + dir_store = FS2DS(file_system); + dir_store->op_count++; + *existp = false; + + if ((ret = dir_store_stat(file_system, session, name, "ss_exist", false, &sb)) == 0) + *existp = true; + else if (ret == ENOENT) + ret = 0; + + return (ret); +} + +/* + * dir_store_file_copy -- + * Copy a file. + */ +static int +dir_store_file_copy(DIR_STORE *dir_store, WT_SESSION *session, const char *src_path, + const char *dest_path, WT_FS_OPEN_FILE_TYPE type) +{ + WT_FILE_HANDLE *dest, *src; + WT_FILE_SYSTEM *wt_fs; + wt_off_t copy_size, file_size, left; + ssize_t pos; + size_t pathlen; + int ret, t_ret; + char buffer[1024 * 64], *tmp_path; + + dest = src = NULL; + pathlen = strlen(dest_path) + 10; + if ((tmp_path = malloc(pathlen)) != NULL) + if (snprintf(tmp_path, pathlen, "%s.TMP", dest_path) >= (int)pathlen) { + ret = dir_store_err(dir_store, session, EINVAL, "overflow snprintf"); + goto err; + } + + if ((ret = dir_store->wt_api->file_system_get(dir_store->wt_api, session, &wt_fs)) != 0) { + ret = dir_store_err( + dir_store, session, ret, "dir_store_file_system: cannot get WiredTiger file system"); + goto err; + } + if ((ret = wt_fs->fs_open_file(wt_fs, session, src_path, type, WT_FS_OPEN_READONLY, &src)) != + 0) { + ret = dir_store_err(dir_store, session, ret, "%s: cannot open for read", src_path); + goto err; + } + + if ((ret = wt_fs->fs_open_file(wt_fs, session, tmp_path, type, WT_FS_OPEN_CREATE, &dest)) != + 0) { + ret = dir_store_err(dir_store, session, ret, "%s: cannot create", tmp_path); + goto err; + } + if ((ret = wt_fs->fs_size(wt_fs, session, src_path, &file_size)) != 0) { + ret = dir_store_err(dir_store, session, ret, "%s: cannot get size", src_path); + goto err; + } + for (pos = 0, left = file_size; left > 0; pos += copy_size, left -= copy_size) { + copy_size = left < (wt_off_t)sizeof(buffer) ? left : (wt_off_t)sizeof(buffer); + if ((ret = src->fh_read(src, session, pos, (size_t)copy_size, buffer)) != 0) { + ret = dir_store_err(dir_store, session, ret, "%s: cannot read", src_path); + goto err; + } + if ((ret = dest->fh_write(dest, session, pos, (size_t)copy_size, buffer)) != 0) { + ret = dir_store_err(dir_store, session, ret, "%s: cannot write", tmp_path); + goto err; + } + } + if (ret == 0 && (ret = chmod(tmp_path, 0444)) < 0) + ret = dir_store_err(dir_store, session, errno, "%s: file_copy chmod failed", tmp_path); + if ((ret = rename(tmp_path, dest_path)) != 0) { + ret = dir_store_err( + dir_store, session, errno, "%s: cannot rename from %s", dest_path, tmp_path); + goto err; + } +err: + if (src != NULL && (t_ret = src->close(src, session)) != 0) + if (ret == 0) + ret = t_ret; + if (dest != NULL && (t_ret = dest->close(dest, session)) != 0) + if (ret == 0) + ret = t_ret; + if (ret != 0) + (void)unlink(tmp_path); + free(tmp_path); + + return (ret); +} + +/* + * dir_store_flush -- + * Return when the file has been flushed. + */ +static int +dir_store_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, WT_FILE_SYSTEM *file_system, + const char *source, const char *object, const char *config) +{ + DIR_STORE *dir_store; + int ret; + char *dest_path, *src_path; + + (void)config; /* unused */ + dest_path = src_path = NULL; + dir_store = (DIR_STORE *)storage_source; + ret = 0; + + if (file_system == NULL || source == NULL || object == NULL) + return dir_store_err(dir_store, session, EINVAL, "ss_flush: required arguments missing"); + + if ((ret = dir_store_home_path(file_system, source, &src_path)) != 0) + goto err; + + if ((ret = dir_store_bucket_path(file_system, object, &dest_path)) != 0) + goto err; + + if ((ret = dir_store_delay(dir_store)) != 0) + goto err; + + if ((ret = dir_store_file_copy( + dir_store, session, src_path, dest_path, WT_FS_OPEN_FILE_TYPE_DATA)) != 0) + goto err; + + dir_store->object_writes++; + +err: + free(dest_path); + free(src_path); + return (ret); +} + +/* + * dir_store_flush_finish -- + * Cache a file in the new file system. + */ +static int +dir_store_flush_finish(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_FILE_SYSTEM *file_system, const char *source, const char *object, const char *config) +{ + DIR_STORE *dir_store; + int ret; + char *dest_path, *src_path; + + (void)config; /* unused */ + dest_path = src_path = NULL; + dir_store = (DIR_STORE *)storage_source; + ret = 0; + + if (file_system == NULL || source == NULL || object == NULL) + return dir_store_err( + dir_store, session, EINVAL, "ss_flush_finish: required arguments missing"); + + if ((ret = dir_store_home_path(file_system, source, &src_path)) != 0) + goto err; + + if ((ret = dir_store_cache_path(file_system, object, &dest_path)) != 0) + goto err; + + dir_store->op_count++; + /* + * Link the object with the original dir_store object. The could be replaced by a file copy if + * portability is an issue. + */ + if ((ret = link(src_path, dest_path)) != 0) { + ret = dir_store_err( + dir_store, session, errno, "ss_flush_finish link %s to %s failed", source, dest_path); + goto err; + } + /* Set the file to readonly in the cache. */ + if (ret == 0 && (ret = chmod(dest_path, 0444)) < 0) + ret = + dir_store_err(dir_store, session, errno, "%s: ss_flush_finish chmod failed", dest_path); +err: + free(dest_path); + free(src_path); + return (ret); +} + +/* + * dir_store_directory_list -- + * Return a list of object names for the given location. + */ +static int +dir_store_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *directory, + const char *prefix, char ***dirlistp, uint32_t *countp) +{ + FS2DS(file_system)->op_count++; + return (dir_store_directory_list_internal( + file_system, session, directory, prefix, 0, dirlistp, countp)); +} + +/* + * dir_store_directory_list_single -- + * Return a single file name for the given location. + */ +static int +dir_store_directory_list_single(WT_FILE_SYSTEM *file_system, WT_SESSION *session, + const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp) +{ + FS2DS(file_system)->op_count++; + return (dir_store_directory_list_internal( + file_system, session, directory, prefix, 1, dirlistp, countp)); +} + +/* + * dir_store_location_list_free -- + * Free memory allocated by dir_store_location_list. + */ +static int +dir_store_directory_list_free( + WT_FILE_SYSTEM *file_system, WT_SESSION *session, char **dirlist, uint32_t count) +{ + (void)session; + + FS2DS(file_system)->op_count++; + if (dirlist != NULL) { + while (count > 0) + free(dirlist[--count]); + free(dirlist); + } + return (0); +} + +/* + * dir_store_directory_list_add -- + * Add an entry to the directory list, growing as needed. + */ +static int +dir_store_directory_list_add( + DIR_STORE *dir_store, char ***entriesp, const char *s, uint32_t count, uint32_t *allocatedp) +{ + size_t alloc_sz; + char **entries, **new_entries; + + entries = *entriesp; + if (count >= *allocatedp) { + *allocatedp += 10; + alloc_sz = sizeof(char *) * (*allocatedp); + if ((new_entries = realloc(entries, alloc_sz)) == NULL) + return (dir_store_err(dir_store, NULL, ENOMEM, "cannot grow directory list")); + entries = new_entries; + *entriesp = entries; + } + if ((entries[count] = strdup(s)) == NULL) + return (dir_store_err(dir_store, NULL, ENOMEM, "cannot grow directory list")); + + return (0); +} + +/* + * dir_store_location_list_internal -- + * Return a list of object names for the given location. + */ +static int +dir_store_directory_list_internal(WT_FILE_SYSTEM *file_system, WT_SESSION *session, + const char *directory, const char *prefix, uint32_t limit, char ***dirlistp, uint32_t *countp) +{ + struct dirent *dp; + DIR *dirp; + DIR_STORE_FILE_SYSTEM *dir_store_fs; + DIR_STORE *dir_store; + size_t dir_len, prefix_len; + uint32_t allocated, count; + int ret, t_ret; + char **entries; + const char *basename; + + dir_store_fs = (DIR_STORE_FILE_SYSTEM *)file_system; + dir_store = dir_store_fs->dir_store; + entries = NULL; + allocated = count = 0; + dir_len = (directory == NULL ? 0 : strlen(directory)); + prefix_len = (prefix == NULL ? 0 : strlen(prefix)); + ret = 0; + + *dirlistp = NULL; + *countp = 0; + + /* + * The items in the bucket directory represent the definitive list. + */ + if ((dirp = opendir(dir_store_fs->bucket_dir)) == NULL) { + ret = errno; + if (ret == 0) + ret = EINVAL; + return (dir_store_err( + dir_store, session, ret, "%s: ss_directory_list: opendir", dir_store_fs->bucket_dir)); + } + + for (count = 0; (dp = readdir(dirp)) != NULL && (limit == 0 || count < limit);) { + /* Skip . and .. */ + basename = dp->d_name; + if (strcmp(basename, ".") == 0 || strcmp(basename, "..") == 0) + continue; + + /* Match only the indicated directory files. */ + if (directory != NULL && strncmp(basename, directory, dir_len) != 0) + continue; + basename += dir_len; + + /* The list of files is optionally filtered by a prefix. */ + if (prefix != NULL && strncmp(basename, prefix, prefix_len) != 0) + continue; + + if ((ret = dir_store_directory_list_add( + dir_store, &entries, basename, count, &allocated)) != 0) + goto err; + count++; + } + + *dirlistp = entries; + *countp = count; + +err: + if (closedir(dirp) != 0) { + t_ret = dir_store_err( + dir_store, session, errno, "%s: ss_directory_list: closedir", dir_store_fs->cache_dir); + if (ret == 0) + ret = t_ret; + } + if (ret == 0) + return (0); + + if (entries != NULL) { + while (count > 0) + free(entries[--count]); + free(entries); + } + return (ret); +} + +/* + * dir_store_fs_terminate -- + * Discard any resources on termination of the file system + */ +static int +dir_store_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) +{ + DIR_STORE_FILE_SYSTEM *dir_store_fs; + + (void)session; /* unused */ + + dir_store_fs = (DIR_STORE_FILE_SYSTEM *)file_system; + FS2DS(file_system)->op_count++; + free(dir_store_fs->auth_token); + free(dir_store_fs->bucket_dir); + free(dir_store_fs->cache_dir); + free(file_system); + + return (0); +} + +/* + * dir_store_open -- + * fopen for our dir_store storage source + */ +static int +dir_store_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, + WT_FS_OPEN_FILE_TYPE file_type, uint32_t flags, WT_FILE_HANDLE **file_handlep) +{ + DIR_STORE_FILE_HANDLE *dir_store_fh; + DIR_STORE_FILE_SYSTEM *dir_store_fs; + DIR_STORE *dir_store; + WT_FILE_HANDLE *file_handle, *wt_fh; + WT_FILE_SYSTEM *wt_fs; + struct stat sb; + int ret; + char *bucket_path, *cache_path; + + (void)flags; /* Unused */ + + ret = 0; + *file_handlep = NULL; + dir_store_fh = NULL; + dir_store_fs = (DIR_STORE_FILE_SYSTEM *)file_system; + dir_store = dir_store_fs->dir_store; + wt_fs = dir_store_fs->wt_fs; + bucket_path = cache_path = NULL; + + if ((flags & WT_FS_OPEN_READONLY) == 0 || (flags & WT_FS_OPEN_CREATE) != 0) + return (dir_store_err( + dir_store, session, EINVAL, "ss_open_object: readonly access required: %s", name)); + + /* + * We expect that the dir_store file system will be used narrowly, like when creating or opening + * a data file or turtle file. It would be unexpected to try to open a non-data file (like a log + * file) in that narrow part of code, so we make it an error here. + * + * Relaxing this constraint to allow opening of, say, log files, would be straightforward - we + * would not translate the path or do any tracking for flushing. But there's a catch. Other + * parts of the API, like remove and rename, have no flag indicating that they are operating on + * a log file, so we wouldn't know whether to do path translation. Of course, we could peek at + * the name, but that would be bad form. + */ + if (file_type != WT_FS_OPEN_FILE_TYPE_DATA && file_type != WT_FS_OPEN_FILE_TYPE_REGULAR) + return (dir_store_err(dir_store, session, EINVAL, + "%s: open: only data file and regular types supported", name)); + + /* Create a new handle. */ + if ((dir_store_fh = calloc(1, sizeof(DIR_STORE_FILE_HANDLE))) == NULL) { + ret = ENOMEM; + goto err; + } + if ((ret = dir_store_cache_path(file_system, name, &cache_path)) != 0) + goto err; + ret = stat(cache_path, &sb); + if (ret != 0) { + if (errno != ENOENT) { + ret = dir_store_err(dir_store, session, errno, "%s: dir_store_open stat", cache_path); + goto err; + } + + /* + * The file doesn't exist locally, make a copy of it from the cloud. + */ + if ((ret = dir_store_bucket_path(file_system, name, &bucket_path)) != 0) + goto err; + + if ((ret = dir_store_delay(dir_store)) != 0) + goto err; + + if ((ret = dir_store_file_copy( + dir_store, session, bucket_path, cache_path, WT_FS_OPEN_FILE_TYPE_DATA)) != 0) + goto err; + + dir_store->object_reads++; + } + if ((ret = wt_fs->fs_open_file(wt_fs, session, cache_path, file_type, flags, &wt_fh)) != 0) { + ret = dir_store_err(dir_store, session, ret, "ss_open_object: open: %s", name); + goto err; + } + dir_store_fh->fh = wt_fh; + dir_store_fh->dir_store = dir_store; + + /* Initialize public information. */ + file_handle = (WT_FILE_HANDLE *)dir_store_fh; + + /* + * Setup the function call table for our custom storage source. Set the function pointer to NULL + * where our implementation doesn't support the functionality. + */ + file_handle->close = dir_store_file_close; + file_handle->fh_advise = NULL; + file_handle->fh_extend = NULL; + file_handle->fh_extend_nolock = NULL; + file_handle->fh_lock = dir_store_file_lock; + file_handle->fh_map = NULL; + file_handle->fh_map_discard = NULL; + file_handle->fh_map_preload = NULL; + file_handle->fh_unmap = NULL; + file_handle->fh_read = dir_store_file_read; + file_handle->fh_size = dir_store_file_size; + file_handle->fh_sync = dir_store_file_sync; + file_handle->fh_sync_nowait = NULL; + file_handle->fh_truncate = NULL; + file_handle->fh_write = dir_store_file_write; + if ((file_handle->name = strdup(name)) == NULL) { + ret = ENOMEM; + goto err; + } + + if ((ret = pthread_rwlock_wrlock(&dir_store->file_handle_lock)) != 0) { + (void)dir_store_err(dir_store, session, ret, "ss_open_object: pthread_rwlock_wrlock"); + goto err; + } + TAILQ_INSERT_HEAD(&dir_store->fileq, dir_store_fh, q); + if ((ret = pthread_rwlock_unlock(&dir_store->file_handle_lock)) != 0) { + (void)dir_store_err(dir_store, session, ret, "ss_open_object: pthread_rwlock_unlock"); + goto err; + } + + *file_handlep = file_handle; + + VERBOSE_LS(dir_store, "File opened: %s final path=%s\n", SHOW_STRING(name), + SHOW_STRING(dir_store_fh->fh->name)); + +err: + free(bucket_path); + free(cache_path); + if (ret != 0) { + if (dir_store_fh != NULL) + dir_store_file_close_internal(dir_store, session, dir_store_fh); + } + return (ret); +} + +/* + * dir_store_rename -- + * POSIX rename, not supported for cloud objects. + */ +static int +dir_store_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *from, const char *to, + uint32_t flags) +{ + (void)to; /* unused */ + (void)flags; /* unused */ + + return (dir_store_err( + FS2DS(file_system), session, ENOTSUP, "%s: rename of file not supported", from)); +} + +/* + * dir_store_remove -- + * POSIX remove, not supported for cloud objects. + */ +static int +dir_store_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, uint32_t flags) +{ + (void)flags; /* unused */ + + return (dir_store_err( + FS2DS(file_system), session, ENOTSUP, "%s: remove of file not supported", name)); +} + +/* + * dir_store_size -- + * Get the size of a file in bytes, by file name. + */ +static int +dir_store_size(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, wt_off_t *sizep) +{ + struct stat sb; + DIR_STORE *dir_store; + int ret; + + dir_store = FS2DS(file_system); + dir_store->op_count++; + *sizep = 0; + + if ((ret = dir_store_stat(file_system, session, name, "ss_size", true, &sb)) != 0) + goto err; + + *sizep = sb.st_size; +err: + return (ret); +} + +/* + * dir_store_terminate -- + * Discard any resources on termination + */ +static int +dir_store_terminate(WT_STORAGE_SOURCE *storage, WT_SESSION *session) +{ + DIR_STORE_FILE_HANDLE *dir_store_fh, *safe_fh; + DIR_STORE *dir_store; + int ret; + + ret = 0; + dir_store = (DIR_STORE *)storage; + + if (--dir_store->reference_count != 0) + return (0); + + dir_store->op_count++; + + /* + * We should be single threaded at this point, so it is safe to destroy the lock and access the + * file handle list without locking it. + */ + if ((ret = pthread_rwlock_destroy(&dir_store->file_handle_lock)) != 0) + (void)dir_store_err(dir_store, session, ret, "terminate: pthread_rwlock_destroy"); + + TAILQ_FOREACH_SAFE(dir_store_fh, &dir_store->fileq, q, safe_fh) + dir_store_file_close_internal(dir_store, session, dir_store_fh); + + free(dir_store); + return (ret); +} + +/* + * dir_store_file_close -- + * ANSI C close. + */ +static int +dir_store_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session) +{ + DIR_STORE *dir_store; + DIR_STORE_FILE_HANDLE *dir_store_fh; + int ret, t_ret; + + ret = 0; + dir_store_fh = (DIR_STORE_FILE_HANDLE *)file_handle; + dir_store = dir_store_fh->dir_store; + + dir_store->fh_ops++; + if ((ret = pthread_rwlock_wrlock(&dir_store->file_handle_lock)) != 0) + /* There really isn't anything more we can do. It will get cleaned up on terminate. */ + return (dir_store_err(dir_store, session, ret, "file handle close: pthread_rwlock_wrlock")); + + TAILQ_REMOVE(&dir_store->fileq, dir_store_fh, q); + + if ((ret = pthread_rwlock_unlock(&dir_store->file_handle_lock)) != 0) + (void)dir_store_err(dir_store, session, ret, "file handle close: pthread_rwlock_unlock"); + + if ((t_ret = dir_store_file_close_internal(dir_store, session, dir_store_fh)) != 0) { + if (ret == 0) + ret = t_ret; + } + + return (ret); +} + +/* + * dir_store_file_close_internal -- + * Internal file handle close. + */ +static int +dir_store_file_close_internal( + DIR_STORE *dir_store, WT_SESSION *session, DIR_STORE_FILE_HANDLE *dir_store_fh) +{ + int ret; + WT_FILE_HANDLE *wt_fh; + + ret = 0; + wt_fh = dir_store_fh->fh; + if (wt_fh != NULL && (ret = wt_fh->close(wt_fh, session)) != 0) + ret = dir_store_err(dir_store, session, ret, "WT_FILE_HANDLE->close: close"); + + free(dir_store_fh->iface.name); + free(dir_store_fh); + + return (ret); +} + +/* + * dir_store_file_lock -- + * Lock/unlock a file. + */ +static int +dir_store_file_lock(WT_FILE_HANDLE *file_handle, WT_SESSION *session, bool lock) +{ + /* Locks are always granted. */ + + (void)session; /* Unused */ + (void)lock; /* Unused */ + + ((DIR_STORE_FILE_HANDLE *)file_handle)->dir_store->fh_ops++; + return (0); +} + +/* + * dir_store_file_read -- + * POSIX pread. + */ +static int +dir_store_file_read( + WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, void *buf) +{ + DIR_STORE_FILE_HANDLE *dir_store_fh; + WT_FILE_HANDLE *wt_fh; + + dir_store_fh = (DIR_STORE_FILE_HANDLE *)file_handle; + wt_fh = dir_store_fh->fh; + + dir_store_fh->dir_store->read_ops++; + return (wt_fh->fh_read(wt_fh, session, offset, len, buf)); +} + +/* + * dir_store_file_size -- + * Get the size of a file in bytes, by file handle. + */ +static int +dir_store_file_size(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t *sizep) +{ + DIR_STORE_FILE_HANDLE *dir_store_fh; + WT_FILE_HANDLE *wt_fh; + + dir_store_fh = (DIR_STORE_FILE_HANDLE *)file_handle; + wt_fh = dir_store_fh->fh; + + dir_store_fh->dir_store->fh_ops++; + return (wt_fh->fh_size(wt_fh, session, sizep)); +} + +/* + * dir_store_file_sync -- + * Ensure the content of the dir_store file is stable. + */ +static int +dir_store_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *session) +{ + /* This is a no-op. We could also disallow it. */ + (void)file_handle; + (void)session; + return (0); +} + +/* + * dir_store_file_write -- + * POSIX pwrite. + */ +static int +dir_store_file_write( + WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, const void *buf) +{ + DIR_STORE_FILE_HANDLE *dir_store_fh; + + (void)offset; + (void)len; + (void)buf; + + dir_store_fh = (DIR_STORE_FILE_HANDLE *)file_handle; + return (dir_store_err(dir_store_fh->dir_store, session, ENOTSUP, + "ss_open_object: write not supported: %s", dir_store_fh->iface.name)); +} + +/* + * wiredtiger_extension_init -- + * A simple shared library encryption example. + */ +int +wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) +{ + DIR_STORE *dir_store; + int ret; + + if ((dir_store = calloc(1, sizeof(DIR_STORE))) == NULL) + return (errno); + dir_store->wt_api = connection->get_extension_api(connection); + if ((ret = pthread_rwlock_init(&dir_store->file_handle_lock, NULL)) != 0) { + (void)dir_store_err(dir_store, NULL, ret, "pthread_rwlock_init"); + free(dir_store); + return (ret); + } + + /* + * Allocate a dir_store storage structure, with a WT_STORAGE structure as the first field, + * allowing us to treat references to either type of structure as a reference to the other type. + */ + dir_store->storage_source.ss_add_reference = dir_store_add_reference; + dir_store->storage_source.ss_customize_file_system = dir_store_customize_file_system; + dir_store->storage_source.ss_flush = dir_store_flush; + dir_store->storage_source.ss_flush_finish = dir_store_flush_finish; + dir_store->storage_source.terminate = dir_store_terminate; + + /* + * The first reference is implied by the call to add_storage_source. + */ + dir_store->reference_count = 1; + + if ((ret = dir_store_configure(dir_store, config)) != 0) { + free(dir_store); + return (ret); + } + + /* Load the storage */ + if ((ret = connection->add_storage_source( + connection, "dir_store", &dir_store->storage_source, NULL)) != 0) { + (void)dir_store_err(dir_store, NULL, ret, "WT_CONNECTION->add_storage_source"); + free(dir_store); + } + return (ret); +} diff --git a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c deleted file mode 100644 index fc1984b2547..00000000000 --- a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c +++ /dev/null @@ -1,1331 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "queue.h" - -/* - * This storage source implementation is used for demonstration and testing. All objects are stored - * as local files. - */ - -#ifdef __GNUC__ -#if __GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ > 0) -/* - * !!! - * GCC with -Wformat-truncation complains about calls to snprintf in this file. - * There's nothing wrong, this makes the warning go away. - */ -#pragma GCC diagnostic ignored "-Wformat-truncation" -#endif -#endif - -/* Local storage source structure. */ -typedef struct { - WT_STORAGE_SOURCE storage_source; /* Must come first */ - - WT_EXTENSION_API *wt_api; /* Extension API */ - - /* - * Locks are used to protect the file handle queue and flush queue. - */ - pthread_rwlock_t file_handle_lock; - - /* - * Keep the number of references to this storage source. - */ - uint32_t reference_count; - - /* - * Configuration values are set at startup. - */ - uint32_t delay_ms; /* Average length of delay when simulated */ - uint32_t force_delay; /* Force a simulated network delay every N operations */ - uint32_t force_error; /* Force a simulated network error every N operations */ - uint32_t verbose; /* Verbose level */ - - /* - * Statistics are collected but not yet exposed. - */ - uint64_t fh_ops; /* Non-read/write operations in file handles */ - uint64_t object_writes; /* (What would be) writes to the cloud */ - uint64_t object_reads; /* (What would be) reads to the cloud */ - uint64_t op_count; /* Number of operations done on local */ - uint64_t read_ops; - uint64_t write_ops; - - /* Queue of file handles */ - TAILQ_HEAD(local_file_handle_qh, local_file_handle) fileq; - -} LOCAL_STORAGE; - -typedef struct { - /* Must come first - this is the interface for the file system we are implementing. */ - WT_FILE_SYSTEM file_system; - LOCAL_STORAGE *local_storage; - - /* This is WiredTiger's file system, it is used in implementing the local file system. */ - WT_FILE_SYSTEM *wt_fs; - - char *auth_token; /* Identifier for key management system */ - char *bucket_dir; /* Directory that stands in for cloud storage bucket */ - char *cache_dir; /* Directory for cached objects */ - const char *home_dir; /* Owned by the connection */ -} LOCAL_FILE_SYSTEM; - -typedef struct local_file_handle { - WT_FILE_HANDLE iface; /* Must come first */ - - LOCAL_STORAGE *local; /* Enclosing storage source */ - WT_FILE_HANDLE *fh; /* File handle */ - - TAILQ_ENTRY(local_file_handle) q; /* Queue of handles */ -} LOCAL_FILE_HANDLE; - -/* - * Forward function declarations for internal functions - */ -static int local_bucket_path(WT_FILE_SYSTEM *, const char *, char **); -static int local_cache_path(WT_FILE_SYSTEM *, const char *, char **); -static int local_home_path(WT_FILE_SYSTEM *, const char *, char **); -static int local_configure(LOCAL_STORAGE *, WT_CONFIG_ARG *); -static int local_configure_int(LOCAL_STORAGE *, WT_CONFIG_ARG *, const char *, uint32_t *); -static int local_delay(LOCAL_STORAGE *); -static int local_err(LOCAL_STORAGE *, WT_SESSION *, int, const char *, ...); -static int local_file_copy( - LOCAL_STORAGE *, WT_SESSION *, const char *, const char *, WT_FS_OPEN_FILE_TYPE); -static int local_get_directory(const char *, const char *, ssize_t len, bool, char **); -static int local_path(WT_FILE_SYSTEM *, const char *, const char *, char **); -static int local_stat( - WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, bool, struct stat *); - -/* - * Forward function declarations for storage source API implementation - */ -static int local_add_reference(WT_STORAGE_SOURCE *); -static int local_customize_file_system( - WT_STORAGE_SOURCE *, WT_SESSION *, const char *, const char *, const char *, WT_FILE_SYSTEM **); -static int local_flush( - WT_STORAGE_SOURCE *, WT_SESSION *, WT_FILE_SYSTEM *, const char *, const char *, const char *); -static int local_flush_finish( - WT_STORAGE_SOURCE *, WT_SESSION *, WT_FILE_SYSTEM *, const char *, const char *, const char *); -static int local_terminate(WT_STORAGE_SOURCE *, WT_SESSION *); - -/* - * Forward function declarations for file system API implementation - */ -static int local_directory_list( - WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, char ***, uint32_t *); -static int local_directory_list_add(LOCAL_STORAGE *, char ***, const char *, uint32_t, uint32_t *); -static int local_directory_list_internal( - WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, uint32_t, char ***, uint32_t *); -static int local_directory_list_single( - WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, char ***, uint32_t *); -static int local_directory_list_free(WT_FILE_SYSTEM *, WT_SESSION *, char **, uint32_t); -static int local_exist(WT_FILE_SYSTEM *, WT_SESSION *, const char *, bool *); -static int local_fs_terminate(WT_FILE_SYSTEM *, WT_SESSION *); -static int local_open(WT_FILE_SYSTEM *, WT_SESSION *, const char *, WT_FS_OPEN_FILE_TYPE file_type, - uint32_t, WT_FILE_HANDLE **); -static int local_remove(WT_FILE_SYSTEM *, WT_SESSION *, const char *, uint32_t); -static int local_rename(WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, uint32_t); -static int local_size(WT_FILE_SYSTEM *, WT_SESSION *, const char *, wt_off_t *); - -/* - * Forward function declarations for file handle API implementation - */ -static int local_file_close(WT_FILE_HANDLE *, WT_SESSION *); -static int local_file_close_internal(LOCAL_STORAGE *, WT_SESSION *, LOCAL_FILE_HANDLE *); -static int local_file_lock(WT_FILE_HANDLE *, WT_SESSION *, bool); -static int local_file_read(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, void *); -static int local_file_size(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); -static int local_file_sync(WT_FILE_HANDLE *, WT_SESSION *); -static int local_file_write(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, const void *); - -#define FS2LOCAL(fs) (((LOCAL_FILE_SYSTEM *)(fs))->local_storage) -#define SHOW_STRING(s) (((s) == NULL) ? "" : (s)) -#define VERBOSE_LS(local, ...) \ - do { \ - if ((local)->verbose > 0) \ - fprintf(stderr, __VA_ARGS__); \ - } while (0); - -/* - * local_configure - * Parse the configuration for the keys we care about. - */ -static int -local_configure(LOCAL_STORAGE *local, WT_CONFIG_ARG *config) -{ - int ret; - - if ((ret = local_configure_int(local, config, "delay_ms", &local->delay_ms)) != 0) - return (ret); - if ((ret = local_configure_int(local, config, "force_delay", &local->force_delay)) != 0) - return (ret); - if ((ret = local_configure_int(local, config, "force_error", &local->force_error)) != 0) - return (ret); - if ((ret = local_configure_int(local, config, "verbose", &local->verbose)) != 0) - return (ret); - - return (0); -} - -/* - * local_configure_int - * Look for a particular configuration key, and return its integer value. - */ -static int -local_configure_int(LOCAL_STORAGE *local, WT_CONFIG_ARG *config, const char *key, uint32_t *valuep) -{ - WT_CONFIG_ITEM v; - int ret; - - ret = 0; - - if ((ret = local->wt_api->config_get(local->wt_api, NULL, config, key, &v)) == 0) { - if (v.len == 0 || v.type != WT_CONFIG_ITEM_NUM) - ret = local_err(local, NULL, EINVAL, "force_error config arg: integer required"); - else - *valuep = (uint32_t)v.val; - } else if (ret == WT_NOTFOUND) - ret = 0; - else - ret = local_err(local, NULL, EINVAL, "WT_API->config_get"); - - return (ret); -} - -/* - * local_delay -- - * Add any artificial delay or simulated network error during an object transfer. - */ -static int -local_delay(LOCAL_STORAGE *local) -{ - struct timeval tv; - int ret; - - ret = 0; - if (local->force_delay != 0 && - (local->object_reads + local->object_writes) % local->force_delay == 0) { - VERBOSE_LS(local, - "Artificial delay %" PRIu32 " milliseconds after %" PRIu64 " object reads, %" PRIu64 - " object writes\n", - local->delay_ms, local->object_reads, local->object_writes); - /* - * tv_usec has type suseconds_t, which is signed (hence the s), but ->delay_ms is unsigned. - * In both gcc8 and gcc10 with -Wsign-conversion enabled (as we do) this causes a spurious - * warning about the implicit conversion possibly changing the value. Hence the explicit - * cast. (both struct timeval and suseconds_t are POSIX) - */ - tv.tv_sec = local->delay_ms / 1000; - tv.tv_usec = (suseconds_t)(local->delay_ms % 1000) * 1000; - (void)select(0, NULL, NULL, NULL, &tv); - } - if (local->force_error != 0 && - (local->object_reads + local->object_writes) % local->force_error == 0) { - VERBOSE_LS(local, - "Artificial error returned after %" PRIu64 " object reads, %" PRIu64 " object writes\n", - local->object_reads, local->object_writes); - ret = ENETUNREACH; - } - - return (ret); -} - -/* - * local_err -- - * Print errors from the interface. Returns "ret", the third argument. - */ -static int -local_err(LOCAL_STORAGE *local, WT_SESSION *session, int ret, const char *format, ...) -{ - va_list ap; - WT_EXTENSION_API *wt_api; - char buf[1000]; - - va_start(ap, format); - wt_api = local->wt_api; - if (vsnprintf(buf, sizeof(buf), format, ap) >= (int)sizeof(buf)) - wt_api->err_printf(wt_api, session, "local_storage: error overflow"); - wt_api->err_printf( - wt_api, session, "local_storage: %s: %s", wt_api->strerror(wt_api, session, ret), buf); - va_end(ap); - - return (ret); -} - -/* - * local_get_directory -- - * Return a copy of a directory name after verifying that it is a directory. - */ -static int -local_get_directory(const char *home, const char *s, ssize_t len, bool create, char **copy) -{ - struct stat sb; - size_t buflen; - int ret; - char *dirname; - - *copy = NULL; - - if (len == -1) - len = (ssize_t)strlen(s); - - /* For relative pathnames, the path is considered to be relative to the home directory. */ - if (*s == '/') - dirname = strndup(s, (size_t)len + 1); /* Room for null */ - else { - buflen = (size_t)len + strlen(home) + 2; /* Room for slash, null */ - if ((dirname = malloc(buflen)) != NULL) - if (snprintf(dirname, buflen, "%s/%.*s", home, (int)len, s) >= (int)buflen) - return (EINVAL); - } - if (dirname == NULL) - return (ENOMEM); - - ret = stat(dirname, &sb); - if (ret != 0 && errno == ENOENT && create) { - (void)mkdir(dirname, 0777); - ret = stat(dirname, &sb); - } - if (ret != 0) - ret = errno; - else if ((sb.st_mode & S_IFMT) != S_IFDIR) - ret = EINVAL; - if (ret != 0) - free(dirname); - else - *copy = dirname; - return (ret); -} - -/* - * local_bucket_path -- - * Construct the bucket pathname from the file system and local name. - */ -static int -local_bucket_path(WT_FILE_SYSTEM *file_system, const char *name, char **pathp) -{ - return (local_path(file_system, ((LOCAL_FILE_SYSTEM *)file_system)->bucket_dir, name, pathp)); -} - -/* - * local_cache_path -- - * Construct the cache pathname from the file system and local name. - */ -static int -local_cache_path(WT_FILE_SYSTEM *file_system, const char *name, char **pathp) -{ - return (local_path(file_system, ((LOCAL_FILE_SYSTEM *)file_system)->cache_dir, name, pathp)); -} - -/* - * local_home_path -- - * Construct the source pathname from the file system and local name. - */ -static int -local_home_path(WT_FILE_SYSTEM *file_system, const char *name, char **pathp) -{ - return (local_path(file_system, ((LOCAL_FILE_SYSTEM *)file_system)->home_dir, name, pathp)); -} - -/* - * local_path -- - * Construct a pathname from the file system and local name. - */ -static int -local_path(WT_FILE_SYSTEM *file_system, const char *dir, const char *name, char **pathp) -{ - size_t len; - int ret; - char *p; - - ret = 0; - - /* Skip over "./" and variations (".//", ".///./././//") at the beginning of the name. */ - while (*name == '.') { - if (name[1] != '/') - break; - name += 2; - while (*name == '/') - name++; - } - len = strlen(dir) + strlen(name) + 2; - if ((p = malloc(len)) == NULL) - return (local_err(FS2LOCAL(file_system), NULL, ENOMEM, "local_path")); - if (snprintf(p, len, "%s/%s", dir, name) >= (int)len) - return (local_err(FS2LOCAL(file_system), NULL, EINVAL, "overflow sprintf")); - *pathp = p; - return (ret); -} - -/* - * local_stat -- - * Perform the stat system call for a name in the file system. - */ -static int -local_stat(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, const char *caller, - bool must_exist, struct stat *statp) -{ - int ret; - char *path; - - path = NULL; - - /* - * We check to see if the file exists in the cache first, and if not the bucket directory. This - * maps what a real cloud implementation would do. This will allow us to instrument this code to - * try out and measure caching implementations. - */ - if ((ret = local_cache_path(file_system, name, &path)) != 0) - goto err; - - ret = stat(path, statp); - if (ret != 0 && errno == ENOENT) { - /* It's not in the cache, try the bucket directory. */ - free(path); - if ((ret = local_bucket_path(file_system, name, &path)) != 0) - goto err; - ret = stat(path, statp); - } - if (ret != 0) { - /* - * If the file must exist, report the error no matter what. - */ - if (must_exist || errno != ENOENT) - ret = local_err(FS2LOCAL(file_system), session, errno, "%s: %s stat", path, caller); - else - ret = errno; - } -err: - free(path); - return (ret); -} - -/* - * local_add_reference -- - * Add a reference to the storage source so we can reference count to know when to really - * terminate. - */ -static int -local_add_reference(WT_STORAGE_SOURCE *storage_source) -{ - LOCAL_STORAGE *local; - - local = (LOCAL_STORAGE *)storage_source; - - /* - * Missing reference or overflow? - */ - if (local->reference_count == 0 || local->reference_count + 1 == 0) - return (EINVAL); - ++local->reference_count; - return (0); -} - -/* - * local_customize_file_system -- - * Return a customized file system to access the local storage source objects. - */ -static int -local_customize_file_system(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - const char *bucket_name, const char *auth_token, const char *config, - WT_FILE_SYSTEM **file_systemp) -{ - LOCAL_STORAGE *local; - LOCAL_FILE_SYSTEM *fs; - WT_CONFIG_ITEM cachedir; - WT_FILE_SYSTEM *wt_fs; - int ret; - const char *p; - char buf[1024]; - - local = (LOCAL_STORAGE *)storage_source; - - fs = NULL; - ret = 0; - - /* Parse configuration string. */ - if ((ret = local->wt_api->config_get_string( - local->wt_api, session, config, "cache_directory", &cachedir)) != 0) { - if (ret == WT_NOTFOUND) { - ret = 0; - cachedir.len = 0; - } else { - ret = local_err(local, session, ret, "customize_file_system: config parsing"); - goto err; - } - } - - if ((ret = local->wt_api->file_system_get(local->wt_api, session, &wt_fs)) != 0) { - ret = - local_err(local, session, ret, "local_file_system: cannot get WiredTiger file system"); - goto err; - } - if ((fs = calloc(1, sizeof(LOCAL_FILE_SYSTEM))) == NULL) { - ret = local_err(local, session, ENOMEM, "local_file_system"); - goto err; - } - fs->local_storage = local; - fs->wt_fs = wt_fs; - - if ((fs->auth_token = strdup(auth_token)) == NULL) { - ret = local_err(local, session, ENOMEM, "local_file_system.auth_token"); - goto err; - } - - /* - * The home directory owned by the connection will not change, and will be valid memory, for as - * long as the connection is open. That is longer than this file system will be open, so we can - * use the string without copying. - */ - fs->home_dir = session->connection->get_home(session->connection); - - /* - * Get the bucket directory and the cache directory. - */ - if ((ret = local_get_directory(fs->home_dir, bucket_name, -1, false, &fs->bucket_dir)) != 0) { - ret = local_err(local, session, ret, "%s: bucket directory", bucket_name); - goto err; - } - - /* - * The default cache directory is named "cache-", where name is the last component of the - * bucket name's path. We'll create it if it doesn't exist. - */ - if (cachedir.len == 0) { - if ((p = strrchr(bucket_name, '/')) != NULL) - p++; - else - p = bucket_name; - if (snprintf(buf, sizeof(buf), "cache-%s", p) >= (int)sizeof(buf)) { - ret = local_err(local, session, EINVAL, "overflow snprintf"); - goto err; - } - cachedir.str = buf; - cachedir.len = strlen(buf); - } - if ((ret = local_get_directory( - fs->home_dir, cachedir.str, (ssize_t)cachedir.len, true, &fs->cache_dir)) != 0) { - ret = - local_err(local, session, ret, "%*s: cache directory", (int)cachedir.len, cachedir.str); - goto err; - } - fs->file_system.fs_directory_list = local_directory_list; - fs->file_system.fs_directory_list_single = local_directory_list_single; - fs->file_system.fs_directory_list_free = local_directory_list_free; - fs->file_system.fs_exist = local_exist; - fs->file_system.fs_open_file = local_open; - fs->file_system.fs_remove = local_remove; - fs->file_system.fs_rename = local_rename; - fs->file_system.fs_size = local_size; - fs->file_system.terminate = local_fs_terminate; - -err: - if (ret == 0) - *file_systemp = &fs->file_system; - else if (fs != NULL) { - free(fs->auth_token); - free(fs->bucket_dir); - free(fs->cache_dir); - free(fs); - } - return (ret); -} - -/* - * local_exist -- - * Return if the file exists. - */ -static int -local_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, bool *existp) -{ - struct stat sb; - LOCAL_STORAGE *local; - int ret; - - local = FS2LOCAL(file_system); - local->op_count++; - *existp = false; - - if ((ret = local_stat(file_system, session, name, "ss_exist", false, &sb)) == 0) - *existp = true; - else if (ret == ENOENT) - ret = 0; - - return (ret); -} - -/* - * local_file_copy -- - * Copy a file. - */ -static int -local_file_copy(LOCAL_STORAGE *local, WT_SESSION *session, const char *src_path, - const char *dest_path, WT_FS_OPEN_FILE_TYPE type) -{ - WT_FILE_HANDLE *dest, *src; - WT_FILE_SYSTEM *wt_fs; - wt_off_t copy_size, file_size, left; - ssize_t pos; - size_t pathlen; - int ret, t_ret; - char buffer[1024 * 64], *tmp_path; - - dest = src = NULL; - pathlen = strlen(dest_path) + 10; - if ((tmp_path = malloc(pathlen)) != NULL) - if (snprintf(tmp_path, pathlen, "%s.TMP", dest_path) >= (int)pathlen) { - ret = local_err(local, session, EINVAL, "overflow snprintf"); - goto err; - } - - if ((ret = local->wt_api->file_system_get(local->wt_api, session, &wt_fs)) != 0) { - ret = - local_err(local, session, ret, "local_file_system: cannot get WiredTiger file system"); - goto err; - } - if ((ret = wt_fs->fs_open_file(wt_fs, session, src_path, type, WT_FS_OPEN_READONLY, &src)) != - 0) { - ret = local_err(local, session, ret, "%s: cannot open for read", src_path); - goto err; - } - - if ((ret = wt_fs->fs_open_file(wt_fs, session, tmp_path, type, WT_FS_OPEN_CREATE, &dest)) != - 0) { - ret = local_err(local, session, ret, "%s: cannot create", tmp_path); - goto err; - } - if ((ret = wt_fs->fs_size(wt_fs, session, src_path, &file_size)) != 0) { - ret = local_err(local, session, ret, "%s: cannot get size", src_path); - goto err; - } - for (pos = 0, left = file_size; left > 0; pos += copy_size, left -= copy_size) { - copy_size = left < (wt_off_t)sizeof(buffer) ? left : (wt_off_t)sizeof(buffer); - if ((ret = src->fh_read(src, session, pos, (size_t)copy_size, buffer)) != 0) { - ret = local_err(local, session, ret, "%s: cannot read", src_path); - goto err; - } - if ((ret = dest->fh_write(dest, session, pos, (size_t)copy_size, buffer)) != 0) { - ret = local_err(local, session, ret, "%s: cannot write", tmp_path); - goto err; - } - } - if (ret == 0 && (ret = chmod(tmp_path, 0444)) < 0) - ret = local_err(local, session, errno, "%s: file_copy chmod failed", tmp_path); - if ((ret = rename(tmp_path, dest_path)) != 0) { - ret = local_err(local, session, errno, "%s: cannot rename from %s", dest_path, tmp_path); - goto err; - } -err: - if (src != NULL && (t_ret = src->close(src, session)) != 0) - if (ret == 0) - ret = t_ret; - if (dest != NULL && (t_ret = dest->close(dest, session)) != 0) - if (ret == 0) - ret = t_ret; - if (ret != 0) - (void)unlink(tmp_path); - free(tmp_path); - - return (ret); -} - -/* - * local_flush -- - * Return when the file has been flushed. - */ -static int -local_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, WT_FILE_SYSTEM *file_system, - const char *source, const char *object, const char *config) -{ - LOCAL_STORAGE *local; - int ret; - char *dest_path, *src_path; - - (void)config; /* unused */ - dest_path = src_path = NULL; - local = (LOCAL_STORAGE *)storage_source; - ret = 0; - - if (file_system == NULL || source == NULL || object == NULL) - return local_err(local, session, EINVAL, "ss_flush: required arguments missing"); - - if ((ret = local_home_path(file_system, source, &src_path)) != 0) - goto err; - - if ((ret = local_bucket_path(file_system, object, &dest_path)) != 0) - goto err; - - if ((ret = local_delay(local)) != 0) - goto err; - - if ((ret = local_file_copy(local, session, src_path, dest_path, WT_FS_OPEN_FILE_TYPE_DATA)) != - 0) - goto err; - - local->object_writes++; - -err: - free(dest_path); - free(src_path); - return (ret); -} - -/* - * local_flush_finish -- - * Cache a file in the new file system. - */ -static int -local_flush_finish(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_FILE_SYSTEM *file_system, const char *source, const char *object, const char *config) -{ - LOCAL_STORAGE *local; - int ret; - char *dest_path, *src_path; - - (void)config; /* unused */ - dest_path = src_path = NULL; - local = (LOCAL_STORAGE *)storage_source; - ret = 0; - - if (file_system == NULL || source == NULL || object == NULL) - return local_err(local, session, EINVAL, "ss_flush_finish: required arguments missing"); - - if ((ret = local_home_path(file_system, source, &src_path)) != 0) - goto err; - - if ((ret = local_cache_path(file_system, object, &dest_path)) != 0) - goto err; - - local->op_count++; - /* - * Link the object with the original local object. The could be replaced by a file copy if - * portability is an issue. - */ - if ((ret = link(src_path, dest_path)) != 0) { - ret = local_err( - local, session, errno, "ss_flush_finish link %s to %s failed", source, dest_path); - goto err; - } - /* Set the file to readonly in the cache. */ - if (ret == 0 && (ret = chmod(dest_path, 0444)) < 0) - ret = local_err(local, session, errno, "%s: ss_flush_finish chmod failed", dest_path); -err: - free(dest_path); - free(src_path); - return (ret); -} - -/* - * local_directory_list -- - * Return a list of object names for the given location. - */ -static int -local_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *directory, - const char *prefix, char ***dirlistp, uint32_t *countp) -{ - FS2LOCAL(file_system)->op_count++; - return ( - local_directory_list_internal(file_system, session, directory, prefix, 0, dirlistp, countp)); -} - -/* - * local_directory_list_single -- - * Return a single file name for the given location. - */ -static int -local_directory_list_single(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *directory, - const char *prefix, char ***dirlistp, uint32_t *countp) -{ - FS2LOCAL(file_system)->op_count++; - return ( - local_directory_list_internal(file_system, session, directory, prefix, 1, dirlistp, countp)); -} - -/* - * local_location_list_free -- - * Free memory allocated by local_location_list. - */ -static int -local_directory_list_free( - WT_FILE_SYSTEM *file_system, WT_SESSION *session, char **dirlist, uint32_t count) -{ - (void)session; - - FS2LOCAL(file_system)->op_count++; - if (dirlist != NULL) { - while (count > 0) - free(dirlist[--count]); - free(dirlist); - } - return (0); -} - -/* - * local_directory_list_add -- - * Add an entry to the directory list, growing as needed. - */ -static int -local_directory_list_add( - LOCAL_STORAGE *local, char ***entriesp, const char *s, uint32_t count, uint32_t *allocatedp) -{ - size_t alloc_sz; - char **entries, **new_entries; - - entries = *entriesp; - if (count >= *allocatedp) { - *allocatedp += 10; - alloc_sz = sizeof(char *) * (*allocatedp); - if ((new_entries = realloc(entries, alloc_sz)) == NULL) - return (local_err(local, NULL, ENOMEM, "cannot grow directory list")); - entries = new_entries; - *entriesp = entries; - } - if ((entries[count] = strdup(s)) == NULL) - return (local_err(local, NULL, ENOMEM, "cannot grow directory list")); - - return (0); -} - -/* - * local_location_list_internal -- - * Return a list of object names for the given location. - */ -static int -local_directory_list_internal(WT_FILE_SYSTEM *file_system, WT_SESSION *session, - const char *directory, const char *prefix, uint32_t limit, char ***dirlistp, uint32_t *countp) -{ - struct dirent *dp; - DIR *dirp; - LOCAL_FILE_SYSTEM *local_fs; - LOCAL_STORAGE *local; - size_t dir_len, prefix_len; - uint32_t allocated, count; - int ret, t_ret; - char **entries; - const char *basename; - - local_fs = (LOCAL_FILE_SYSTEM *)file_system; - local = local_fs->local_storage; - entries = NULL; - allocated = count = 0; - dir_len = (directory == NULL ? 0 : strlen(directory)); - prefix_len = (prefix == NULL ? 0 : strlen(prefix)); - ret = 0; - - *dirlistp = NULL; - *countp = 0; - - /* - * The items in the bucket directory represent the definitive list. - */ - if ((dirp = opendir(local_fs->bucket_dir)) == NULL) { - ret = errno; - if (ret == 0) - ret = EINVAL; - return ( - local_err(local, session, ret, "%s: ss_directory_list: opendir", local_fs->bucket_dir)); - } - - for (count = 0; (dp = readdir(dirp)) != NULL && (limit == 0 || count < limit);) { - /* Skip . and .. */ - basename = dp->d_name; - if (strcmp(basename, ".") == 0 || strcmp(basename, "..") == 0) - continue; - - /* Match only the indicated directory files. */ - if (directory != NULL && strncmp(basename, directory, dir_len) != 0) - continue; - basename += dir_len; - - /* The list of files is optionally filtered by a prefix. */ - if (prefix != NULL && strncmp(basename, prefix, prefix_len) != 0) - continue; - - if ((ret = local_directory_list_add(local, &entries, basename, count, &allocated)) != 0) - goto err; - count++; - } - - *dirlistp = entries; - *countp = count; - -err: - if (closedir(dirp) != 0) { - t_ret = - local_err(local, session, errno, "%s: ss_directory_list: closedir", local_fs->cache_dir); - if (ret == 0) - ret = t_ret; - } - if (ret == 0) - return (0); - - if (entries != NULL) { - while (count > 0) - free(entries[--count]); - free(entries); - } - return (ret); -} - -/* - * local_fs_terminate -- - * Discard any resources on termination of the file system - */ -static int -local_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) -{ - LOCAL_FILE_SYSTEM *local_fs; - - (void)session; /* unused */ - - local_fs = (LOCAL_FILE_SYSTEM *)file_system; - FS2LOCAL(file_system)->op_count++; - free(local_fs->auth_token); - free(local_fs->bucket_dir); - free(local_fs->cache_dir); - free(file_system); - - return (0); -} - -/* - * local_open -- - * fopen for our local storage source - */ -static int -local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, - WT_FS_OPEN_FILE_TYPE file_type, uint32_t flags, WT_FILE_HANDLE **file_handlep) -{ - LOCAL_FILE_HANDLE *local_fh; - LOCAL_FILE_SYSTEM *local_fs; - LOCAL_STORAGE *local; - WT_FILE_HANDLE *file_handle, *wt_fh; - WT_FILE_SYSTEM *wt_fs; - struct stat sb; - int ret; - char *bucket_path, *cache_path; - - (void)flags; /* Unused */ - - ret = 0; - *file_handlep = NULL; - local_fh = NULL; - local_fs = (LOCAL_FILE_SYSTEM *)file_system; - local = local_fs->local_storage; - wt_fs = local_fs->wt_fs; - bucket_path = cache_path = NULL; - - if ((flags & WT_FS_OPEN_READONLY) == 0 || (flags & WT_FS_OPEN_CREATE) != 0) - return ( - local_err(local, session, EINVAL, "ss_open_object: readonly access required: %s", name)); - - /* - * We expect that the local file system will be used narrowly, like when creating or opening a - * data file or turtle file. It would be unexpected to try to open a non-data file (like a log - * file) in that narrow part of code, so we make it an error here. - * - * Relaxing this constraint to allow opening of, say, log files, would be straightforward - we - * would not translate the path or do any tracking for flushing. But there's a catch. Other - * parts of the API, like remove and rename, have no flag indicating that they are operating on - * a log file, so we wouldn't know whether to do path translation. Of course, we could peek at - * the name, but that would be bad form. - */ - if (file_type != WT_FS_OPEN_FILE_TYPE_DATA && file_type != WT_FS_OPEN_FILE_TYPE_REGULAR) - return (local_err( - local, session, EINVAL, "%s: open: only data file and regular types supported", name)); - - /* Create a new handle. */ - if ((local_fh = calloc(1, sizeof(LOCAL_FILE_HANDLE))) == NULL) { - ret = ENOMEM; - goto err; - } - if ((ret = local_cache_path(file_system, name, &cache_path)) != 0) - goto err; - ret = stat(cache_path, &sb); - if (ret != 0) { - if (errno != ENOENT) { - ret = local_err(local, session, errno, "%s: local_open stat", cache_path); - goto err; - } - - /* - * The file doesn't exist locally, make a copy of it from the cloud. - */ - if ((ret = local_bucket_path(file_system, name, &bucket_path)) != 0) - goto err; - - if ((ret = local_delay(local)) != 0) - goto err; - - if ((ret = local_file_copy( - local, session, bucket_path, cache_path, WT_FS_OPEN_FILE_TYPE_DATA)) != 0) - goto err; - - local->object_reads++; - } - if ((ret = wt_fs->fs_open_file(wt_fs, session, cache_path, file_type, flags, &wt_fh)) != 0) { - ret = local_err(local, session, ret, "ss_open_object: open: %s", name); - goto err; - } - local_fh->fh = wt_fh; - local_fh->local = local; - - /* Initialize public information. */ - file_handle = (WT_FILE_HANDLE *)local_fh; - - /* - * Setup the function call table for our custom storage source. Set the function pointer to NULL - * where our implementation doesn't support the functionality. - */ - file_handle->close = local_file_close; - file_handle->fh_advise = NULL; - file_handle->fh_extend = NULL; - file_handle->fh_extend_nolock = NULL; - file_handle->fh_lock = local_file_lock; - file_handle->fh_map = NULL; - file_handle->fh_map_discard = NULL; - file_handle->fh_map_preload = NULL; - file_handle->fh_unmap = NULL; - file_handle->fh_read = local_file_read; - file_handle->fh_size = local_file_size; - file_handle->fh_sync = local_file_sync; - file_handle->fh_sync_nowait = NULL; - file_handle->fh_truncate = NULL; - file_handle->fh_write = local_file_write; - if ((file_handle->name = strdup(name)) == NULL) { - ret = ENOMEM; - goto err; - } - - if ((ret = pthread_rwlock_wrlock(&local->file_handle_lock)) != 0) { - (void)local_err(local, session, ret, "ss_open_object: pthread_rwlock_wrlock"); - goto err; - } - TAILQ_INSERT_HEAD(&local->fileq, local_fh, q); - if ((ret = pthread_rwlock_unlock(&local->file_handle_lock)) != 0) { - (void)local_err(local, session, ret, "ss_open_object: pthread_rwlock_unlock"); - goto err; - } - - *file_handlep = file_handle; - - VERBOSE_LS( - local, "File opened: %s final path=%s\n", SHOW_STRING(name), SHOW_STRING(local_fh->fh->name)); - -err: - free(bucket_path); - free(cache_path); - if (ret != 0) { - if (local_fh != NULL) - local_file_close_internal(local, session, local_fh); - } - return (ret); -} - -/* - * local_rename -- - * POSIX rename, not supported for cloud objects. - */ -static int -local_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *from, const char *to, - uint32_t flags) -{ - (void)to; /* unused */ - (void)flags; /* unused */ - - return ( - local_err(FS2LOCAL(file_system), session, ENOTSUP, "%s: rename of file not supported", from)); -} - -/* - * local_remove -- - * POSIX remove, not supported for cloud objects. - */ -static int -local_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, uint32_t flags) -{ - (void)flags; /* unused */ - - return ( - local_err(FS2LOCAL(file_system), session, ENOTSUP, "%s: remove of file not supported", name)); -} - -/* - * local_size -- - * Get the size of a file in bytes, by file name. - */ -static int -local_size(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, wt_off_t *sizep) -{ - struct stat sb; - LOCAL_STORAGE *local; - int ret; - - local = FS2LOCAL(file_system); - local->op_count++; - *sizep = 0; - - if ((ret = local_stat(file_system, session, name, "ss_size", true, &sb)) != 0) - goto err; - - *sizep = sb.st_size; -err: - return (ret); -} - -/* - * local_terminate -- - * Discard any resources on termination - */ -static int -local_terminate(WT_STORAGE_SOURCE *storage, WT_SESSION *session) -{ - LOCAL_FILE_HANDLE *local_fh, *safe_fh; - LOCAL_STORAGE *local; - int ret; - - ret = 0; - local = (LOCAL_STORAGE *)storage; - - if (--local->reference_count != 0) - return (0); - - local->op_count++; - - /* - * We should be single threaded at this point, so it is safe to destroy the lock and access the - * file handle list without locking it. - */ - if ((ret = pthread_rwlock_destroy(&local->file_handle_lock)) != 0) - (void)local_err(local, session, ret, "terminate: pthread_rwlock_destroy"); - - TAILQ_FOREACH_SAFE(local_fh, &local->fileq, q, safe_fh) - local_file_close_internal(local, session, local_fh); - - free(local); - return (ret); -} - -/* - * local_file_close -- - * ANSI C close. - */ -static int -local_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session) -{ - LOCAL_STORAGE *local; - LOCAL_FILE_HANDLE *local_fh; - int ret, t_ret; - - ret = 0; - local_fh = (LOCAL_FILE_HANDLE *)file_handle; - local = local_fh->local; - - local->fh_ops++; - if ((ret = pthread_rwlock_wrlock(&local->file_handle_lock)) != 0) - /* There really isn't anything more we can do. It will get cleaned up on terminate. */ - return (local_err(local, session, ret, "file handle close: pthread_rwlock_wrlock")); - - TAILQ_REMOVE(&local->fileq, local_fh, q); - - if ((ret = pthread_rwlock_unlock(&local->file_handle_lock)) != 0) - (void)local_err(local, session, ret, "file handle close: pthread_rwlock_unlock"); - - if ((t_ret = local_file_close_internal(local, session, local_fh)) != 0) { - if (ret == 0) - ret = t_ret; - } - - return (ret); -} - -/* - * local_file_close_internal -- - * Internal file handle close. - */ -static int -local_file_close_internal(LOCAL_STORAGE *local, WT_SESSION *session, LOCAL_FILE_HANDLE *local_fh) -{ - int ret; - WT_FILE_HANDLE *wt_fh; - - ret = 0; - wt_fh = local_fh->fh; - if (wt_fh != NULL && (ret = wt_fh->close(wt_fh, session)) != 0) - ret = local_err(local, session, ret, "WT_FILE_HANDLE->close: close"); - - free(local_fh->iface.name); - free(local_fh); - - return (ret); -} - -/* - * local_file_lock -- - * Lock/unlock a file. - */ -static int -local_file_lock(WT_FILE_HANDLE *file_handle, WT_SESSION *session, bool lock) -{ - /* Locks are always granted. */ - - (void)session; /* Unused */ - (void)lock; /* Unused */ - - ((LOCAL_FILE_HANDLE *)file_handle)->local->fh_ops++; - return (0); -} - -/* - * local_file_read -- - * POSIX pread. - */ -static int -local_file_read( - WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, void *buf) -{ - LOCAL_FILE_HANDLE *local_fh; - WT_FILE_HANDLE *wt_fh; - - local_fh = (LOCAL_FILE_HANDLE *)file_handle; - wt_fh = local_fh->fh; - - local_fh->local->read_ops++; - return (wt_fh->fh_read(wt_fh, session, offset, len, buf)); -} - -/* - * local_file_size -- - * Get the size of a file in bytes, by file handle. - */ -static int -local_file_size(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t *sizep) -{ - LOCAL_FILE_HANDLE *local_fh; - WT_FILE_HANDLE *wt_fh; - - local_fh = (LOCAL_FILE_HANDLE *)file_handle; - wt_fh = local_fh->fh; - - local_fh->local->fh_ops++; - return (wt_fh->fh_size(wt_fh, session, sizep)); -} - -/* - * local_file_sync -- - * Ensure the content of the local file is stable. - */ -static int -local_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *session) -{ - /* This is a no-op. We could also disallow it. */ - (void)file_handle; - (void)session; - return (0); -} - -/* - * local_file_write -- - * POSIX pwrite. - */ -static int -local_file_write( - WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, const void *buf) -{ - LOCAL_FILE_HANDLE *local_fh; - - (void)offset; - (void)len; - (void)buf; - - local_fh = (LOCAL_FILE_HANDLE *)file_handle; - return (local_err(local_fh->local, session, ENOTSUP, "ss_open_object: write not supported: %s", - local_fh->iface.name)); -} - -/* - * wiredtiger_extension_init -- - * A simple shared library encryption example. - */ -int -wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) -{ - LOCAL_STORAGE *local; - int ret; - - if ((local = calloc(1, sizeof(LOCAL_STORAGE))) == NULL) - return (errno); - local->wt_api = connection->get_extension_api(connection); - if ((ret = pthread_rwlock_init(&local->file_handle_lock, NULL)) != 0) { - (void)local_err(local, NULL, ret, "pthread_rwlock_init"); - free(local); - return (ret); - } - - /* - * Allocate a local storage structure, with a WT_STORAGE structure as the first field, allowing - * us to treat references to either type of structure as a reference to the other type. - */ - local->storage_source.ss_add_reference = local_add_reference; - local->storage_source.ss_customize_file_system = local_customize_file_system; - local->storage_source.ss_flush = local_flush; - local->storage_source.ss_flush_finish = local_flush_finish; - local->storage_source.terminate = local_terminate; - - /* - * The first reference is implied by the call to add_storage_source. - */ - local->reference_count = 1; - - if ((ret = local_configure(local, config)) != 0) { - free(local); - return (ret); - } - - /* Load the storage */ - if ((ret = connection->add_storage_source( - connection, "local_store", &local->storage_source, NULL)) != 0) { - (void)local_err(local, NULL, ret, "WT_CONNECTION->add_storage_source"); - free(local); - } - return (ret); -} diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 563a013a36a..e9070bec2d5 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "f1c6393e01d46d1afe941187a9a1629e056100f2" + "commit": "fadba1cd56e27dc8ccc97dcccd1883e1ecdaf6ce" } diff --git a/src/third_party/wiredtiger/src/docs/custom-storage-sources.dox b/src/third_party/wiredtiger/src/docs/custom-storage-sources.dox index ffa8ef11783..275d4f32574 100644 --- a/src/third_party/wiredtiger/src/docs/custom-storage-sources.dox +++ b/src/third_party/wiredtiger/src/docs/custom-storage-sources.dox @@ -63,10 +63,10 @@ It must always be provided when WiredTiger is reopened (again, with the ::wiredt @section storage_examples Storage source examples -An example of a storage source exists in \c ext/storage_sources/local_store/local_store.c. +An example of a storage source exists in \c ext/storage_sources/dir_store/dir_store.c. This storage source emulates cloud storage by storing all objects on the local file system. This example does not include application level code to call it. By default, WiredTiger builds it as a loadable shared library, and it can be loaded during a ::wiredtiger_open call as with -any other extension, and \c local_store can be specified to be used with tiered storage system. +any other extension, and \c dir_store can be specified to be used with tiered storage system. */ diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 861701612e7..5ee6ec9d409 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -285,6 +285,7 @@ dev dhandle dhandle's dhandles +dir disjunction disjunctions distclean diff --git a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c index 815e12a2a14..de71536c789 100644 --- a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c @@ -71,7 +71,7 @@ static char home[1024]; /* Program working dir */ /* Include worker threads and extra sessions */ #define SESSION_MAX (MAX_TH + 4) #ifndef WT_STORAGE_LIB -#define WT_STORAGE_LIB "ext/storage_sources/local_store/.libs/libwiredtiger_local_store.so" +#define WT_STORAGE_LIB "ext/storage_sources/dir_store/.libs/libwiredtiger_dir_store.so" #endif static const char *table_pfx = "table"; @@ -100,7 +100,7 @@ static uint32_t flush_calls = 1; "eviction_updates_target=20,eviction_updates_trigger=90," \ "log=(enabled,file_max=10M,remove=true),session_max=%d," \ "statistics=(fast),statistics_log=(wait=1,json=true)," \ - "tiered_storage=(bucket=%s,bucket_prefix=pfx,local_retention=%d,name=local_store)" + "tiered_storage=(bucket=%s,bucket_prefix=pfx,local_retention=%d,name=dir_store)" #define ENV_CONFIG_TXNSYNC \ ENV_CONFIG_DEF \ ",eviction_dirty_target=20,eviction_dirty_trigger=90" \ diff --git a/src/third_party/wiredtiger/test/suite/helper_tiered.py b/src/third_party/wiredtiger/test/suite/helper_tiered.py old mode 100755 new mode 100644 index 9587bd4a28a..64406130d63 --- a/src/third_party/wiredtiger/test/suite/helper_tiered.py +++ b/src/third_party/wiredtiger/test/suite/helper_tiered.py @@ -35,7 +35,7 @@ import datetime, inspect, os, random # Generate a storage store specific authentication token. def get_auth_token(storage_source): auth_token = None - if storage_source == 'local_store': + if storage_source == 'dir_store': # Fake a secret token. auth_token = "Secret" if storage_source == 's3_store': @@ -59,7 +59,7 @@ local_buckets = ['bucket1', 'bucket2'] def get_bucket1_name(storage_source): if storage_source == 's3_store': return s3_buckets[0] - if storage_source == 'local_store': + if storage_source == 'dir_store': return local_buckets[0] return None @@ -67,7 +67,7 @@ def get_bucket1_name(storage_source): def get_bucket2_name(storage_source): if storage_source == 's3_store': return s3_buckets[1] - if storage_source == 'local_store': + if storage_source == 'dir_store': return local_buckets[1] return None diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py old mode 100755 new mode 100644 index 52b9c6bbb6a..a3d922b6120 --- a/src/third_party/wiredtiger/test/suite/hook_tiered.py +++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py @@ -65,7 +65,7 @@ from wttest import WiredTigerTestCase def wiredtiger_open_tiered(ignored_self, args): auth_token = "test_token" bucket = "mybucket" - extension_name = "local_store" + extension_name = "dir_store" prefix = "pfx-" extension_libs = WiredTigerTestCase.findExtension('storage_sources', extension_name) if len(extension_libs) == 0: diff --git a/src/third_party/wiredtiger/test/suite/test_tiered02.py b/src/third_party/wiredtiger/test/suite/test_tiered02.py index 9887de11055..8a1e5e3cb3b 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered02.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered02.py @@ -35,10 +35,10 @@ from wtscenario import make_scenarios # Test tiered tree class test_tiered02(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dirstore', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), bucket_prefix = "pfx_", - ss_name = 'local_store')), + ss_name = 'dir_store')), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket = get_bucket1_name('s3_store'), bucket_prefix = generate_s3_prefix(), @@ -50,7 +50,7 @@ class test_tiered02(wttest.WiredTigerTestCase): uri = "table:test_tiered02" def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) return \ 'debug_mode=(flush_checkpoint=true),' + \ @@ -66,7 +66,7 @@ class test_tiered02(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': @@ -78,9 +78,9 @@ class test_tiered02(wttest.WiredTigerTestCase): self.pr(s) def confirm_flush(self, increase=True): - # Without directly using the filesystem API, directory listing is only supported on the - # local store. Limit this check to the local store. - if self.ss_name != 'local_store': + # Without directly using the filesystem API, directory listing is only supported on + # the directory store. Limit this check to the directory store. + if self.ss_name != 'dir_store': return got = sorted(list(os.listdir(self.bucket))) diff --git a/src/third_party/wiredtiger/test/suite/test_tiered03.py b/src/third_party/wiredtiger/test/suite/test_tiered03.py index 3b7dacabf71..1198e45a8f6 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered03.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered03.py @@ -43,10 +43,10 @@ class test_tiered03(wttest.WiredTigerTestCase): uri = 'file:test_tiered03' storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dirstore', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), bucket_prefix = "pfx_", - ss_name = 'local_store')), + ss_name = 'dir_store')), # FIXME-WT-8896 The S3 extension gets stuck during initialization if more than one # simultaneous WT connection is created. Enable once we have fixed this issue. #('s3', dict(auth_token = get_auth_token('s3_store'), @@ -71,12 +71,12 @@ class test_tiered03(wttest.WiredTigerTestCase): cache_dir = self.bucket + '-cache' # We have multiple connections that want to share a bucket. - # For the local store, the first time this function is called, we'll + # For the directory store, the first time this function is called, we'll # establish the absolute path for the bucket, and always use that for # the bucket name. # The cache directory name is a relative one, so it won't be shared # between connections. - if self.ss_name == 'local_store': + if self.ss_name == 'dir_store': if self.absolute_bucket_dir == None: self.absolute_bucket_dir = os.path.join(os.getcwd(), self.bucket) os.mkdir(self.absolute_bucket_dir) @@ -96,7 +96,7 @@ class test_tiered03(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py old mode 100755 new mode 100644 index b5a00a26b21..dc6ac853454 --- a/src/third_party/wiredtiger/test/suite/test_tiered04.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py @@ -37,12 +37,12 @@ StorageSource = wiredtiger.StorageSource # easy access to constants # Basic tiered storage API test. class test_tiered04(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), - bucket1 = get_bucket2_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), + bucket1 = get_bucket2_name('dir_store'), prefix = "pfx_", prefix1 = "pfx1_", - ss_name = 'local_store')), + ss_name = 'dir_store')), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket = get_bucket1_name('s3_store'), bucket1 = get_bucket2_name('s3_store'), @@ -72,7 +72,7 @@ class test_tiered04(wttest.WiredTigerTestCase): retention = 3 retention1 = 600 def conn_config(self): - if self.ss_name == 'local_store': + if self.ss_name == 'dir_store': os.mkdir(self.bucket) os.mkdir(self.bucket1) self.saved_conn = \ @@ -93,7 +93,7 @@ class test_tiered04(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py old mode 100755 new mode 100644 index 02fcb60d8ce..8b638788736 --- a/src/third_party/wiredtiger/test/suite/test_tiered05.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered05.py @@ -35,10 +35,10 @@ StorageSource = wiredtiger.StorageSource # easy access to constants # Basic tiered storage API test error for tiered manager and flush_tier. class test_tiered05(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), bucket_prefix = "pfx_", - ss_name = 'local_store')), + ss_name = 'dir_store')), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket = get_bucket1_name('s3_store'), bucket_prefix = generate_s3_prefix(), @@ -56,7 +56,7 @@ class test_tiered05(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': @@ -64,7 +64,7 @@ class test_tiered05(wttest.WiredTigerTestCase): extlist.extension('storage_sources', self.ss_name + config) def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) return \ 'tiered_manager=(wait=%d),' % self.wait + \ diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index 3c996bdcba9..c4e931c7a3f 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -39,11 +39,11 @@ FileSystem = wiredtiger.FileSystem # easy access to constants class test_tiered06(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket1 = get_bucket1_name('local_store'), - bucket2 = get_bucket2_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket1 = get_bucket1_name('dir_store'), + bucket2 = get_bucket2_name('dir_store'), bucket_prefix_base = "pfx_", - ss_name = 'local_store')), + ss_name = 'dir_store')), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket1 = get_bucket1_name('s3_store'), bucket2 = get_bucket2_name('s3_store'), @@ -60,7 +60,7 @@ class test_tiered06(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': @@ -96,8 +96,8 @@ class test_tiered06(wttest.WiredTigerTestCase): # avoid namespace collison. 0th element on the stack is the current function. prefix = self.bucket_prefix_base + inspect.stack()[0][3] + '/' - # Local store needs the bucket created as a directory on the filesystem. - if self.ss_name == 'local_store': + # The directory store needs the bucket created as a directory on the filesystem. + if self.ss_name == 'dir_store': os.mkdir(self.bucket1) fs = ss.ss_customize_file_system(session, self.bucket1, self.auth_token, @@ -144,7 +144,7 @@ class test_tiered06(wttest.WiredTigerTestCase): self.assertEquals(fh.fh_size(session), len(outbytes)) fh.close(session) - # The fh_lock call doesn't do anything in the local and S3 store implementation. + # The fh_lock call doesn't do anything in the directory and S3 store implementation. fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_readonly) fh.fh_lock(session, True) fh.fh_lock(session, False) @@ -178,8 +178,8 @@ class test_tiered06(wttest.WiredTigerTestCase): cachedir = self.bucket1 + '_cache' os.mkdir(cachedir) - # Local store needs the bucket created as a directory on the filesystem. - if self.ss_name == 'local_store': + # Directory store needs the bucket created as a directory on the filesystem. + if self.ss_name == 'dir_store': os.mkdir(self.bucket1) fs = ss.ss_customize_file_system(session, self.bucket1, self.auth_token, @@ -279,11 +279,11 @@ class test_tiered06(wttest.WiredTigerTestCase): expect = sorted(self.suffix(expect, 'wt')) self.assertEquals(got, expect) - # Check that objects are "in the cloud" for the local store after a flush. - # Using the local storage module, they are actually going to be in either + # Check that objects are "in the cloud" for the directory store after a flush. + # Using the directory storage module, they are actually going to be in either # bucket1 or bucket2. def check_local_objects(self, expect1, expect2): - if self.ss_name != 'local_store': + if self.ss_name != 'dir_store': return got = sorted(list(os.listdir(self.bucket1))) @@ -316,8 +316,8 @@ class test_tiered06(wttest.WiredTigerTestCase): # avoid namespace collison. 0th element on the stack is the current function. prefix = self.bucket_prefix_base + inspect.stack()[0][3] + '/' - # Local store needs the bucket created as a directory on the filesystem. - if self.ss_name == 'local_store': + # Directory store needs the bucket created as a directory on the filesystem. + if self.ss_name == 'dir_store': os.mkdir(self.bucket1) os.mkdir(self.bucket2) @@ -340,8 +340,8 @@ class test_tiered06(wttest.WiredTigerTestCase): lambda: ss.ss_customize_file_system(session, bad_bucket, self.auth_token, self.get_fs_config(prefix, self.cachedir1)), errmsg) - # For local store - Create an empty file, try to use it as a directory. - if self.ss_name == 'local_store': + # For directory store - Create an empty file, try to use it as a directory. + if self.ss_name == 'dir_store': with open("some_file", "w"): pass errmsg = '/Invalid argument/' diff --git a/src/third_party/wiredtiger/test/suite/test_tiered07.py b/src/third_party/wiredtiger/test/suite/test_tiered07.py old mode 100755 new mode 100644 index b22b8fd158a..c13a47ff9ca --- a/src/third_party/wiredtiger/test/suite/test_tiered07.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered07.py @@ -35,10 +35,10 @@ StorageSource = wiredtiger.StorageSource # easy access to constants # Basic tiered storage API for schema operations. class test_tiered07(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), bucket_prefix = "pfx_", - ss_name = 'local_store')), + ss_name = 'dir_store')), # FIXME-WT-8897 Disabled as S3 directory listing is interpreting a directory to end in a '/', # whereas the code in the tiered storage doesn't expect that. Enable when fixed. #('s3', dict(auth_token = get_auth_token('s3_store'), @@ -61,7 +61,7 @@ class test_tiered07(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': @@ -69,7 +69,7 @@ class test_tiered07(wttest.WiredTigerTestCase): extlist.extension('storage_sources', self.ss_name + config) def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) # 'verbose=(tiered),' + \ diff --git a/src/third_party/wiredtiger/test/suite/test_tiered08.py b/src/third_party/wiredtiger/test/suite/test_tiered08.py old mode 100755 new mode 100644 index 384b9a4d527..50b80f6b54e --- a/src/third_party/wiredtiger/test/suite/test_tiered08.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered08.py @@ -42,10 +42,10 @@ from wtthread import checkpoint_thread, flush_tier_thread # data into a table from another thread. class test_tiered08(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), bucket_prefix = "pfx_", - ss_name = 'local_store')), + ss_name = 'dir_store')), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket = get_bucket1_name('s3_store'), bucket_prefix = generate_s3_prefix(), @@ -62,7 +62,7 @@ class test_tiered08(wttest.WiredTigerTestCase): uri = "table:test_tiered08" def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) return \ 'debug_mode=(flush_checkpoint=true),' + \ @@ -79,7 +79,7 @@ class test_tiered08(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': diff --git a/src/third_party/wiredtiger/test/suite/test_tiered09.py b/src/third_party/wiredtiger/test/suite/test_tiered09.py index 8b411623234..288a9f2f14d 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered09.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered09.py @@ -35,12 +35,12 @@ StorageSource = wiredtiger.StorageSource # easy access to constants # Test tiered storage with sequential connections with different prefixes. class test_tiered09(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), prefix1 = '1_', prefix2 = '2_', prefix3 = '3_', - ss_name = 'local_store')), + ss_name = 'dir_store')), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket = get_bucket1_name('s3_store'), prefix1 = generate_s3_prefix(), @@ -63,7 +63,7 @@ class test_tiered09(wttest.WiredTigerTestCase): retention = 1 saved_conn = '' def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) self.saved_conn = \ 'debug_mode=(flush_checkpoint=true),' + \ @@ -82,7 +82,7 @@ class test_tiered09(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': @@ -118,8 +118,8 @@ class test_tiered09(wttest.WiredTigerTestCase): self.session.flush_tier(None) self.close_conn() - # For local store, check if the path exists. - if self.ss_name == 'local_store': + # For directory store, check if the path exists. + if self.ss_name == 'dir_store': self.assertTrue(os.path.exists(self.obj1file)) self.assertTrue(os.path.exists(self.obj2file)) bucket_obj = os.path.join(self.bucket, self.prefix1 + self.obj1file) @@ -152,8 +152,8 @@ class test_tiered09(wttest.WiredTigerTestCase): self.session.flush_tier(None) self.close_conn() - # For local store, Check each table was created with the correct prefix. - if self.ss_name == 'local_store': + # For directory store, Check each table was created with the correct prefix. + if self.ss_name == 'dir_store': bucket_obj = os.path.join(self.bucket, self.prefix2 + self.obj1second) self.assertTrue(os.path.exists(bucket_obj)) bucket_obj = os.path.join(self.bucket, self.prefix1 + self.obj2file) diff --git a/src/third_party/wiredtiger/test/suite/test_tiered10.py b/src/third_party/wiredtiger/test/suite/test_tiered10.py index 06ca883c19e..2f928910b39 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered10.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered10.py @@ -36,11 +36,11 @@ StorageSource = wiredtiger.StorageSource # easy access to constants # prefixes to the same bucket directory but different local databases. class test_tiered10(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), prefix1 = '1_', prefix2 = '2_', - ss_name = 'local_store')), + ss_name = 'dir_store')), # FIXME-WT-8896 The S3 extension gets stuck during initialization if more than one # simultaneous WT connection is created. Enable once we have fixed this issue. #('s3', dict(auth_token = get_auth_token('s3_store'), @@ -62,16 +62,16 @@ class test_tiered10(wttest.WiredTigerTestCase): retention = 1 saved_conn = '' def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) os.mkdir(self.conn1_dir) os.mkdir(self.conn2_dir) # Use this to create the directories and set up for the others. dummy_conn = 'create,statistics=(all),' - # For local store, the bucket is a directory one level up from database directories. + # For directory store, the bucket is a directory one level up from database directories. bucket = '' - if self.ss_name == 'local_store': + if self.ss_name == 'dir_store': bucket = '../' bucket += self.bucket @@ -91,7 +91,7 @@ class test_tiered10(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': diff --git a/src/third_party/wiredtiger/test/suite/test_tiered11.py b/src/third_party/wiredtiger/test/suite/test_tiered11.py old mode 100755 new mode 100644 index 5afd06301f6..3379e8edc64 --- a/src/third_party/wiredtiger/test/suite/test_tiered11.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered11.py @@ -35,10 +35,10 @@ StorageSource = wiredtiger.StorageSource # easy access to constants # Test flush time and flush timestamp in metadata. class test_tiered11(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), bucket_prefix = "pfx_", - ss_name = 'local_store')), + ss_name = 'dir_store')), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket = get_bucket1_name('s3_store'), bucket_prefix = generate_s3_prefix(), @@ -55,7 +55,7 @@ class test_tiered11(wttest.WiredTigerTestCase): uri = "table:test_tiered11" def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) self.saved_conn = \ 'debug_mode=(flush_checkpoint=true),' + \ @@ -73,7 +73,7 @@ class test_tiered11(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': diff --git a/src/third_party/wiredtiger/test/suite/test_tiered12.py b/src/third_party/wiredtiger/test/suite/test_tiered12.py index 5f53f79e00b..bc415984592 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered12.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered12.py @@ -35,10 +35,10 @@ StorageSource = wiredtiger.StorageSource # easy access to constants # Test tiered storage with tiered flush finish timing delay. class test_tiered12(wttest.WiredTigerTestCase): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), bucket_prefix = "pfx_", - ss_name = 'local_store')), + ss_name = 'dir_store')), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket = get_bucket1_name('s3_store'), bucket_prefix = generate_s3_prefix(), @@ -55,7 +55,7 @@ class test_tiered12(wttest.WiredTigerTestCase): retention = 1 saved_conn = '' def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) self.saved_conn = \ 'debug_mode=(flush_checkpoint=true),' + \ @@ -74,7 +74,7 @@ class test_tiered12(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': @@ -117,8 +117,8 @@ class test_tiered12(wttest.WiredTigerTestCase): cache_obj = os.path.join(cache, self.bucket_prefix + self.obj1file) self.assertFalse(os.path.exists(cache_obj)) - # On local store, the bucket object should exist. - if self.ss_name == 'local_store': + # On directory store, the bucket object should exist. + if self.ss_name == 'dir_store': bucket_obj = os.path.join(self.bucket, self.bucket_prefix + self.obj1file) self.assertTrue(os.path.exists(bucket_obj)) diff --git a/src/third_party/wiredtiger/test/suite/test_tiered13.py b/src/third_party/wiredtiger/test/suite/test_tiered13.py old mode 100755 new mode 100644 index cc93d3dc9bc..bdd4c7bf265 --- a/src/third_party/wiredtiger/test/suite/test_tiered13.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered13.py @@ -36,10 +36,10 @@ from test_import01 import test_import_base class test_tiered13(test_import_base): storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), bucket_prefix = "pfx_", - ss_name = 'local_store')), + ss_name = 'dir_store')), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket = get_bucket1_name('s3_store'), bucket_prefix = generate_s3_prefix(), @@ -65,7 +65,7 @@ class test_tiered13(test_import_base): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': @@ -73,7 +73,7 @@ class test_tiered13(test_import_base): extlist.extension('storage_sources', self.ss_name + config) def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) self.saved_conn = \ 'debug_mode=(flush_checkpoint=true),' + \ @@ -133,7 +133,7 @@ class test_tiered13(test_import_base): shutil.rmtree(newdir, ignore_errors=True) os.mkdir(newdir) newbucket = os.path.join(newdir, self.bucket) - if self.ss_name == 'local_store': + if self.ss_name == 'dir_store': os.mkdir(newbucket) # It is tricky to work around the extension and connection bucket setup for # creating the new import directory that is tiered-enabled. diff --git a/src/third_party/wiredtiger/test/suite/test_tiered14.py b/src/third_party/wiredtiger/test/suite/test_tiered14.py old mode 100755 new mode 100644 index 5d36b8667e2..61e3a54cf91 --- a/src/third_party/wiredtiger/test/suite/test_tiered14.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered14.py @@ -58,11 +58,11 @@ class test_tiered14(wttest.WiredTigerTestCase): #('complex', dict(dataset='complex', long_only=True)), ] storage_sources = [ - ('local', dict(auth_token = get_auth_token('local_store'), - bucket = get_bucket1_name('local_store'), + ('dir_store', dict(auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), bucket_prefix = "pfx_", num_ops = 100, - ss_name = 'local_store',)), + ss_name = 'dir_store',)), ('s3', dict(auth_token = get_auth_token('s3_store'), bucket = get_bucket1_name('s3_store'), bucket_prefix = generate_s3_prefix(), @@ -72,7 +72,7 @@ class test_tiered14(wttest.WiredTigerTestCase): scenarios = wtscenario.make_scenarios(multiplier, keyfmt, dataset, storage_sources) def conn_config(self): - if self.ss_name == 'local_store' and not os.path.exists(self.bucket): + if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): os.mkdir(self.bucket) return \ 'debug_mode=(flush_checkpoint=true),' + \ @@ -88,7 +88,7 @@ class test_tiered14(wttest.WiredTigerTestCase): if self.ss_name == 's3_store': #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True - #if self.ss_name == 'local_store': + #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': -- cgit v1.2.1