diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-04-19 15:06:48 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-04-20 22:19:00 +0000 |
| commit | 6e9b3bf294a08a1e2b78147d4aae34217cd5245a (patch) | |
| tree | d27b0e036aa087af9ea37c8aba432a6a5c15de69 | |
| parent | e71e821acd084d3010b59bb65a2d82ef7a7fdba4 (diff) | |
| download | mongo-6e9b3bf294a08a1e2b78147d4aae34217cd5245a.tar.gz | |
Import wiredtiger: 07fef8b2b80bf1e07104e8627a6bf76e2f998eac from branch mongodb-5.0
ref: 716858f654..07fef8b2b8
for: 4.9.0-rc1
WT-7105 Add recovery error messages to include the URI
WT-7266 Test to validate re-reading files that were closed with active history
WT-7312 Keys/Values updated to String type and save the created keys
WT-7315 Implementation of the update thread operation in the test framework
WT-7329 Add hook capability to Python testing
WT-7367 Do not remove unstable updates of an in-memory database btree page
WT-7368 Add WT_STORAGE_SOURCE.customize_file_system in place of locations
WT-7376 Initialize tiered cursor name
WT-7380 Fix wiredtiger connection string to clear statistics
WT-7384 Fix an assert fire when inserting to the history store
WT-7385 Remove 'auth_token' from being reconfigurable
WT-7387 Replace cluster/member with hostid
WT-7388 Add parens to assignment in conditional
WT-7390 Add --noremove flag to Python test runner
WT-7395 Coverity analysis defect 118042: Dereference after null check
49 files changed, 1966 insertions, 2287 deletions
diff --git a/src/third_party/wiredtiger/bench/workgen/runner/many-dhandle-stress.py b/src/third_party/wiredtiger/bench/workgen/runner/many-dhandle-stress.py index 7fae0f9799d..9d131c16148 100644 --- a/src/third_party/wiredtiger/bench/workgen/runner/many-dhandle-stress.py +++ b/src/third_party/wiredtiger/bench/workgen/runner/many-dhandle-stress.py @@ -38,7 +38,7 @@ sample_rate=1 context = Context() conn_config = "" -conn_config += ",cache_size=10G,eviction=(threads_min=4,threads_max=4),file_manager=(close_idle_time=30),session_max=1000" # explicitly added +conn_config += ",cache_size=10G,eviction=(threads_min=4,threads_max=4),file_manager=(close_idle_time=30),session_max=1000,statistics=[all,clear],statistics_log=(wait=1,json=false,on_close=true)" # explicitly added conn = context.wiredtiger_open("create," + conn_config) s = conn.open_session("") diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx index 79552218483..cc2c93d3f6b 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx +++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx @@ -178,7 +178,7 @@ int WorkloadRunner::start_table_idle_cycle(WT_CONNECTION *conn) { char uri[BUF_SIZE]; cycle_count = 0; - if (ret = conn->open_session(conn, NULL, NULL, &session) != 0) { + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { THROW("Error Opening a Session."); } diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/many-dhandle-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/many-dhandle-stress.wtperf index 160cc429896..2c6e13a486c 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/many-dhandle-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/many-dhandle-stress.wtperf @@ -1,7 +1,7 @@ # This workload uses several tens of thousands of tables and the workload is evenly distributed # among them. The workload creates, opens and drop tables, and it generates warning if the time # taken is more than the configured max_idle_table_cycle. -conn_config="cache_size=10G,eviction=(threads_min=4,threads_max=4),file_manager=(close_idle_time=30),session_max=1000" +conn_config="cache_size=10G,eviction=(threads_min=4,threads_max=4),file_manager=(close_idle_time=30),session_max=1000,statistics=[all,clear],statistics_log=(wait=1,json=false,on_close=true)" table_config="type=file" table_count=15000 max_idle_table_cycle=2 diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 2027298bb2e..7a8cdbc272c 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -951,8 +951,6 @@ connection_reconfigure_statistics_log_configuration = [ ] tiered_storage_configuration_common = [ - Config('auth_token', '', r''' - authentication token string'''), Config('local_retention', '300', r''' time in seconds to retain data on tiered storage on the local tier for faster read access''', @@ -979,14 +977,10 @@ wiredtiger_open_tiered_storage_configuration = [ authentication string identifier'''), Config('bucket', '', r''' bucket string identifier where the objects should reside'''), - Config('cluster', '', r''' - unique string identifier identifying the cluster owning these objects. - This identifier is used in naming since objects multiple instances can share - the object storage bucket'''), - Config('member', '', r''' - unique string identifier identifying the member within a cluster. - This identifier is used in naming objects since multiple nodes in a - cluster could write to the same table in the object storage bucket'''), + Config('bucket_prefix', '', r''' + unique string prefix to identify our objects in the bucket. + Multiple instances can share the storage bucket and this + identifier is used in naming objects'''), Config('name', 'none', r''' Permitted values are \c "none" or custom storage name created with diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 98e39d86683..3e0f6af6581 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -70,8 +70,6 @@ WT_SESSION_LOCKED_TURTLE WT_SINGLE_THREAD_CHECK_START WT_SINGLE_THREAD_CHECK_STOP WT_SIZE_CHECK -WT_SS_OPEN_CREATE -WT_SS_OPEN_READONLY WT_STATS_FIELD_TO_OFFSET WT_STATS_SLOT_ID WT_STAT_CONN_DATA_DECRV diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index 8e823c1f229..2102b355c8c 100755 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -105,11 +105,11 @@ func_ok() -e '/int index_compare_primary$/d' \ -e '/int index_compare_u$/d' \ -e '/int index_extractor_u$/d' \ + -e '/int local_directory_list_free$/d' \ -e '/int local_err$/d' \ -e '/int local_file_lock$/d' \ -e '/int local_file_sync$/d' \ - -e '/int local_location_handle_close$/d' \ - -e '/int local_location_list_free$/d' \ + -e '/int local_fs_terminate$/d' \ -e '/int log_print_err$/d' \ -e '/int lz4_error$/d' \ -e '/int lz4_pre_size$/d' \ diff --git a/src/third_party/wiredtiger/dist/test_data.py b/src/third_party/wiredtiger/dist/test_data.py index 863d58bb358..a34f75d4578 100644 --- a/src/third_party/wiredtiger/dist/test_data.py +++ b/src/third_party/wiredtiger/dist/test_data.py @@ -48,12 +48,8 @@ class Config: record_config = [ Config('key_size', 0, r''' The size of the keys created''', min=0, max=10000), - Config('key_format', 'i', r''' - The format of the keys in the database'''), Config('value_size', 0, r''' The size of the values created''', min=0, max=1000000000), - Config('value_format', 'S', r''' - The format of the values stored in the database.''') ] # @@ -175,5 +171,6 @@ test_config = [ ] methods = { + 'example_test' : Method(test_config), 'poc_test' : Method(test_config), } diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index 664e0f33ce7..6c20b31a146 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -23,7 +23,6 @@ noinst_PROGRAMS = \ ex_pack \ ex_process \ ex_schema \ - ex_storage_source \ ex_smoke \ ex_stat \ ex_sync \ @@ -31,7 +30,6 @@ noinst_PROGRAMS = \ ex_encrypt_LDFLAGS = -rdynamic ex_file_system_LDFLAGS = -rdynamic -ex_storage_source_LDFLAGS = -rdynamic # The examples can be run with no arguments as simple smoke tests TESTS = $(noinst_PROGRAMS) diff --git a/src/third_party/wiredtiger/examples/c/ex_storage_source.c b/src/third_party/wiredtiger/examples/c/ex_storage_source.c deleted file mode 100644 index 6cfd0cb3e07..00000000000 --- a/src/third_party/wiredtiger/examples/c/ex_storage_source.c +++ /dev/null @@ -1,1203 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * ex_storage_source.c - * demonstrates how to use the custom storage source interface - */ -#include <test_util.h> - -#ifdef __GNUC__ -#if __GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ > 0) -/* - * !!! - * GCC with -Wformat-truncation complains about calls to snprintf in this file. - * There's nothing wrong, this makes the warning go away. - */ -#pragma GCC diagnostic ignored "-Wformat-truncation" -#endif -#endif - -/* - * This example code uses pthread functions for portable locking, we ignore errors for simplicity. - */ -static void -allocate_storage_source_lock(pthread_rwlock_t *lockp) -{ - error_check(pthread_rwlock_init(lockp, NULL)); -} - -static void -destroy_storage_source_lock(pthread_rwlock_t *lockp) -{ - error_check(pthread_rwlock_destroy(lockp)); -} - -static void -lock_storage_source(pthread_rwlock_t *lockp) -{ - error_check(pthread_rwlock_wrlock(lockp)); -} - -static void -unlock_storage_source(pthread_rwlock_t *lockp) -{ - error_check(pthread_rwlock_unlock(lockp)); -} - -/* - * Example storage source implementation, using memory buffers to represent objects. - */ -typedef struct { - WT_STORAGE_SOURCE iface; - - /* - * WiredTiger performs schema and I/O operations in parallel, all storage sources and file - * handle access must be thread-safe. This example uses a single, global storage source lock for - * simplicity; real applications might require finer granularity, for example, a single lock for - * the storage source handle list and per-handle locks serializing I/O. - */ - pthread_rwlock_t lock; /* Lock */ - - int closed_object_count; - int opened_object_count; - int opened_unique_object_count; - int read_ops; - int write_ops; - - /* Queue of file handles */ - TAILQ_HEAD(demo_file_handle_qh, demo_file_handle) fileq; - - WT_EXTENSION_API *wtext; /* Extension functions */ - -} DEMO_STORAGE_SOURCE; - -typedef struct demo_file_handle { - WT_FILE_HANDLE iface; - - /* - * Add custom file handle fields after the interface. - */ - DEMO_STORAGE_SOURCE *demo_ss; /* Enclosing storage source */ - - TAILQ_ENTRY(demo_file_handle) q; /* Queue of handles */ - uint32_t ref; /* Reference count */ - - char *buf; /* In-memory contents */ - size_t bufsize; /* In-memory buffer size */ - - size_t size; /* Read/write data size */ -} DEMO_FILE_HANDLE; - -typedef struct demo_location_handle { - WT_LOCATION_HANDLE iface; - - char *loc_string; /* location as a string. */ -} DEMO_LOCATION_HANDLE; - -#define LOCATION_STRING(lh) (((DEMO_LOCATION_HANDLE *)lh)->loc_string) - -/* - * Extension initialization function. - */ -#ifdef _WIN32 -/* - * Explicitly export this function so it is visible when loading extensions. - */ -__declspec(dllexport) -#endif - int demo_storage_source_create(WT_CONNECTION *, WT_CONFIG_ARG *); - -/* - * Forward function declarations for storage source API implementation. - */ -static int demo_ss_exist( - WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, bool *); -static int demo_ss_location_handle( - WT_STORAGE_SOURCE *, WT_SESSION *, const char *, WT_LOCATION_HANDLE **); -static int demo_ss_location_list(WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, - const char *, uint32_t, char ***, uint32_t *); -static int demo_ss_location_list_free(WT_STORAGE_SOURCE *, WT_SESSION *, char **, uint32_t); -static int demo_ss_open(WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, - uint32_t, WT_FILE_HANDLE **); -static int demo_ss_remove( - WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, uint32_t); -static int demo_ss_size( - WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, wt_off_t *); -static int demo_ss_terminate(WT_STORAGE_SOURCE *, WT_SESSION *); - -/* - * Forward function declarations for location API implementation. - */ -static int demo_location_close(WT_LOCATION_HANDLE *, WT_SESSION *); - -/* - * Forward function declarations for file handle API implementation. - */ -static int demo_file_close(WT_FILE_HANDLE *, WT_SESSION *); -static int demo_file_lock(WT_FILE_HANDLE *, WT_SESSION *, bool); -static int demo_file_read(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, void *); -static int demo_file_size(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); -static int demo_file_sync(WT_FILE_HANDLE *, WT_SESSION *); -static int demo_file_truncate(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t); -static int demo_file_write(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, const void *); - -/* - * Forward function declarations for internal functions. - */ -static int demo_handle_remove(WT_SESSION *, DEMO_FILE_HANDLE *); -static DEMO_FILE_HANDLE *demo_handle_search( - WT_STORAGE_SOURCE *, WT_LOCATION_HANDLE *, const char *); - -#define DEMO_FILE_SIZE_INCREMENT 32768 - -/* - * string_match -- - * Return if a string matches a byte string of len bytes. - */ -static bool -byte_string_match(const char *str, const char *bytes, size_t len) -{ - return (strncmp(str, bytes, len) == 0 && (str)[(len)] == '\0'); -} - -/* - * demo_storage_source_create -- - * Initialize the demo storage source. - */ -int -demo_storage_source_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) -{ - DEMO_STORAGE_SOURCE *demo_ss; - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; - WT_EXTENSION_API *wtext; - WT_STORAGE_SOURCE *storage_source; - int ret = 0; - - wtext = conn->get_extension_api(conn); - - if ((demo_ss = calloc(1, sizeof(DEMO_STORAGE_SOURCE))) == NULL) { - (void)wtext->err_printf( - wtext, NULL, "demo_storage_source_create: %s", wtext->strerror(wtext, NULL, ENOMEM)); - return (ENOMEM); - } - demo_ss->wtext = wtext; - storage_source = (WT_STORAGE_SOURCE *)demo_ss; - - /* - * Applications may have their own configuration information to pass to the underlying - * filesystem implementation. See the main function for the setup of those configuration - * strings; here we parse configuration information as passed in by main, through WiredTiger. - */ - if ((ret = wtext->config_parser_open_arg(wtext, NULL, config, &config_parser)) != 0) { - (void)wtext->err_printf(wtext, NULL, "WT_EXTENSION_API.config_parser_open: config: %s", - wtext->strerror(wtext, NULL, ret)); - goto err; - } - - /* Step through our configuration values. */ - printf("Custom storage source configuration\n"); - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { - if (byte_string_match("config_string", k.str, k.len)) { - printf( - "\t" - "key %.*s=\"%.*s\"\n", - (int)k.len, k.str, (int)v.len, v.str); - continue; - } - if (byte_string_match("config_value", k.str, k.len)) { - printf( - "\t" - "key %.*s=%" PRId64 "\n", - (int)k.len, k.str, v.val); - continue; - } - ret = EINVAL; - (void)wtext->err_printf(wtext, NULL, - "WT_CONFIG_PARSER.next: unexpected configuration " - "information: %.*s=%.*s: %s", - (int)k.len, k.str, (int)v.len, v.str, wtext->strerror(wtext, NULL, ret)); - goto err; - } - - /* Check for expected parser termination and close the parser. */ - if (ret != WT_NOTFOUND) { - (void)wtext->err_printf( - wtext, NULL, "WT_CONFIG_PARSER.next: config: %s", wtext->strerror(wtext, NULL, ret)); - goto err; - } - if ((ret = config_parser->close(config_parser)) != 0) { - (void)wtext->err_printf( - wtext, NULL, "WT_CONFIG_PARSER.close: config: %s", wtext->strerror(wtext, NULL, ret)); - goto err; - } - - allocate_storage_source_lock(&demo_ss->lock); - - /* Initialize the in-memory jump table. */ - storage_source->ss_exist = demo_ss_exist; - storage_source->ss_location_handle = demo_ss_location_handle; - storage_source->ss_location_list = demo_ss_location_list; - storage_source->ss_location_list_free = demo_ss_location_list_free; - storage_source->ss_open_object = demo_ss_open; - storage_source->ss_remove = demo_ss_remove; - storage_source->ss_size = demo_ss_size; - storage_source->terminate = demo_ss_terminate; - - if ((ret = conn->add_storage_source(conn, "demo", storage_source, NULL)) != 0) { - (void)wtext->err_printf( - wtext, NULL, "WT_CONNECTION.set_storage_source: %s", wtext->strerror(wtext, NULL, ret)); - goto err; - } - - return (0); - -err: - free(demo_ss); - /* An error installing the storage source is fatal. */ - exit(1); -} - -/* - * demo_ss_open -- - * fopen for our demo storage source. - */ -static int -demo_ss_open(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags, - WT_FILE_HANDLE **file_handlep) -{ - DEMO_FILE_HANDLE *demo_fh; - DEMO_STORAGE_SOURCE *demo_ss; - WT_EXTENSION_API *wtext; - WT_FILE_HANDLE *file_handle; - const char *location; - char *full_name; - size_t name_len; - int ret = 0; - - (void)flags; /* Unused */ - - *file_handlep = NULL; - - demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; - demo_fh = NULL; - wtext = demo_ss->wtext; - - lock_storage_source(&demo_ss->lock); - ++demo_ss->opened_object_count; - - /* - * First search the file queue, if we find it, assert there's only a single reference, we only - * support a single handle on any file. - */ - demo_fh = demo_handle_search(storage_source, location_handle, name); - if (demo_fh != NULL) { - if (demo_fh->ref != 0) { - (void)wtext->err_printf(wtext, session, "demo_ss_open: %s: file already open", name); - ret = EBUSY; - goto err; - } - - demo_fh->ref = 1; - *file_handlep = (WT_FILE_HANDLE *)demo_fh; - unlock_storage_source(&demo_ss->lock); - return (0); - } - - /* The file hasn't been opened before, create a new one. */ - if ((demo_fh = calloc(1, sizeof(DEMO_FILE_HANDLE))) == NULL) { - ret = ENOMEM; - goto err; - } - - /* Initialize private information. */ - demo_fh->demo_ss = demo_ss; - demo_fh->ref = 1; - if ((demo_fh->buf = calloc(1, DEMO_FILE_SIZE_INCREMENT)) == NULL) { - ret = ENOMEM; - goto err; - } - demo_fh->bufsize = DEMO_FILE_SIZE_INCREMENT; - demo_fh->size = 0; - - /* Construct the public name. */ - location = LOCATION_STRING(location_handle); - name_len = strlen(location) + strlen(name) + 1; - full_name = calloc(1, name_len); - if (snprintf(full_name, name_len, "%s%s", location, name) != (ssize_t)(name_len - 1)) { - ret = ENOMEM; - goto err; - } - - /* Initialize public information. */ - file_handle = (WT_FILE_HANDLE *)demo_fh; - file_handle->name = full_name; - - /* - * Setup the function call table for our custom storage source. Set the function pointer to NULL - * where our implementation doesn't support the functionality. - */ - file_handle->close = demo_file_close; - file_handle->fh_advise = NULL; - file_handle->fh_extend = NULL; - file_handle->fh_extend_nolock = NULL; - file_handle->fh_lock = demo_file_lock; - file_handle->fh_map = NULL; - file_handle->fh_map_discard = NULL; - file_handle->fh_map_preload = NULL; - file_handle->fh_read = demo_file_read; - file_handle->fh_size = demo_file_size; - file_handle->fh_sync = demo_file_sync; - file_handle->fh_sync_nowait = NULL; - file_handle->fh_truncate = demo_file_truncate; - file_handle->fh_unmap = NULL; - file_handle->fh_write = demo_file_write; - - TAILQ_INSERT_HEAD(&demo_ss->fileq, demo_fh, q); - ++demo_ss->opened_unique_object_count; - - *file_handlep = file_handle; - - if (0) { -err: - free(demo_fh->buf); - free(demo_fh); - } - - unlock_storage_source(&demo_ss->lock); - return (ret); -} - -/* - * demo_ss_location_handle -- - * Return a location handle from a location string. - */ -static int -demo_ss_location_handle(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - const char *location_info, WT_LOCATION_HANDLE **location_handlep) -{ - DEMO_LOCATION_HANDLE *demo_loc; - size_t len; - int ret; - char *p; - - (void)storage_source; /* Unused */ - (void)session; /* Unused */ - - ret = 0; - p = NULL; - demo_loc = NULL; - - /* - * We save the location string we're given followed by a slash delimiter. We won't allow slashes - * in the location info parameter. - */ - if (strchr(location_info, '/') != NULL) - return (EINVAL); - len = strlen(location_info) + 2; - p = malloc(len); - if (snprintf(p, len, "%s/", location_info) != (ssize_t)(len - 1)) { - ret = ENOMEM; - goto err; - } - - /* - * Now create the location handle and save the string. - */ - if ((demo_loc = calloc(1, sizeof(DEMO_LOCATION_HANDLE))) == NULL) { - ret = ENOMEM; - goto err; - } - - /* Initialize private information. */ - demo_loc->loc_string = p; - - /* Initialize public information. */ - demo_loc->iface.close = demo_location_close; - - *location_handlep = &demo_loc->iface; - -err: - if (ret != 0) { - free(p); - free(demo_loc); - return (ret); - } - return (0); -} - -/* - * demo_ss_location_list -- - * Return a list of object names for the given location. - */ -static int -demo_ss_location_list(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *prefix, uint32_t limit, char ***dirlistp, - uint32_t *countp) -{ - DEMO_FILE_HANDLE *demo_fh; - DEMO_STORAGE_SOURCE *demo_ss; - size_t location_len, prefix_len; - uint32_t allocated, count; - int ret = 0; - const char *location; - char **entries, *name; - void *p; - - (void)session; /* Unused */ - - demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; - - *dirlistp = NULL; - *countp = 0; - - entries = NULL; - allocated = count = 0; - location = LOCATION_STRING(location_handle); - location_len = strlen(location); - prefix_len = (prefix == NULL ? 0 : strlen(prefix)); - - lock_storage_source(&demo_ss->lock); - TAILQ_FOREACH (demo_fh, &demo_ss->fileq, q) { - name = demo_fh->iface.name; - if (strncmp(name, location, location_len) != 0) - continue; - name += location_len; - if (prefix != NULL && strncmp(name, prefix, prefix_len) != 0) - continue; - - /* - * Increase the list size in groups of 10, it doesn't matter if the list is a bit longer - * than necessary. - */ - if (count >= allocated) { - p = realloc(entries, (allocated + 10) * sizeof(*entries)); - if (p == NULL) { - ret = ENOMEM; - goto err; - } - - entries = p; - memset(entries + allocated * sizeof(*entries), 0, 10 * sizeof(*entries)); - allocated += 10; - } - entries[count++] = strdup(name); - if (limit > 0 && count >= limit) - break; - } - - *dirlistp = entries; - *countp = count; - -err: - unlock_storage_source(&demo_ss->lock); - if (ret == 0) - return (0); - - if (entries != NULL) { - while (count > 0) - free(entries[--count]); - free(entries); - } - - return (ret); -} - -/* - * demo_ss_location_list_free -- - * Free memory allocated by demo_ss_location_list. - */ -static int -demo_ss_location_list_free( - WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, char **dirlist, uint32_t count) -{ - (void)storage_source; - (void)session; - - if (dirlist != NULL) { - while (count > 0) - free(dirlist[--count]); - free(dirlist); - } - return (0); -} - -/* - * demo_ss_exist -- - * Return if the file exists. - */ -static int -demo_ss_exist(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, bool *existp) -{ - DEMO_STORAGE_SOURCE *demo_ss; - - (void)session; /* Unused */ - - demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; - - lock_storage_source(&demo_ss->lock); - *existp = demo_handle_search(storage_source, location_handle, name) != NULL; - unlock_storage_source(&demo_ss->lock); - - return (0); -} - -/* - * demo_ss_remove -- - * POSIX remove. - */ -static int -demo_ss_remove(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags) -{ - DEMO_STORAGE_SOURCE *demo_ss; - DEMO_FILE_HANDLE *demo_fh; - int ret = 0; - - (void)session; /* Unused */ - (void)flags; /* Unused */ - - demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; - - ret = ENOENT; - lock_storage_source(&demo_ss->lock); - if ((demo_fh = demo_handle_search(storage_source, location_handle, name)) != NULL) - ret = demo_handle_remove(session, demo_fh); - unlock_storage_source(&demo_ss->lock); - - return (ret); -} - -/* - * demo_ss_size -- - * Get the size of a file in bytes, by file name. - */ -static int -demo_ss_size(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, wt_off_t *sizep) -{ - DEMO_STORAGE_SOURCE *demo_ss; - DEMO_FILE_HANDLE *demo_fh; - int ret = 0; - - demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; - - ret = ENOENT; - lock_storage_source(&demo_ss->lock); - if ((demo_fh = demo_handle_search(storage_source, location_handle, name)) != NULL) - ret = demo_file_size((WT_FILE_HANDLE *)demo_fh, session, sizep); - unlock_storage_source(&demo_ss->lock); - - return (ret); -} - -/* - * demo_ss_terminate -- - * Discard any resources on termination. - */ -static int -demo_ss_terminate(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session) -{ - DEMO_FILE_HANDLE *demo_fh, *demo_fh_tmp; - DEMO_STORAGE_SOURCE *demo_ss; - int ret = 0, tret; - - demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; - - TAILQ_FOREACH_SAFE(demo_fh, &demo_ss->fileq, q, demo_fh_tmp) - if ((tret = demo_handle_remove(session, demo_fh)) != 0 && ret == 0) - ret = tret; - - printf("Custom storage source\n"); - printf("\t%d unique object opens\n", demo_ss->opened_unique_object_count); - printf("\t%d objects opened\n", demo_ss->opened_object_count); - printf("\t%d objects closed\n", demo_ss->closed_object_count); - printf("\t%d reads, %d writes\n", demo_ss->read_ops, demo_ss->write_ops); - - destroy_storage_source_lock(&demo_ss->lock); - free(demo_ss); - - return (ret); -} - -/* - * demo_location_close -- - * Free a location handle created by ss_location_handle. - */ -static int -demo_location_close(WT_LOCATION_HANDLE *location_handle, WT_SESSION *session) -{ - (void)session; /* Unused */ - - free(LOCATION_STRING(location_handle)); - free(location_handle); - return (0); -} - -/* - * demo_file_close -- - * ANSI C close. - */ -static int -demo_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session) -{ - DEMO_FILE_HANDLE *demo_fh; - DEMO_STORAGE_SOURCE *demo_ss; - - (void)session; /* Unused */ - - demo_fh = (DEMO_FILE_HANDLE *)file_handle; - demo_ss = demo_fh->demo_ss; - - lock_storage_source(&demo_ss->lock); - if (--demo_fh->ref == 0) - ++demo_ss->closed_object_count; - unlock_storage_source(&demo_ss->lock); - - return (0); -} - -/* - * demo_file_lock -- - * Lock/unlock a file. - */ -static int -demo_file_lock(WT_FILE_HANDLE *file_handle, WT_SESSION *session, bool lock) -{ - /* Locks are always granted. */ - (void)file_handle; /* Unused */ - (void)session; /* Unused */ - (void)lock; /* Unused */ - return (0); -} - -/* - * demo_file_read -- - * POSIX pread. - */ -static int -demo_file_read( - WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, void *buf) -{ - DEMO_FILE_HANDLE *demo_fh; - DEMO_STORAGE_SOURCE *demo_ss; - WT_EXTENSION_API *wtext; - size_t off; - int ret = 0; - - demo_fh = (DEMO_FILE_HANDLE *)file_handle; - demo_ss = demo_fh->demo_ss; - wtext = demo_ss->wtext; - off = (size_t)offset; - - lock_storage_source(&demo_ss->lock); - ++demo_ss->read_ops; - if (off < demo_fh->size) { - if (len > demo_fh->size - off) - len = demo_fh->size - off; - memcpy(buf, (uint8_t *)demo_fh->buf + off, len); - } else - ret = EIO; /* EOF */ - unlock_storage_source(&demo_ss->lock); - if (ret == 0) - return (0); - - (void)wtext->err_printf(wtext, session, - "%s: handle-read: failed to read %zu bytes at offset %zu: %s", demo_fh->iface.name, len, off, - wtext->strerror(wtext, NULL, ret)); - return (ret); -} - -/* - * demo_file_size -- - * Get the size of a file in bytes, by file handle. - */ -static int -demo_file_size(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t *sizep) -{ - DEMO_FILE_HANDLE *demo_fh; - DEMO_STORAGE_SOURCE *demo_ss; - - (void)session; /* Unused */ - - demo_fh = (DEMO_FILE_HANDLE *)file_handle; - demo_ss = demo_fh->demo_ss; - - lock_storage_source(&demo_ss->lock); - *sizep = (wt_off_t)demo_fh->size; - unlock_storage_source(&demo_ss->lock); - return (0); -} - -/* - * demo_file_sync -- - * Ensure the content of the file is stable. This is a no-op in our memory backed storage - * source. - */ -static int -demo_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *session) -{ - (void)file_handle; /* Unused */ - (void)session; /* Unused */ - - return (0); -} - -/* - * demo_buffer_resize -- - * Resize the write buffer. - */ -static int -demo_buffer_resize(WT_SESSION *session, DEMO_FILE_HANDLE *demo_fh, wt_off_t offset) -{ - DEMO_STORAGE_SOURCE *demo_ss; - WT_EXTENSION_API *wtext; - size_t off; - void *p; - - demo_ss = demo_fh->demo_ss; - wtext = demo_ss->wtext; - off = (size_t)offset; - - /* Grow the buffer as necessary and clear any new space in the file. */ - if (demo_fh->bufsize >= off) - return (0); - - if ((p = realloc(demo_fh->buf, off)) == NULL) { - (void)wtext->err_printf(wtext, session, "%s: failed to resize buffer", demo_fh->iface.name, - wtext->strerror(wtext, NULL, ENOMEM)); - return (ENOMEM); - } - memset((uint8_t *)p + demo_fh->bufsize, 0, off - demo_fh->bufsize); - demo_fh->buf = p; - demo_fh->bufsize = off; - - return (0); -} - -/* - * demo_file_truncate -- - * POSIX ftruncate. - */ -static int -demo_file_truncate(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset) -{ - DEMO_FILE_HANDLE *demo_fh; - DEMO_STORAGE_SOURCE *demo_ss; - WT_EXTENSION_API *wtext; - - (void)file_handle; /* Unused */ - (void)session; /* Unused */ - (void)offset; /* Unused */ - - demo_fh = (DEMO_FILE_HANDLE *)file_handle; - demo_ss = demo_fh->demo_ss; - wtext = demo_ss->wtext; - - (void)wtext->err_printf(wtext, session, "%s: truncate not supported in storage source", - demo_fh->iface.name, wtext->strerror(wtext, NULL, ENOTSUP)); - return (ENOTSUP); -} - -/* - * demo_file_write -- - * POSIX pwrite. - */ -static int -demo_file_write( - WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, const void *buf) -{ - DEMO_FILE_HANDLE *demo_fh; - DEMO_STORAGE_SOURCE *demo_ss; - WT_EXTENSION_API *wtext; - size_t off; - int ret = 0; - - demo_fh = (DEMO_FILE_HANDLE *)file_handle; - demo_ss = demo_fh->demo_ss; - wtext = demo_ss->wtext; - off = (size_t)offset; - - lock_storage_source(&demo_ss->lock); - ++demo_ss->write_ops; - if ((ret = demo_buffer_resize( - session, demo_fh, offset + (wt_off_t)(len + DEMO_FILE_SIZE_INCREMENT))) == 0) { - memcpy((uint8_t *)demo_fh->buf + off, buf, len); - if (off + len > demo_fh->size) - demo_fh->size = off + len; - } - unlock_storage_source(&demo_ss->lock); - if (ret == 0) - return (0); - - (void)wtext->err_printf(wtext, session, - "%s: handle-write: failed to write %zu bytes at offset %zu: %s", demo_fh->iface.name, len, - off, wtext->strerror(wtext, NULL, ret)); - return (ret); -} - -/* - * demo_handle_remove -- - * Destroy an in-memory file handle. Should only happen on remove or shutdown. - */ -static int -demo_handle_remove(WT_SESSION *session, DEMO_FILE_HANDLE *demo_fh) -{ - DEMO_STORAGE_SOURCE *demo_ss; - WT_EXTENSION_API *wtext; - - demo_ss = demo_fh->demo_ss; - wtext = demo_ss->wtext; - - if (demo_fh->ref != 0) { - (void)wtext->err_printf(wtext, session, "demo_handle_remove: %s: file is currently open", - demo_fh->iface.name, wtext->strerror(wtext, NULL, EBUSY)); - return (EBUSY); - } - - TAILQ_REMOVE(&demo_ss->fileq, demo_fh, q); - - /* Clean up private information. */ - free(demo_fh->buf); - - /* Clean up public information. */ - free(demo_fh->iface.name); - - free(demo_fh); - - return (0); -} - -/* - * demo_handle_search -- - * Return a matching handle, if one exists. - */ -static DEMO_FILE_HANDLE * -demo_handle_search( - WT_STORAGE_SOURCE *storage_source, WT_LOCATION_HANDLE *location_handle, const char *name) -{ - DEMO_FILE_HANDLE *demo_fh; - DEMO_STORAGE_SOURCE *demo_ss; - size_t len; - const char *location; - - demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; - location = LOCATION_STRING(location_handle); - len = strlen(location); - - TAILQ_FOREACH (demo_fh, &demo_ss->fileq, q) - if (strncmp(demo_fh->iface.name, location, len) == 0 && - strcmp(&demo_fh->iface.name[len], name) == 0) - break; - return (demo_fh); -} - -static const char *home; - -static int -demo_test_create(WT_STORAGE_SOURCE *ss, WT_SESSION *session, WT_LOCATION_HANDLE *location, - const char *objname, const char *content) -{ - WT_FILE_HANDLE *fh; - const char *op; - size_t len; - int ret, t_ret; - - fh = NULL; - len = strlen(content) + 1; - op = "open"; - if ((ret = ss->ss_open_object(ss, session, location, objname, WT_SS_OPEN_CREATE, &fh)) != 0) - goto err; - op = "write"; - if ((ret = fh->fh_write(fh, session, 0, len, content)) != 0) - goto err; - -err: - if (fh != NULL && (t_ret = fh->close(fh, session)) != 0 && ret == 0) { - op = "close"; - ret = t_ret; - } - if (ret != 0) - fprintf(stderr, "demo failed during %s: %s\n", op, wiredtiger_strerror(ret)); - else - printf("demo succeeded create %s\n", objname); - - return (ret); -} - -static int -demo_test_read(WT_STORAGE_SOURCE *ss, WT_SESSION *session, WT_LOCATION_HANDLE *location, - const char *objname, const char *content) -{ - WT_FILE_HANDLE *fh; - char buf[100]; - const char *op; - size_t len; - wt_off_t size; - int ret, t_ret; - - fh = NULL; - len = strlen(content) + 1; - - /* Set the op string so that on error we know what failed. */ - op = "open"; - if ((ret = ss->ss_open_object(ss, session, location, objname, WT_SS_OPEN_READONLY, &fh)) != 0) - goto err; - op = "size"; - if ((ret = fh->fh_size(fh, session, &size)) != 0) - goto err; - op = "size-compare"; - if ((size_t)size != len || (size_t)size > sizeof(buf)) { - ret = EINVAL; - goto err; - } - op = "read"; - if ((ret = fh->fh_read(fh, session, 0, len, buf)) != 0) - goto err; - op = "read-compare"; - if (strncmp(buf, content, len) != 0) { - ret = EINVAL; - goto err; - } - -err: - if (fh != NULL && (t_ret = fh->close(fh, session)) != 0 && ret == 0) { - op = "close"; - ret = t_ret; - } - if (ret != 0) - fprintf(stderr, "demo failed during %s: %s\n", op, wiredtiger_strerror(ret)); - else - printf("demo succeeded read %s\n", objname); - - return (ret); -} - -static int -demo_test_list(WT_STORAGE_SOURCE *ss, WT_SESSION *session, const char *description, - WT_LOCATION_HANDLE *location, const char *prefix, uint32_t limit, uint32_t expect) -{ - char **obj_list; - const char *op; - uint32_t i, obj_count; - int ret, t_ret; - - obj_list = NULL; - /* Set the op string so that on error we know what failed. */ - op = "location_list"; - if ((ret = ss->ss_location_list(ss, session, location, prefix, limit, &obj_list, &obj_count)) != - 0) - goto err; - op = "location_list count"; - if (obj_count != expect) { - ret = EINVAL; - goto err; - } - printf("list: %s:\n", description); - for (i = 0; i < obj_count; i++) { - printf(" %s\n", obj_list[i]); - } - -err: - if (obj_list != NULL && - (t_ret = ss->ss_location_list_free(ss, session, obj_list, obj_count)) != 0 && ret == 0) { - op = "location_list_free"; - ret = t_ret; - } - if (ret != 0) - fprintf(stderr, "demo failed during %s: %s\n", op, wiredtiger_strerror(ret)); - else - printf("demo succeeded location_list %s\n", description); - - return (ret); -} - -static int -demo_test_storage_source(WT_STORAGE_SOURCE *ss, WT_SESSION *session) -{ - WT_LOCATION_HANDLE *location1, *location2; - const char *op; - int ret, t_ret; - bool exist; - - location1 = location2 = NULL; - - /* Create two locations. Set the op string so that on error we know what failed. */ - op = "location_handle"; - if ((ret = ss->ss_location_handle(ss, session, "location-one", &location1)) != 0) - goto err; - if ((ret = ss->ss_location_handle(ss, session, "location-two", &location2)) != 0) - goto err; - - /* - * Create and existence checks. In location-one, create "A". In location-two, create "A", "B", - * "AA". We'll do simple lists of both locations, and a list of location-two with a prefix. - */ - op = "create/exist checks"; - if ((ret = demo_test_create(ss, session, location1, "A", "location-one-A")) != 0) - goto err; - - if ((ret = ss->ss_exist(ss, session, location1, "A", &exist)) != 0) - goto err; - if (!exist) { - fprintf(stderr, "Exist test failed for A\n"); - ret = EINVAL; - goto err; - } - if ((ret = ss->ss_exist(ss, session, location2, "A", &exist)) != 0) - goto err; - if (exist) { - fprintf(stderr, "Exist test failed for A in location2\n"); - ret = EINVAL; - goto err; - } - - if ((ret = demo_test_create(ss, session, location2, "A", "location-two-A")) != 0) - goto err; - if ((ret = demo_test_create(ss, session, location2, "B", "location-two-B")) != 0) - goto err; - if ((ret = demo_test_create(ss, session, location2, "AA", "location-two-AA")) != 0) - goto err; - - /* Make sure the objects contain the expected data. */ - op = "read checks"; - if ((ret = demo_test_read(ss, session, location1, "A", "location-one-A")) != 0) - goto err; - if ((ret = demo_test_read(ss, session, location2, "A", "location-two-A")) != 0) - goto err; - if ((ret = demo_test_read(ss, session, location2, "B", "location-two-B")) != 0) - goto err; - - /* - * List the locations. For location-one, we expect just one object. - */ - op = "list checks"; - if ((ret = demo_test_list(ss, session, "location1", location1, NULL, 0, 1)) != 0) - goto err; - - /* - * For location-two, we expect three objects. - */ - if ((ret = demo_test_list(ss, session, "location2", location2, NULL, 0, 3)) != 0) - goto err; - - /* - * If we limit the number of objects received to 2, we should only see 2. - */ - if ((ret = demo_test_list(ss, session, "location2, limit:2", location2, NULL, 2, 2)) != 0) - goto err; - - /* - * With a prefix of "A", and no limit, we'll see two objects. - */ - if ((ret = demo_test_list(ss, session, "location2: A", location2, "A", 0, 2)) != 0) - goto err; - - /* - * With a prefix of "A", and a limit of one, we'll see just one object. - */ - if ((ret = demo_test_list(ss, session, "location2: A, limit:1", location2, "A", 1, 1)) != 0) - goto err; - -err: - if (location1 != NULL && (t_ret = location1->close(location1, session)) != 0 && ret == 0) - ret = t_ret; - if (location2 != NULL && (t_ret = location2->close(location2, session)) != 0 && ret == 0) - ret = t_ret; - if (ret != 0) - fprintf(stderr, "demo failed during %s: %s\n", op, wiredtiger_strerror(ret)); - - return (ret); -} - -int -main(void) -{ - WT_CONNECTION *conn; - WT_SESSION *session; - WT_STORAGE_SOURCE *storage_source; - const char *open_config; - int ret; - - fprintf(stderr, "ex_storage_source: starting\n"); - /* - * Create a clean test directory for this run of the test program if the environment variable - * isn't already set (as is done by make check). - */ - if (getenv("WIREDTIGER_HOME") == NULL) { - home = "WT_HOME"; - if ((ret = system("rm -rf WT_HOME && mkdir WT_HOME")) != 0) { - fprintf(stderr, "system: directory recreate failed: %s\n", strerror(ret)); - return (EXIT_FAILURE); - } - } else - home = NULL; - - /*! [WT_STORAGE_SOURCE register] */ - /* - * Setup a configuration string that will load our custom storage source. Use the special local - * extension to indicate that the entry point is in the same executable. Finally, pass in two - * pieces of configuration information to our initialization function as the "config" value. - */ - open_config = - "create,log=(enabled=true),extensions=(local={entry=demo_storage_source_create," - "config={config_string=\"demo-storage-source\",config_value=37}})"; - /* Open a connection to the database, creating it if necessary. */ - if ((ret = wiredtiger_open(home, NULL, open_config, &conn)) != 0) { - fprintf(stderr, "Error connecting to %s: %s\n", home == NULL ? "." : home, - wiredtiger_strerror(ret)); - return (EXIT_FAILURE); - } - /*! [WT_STORAGE_SOURCE register] */ - - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { - fprintf(stderr, "WT_CONNECTION.open_session: %s\n", wiredtiger_strerror(ret)); - return (EXIT_FAILURE); - } - - if ((ret = conn->get_storage_source(conn, "demo", &storage_source)) != 0) { - fprintf(stderr, "WT_CONNECTION.get_storage_source: %s\n", wiredtiger_strerror(ret)); - return (EXIT_FAILURE); - } - /* - * At the moment, the infrastructure within WiredTiger that would use the storage source - * extension does not exist. So call the interface directly as a demonstration. - */ - if ((ret = demo_test_storage_source(storage_source, session)) != 0) { - fprintf(stderr, "storage source test failed: %s\n", wiredtiger_strerror(ret)); - return (EXIT_FAILURE); - } - if ((ret = conn->close(conn, NULL)) != 0) { - fprintf(stderr, "Error closing connection to %s: %s\n", home == NULL ? "." : home, - wiredtiger_strerror(ret)); - return (EXIT_FAILURE); - } - - return (EXIT_SUCCESS); -} diff --git a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c index 0161ba928fb..77cadec8a38 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c +++ b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c @@ -91,6 +91,15 @@ typedef struct { } LOCAL_STORAGE; +typedef struct { + WT_FILE_SYSTEM file_system; /* Must come first */ + LOCAL_STORAGE *local_storage; + + char *auth_token; /* Identifier for key management system */ + char *bucket; /* Actually a directory path for local implementation */ + char *fs_prefix; /* File system prefix, allowing for a "directory" within a bucket */ +} LOCAL_FILE_SYSTEM; + /* * Indicates a object that has not yet been flushed. */ @@ -119,53 +128,45 @@ typedef struct local_file_handle { TAILQ_ENTRY(local_file_handle) q; /* Queue of handles */ } LOCAL_FILE_HANDLE; -typedef struct local_location { - WT_LOCATION_HANDLE iface; /* Must come first */ - - char *cluster_prefix; /* Cluster prefix */ - char *auth_token; /* Identifier for key management system */ - char *bucket; /* Actually a directory path for local implementation */ -} LOCAL_LOCATION; - /* * Forward function declarations for internal functions */ -static int local_config_dup( - LOCAL_STORAGE *, WT_SESSION *, WT_CONFIG_ITEM *, const char *, const char *, char **); static int local_configure(LOCAL_STORAGE *, WT_CONFIG_ARG *); static int local_configure_int(LOCAL_STORAGE *, WT_CONFIG_ARG *, const char *, uint32_t *); static int local_delay(LOCAL_STORAGE *); static int local_err(LOCAL_STORAGE *, WT_SESSION *, int, const char *, ...); static void local_flush_free(LOCAL_FLUSH_ITEM *); -static int local_location_decode(LOCAL_STORAGE *, WT_LOCATION_HANDLE *, char **, char **, char **); -static int local_location_path( - LOCAL_STORAGE *, WT_LOCATION_HANDLE *, const char *, const char *, char **); +static int local_location_path(WT_FILE_SYSTEM *, const char *, const char *, char **); /* * Forward function declarations for storage source API implementation */ -static int local_exist( - WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, bool *); +static int local_exist(WT_FILE_SYSTEM *, WT_SESSION *, const char *, bool *); +static int local_customize_file_system(WT_STORAGE_SOURCE *, WT_SESSION *, const char *, + const char *, const char *, const char *, WT_FILE_SYSTEM **); static int local_flush( - WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, const char *); + WT_STORAGE_SOURCE *, WT_SESSION *, WT_FILE_SYSTEM *, const char *, const char *); static int local_flush_one(LOCAL_STORAGE *, WT_SESSION *, LOCAL_FLUSH_ITEM *); -static int local_location_handle( - WT_STORAGE_SOURCE *, WT_SESSION *, const char *, WT_LOCATION_HANDLE **); -static int local_location_handle_close(WT_LOCATION_HANDLE *, WT_SESSION *); -static int local_location_list(WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, - const char *, uint32_t, char ***, uint32_t *); -static int local_location_list_free(WT_STORAGE_SOURCE *, WT_SESSION *, char **, uint32_t); -static int local_location_list_internal(WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, - const char *, const char *, uint32_t, char ***, uint32_t *); -static int local_open(WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, - uint32_t, WT_FILE_HANDLE **); -static int local_remove( - WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, uint32_t); -static int local_size( - WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, wt_off_t *); static int local_terminate(WT_STORAGE_SOURCE *, WT_SESSION *); /* + * Forward function declarations for file system API implementation + */ +static int local_directory_list( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, char ***, uint32_t *); +static int local_directory_list_internal( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, uint32_t, char ***, uint32_t *); +static int local_directory_list_single( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, char ***, uint32_t *); +static int local_directory_list_free(WT_FILE_SYSTEM *, WT_SESSION *, char **, uint32_t); +static int local_fs_terminate(WT_FILE_SYSTEM *, WT_SESSION *); +static int local_open(WT_FILE_SYSTEM *, WT_SESSION *, const char *, WT_FS_OPEN_FILE_TYPE file_type, + uint32_t, WT_FILE_HANDLE **); +static int local_remove(WT_FILE_SYSTEM *, WT_SESSION *, const char *, uint32_t); +static int local_rename(WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, uint32_t); +static int local_size(WT_FILE_SYSTEM *, WT_SESSION *, const char *, wt_off_t *); + +/* * Forward function declarations for file handle API implementation */ static int local_file_close(WT_FILE_HANDLE *, WT_SESSION *); @@ -180,6 +181,7 @@ static int local_file_write(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, co * Report an error for a file operation. Note that local_err returns its third argument, and this * macro will too. */ +#define FS2LOCAL(fs) (((LOCAL_FILE_SYSTEM *)(fs))->local_storage) #define local_file_err(fh, session, ret, str) \ local_err((fh)->local, session, ret, "\"%s\": %s", fh->iface.name, str) @@ -207,40 +209,6 @@ static const char *MARKER_NEED_FLUSH = "FLUSH_"; static const char *MARKER_TEMPORARY = "TEMP_"; /* - * local_config_dup -- - * Make a copy of a configuration string as an allocated C string. - */ -static int -local_config_dup(LOCAL_STORAGE *local, WT_SESSION *session, WT_CONFIG_ITEM *v, const char *suffix, - const char *disallowed, char **result) -{ - size_t len; - int ret; - char *p; - - if (suffix == NULL) - suffix = ""; - len = v->len + strlen(suffix) + 1; - if ((p = malloc(len)) == NULL) - return (local_err(local, session, ENOMEM, "configuration parsing")); - (void)snprintf(p, len, "%.*s", (int)v->len, v->str); - - /* - * Check for illegal characters before adding the suffix, as the suffix may contain such - * characters. - */ - if (disallowed != NULL && strstr(p, disallowed) != NULL) { - ret = local_err(local, session, EINVAL, - "characters \"%s\" disallowed in configuration string \"%s\"", disallowed, p); - free(p); - return (ret); - } - (void)strcat(p, suffix); - *result = p; - return (0); -} - -/* * local_configure * Parse the configuration for the keys we care about. */ @@ -353,83 +321,105 @@ local_flush_free(LOCAL_FLUSH_ITEM *flush) } /* - * local_location_decode -- - * Break down a location into component parts. - */ -static int -local_location_decode(LOCAL_STORAGE *local, WT_LOCATION_HANDLE *location_handle, char **bucket_name, - char **cluster_prefix, char **auth_token) -{ - LOCAL_LOCATION *location; - char *p; - - location = (LOCAL_LOCATION *)location_handle; - - if (bucket_name != NULL) { - if ((p = strdup(location->bucket)) == NULL) - return (local_err(local, NULL, ENOMEM, "local_location_decode")); - *bucket_name = p; - } - if (cluster_prefix != NULL) { - if ((p = strdup(location->cluster_prefix)) == NULL) - return (local_err(local, NULL, ENOMEM, "local_location_decode")); - *cluster_prefix = p; - } - if (auth_token != NULL) { - if ((p = strdup(location->auth_token)) == NULL) - return (local_err(local, NULL, ENOMEM, "local_location_decode")); - *auth_token = p; - } - - return (0); -} - -/* * local_location_path -- - * Construct a pathname from the location and local name. + * Construct a pathname from the file system and local name. */ int -local_location_path(LOCAL_STORAGE *local, WT_LOCATION_HANDLE *location_handle, const char *name, - const char *marker, char **pathp) +local_location_path(WT_FILE_SYSTEM *file_system, const char *name, const char *marker, char **pathp) { - LOCAL_LOCATION *location; + LOCAL_FILE_SYSTEM *local_fs; size_t len; int ret; char *p; ret = 0; - location = (LOCAL_LOCATION *)location_handle; + local_fs = (LOCAL_FILE_SYSTEM *)file_system; /* If this is a marker file, it will be hidden from all namespaces. */ if (marker == NULL) marker = ""; - len = strlen(location->bucket) + strlen(marker) + strlen(location->cluster_prefix) + - strlen(name) + 2; + len = + strlen(local_fs->bucket) + strlen(marker) + strlen(local_fs->fs_prefix) + strlen(name) + 2; if ((p = malloc(len)) == NULL) - return (local_err(local, NULL, ENOMEM, "local_location_path")); - snprintf(p, len, "%s/%s%s%s", location->bucket, marker, location->cluster_prefix, name); + return (local_err(FS2LOCAL(file_system), NULL, ENOMEM, "local_location_path")); + snprintf(p, len, "%s/%s%s%s", local_fs->bucket, marker, local_fs->fs_prefix, name); *pathp = p; return (ret); } /* + * local_customize_file_system -- + * Return a customized file system to access the local storage source objects. + */ +static int +local_customize_file_system(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + const char *bucket_name, const char *prefix, const char *auth_token, const char *config, + WT_FILE_SYSTEM **file_systemp) +{ + LOCAL_STORAGE *local; + LOCAL_FILE_SYSTEM *fs; + int ret; + + local = (LOCAL_STORAGE *)storage_source; + + fs = NULL; + ret = 0; + if (config != NULL) + return local_err(local, session, EINVAL, "customize file system: config must be NULL"); + + if ((fs = calloc(1, sizeof(LOCAL_FILE_SYSTEM))) == NULL) { + ret = local_err(local, session, ENOMEM, "local_file_system"); + goto err; + } + fs->local_storage = local; + + if ((fs->auth_token = strdup(auth_token)) == NULL) { + ret = local_err(local, session, ENOMEM, "local_file_system.auth_token"); + goto err; + } + if ((fs->bucket = strdup(bucket_name)) == NULL) { + ret = local_err(local, session, ENOMEM, "local_file_system.bucket_name"); + goto err; + } + if ((fs->fs_prefix = strdup(prefix)) == NULL) { + ret = local_err(local, session, ENOMEM, "local_file_system.prefix"); + goto err; + } + fs->file_system.fs_directory_list = local_directory_list; + fs->file_system.fs_directory_list_single = local_directory_list_single; + fs->file_system.fs_directory_list_free = local_directory_list_free; + fs->file_system.fs_exist = local_exist; + fs->file_system.fs_open_file = local_open; + fs->file_system.fs_remove = local_remove; + fs->file_system.fs_rename = local_rename; + fs->file_system.fs_size = local_size; + fs->file_system.terminate = local_fs_terminate; + +err: + if (ret != 0) + free(fs); + else + *file_systemp = &fs->file_system; + return (ret); +} + +/* * local_exist -- * Return if the file exists. */ static int -local_exist(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, bool *existp) +local_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, bool *existp) { struct stat sb; LOCAL_STORAGE *local; int ret; char *path; - local = (LOCAL_STORAGE *)storage_source; + local = FS2LOCAL(file_system); path = NULL; local->op_count++; - if ((ret = local_location_path(local, location_handle, name, NULL, &path)) != 0) + if ((ret = local_location_path(file_system, name, NULL, &path)) != 0) goto err; ret = stat(path, &sb); @@ -451,8 +441,8 @@ err: * Return when the files have been flushed. */ static int -local_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, const char *config) +local_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, WT_FILE_SYSTEM *file_system, + const char *name, const char *config) { LOCAL_STORAGE *local; LOCAL_FLUSH_ITEM *flush, *safe_flush; @@ -469,13 +459,12 @@ local_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, local = (LOCAL_STORAGE *)storage_source; match = NULL; - if (location_handle == NULL && name != NULL) - return local_err(local, session, EINVAL, "flush: cannot specify name without location"); + if (file_system == NULL && name != NULL) + return local_err(local, session, EINVAL, "flush: cannot specify name without file system"); local->op_count++; - if (location_handle != NULL) { - if ((ret = local_location_path( - local, location_handle, name == NULL ? "" : name, NULL, &match)) != 0) + if (file_system != NULL) { + if ((ret = local_location_path(file_system, name == NULL ? "" : name, NULL, &match)) != 0) goto err; } VERBOSE(local, "Flush: match=%s\n", SHOW_STRING(match)); @@ -496,7 +485,7 @@ local_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, /* * We must match against the bucket and the name if given. * Our match string is of the form: - * <bucket_name>/<cluster_prefix><name> + * <bucket_name>/<fs_prefix><name> * * If name is given, we must match the entire path. * If name is not given, we must match up to the beginning @@ -564,117 +553,29 @@ local_flush_one(LOCAL_STORAGE *local, WT_SESSION *session, LOCAL_FLUSH_ITEM *flu } /* - * local_location_handle -- - * Return a location handle from a location string. - */ -static int -local_location_handle(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - const char *location_info, WT_LOCATION_HANDLE **location_handlep) -{ - LOCAL_LOCATION *location; - LOCAL_STORAGE *local; - WT_CONFIG_ITEM value; - WT_CONFIG_PARSER *parser; - WT_EXTENSION_API *wt_api; - int ret, t_ret; - - ret = 0; - location = NULL; - local = (LOCAL_STORAGE *)storage_source; - wt_api = local->wt_api; - parser = NULL; - - local->op_count++; - if ((ret = wt_api->config_parser_open( - wt_api, session, location_info, strlen(location_info), &parser)) != 0) - return (ret); - - if ((location = calloc(1, sizeof(*location))) == NULL) { - ret = ENOMEM; - goto err; - } - - if ((ret = parser->get(parser, "bucket", &value)) != 0) { - if (ret == WT_NOTFOUND) - ret = local_err(local, session, EINVAL, "ss_location_handle: missing bucket parameter"); - goto err; - } - if (value.len == 0) { - ret = - local_err(local, session, EINVAL, "ss_location_handle: bucket_name must be non-empty"); - goto err; - } - if ((ret = local_config_dup(local, session, &value, NULL, NULL, &location->bucket)) != 0) - goto err; - - if ((ret = parser->get(parser, "cluster", &value)) != 0) { - if (ret == WT_NOTFOUND) - ret = - local_err(local, session, EINVAL, "ss_location_handle: missing cluster parameter"); - goto err; - } - if ((ret = local_config_dup(local, session, &value, "_", "_/", &location->cluster_prefix)) != 0) - goto err; - - if ((ret = parser->get(parser, "auth_token", &value)) != 0) { - if (ret == WT_NOTFOUND) - ret = - local_err(local, session, EINVAL, "ss_location_handle: missing auth_token parameter"); - goto err; - } - if ((ret = local_config_dup(local, session, &value, NULL, NULL, &location->auth_token)) != 0) - goto err; - - VERBOSE(local, "Location: (bucket=%s,cluster=%s,auth_token=%s)\n", - SHOW_STRING(location->bucket), SHOW_STRING(location->cluster_prefix), - SHOW_STRING(location->auth_token)); - - location->iface.close = local_location_handle_close; - *location_handlep = &location->iface; - - if (0) { -err: - (void)local_location_handle_close(&location->iface, session); - } - - if (parser != NULL) - if ((t_ret = parser->close(parser)) != 0 && ret == 0) - ret = t_ret; - - return (ret); -} - -/* - * local_location_handle_close -- - * Free a location handle created by ss_location_handle. + * local_directory_list -- + * Return a list of object names for the given location. */ static int -local_location_handle_close(WT_LOCATION_HANDLE *location_handle, WT_SESSION *session) +local_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *directory, + const char *prefix, char ***dirlistp, uint32_t *countp) { - LOCAL_LOCATION *location; - - (void)session; /* Unused */ - - location = (LOCAL_LOCATION *)location_handle; - free(location->auth_token); - free(location->bucket); - free(location->cluster_prefix); - free(location); - return (0); + FS2LOCAL(file_system)->op_count++; + return ( + local_directory_list_internal(file_system, session, directory, prefix, 0, dirlistp, countp)); } /* - * local_location_list -- - * Return a list of object names for the given location. + * local_directory_list_single -- + * Return a single file name for the given .... */ static int -local_location_list(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *prefix, uint32_t limit, char ***dirlistp, - uint32_t *countp) +local_directory_list_single(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *directory, + const char *prefix, char ***dirlistp, uint32_t *countp) { - ((LOCAL_STORAGE *)storage_source)->op_count++; - return (local_location_list_internal( - storage_source, session, location_handle, NULL, prefix, limit, dirlistp, countp)); + FS2LOCAL(file_system)->op_count++; + return ( + local_directory_list_internal(file_system, session, directory, prefix, 1, dirlistp, countp)); } /* @@ -682,12 +583,12 @@ local_location_list(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, * Free memory allocated by local_location_list. */ static int -local_location_list_free( - WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, char **dirlist, uint32_t count) +local_directory_list_free( + WT_FILE_SYSTEM *file_system, WT_SESSION *session, char **dirlist, uint32_t count) { (void)session; - ((LOCAL_STORAGE *)storage_source)->op_count++; + FS2LOCAL(file_system)->op_count++; if (dirlist != NULL) { while (count > 0) free(dirlist[--count]); @@ -701,37 +602,36 @@ local_location_list_free( * Return a list of object names for the given location, matching the given marker if needed. */ static int -local_location_list_internal(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *marker, const char *prefix, uint32_t limit, - char ***dirlistp, uint32_t *countp) +local_directory_list_internal(WT_FILE_SYSTEM *file_system, WT_SESSION *session, + const char *directory, const char *prefix, uint32_t limit, char ***dirlistp, uint32_t *countp) { struct dirent *dp; DIR *dirp; - LOCAL_LOCATION *location; + LOCAL_FILE_SYSTEM *local_fs; LOCAL_STORAGE *local; - size_t alloc_sz, cluster_len, marker_len, prefix_len; + size_t alloc_sz, fs_prefix_len, dir_len, prefix_len; uint32_t allocated, count; int ret, t_ret; char **entries, **new_entries; const char *basename; - local = (LOCAL_STORAGE *)storage_source; - location = (LOCAL_LOCATION *)location_handle; + local_fs = (LOCAL_FILE_SYSTEM *)file_system; + local = local_fs->local_storage; entries = NULL; allocated = count = 0; - cluster_len = strlen(location->cluster_prefix); - marker_len = (marker == NULL ? 0 : strlen(marker)); + fs_prefix_len = strlen(local_fs->fs_prefix); + dir_len = (directory == NULL ? 0 : strlen(directory)); prefix_len = (prefix == NULL ? 0 : strlen(prefix)); ret = 0; *dirlistp = NULL; *countp = 0; - if ((dirp = opendir(location->bucket)) == NULL) { + if ((dirp = opendir(local_fs->bucket)) == NULL) { ret = errno; if (ret == 0) ret = EINVAL; - return (local_err(local, session, ret, "%s: ss_location_list: opendir", location->bucket)); + return (local_err(local, session, ret, "%s: ss_directory_list: opendir", local_fs->bucket)); } for (count = 0; (dp = readdir(dirp)) != NULL && (limit == 0 || count < limit);) { @@ -739,23 +639,22 @@ local_location_list_internal(WT_STORAGE_SOURCE *storage_source, WT_SESSION *sess basename = dp->d_name; if (strcmp(basename, ".") == 0 || strcmp(basename, "..") == 0) continue; - if (marker_len == 0) { - /* Skip over any marker files. */ - if (strncmp(basename, MARKER_TEMPORARY, strlen(MARKER_TEMPORARY)) == 0 || - strncmp(basename, MARKER_NEED_FLUSH, strlen(MARKER_NEED_FLUSH)) == 0) - continue; - } else { - /* Match only the indicated marker files. */ - if (strncmp(basename, marker, marker_len) != 0) - continue; - basename += marker_len; - } - /* Skip files not associated with our cluster. */ - if (strncmp(basename, location->cluster_prefix, cluster_len) != 0) + /* Skip over any marker files. */ + if (strncmp(basename, MARKER_TEMPORARY, strlen(MARKER_TEMPORARY)) == 0 || + strncmp(basename, MARKER_NEED_FLUSH, strlen(MARKER_NEED_FLUSH)) == 0) + continue; + + /* Match only the indicated directory files. */ + if (directory != NULL && strncmp(basename, directory, dir_len) != 0) + continue; + basename += dir_len; + + /* Skip files not associated with our file system prefix. */ + if (strncmp(basename, local_fs->fs_prefix, fs_prefix_len) != 0) continue; - basename += cluster_len; + basename += fs_prefix_len; /* The list of files is optionally filtered by a prefix. */ if (prefix != NULL && strncmp(basename, prefix, prefix_len) != 0) continue; @@ -782,7 +681,7 @@ local_location_list_internal(WT_STORAGE_SOURCE *storage_source, WT_SESSION *sess err: if (closedir(dirp) != 0) { t_ret = - local_err(local, session, errno, "%s: ss_location_list: closedir", location->bucket); + local_err(local, session, errno, "%s: ss_directory_list: closedir", local_fs->bucket); if (ret == 0) ret = t_ret; } @@ -798,15 +697,36 @@ err: } /* + * local_fs_terminate -- + * Discard any resources on termination of the file system + */ +static int +local_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) +{ + LOCAL_FILE_SYSTEM *local_fs; + + (void)session; /* unused */ + + local_fs = (LOCAL_FILE_SYSTEM *)file_system; + FS2LOCAL(file_system)->op_count++; + free(local_fs->auth_token); + free(local_fs->bucket); + free(local_fs->fs_prefix); + free(file_system); + + return (0); +} + +/* * local_open -- * fopen for our local storage source */ static int -local_open(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags, - WT_FILE_HANDLE **file_handlep) +local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, + WT_FS_OPEN_FILE_TYPE file_type, uint32_t flags, WT_FILE_HANDLE **file_handlep) { LOCAL_FILE_HANDLE *local_fh; + LOCAL_FILE_SYSTEM *local_fs; LOCAL_FLUSH_ITEM *flush; LOCAL_STORAGE *local; WT_FILE_HANDLE *file_handle; @@ -818,26 +738,29 @@ local_open(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, fd = oflags = ret = 0; *file_handlep = NULL; local_fh = NULL; - local = (LOCAL_STORAGE *)storage_source; + local_fs = (LOCAL_FILE_SYSTEM *)file_system; + local = local_fs->local_storage; + + if (file_type != WT_FS_OPEN_FILE_TYPE_DATA) + return ( + local_err(local, session, EINVAL, "%s: open: only data file types supported", name)); local->op_count++; - if (flags == WT_SS_OPEN_CREATE) + if (flags == WT_FS_OPEN_CREATE) oflags = O_WRONLY | O_CREAT; - else if (flags == WT_SS_OPEN_READONLY) + else if (flags == WT_FS_OPEN_READONLY) oflags = O_RDONLY; - else { - ret = local_err(local, session, EINVAL, "open: invalid flags: 0x%x", flags); - goto err; - } + else + return (local_err(local, session, EINVAL, "open: invalid flags: 0x%x", flags)); /* Create a new handle. */ if ((local_fh = calloc(1, sizeof(LOCAL_FILE_HANDLE))) == NULL) { ret = ENOMEM; goto err; } - if ((ret = local_location_path(local, location_handle, name, NULL, &local_fh->path)) != 0) + if ((ret = local_location_path(file_system, name, NULL, &local_fh->path)) != 0) goto err; - if (flags == WT_SS_OPEN_CREATE) { + if (flags == WT_FS_OPEN_CREATE) { if ((flush = calloc(1, sizeof(LOCAL_FLUSH_ITEM))) == NULL) { ret = ENOMEM; goto err; @@ -848,7 +771,7 @@ local_open(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, * Create a marker file that indicates that the file will need to be flushed. */ if ((ret = local_location_path( - local, location_handle, name, MARKER_NEED_FLUSH, &flush->marker_path)) != 0) + file_system, name, MARKER_NEED_FLUSH, &flush->marker_path)) != 0) goto err; if ((fd = open(flush->marker_path, O_WRONLY | O_CREAT, 0666)) < 0) { ret = local_err(local, session, errno, "ss_open_object: open: %s", flush->marker_path); @@ -858,16 +781,21 @@ local_open(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, ret = local_err(local, session, errno, "ss_open_object: close: %s", flush->marker_path); goto err; } - if ((ret = local_location_decode( - local, location_handle, &flush->bucket, NULL, &flush->auth_token)) != 0) + if ((flush->auth_token = strdup(local_fs->auth_token)) == NULL) { + ret = local_err(local, session, ENOMEM, "open.auth_token"); + goto err; + } + if ((flush->bucket = strdup(local_fs->bucket)) == NULL) { + ret = local_err(local, session, ENOMEM, "open.bucket"); goto err; + } /* * For the file handle, we will be writing into a file marked as temporary. When the handle * is closed, we'll move it to its final name. */ if ((ret = local_location_path( - local, location_handle, name, MARKER_TEMPORARY, &local_fh->temp_path)) != 0) + file_system, name, MARKER_TEMPORARY, &local_fh->temp_path)) != 0) goto err; open_name = local_fh->temp_path; @@ -924,8 +852,8 @@ local_open(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, VERBOSE(local, "File opened: %s final path=%s, temp path=%s\n", SHOW_STRING(name), SHOW_STRING(local_fh->path), SHOW_STRING(local_fh->temp_path)); - if (0) { err: + if (ret != 0) { if (local_fh != NULL) local_file_close_internal(local, session, local_fh, true); } @@ -933,12 +861,27 @@ err: } /* + * local_rename -- + * POSIX rename. Currently not implemented, as cloud implementations may not support it. + */ +static int +local_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *from, const char *to, + uint32_t flags) +{ + + (void)from; /* Unused */ + (void)to; /* Unused */ + (void)flags; /* Unused */ + + return (local_err(FS2LOCAL(file_system), session, ENOTSUP, "local remove not supported")); +} + +/* * local_remove -- * POSIX remove. */ static int -local_remove(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags) +local_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, uint32_t flags) { LOCAL_STORAGE *local; int ret; @@ -946,11 +889,11 @@ local_remove(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, (void)flags; /* Unused */ - local = (LOCAL_STORAGE *)storage_source; + local = FS2LOCAL(file_system); path = NULL; local->op_count++; - if ((ret = local_location_path(local, location_handle, name, NULL, &path)) != 0) + if ((ret = local_location_path(file_system, name, NULL, &path)) != 0) goto err; ret = unlink(path); @@ -969,19 +912,18 @@ err: * Get the size of a file in bytes, by file name. */ static int -local_size(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, wt_off_t *sizep) +local_size(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, wt_off_t *sizep) { struct stat sb; LOCAL_STORAGE *local; int ret; char *path; - local = (LOCAL_STORAGE *)storage_source; + local = FS2LOCAL(file_system); path = NULL; local->op_count++; - if ((ret = local_location_path(local, location_handle, name, NULL, &path)) != 0) + if ((ret = local_location_path(file_system, name, NULL, &path)) != 0) goto err; ret = stat(path, &sb); @@ -1013,7 +955,7 @@ local_terminate(WT_STORAGE_SOURCE *storage, WT_SESSION *session) /* * We should be single threaded at this point, so it is safe to destroy the lock and access the - * file handle list without it. + * file handle list without locking it. */ if ((ret = pthread_rwlock_destroy(&local->file_handle_lock)) != 0) (void)local_err(local, session, ret, "terminate: pthread_rwlock_destroy"); @@ -1255,14 +1197,8 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) * Allocate a local storage structure, with a WT_STORAGE structure as the first field, allowing * us to treat references to either type of structure as a reference to the other type. */ - local->storage_source.ss_exist = local_exist; + local->storage_source.ss_customize_file_system = local_customize_file_system; local->storage_source.ss_flush = local_flush; - local->storage_source.ss_location_handle = local_location_handle; - local->storage_source.ss_location_list = local_location_list; - local->storage_source.ss_location_list_free = local_location_list_free; - local->storage_source.ss_open_object = local_open; - local->storage_source.ss_remove = local_remove; - local->storage_source.ss_size = local_size; local->storage_source.terminate = local_terminate; if ((ret = local_configure(local, config)) != 0) { diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 5aa4b2708e5..1c1da7bc8ab 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "716858f654e00175114f754b5c299bcce91bf276" + "commit": "07fef8b2b80bf1e07104e8627a6bf76e2f998eac" } diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i index 829f58d6ac3..3dbf69d35c2 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i @@ -75,10 +75,10 @@ from packing import pack, unpack $1 = &temp; } %typemap(in, numinputs=0) WT_FILE_HANDLE ** (WT_FILE_HANDLE *temp = NULL) { - $1 = &temp; + $1 = &temp; } -%typemap(in, numinputs=0) WT_LOCATION_HANDLE ** (WT_LOCATION_HANDLE *temp = NULL) { - $1 = &temp; +%typemap(in, numinputs=0) WT_FILE_SYSTEM ** (WT_FILE_SYSTEM *temp = NULL) { + $1 = &temp; } %typemap(in, numinputs=0) WT_STORAGE_SOURCE ** (WT_STORAGE_SOURCE *temp = NULL) { $1 = &temp; @@ -190,12 +190,12 @@ from packing import pack, unpack $1 = &val; } -%typemap(in,numinputs=0) (char ***object_list, int *countp) (char **list, uint32_t nentries) { +%typemap(in,numinputs=0) (char ***dirlist, int *countp) (char **list, uint32_t nentries) { $1 = &list; $2 = &nentries; } -%typemap(argout) (char ***object_list, int *countp) { +%typemap(argout) (char ***dirlist, int *countp) { int i; char **list; @@ -203,8 +203,8 @@ from packing import pack, unpack list = (*$1); /* * When we're done with the individual C strings, free them. - * In theory, we should call the ss_location_list_free() method, - * but that's awkward, since we don't have the storage_source and session. + * In theory, we should call the fs_directory_list_free() method, + * but that's awkward, since we don't have the file system and session. */ for (i = 0; i < *$2; i++) { PyObject *o = PyString_InternFromString(list[i]); @@ -219,8 +219,8 @@ from packing import pack, unpack $result = SWIG_NewPointerObj(SWIG_as_voidptr(*$1), SWIGTYPE_p___wt_file_handle, 0); } -%typemap(argout) WT_LOCATION_HANDLE ** { - $result = SWIG_NewPointerObj(SWIG_as_voidptr(*$1), SWIGTYPE_p___wt_location_handle, 0); +%typemap(argout) WT_FILE_SYSTEM ** { + $result = SWIG_NewPointerObj(SWIG_as_voidptr(*$1), SWIGTYPE_p___wt_file_system, 0); } %typemap(argout) WT_STORAGE_SOURCE ** { @@ -340,8 +340,8 @@ DESTRUCTOR(__wt_connection, close) DESTRUCTOR(__wt_cursor, close) DESTRUCTOR(__wt_file_handle, close) DESTRUCTOR(__wt_session, close) -DESTRUCTOR(__wt_storage_source, close) -DESTRUCTOR(__wt_location_handle, close) +DESTRUCTOR(__wt_storage_source, ss_terminate) +DESTRUCTOR(__wt_file_system, fs_terminate) /* * OVERRIDE_METHOD must be used when overriding or extending an existing @@ -518,7 +518,7 @@ SELFHELPER(struct __wt_connection, connection) SELFHELPER(struct __wt_session, session) SELFHELPER(struct __wt_cursor, cursor) SELFHELPER(struct __wt_file_handle, file_handle) -SELFHELPER(struct __wt_location_handle, location_handle) +SELFHELPER(struct __wt_file_system, file_system) SELFHELPER(struct __wt_storage_source, storage_source) /* @@ -985,40 +985,42 @@ typedef int int_void; }; %enddef -SIDESTEP_METHOD(__wt_storage_source, ss_location_handle, - (WT_SESSION *session, const char *config, WT_LOCATION_HANDLE **handle), - (self, session, config, handle)) +SIDESTEP_METHOD(__wt_storage_source, ss_customize_file_system, + (WT_SESSION *session, const char *bucket_name, const char *prefix, + const char *auth_token, const char *config, WT_FILE_SYSTEM **file_systemp), + (self, session, bucket_name, prefix, auth_token, config, file_systemp)) + +SIDESTEP_METHOD(__wt_storage_source, ss_flush, + (WT_SESSION *session, WT_FILE_SYSTEM *file_system, + const char *name, const char *config), + (self, session, file_system, name, config)) -SIDESTEP_METHOD(__wt_location_handle, close, +SIDESTEP_METHOD(__wt_storage_source, terminate, (WT_SESSION *session), (self, session)) -SIDESTEP_METHOD(__wt_storage_source, ss_exist, - (WT_SESSION *session, WT_LOCATION_HANDLE *location_handle, - const char *name, bool *existp), - (self, session, location_handle, name, existp)) +SIDESTEP_METHOD(__wt_file_system, fs_exist, + (WT_SESSION *session, const char *name, bool *existp), + (self, session, name, existp)) -SIDESTEP_METHOD(__wt_storage_source, ss_flush, - (WT_SESSION *session, WT_LOCATION_HANDLE *location_handle, - const char *name, const char *config), - (self, session, location_handle, name, config)) +SIDESTEP_METHOD(__wt_file_system, fs_open_file, + (WT_SESSION *session, const char *name, WT_FS_OPEN_FILE_TYPE file_type, + uint32_t flags, WT_FILE_HANDLE **file_handlep), + (self, session, name, file_type, flags, file_handlep)) -SIDESTEP_METHOD(__wt_storage_source, ss_open_object, - (WT_SESSION *session, WT_LOCATION_HANDLE *location_handle, - const char *name, uint32_t flags, WT_FILE_HANDLE **file_handlep), - (self, session, location_handle, name, flags, file_handlep)) +SIDESTEP_METHOD(__wt_file_system, fs_remove, + (WT_SESSION *session, const char *name, uint32_t flags), + (self, session, name, flags)) -SIDESTEP_METHOD(__wt_storage_source, ss_remove, - (WT_SESSION *session, WT_LOCATION_HANDLE *location_handle, - const char *name, uint32_t flags), - (self, session, location_handle, name, flags)) +SIDESTEP_METHOD(__wt_file_system, fs_rename, + (WT_SESSION *session, const char *from, const char *to, uint32_t flags), + (self, session, from, to, flags)) -SIDESTEP_METHOD(__wt_storage_source, ss_size, - (WT_SESSION *session, WT_LOCATION_HANDLE *location_handle, - const char *name, wt_off_t *sizep), - (self, session, location_handle, name, sizep)) +SIDESTEP_METHOD(__wt_file_system, fs_size, + (WT_SESSION *session, const char *name, wt_off_t *sizep), + (self, session, name, sizep)) -SIDESTEP_METHOD(__wt_storage_source, terminate, +SIDESTEP_METHOD(__wt_file_system, terminate, (WT_SESSION *session), (self, session)) @@ -1092,20 +1094,26 @@ SIDESTEP_METHOD(__wt_file_handle, fh_write, } }; -%ignore __wt_storage_source::ss_location_list; -%rename (ss_location_list) __wt_storage_source::_ss_location_list; -%extend __wt_storage_source { - int _ss_location_list(WT_SESSION *session, WT_LOCATION_HANDLE *handle, const char *prefix, - uint32_t limit, char ***object_list, int *countp) { - return (self->ss_location_list(self, session, handle, prefix, limit, object_list, countp)); +%ignore __wt_file_system::fs_directory_list; +%ignore __wt_file_system::fs_directory_list_single; +%rename (fs_directory_list) __wt_file_system::_fs_directory_list; +%rename (fs_directory_list_single) __wt_file_system::_fs_directory_list_single; +%extend __wt_file_system { + int _fs_directory_list(WT_SESSION *session, const char *directory, const char *prefix, + char ***dirlist, int *countp) { + return (self->fs_directory_list(self, session, directory, prefix, dirlist, countp)); + } + int _fs_directory_list_single(WT_SESSION *session, const char *directory, const char *prefix, + char ***dirlist, int *countp) { + return (self->fs_directory_list_single(self, session, directory, prefix, dirlist, countp)); } }; /* - * No need for a location_list_free method, as the list and its components - * are freed immediately after the location_list call. + * No need for a directory_list_free method, as the list and its components + * are freed immediately after the directory_list call. */ -%ignore __wt_storage_source::ss_location_list_free; +%ignore __wt_file_system::fs_directory_list_free; %{ int diagnostic_build() { @@ -1164,7 +1172,7 @@ OVERRIDE_METHOD(__wt_session, WT_SESSION, log_printf, (self, msg)) %rename(Connection) __wt_connection; %rename(FileHandle) __wt_file_handle; %rename(StorageSource) __wt_storage_source; -%rename(LocationHandle) __wt_location_handle; +%rename(FileSystem) __wt_file_system; %include "wiredtiger.h" @@ -1441,7 +1449,7 @@ def _rename_with_prefix(prefix, toclass): _rename_with_prefix('WT_STAT_CONN_', stat.conn) _rename_with_prefix('WT_STAT_DSRC_', stat.dsrc) _rename_with_prefix('WT_STAT_SESSION_', stat.session) -_rename_with_prefix('WT_SS_', StorageSource) +_rename_with_prefix('WT_FS_', FileSystem) _rename_with_prefix('WT_FILE_HANDLE_', FileHandle) del _rename_with_prefix %} diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 43bfb1f769f..cdae90b9733 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -96,7 +96,6 @@ static const WT_CONFIG_CHECK confchk_tiered_manager_subconfigs[] = { {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_tiered_storage_subconfigs[] = { - {"auth_token", "string", NULL, NULL, NULL, 0}, {"local_retention", "int", NULL, "min=0,max=10000", NULL, 0}, {"object_target_size", "int", NULL, "min=100K,max=10TB", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -135,7 +134,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5}, {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, {"tiered_storage", "category", NULL, NULL, - confchk_WT_CONNECTION_reconfigure_tiered_storage_subconfigs, 3}, + confchk_WT_CONNECTION_reconfigure_tiered_storage_subconfigs, 2}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -634,10 +633,10 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_statistics_log_subconfigs[] {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_tiered_storage_subconfigs[] = { - {"auth_token", "string", NULL, NULL, NULL, 0}, {"auth_token", "string", NULL, NULL, NULL, 0}, - {"bucket", "string", NULL, NULL, NULL, 0}, {"cluster", "string", NULL, NULL, NULL, 0}, + {"auth_token", "string", NULL, NULL, NULL, 0}, {"bucket", "string", NULL, NULL, NULL, 0}, + {"bucket_prefix", "string", NULL, NULL, NULL, 0}, {"local_retention", "int", NULL, "min=0,max=10000", NULL, 0}, - {"member", "string", NULL, NULL, NULL, 0}, {"name", "string", NULL, NULL, NULL, 0}, + {"name", "string", NULL, NULL, NULL, 0}, {"object_target_size", "int", NULL, "min=100K,max=10TB", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -696,7 +695,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, - {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 8}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -774,7 +773,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, - {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 8}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -849,7 +848,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, - {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 8}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -922,7 +921,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, - {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 8}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -985,8 +984,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "tiered_manager=(threads_max=8,threads_min=1,wait=0)," - "tiered_storage=(auth_token=,local_retention=300," - "object_target_size=10M),timing_stress_for_test=,verbose=[]", + "tiered_storage=(local_retention=300,object_target_size=10M)," + "timing_stress_for_test=,verbose=[]", confchk_WT_CONNECTION_reconfigure, 29}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", @@ -1200,8 +1199,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," - "tiered_storage=(auth_token=,auth_token=,bucket=,cluster=," - "local_retention=300,member=,name=,object_target_size=10M)," + "tiered_storage=(auth_token=,bucket=,bucket_prefix=," + "local_retention=300,name=,object_target_size=10M)," "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),use_environment=true,use_environment_priv=false," "verbose=[],verify_metadata=false,write_through=", @@ -1235,8 +1234,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," - "tiered_storage=(auth_token=,auth_token=,bucket=,cluster=," - "local_retention=300,member=,name=,object_target_size=10M)," + "tiered_storage=(auth_token=,bucket=,bucket_prefix=," + "local_retention=300,name=,object_target_size=10M)," "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),use_environment=true,use_environment_priv=false," "verbose=[],verify_metadata=false,version=(major=0,minor=0)," @@ -1270,8 +1269,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "tiered_manager=(threads_max=8,threads_min=1,wait=0)," - "tiered_storage=(auth_token=,auth_token=,bucket=,cluster=," - "local_retention=300,member=,name=,object_target_size=10M)," + "tiered_storage=(auth_token=,bucket=,bucket_prefix=," + "local_retention=300,name=,object_target_size=10M)," "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),verbose=[],verify_metadata=false,version=(major=0," "minor=0),write_through=", @@ -1304,8 +1303,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "tiered_manager=(threads_max=8,threads_min=1,wait=0)," - "tiered_storage=(auth_token=,auth_token=,bucket=,cluster=," - "local_retention=300,member=,name=,object_target_size=10M)," + "tiered_storage=(auth_token=,bucket=,bucket_prefix=," + "local_retention=300,name=,object_target_size=10M)," "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),verbose=[],verify_metadata=false,write_through=", confchk_wiredtiger_open_usercfg, 51}, diff --git a/src/third_party/wiredtiger/src/config/test_config.c b/src/third_party/wiredtiger/src/config/test_config.c index 1b42bbee4d1..7843c8b5403 100644 --- a/src/third_party/wiredtiger/src/config/test_config.c +++ b/src/third_party/wiredtiger/src/config/test_config.c @@ -17,9 +17,7 @@ static const WT_CONFIG_CHECK confchk_timestamp_manager_subconfigs[] = { {"stable_lag", "int", NULL, "min=0,max=1000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_insert_config_subconfigs[] = { - {"key_format", "string", NULL, NULL, NULL, 0}, {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, - {"value_format", "string", NULL, NULL, NULL, 0}, {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_ops_per_transaction_subconfigs[] = { @@ -27,51 +25,66 @@ static const WT_CONFIG_CHECK confchk_ops_per_transaction_subconfigs[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_update_config_subconfigs[] = { - {"key_format", "string", NULL, NULL, NULL, 0}, {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, - {"value_format", "string", NULL, NULL, NULL, 0}, {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_workload_generator_subconfigs[] = { {"collection_count", "int", NULL, "min=0,max=200000", NULL, 0}, - {"insert_config", "category", NULL, NULL, confchk_insert_config_subconfigs, 4}, + {"insert_config", "category", NULL, NULL, confchk_insert_config_subconfigs, 2}, {"insert_threads", "int", NULL, "min=0,max=20", NULL, 0}, {"key_count", "int", NULL, "min=0,max=1000000", NULL, 0}, - {"key_format", "string", NULL, NULL, NULL, 0}, {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, {"ops_per_transaction", "category", NULL, NULL, confchk_ops_per_transaction_subconfigs, 2}, {"read_threads", "int", NULL, "min=0,max=100", NULL, 0}, - {"update_config", "category", NULL, NULL, confchk_update_config_subconfigs, 4}, + {"update_config", "category", NULL, NULL, confchk_update_config_subconfigs, 2}, {"update_threads", "int", NULL, "min=0,max=20", NULL, 0}, - {"value_format", "string", NULL, NULL, NULL, 0}, {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_workload_tracking_subconfigs[] = { {"enabled", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_example_test[] = { + {"cache_size_mb", "int", NULL, "min=0,max=100000000000", NULL, 0}, + {"duration_seconds", "int", NULL, "min=0,max=1000000", NULL, 0}, + {"enable_logging", "boolean", NULL, NULL, NULL, 0}, + {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 2}, + {"timestamp_manager", "category", NULL, NULL, confchk_timestamp_manager_subconfigs, 3}, + {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 10}, + {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 1}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_poc_test[] = { {"cache_size_mb", "int", NULL, "min=0,max=100000000000", NULL, 0}, {"duration_seconds", "int", NULL, "min=0,max=1000000", NULL, 0}, {"enable_logging", "boolean", NULL, NULL, NULL, 0}, {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 2}, {"timestamp_manager", "category", NULL, NULL, confchk_timestamp_manager_subconfigs, 3}, - {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 12}, + {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 10}, {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 1}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_ENTRY config_entries[] = { + {"example_test", + "cache_size_mb=0,duration_seconds=0,enable_logging=true," + "runtime_monitor=(rate_per_second=1," + "stat_cache_size=(enabled=false,limit=))," + "timestamp_manager=(enabled=false,oldest_lag=0,stable_lag=0)," + "workload_generator=(collection_count=1,insert_config=(key_size=0" + ",value_size=0),insert_threads=0,key_count=0,key_size=0," + "ops_per_transaction=(max=1,min=),read_threads=0," + "update_config=(key_size=0,value_size=0),update_threads=0," + "value_size=0),workload_tracking=(enabled=false)", + confchk_example_test, 7}, {"poc_test", "cache_size_mb=0,duration_seconds=0,enable_logging=true," "runtime_monitor=(rate_per_second=1," "stat_cache_size=(enabled=false,limit=))," "timestamp_manager=(enabled=false,oldest_lag=0,stable_lag=0)," - "workload_generator=(collection_count=1," - "insert_config=(key_format=i,key_size=0,value_format=S," - "value_size=0),insert_threads=0,key_count=0,key_format=i," - "key_size=0,ops_per_transaction=(max=1,min=),read_threads=0," - "update_config=(key_format=i,key_size=0,value_format=S," - "value_size=0),update_threads=0,value_format=S,value_size=0)," - "workload_tracking=(enabled=false)", + "workload_generator=(collection_count=1,insert_config=(key_size=0" + ",value_size=0),insert_threads=0,key_count=0,key_size=0," + "ops_per_transaction=(max=1,min=),read_threads=0," + "update_config=(key_size=0,value_size=0),update_threads=0," + "value_size=0),workload_tracking=(enabled=false)", confchk_poc_test, 7}, {NULL, NULL, NULL, 0}}; diff --git a/src/third_party/wiredtiger/src/conn/conn_tiered.c b/src/third_party/wiredtiger/src/conn/conn_tiered.c index 2c0d95542ce..9771cced268 100644 --- a/src/third_party/wiredtiger/src/conn/conn_tiered.c +++ b/src/third_party/wiredtiger/src/conn/conn_tiered.c @@ -221,19 +221,16 @@ __tiered_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp, bool rec WT_STAT_CONN_SET(session, tiered_object_size, conn->bstorage->object_size); WT_STAT_CONN_SET(session, tiered_retention, conn->bstorage->retain_secs); - /* The strings for unique identification are connection level not per bucket. */ - WT_RET(__wt_config_gets(session, cfg, "tiered_storage.cluster", &cval)); - WT_ERR(__wt_strndup(session, cval.str, cval.len, &conn->tiered_cluster)); - WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.member", &cval)); - WT_ERR(__wt_strndup(session, cval.str, cval.len, &conn->tiered_member)); + /* The strings for unique identification are connection level. */ + WT_RET(__wt_config_gets(session, cfg, "tiered_storage.bucket_prefix", &cval)); + WT_ERR(__wt_strndup(session, cval.str, cval.len, &conn->tiered_prefix)); return (__tiered_manager_config(session, cfg, runp)); err: __wt_free(session, conn->bstorage->auth_token); __wt_free(session, conn->bstorage->bucket); __wt_free(session, conn->bstorage); - __wt_free(session, conn->tiered_cluster); - __wt_free(session, conn->tiered_member); + __wt_free(session, conn->tiered_prefix); return (ret); } @@ -346,8 +343,7 @@ __wt_tiered_storage_destroy(WT_SESSION_IMPL *session) WT_DECL_RET; conn = S2C(session); - __wt_free(session, conn->tiered_cluster); - __wt_free(session, conn->tiered_member); + __wt_free(session, conn->tiered_prefix); /* Stop the server thread. */ FLD_CLR(conn->server_flags, WT_CONN_SERVER_TIERED); diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 1e5d2a5dec7..e4a03c2fa80 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -1113,8 +1113,11 @@ __wt_cursor_init( session = CUR2S(cursor); - if (cursor->internal_uri == NULL) + if (cursor->internal_uri == NULL) { + /* Various cursor code assumes there is an internal URI, so there better be one to set. */ + WT_ASSERT(session, uri != NULL); WT_RET(__wt_strdup(session, uri, &cursor->internal_uri)); + } /* * append The append flag is only relevant to column stores. diff --git a/src/third_party/wiredtiger/src/docs/custom-storage-sources.dox b/src/third_party/wiredtiger/src/docs/custom-storage-sources.dox index e1a0b10644e..ffa8ef11783 100644 --- a/src/third_party/wiredtiger/src/docs/custom-storage-sources.dox +++ b/src/third_party/wiredtiger/src/docs/custom-storage-sources.dox @@ -63,12 +63,10 @@ It must always be provided when WiredTiger is reopened (again, with the ::wiredt @section storage_examples Storage source examples -There are two kinds of example code with overlapping functionality. -A simple, self contained storage source example is in @ex_ref{ex_storage_source.c}. -This example includes a small demo storage source that is a no-op and -simply returns. This example also shows how a storage source is configured -within an application. The second set of examples are in \c ext/storage. These are -storage source only (no application level code), showing how a storage source -might be packaged in a loadable shared library. +An example of a storage source exists in \c ext/storage_sources/local_store/local_store.c. +This storage source emulates cloud storage by storing all objects on the local file system. +This example does not include application level code to call it. By default, WiredTiger builds +it as a loadable shared library, and it can be loaded during a ::wiredtiger_open call as with +any other extension, and \c local_store can be specified to be used with tiered storage system. */ diff --git a/src/third_party/wiredtiger/src/docs/examples.dox b/src/third_party/wiredtiger/src/docs/examples.dox index 26167ab0631..d5a5102be61 100644 --- a/src/third_party/wiredtiger/src/docs/examples.dox +++ b/src/third_party/wiredtiger/src/docs/examples.dox @@ -52,9 +52,6 @@ Shows how to create column-oriented data and access individual columns. @example ex_stat.c Shows how to access database and table statistics. -@example ex_storage_source.c -Shows how to extend WiredTiger with a custom storage source implementation. - @example ex_thread.c Shows how to access a database with multiple threads. diff --git a/src/third_party/wiredtiger/src/history/hs_rec.c b/src/third_party/wiredtiger/src/history/hs_rec.c index 56573d374bd..904956b68cb 100644 --- a/src/third_party/wiredtiger/src/history/hs_rec.c +++ b/src/third_party/wiredtiger/src/history/hs_rec.c @@ -142,13 +142,14 @@ __hs_insert_record(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, &upd_type_full_diag, existing_val)); WT_ERR(__wt_compare(session, NULL, existing_val, hs_value, &cmp)); /* - * We shouldn't be inserting the same value again for the key unless coming from a - * different transaction. If the updates are from the same transaction, the start - * timestamp for each update should be different. + * Same value should not be inserted again unless 1. previous entry is already + * deleted(i.e. the stop timestamp is globally visible), 2. from a different + * transaction 3. with a different timestamp if from the same transaction. */ if (cmp == 0) WT_ASSERT(session, - tw->start_txn == WT_TXN_NONE || + __wt_txn_tw_stop_visible_all(session, &hs_cbt->upd_value->tw) || + tw->start_txn == WT_TXN_NONE || tw->start_txn != hs_cbt->upd_value->tw.start_txn || tw->start_ts != hs_cbt->upd_value->tw.start_ts); counter = hs_counter + 1; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 61bbe022371..16d5b4a596a 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -407,8 +407,7 @@ struct __wt_connection_impl { bool tiered_tid_set; /* Tiered thread set */ WT_CONDVAR *tiered_cond; /* Tiered wait mutex */ - const char *tiered_cluster; /* Tiered storage cluster name */ - const char *tiered_member; /* Tiered storage member name */ + const char *tiered_prefix; /* Tiered storage naming prefix */ WT_TIERED_MANAGER tiered_manager; /* Tiered worker thread information */ bool tiered_server_running; /* Internal tiered server operating */ diff --git a/src/third_party/wiredtiger/src/include/tiered.h b/src/third_party/wiredtiger/src/include/tiered.h index 06a49c20a59..fe2c29f5ffa 100644 --- a/src/third_party/wiredtiger/src/include/tiered.h +++ b/src/third_party/wiredtiger/src/include/tiered.h @@ -55,7 +55,7 @@ struct __wt_cursor_tiered { struct __wt_tiered { WT_DATA_HANDLE iface; - const char *name, *config, *filename; + const char *config, *filename; const char *key_format, *value_format; WT_DATA_HANDLE **tiers; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 188a1a9e530..f658f905b49 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -80,7 +80,6 @@ struct __wt_modify; typedef struct __wt_modify WT_MODIFY; struct __wt_session; typedef struct __wt_session WT_SESSION; #if !defined(DOXYGEN) struct __wt_storage_source; typedef struct __wt_storage_source WT_STORAGE_SOURCE; -struct __wt_location_handle; typedef struct __wt_location_handle WT_LOCATION_HANDLE; #endif #if defined(SWIGJAVA) @@ -2238,14 +2237,12 @@ struct __wt_connection { * @config{tiered_storage = (, enable tiered storage. Enabling tiered storage may use one * session from the configured session_max., a set of related configuration options defined * below.} - * @config{ auth_token, authentication token string., a - * string; default empty.} - * @config{ local_retention, time in seconds - * to retain data on tiered storage on the local tier for faster read access., an integer - * between 0 and 10000; default \c 300.} - * @config{ object_target_size, - * the approximate size of objects before creating them on the tiered storage tier., an - * integer between 100K and 10TB; default \c 10M.} + * @config{ local_retention, time in seconds to retain data + * on tiered storage on the local tier for faster read access., an integer between 0 and + * 10000; default \c 300.} + * @config{ object_target_size, the + * approximate size of objects before creating them on the tiered storage tier., an integer + * between 100K and 10TB; default \c 10M.} * @config{ ),,} * @config{verbose, enable messages for various events. Options are given as a list\, such * as <code>"verbose=[evictserver\,read]"</code>., a list\, with values chosen from the @@ -4232,7 +4229,6 @@ struct __wt_extractor { int (*terminate)(WT_EXTRACTOR *extractor, WT_SESSION *session); }; -#if !defined(SWIG) /*! WT_FILE_SYSTEM::open_file file types */ typedef enum { WT_FS_OPEN_FILE_TYPE_CHECKPOINT,/*!< open a data file checkpoint */ @@ -4444,7 +4440,6 @@ struct __wt_file_system { */ int (*terminate)(WT_FILE_SYSTEM *file_system, WT_SESSION *session); }; -#endif /* !defined(SWIG) */ /*! WT_FILE_HANDLE::fadvise flags: no longer need */ #define WT_FILE_HANDLE_DONTNEED 1 @@ -4735,28 +4730,6 @@ struct __wt_file_handle { #if !defined(DOXYGEN) /* This interface is not yet public. */ -/* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_SS_OPEN_CREATE 0x1u -#define WT_SS_OPEN_READONLY 0x2u -/* AUTOMATIC FLAG VALUE GENERATION STOP */ - -/*! - * A location handle, and its encoding is defined by each implementation - * of the WT_STORAGE_SOURCE interface. - */ -struct __wt_location_handle { - /*! - * Close a location handle, the handle will not be further accessed by - * WiredTiger. - * - * @errors - * - * @param location_handle the WT_LOCATION_HANDLE - * @param session the current WiredTiger session - */ - int (*close)(WT_LOCATION_HANDLE *location_handle, WT_SESSION *session); -}; - /*! * The interface implemented by applications to provide a storage source * implementation. This documentation refers to "object" and "bucket" @@ -4773,66 +4746,56 @@ struct __wt_location_handle { */ struct __wt_storage_source { /*! - * Return a location handle from a location string. - * A location string may encode a bucket name, or the equivalent for this - * storage source, authorization information for that bucket, - * naming prefixes to be used for objects in that bucket, etc. - * - * @errors - * - * @param storage_source the WT_STORAGE_SOURCE - * @param session the current WiredTiger session - * @param location the location string - * @param[out] location_handle the allocated handle - */ - int (*ss_location_handle)(WT_STORAGE_SOURCE *storage_source, - WT_SESSION *session, const char *location, WT_LOCATION_HANDLE **location_handle); - - /*! - * Return a list of object names for the given location. - * - * @errors - * - * @param storage_source the WT_STORAGE_SOURCE - * @param session the current WiredTiger session - * @param location_handle the location to list - * @param prefix if not NULL, only files with names matching the prefix - * are returned - * @param limit if not 0, limits the number of objects listed to this number. - * @param[out] object_list the method returns an allocated array of - * individually allocated strings, one for each object in the location. - * @param[out] countp the number of entries returned - */ - int (*ss_location_list)(WT_STORAGE_SOURCE *storage_source, - WT_SESSION *session, WT_LOCATION_HANDLE *location_handle, const char *prefix, - uint32_t limit, char ***object_list, uint32_t *countp); - - /*! - * Free memory allocated by WT_STORAGE_SOURCE::location_list. - * - * @errors - * - * @param storage_source the WT_STORAGE_SOURCE - * @param session the current WiredTiger session - * @param object_list array returned by WT_STORAGE_SOURCE::location_list - * @param count count returned by WT_STORAGE_SOURCE::location_list - */ - int (*ss_location_list_free)(WT_STORAGE_SOURCE *storage_source, - WT_SESSION *session, char **object_list, uint32_t count); - - /*! - * Return if the named object exists in the location. + * Create a customized file system to access the storage source + * objects. + * + * The file system returned behaves as if objects in the specified buckets are + * files in the file system. In particular, the fs_open_file method requires + * its flags argument to include either WT_FS_OPEN_CREATE or WT_FS_OPEN_READONLY. + * Objects being created are not deemed to "exist" and be visible to + * WT_FILE_SYSTEM::fs_exist and other file system methods until the new handle has + * been closed. Objects once created are immutable. That is, only objects that + * do not already exist can be opened with the create flag, and objects that + * already exist can only be opened with the readonly flag. Only objects that + * exist can be transferred to the underlying shared object storage. This can + * happen at any time after an object is created, and can be forced to happen using + * WT_STORAGE_SOURCE::ss_flush. + * + * Additionally file handles returned by the file system behave as file handles to a + * local file. For example, WT_FILE_HANDLE::fh_sync synchronizes writes to the + * local file, and does not imply any transferring of data to the shared object store. + * + * The directory argument to the WT_FILE_SYSTEM::fs_directory_list method is normally + * the empty string as the cloud equivalent (bucket) has already been given when + * customizing the file system. If specified, the directory path is interpreted + * as another prefix, which is removed from the results. + * + * Names used by the file system methods are generally flat. However, in some + * implementations of a file system returned by a storage source, "..", ".", "/" + * may have a particular meaning, as in a POSIX file system. We suggest that + * these constructs be avoided when a caller chooses file names within the returned + * file system; they may be rejected by the implementation. Within a bucket name, + * these characters may or may not be acceptable. That is implementation dependent. + * In the prefix, "/" is specifically allowed, as this may have performance or + * administrative benefits. That said, within a prefix, certain combinations + * involving "/" may be rejected, for example "/../". * * @errors * * @param storage_source the WT_STORAGE_SOURCE * @param session the current WiredTiger session - * @param location_handle the location to search - * @param name the name of the object - * @param[out] existp If the named storage source object exists + * @param bucket_name the name of the bucket. Use of '/' is implementation dependent. + * @param prefix a prefix for each file. If used, the prefix will be added to the + * name of each object created or otherwise accessed in the bucket. Also, only + * objects with this prefix will be visible, and the prefix will be removed when + * listed. Prefixes may contain '/' as a separator. + * @param auth_token the authorization identifier. + * @param config additional configuration, currently must be NULL. + * @param[out] file_system the customized file system returned */ - int (*ss_exist)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, bool *existp); + int (*ss_customize_file_system)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + const char *bucket_name, const char *prefix, const char *auth_token, const char *config, + WT_FILE_SYSTEM **file_system); /*! * Flush any existing objects that match the location and name from @@ -4844,85 +4807,13 @@ struct __wt_storage_source { * * @param storage_source the WT_STORAGE_SOURCE * @param session the current WiredTiger session - * @param location_handle the location to flush (or NULL for all) + * @param file_system if NULL, all objects are considered, otherwise only objects + * managed by the given file system. * @param name the name of the object to flush (or NULL for all) * @param config additional configuration, currently must be NULL */ int (*ss_flush)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, const char *config); - - /*! - * Open a handle for a named storage source object. - * - * Objects created are not deemed to "exist" and be visible to other APIs - * like WT_STORAGE_SOURCE::ss_exist until the new handle has been closed. - * Objects once created are immutable. That is, only objects that do not already - * exist can be opened with the create flag, and objects that already exist can - * only be opened with the readonly flag. - * - * Only objects that exist can be transferred to and made visible in the underlying - * shared object store. However, they don't need to be transferred immediately when - * the created handle is closed. Transfers can be forced with WT_STORAGE_SOURCE::ss_flush. - * - * File handles returned behave as file handles to a local file. For example, - * WT_FILE_HANDLE::fh_sync synchronizes writes to the local file, and does not - * imply any transferring of data to the shared object store. - * - * The method should return ENOENT if the object is not being created and - * does not exist. - * - * The method should return EACCES if the object cannot be opened given - * permissions by the location. - * - * @errors - * - * @param storage_source the WT_STORAGE_SOURCE - * @param session the current WiredTiger session - * @param location_handle the location where the object will be stored. - * @param name the name of the object within the location. - * @param flags flags indicating how to open the object, exactly one of - * ::WT_SS_OPEN_CREATE, ::WT_SS_OPEN_READONLY. - * @param[out] file_handlep the handle to the newly opened object. Storage - * source implementations must allocate memory for the handle and - * the WT_FILE_HANDLE::name field, and fill in the WT_FILE_HANDLE:: - * fields. Applications wanting to associate private information - * with the WT_FILE_HANDLE:: structure should declare and allocate - * their own structure as a superset of a WT_FILE_HANDLE:: structure. - */ - int (*ss_open_object)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags, - WT_FILE_HANDLE **file_handlep); - - /*! - * Remove a named storage source object - * - * This method is not required if storage source is configured readonly - * and should be set to NULL when not required by the storage source implementation. - * - * @errors - * - * @param storage_source the WT_STORAGE_SOURCE - * @param session the current WiredTiger session - * @param location_handle the location containing the object - * @param name the name of the storage source object - * @param flags must be 0 - */ - int (*ss_remove)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags); - - /*! - * Return the size of a named storage source object - * - * @errors - * - * @param storage_source the WT_STORAGE_SOURCE - * @param session the current WiredTiger session - * @param location_handle the location containing the object - * @param name the name of the storage source object - * @param[out] sizep the size of the storage source object - */ - int (*ss_size)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - WT_LOCATION_HANDLE *location_handle, const char *name, wt_off_t *sizep); + WT_FILE_SYSTEM *file_system, const char *name, const char *config); /*! * A callback performed when the storage source is closed and will no diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index 0848660c455..d1e4d909b50 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -216,8 +216,10 @@ __rec_need_save_upd( if (F_ISSET(r, WT_REC_CHECKPOINT) && upd_select->upd == NULL) return (false); - return (!__wt_txn_tw_stop_visible_all(session, &upd_select->tw) && - !__wt_txn_tw_start_visible_all(session, &upd_select->tw)); + if (WT_TIME_WINDOW_HAS_STOP(&upd_select->tw)) + return (!__wt_txn_tw_stop_visible_all(session, &upd_select->tw)); + else + return (!__wt_txn_tw_start_visible_all(session, &upd_select->tw)); } /* diff --git a/src/third_party/wiredtiger/src/tiered/tiered_cursor.c b/src/third_party/wiredtiger/src/tiered/tiered_cursor.c index f0aa30d2023..0737c4f5f68 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_cursor.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_cursor.c @@ -1142,7 +1142,7 @@ __wt_curtiered_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, cursor = (WT_CURSOR *)curtiered; *cursor = iface; cursor->session = (WT_SESSION *)session; - WT_ERR(__wt_strdup(session, tiered->name, &cursor->uri)); + WT_ERR(__wt_strdup(session, tiered->iface.name, &cursor->uri)); cursor->key_format = tiered->key_format; cursor->value_format = tiered->value_format; diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 2baf6131d97..bd19c96e0f7 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -558,6 +558,7 @@ __recovery_correct_write_gen(WT_SESSION_IMPL *session) WT_DECL_RET; char *config, *uri; + uri = NULL; WT_RET(__wt_metadata_cursor(session, &cursor)); while ((ret = cursor->next(cursor)) == 0) { WT_ERR(cursor->get_key(cursor, &uri)); @@ -573,6 +574,8 @@ __recovery_correct_write_gen(WT_SESSION_IMPL *session) WT_ERR_NOTFOUND_OK(ret, false); err: + if (ret != 0 && uri != NULL) + __wt_err(session, ret, "unable to correct write gen for %s", uri); WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); } @@ -586,6 +589,7 @@ static int __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) { WT_CONFIG_ITEM cval; + WT_DECL_RET; WT_LSN lsn; uint32_t fileid, lsnfile, lsnoffset; @@ -606,7 +610,9 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) "metadata corruption: files %s and %s have the same file ID %u", uri, r->files[fileid].uri, fileid); WT_RET(__wt_strdup(r->session, uri, &r->files[fileid].uri)); - WT_RET(__wt_config_getones(r->session, config, "checkpoint_lsn", &cval)); + if ((ret = __wt_config_getones(r->session, config, "checkpoint_lsn", &cval)) != 0) + WT_RET_MSG( + r->session, ret, "Failed recovery setup for %s: cannot parse config '%s'", uri, config); /* If there is no checkpoint logged for the file, apply everything. */ if (cval.type != WT_CONFIG_ITEM_STRUCT) WT_INIT_LSN(&lsn); @@ -614,8 +620,9 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) else if (sscanf(cval.str, "(%" SCNu32 ",%" SCNu32 ")", &lsnfile, &lsnoffset) == 2) WT_SET_LSN(&lsn, lsnfile, lsnoffset); else - WT_RET_MSG( - r->session, EINVAL, "Failed to parse checkpoint LSN '%.*s'", (int)cval.len, cval.str); + WT_RET_MSG(r->session, EINVAL, + "Failed recovery setup for %s: cannot parse checkpoint LSN '%.*s'", uri, (int)cval.len, + cval.str); WT_ASSIGN_LSN(&r->files[fileid].ckpt_lsn, &lsn); __wt_verbose(r->session, WT_VERB_RECOVERY, @@ -627,7 +634,9 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) WT_ASSIGN_LSN(&r->max_ckpt_lsn, &lsn); /* Update the base write gen based on this file's configuration. */ - return (__wt_metadata_update_base_write_gen(r->session, config)); + if ((ret = __wt_metadata_update_base_write_gen(r->session, config)) != 0) + WT_RET_MSG(r->session, ret, "Failed recovery setup for %s: cannot update write gen", uri); + return (0); } /* diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/config_example_test_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/config_example_test_default.txt new file mode 100644 index 00000000000..2ea29310102 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/config_example_test_default.txt @@ -0,0 +1,37 @@ +# Same parameters as config_poc_test_default +duration_seconds=10 +cache_size_mb=1000 +enable_logging=true +runtime_monitor= +{ + rate_per_second=3 + stat_cache_size= + { + enabled=true + limit=100 + } +} +timestamp_manager= +{ + enabled=true + oldest_lag=1 + stable_lag=1 +} +workload_generator= +{ + collection_count=2 + key_count=5 + key_size=1 + ops_per_transaction= + { + min=5 + max=50 + } + read_threads=1 + update_threads=1 + value_size=10 +} +workload_tracking= +{ + enabled=true +} diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_default.txt index 52f4f536876..a8876f05a99 100644 --- a/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_default.txt +++ b/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_default.txt @@ -23,7 +23,6 @@ workload_generator= { collection_count=2 key_count=5 - key_format=i key_size=1 ops_per_transaction= { @@ -31,8 +30,8 @@ workload_generator= max=50 } read_threads=1 + update_threads=1 value_size=10 - value_format=S } workload_tracking= { diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/api_const.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/api_const.h index 46a6a775677..bccb3b9a63d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/api_const.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/api_const.h @@ -45,6 +45,7 @@ static const char *DURATION_SECONDS = "duration_seconds"; static const char *ENABLED = "enabled"; static const char *ENABLE_LOGGING = "enable_logging"; static const char *KEY_COUNT = "key_count"; +static const char *KEY_SIZE = "key_size"; static const char *LIMIT = "limit"; static const char *MAX = "max"; static const char *MIN = "min"; @@ -54,6 +55,7 @@ static const char *RATE_PER_SECOND = "rate_per_second"; static const char *READ_THREADS = "read_threads"; static const char *STABLE_LAG = "stable_lag"; static const char *STAT_CACHE_SIZE = "stat_cache_size"; +static const char *UPDATE_THREADS = "update_threads"; static const char *VALUE_SIZE = "value_size"; /* WiredTiger API consts. */ @@ -63,6 +65,7 @@ static const char *OLDEST_TS = "oldest_timestamp"; static const char *STABLE_TS = "stable_timestamp"; /* Test harness consts. */ +static const char *DEFAULT_FRAMEWORK_SCHEMA = "key_format=S,value_format=S"; static const char *TABLE_OPERATION_TRACKING = "table:operation_tracking"; static const char *TABLE_SCHEMA_TRACKING = "table:schema_tracking"; static const char *STATISTICS_URI = "statistics:"; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration.h index adae5b1b8c5..5cf3f9f1fbe 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration.h @@ -83,7 +83,7 @@ class configuration { { WT_CONFIG_ITEM temp_value; testutil_check(_config_parser->get(_config_parser, key.c_str(), &temp_value)); - if (temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRING || + if (temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRING && temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_ID) return (-1); value = std::string(temp_value.str, temp_value.len); diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/database_model.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/database_model.h new file mode 100644 index 00000000000..07e7c007ea7 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/database_model.h @@ -0,0 +1,89 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef DATABASE_MODEL_H +#define DATABASE_MODEL_H + +#include <map> +#include <string> + +namespace test_harness { + +/* Key/Value type. */ +typedef std::string key_value_t; + +/* Representation of key states. */ +struct key_t { + bool exists; +}; + +/* Iterator type used to iterate over keys that are stored in the data model. */ +typedef std::map<test_harness::key_value_t, test_harness::key_t>::const_iterator keys_iterator_t; + +/* Representation of a value. */ +struct value_t { + key_value_t value; +}; + +/* A collection is made of mapped Key objects. */ +struct collection_t { + std::map<key_value_t, key_t> keys; + std::map<key_value_t, value_t> *values = {nullptr}; +}; + +/* Representation of the collections in memory. */ +class database { + public: + const keys_iterator_t + get_collection_keys_begin(const std::string &collection_name) const + { + return (collections.at(collection_name).keys.begin()); + } + + const keys_iterator_t + get_collection_keys_end(const std::string &collection_name) const + { + return (collections.at(collection_name).keys.end()); + } + + const std::vector<std::string> + get_collection_names() const + { + std::vector<std::string> collection_names; + + for (auto const &it : collections) + collection_names.push_back(it.first); + + return (collection_names); + } + + std::map<std::string, collection_t> collections; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/database_operation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/database_operation.h new file mode 100644 index 00000000000..53e6d2b73a0 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/database_operation.h @@ -0,0 +1,272 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef DATABASE_OPERATION_H +#define DATABASE_OPERATION_H + +#include "database_model.h" +#include "workload_tracking.h" +#include "thread_context.h" + +namespace test_harness { +class database_operation { + public: + /* + * Function that performs the following steps using the configuration that is defined by the + * test: + * - Create the working dir. + * - Open a connection. + * - Open a session. + * - Create n collections as per the configuration. + * - Open a cursor on each collection. + * - Insert m key/value pairs in each collection. Values are random strings which size is + * defined by the configuration. + * - Store in memory the created collections and the generated keys that were inserted. + */ + virtual void + populate(database &database, timestamp_manager *_timestamp_manager, configuration *_config, + workload_tracking *tracking) + { + WT_CURSOR *cursor; + WT_SESSION *session; + wt_timestamp_t ts; + int64_t collection_count, key_count, key_cpt, key_size, value_size; + std::string collection_name, config, home; + key_value_t generated_key, generated_value; + bool ts_enabled = _timestamp_manager->is_enabled(); + + cursor = nullptr; + collection_count = key_count = key_size = value_size = 0; + + /* Get a session. */ + session = connection_manager::instance().create_session(); + /* Create n collections as per the configuration and store each collection name. */ + testutil_check(_config->get_int(COLLECTION_COUNT, collection_count)); + for (int i = 0; i < collection_count; ++i) { + collection_name = "table:collection" + std::to_string(i); + database.collections[collection_name] = {}; + testutil_check( + session->create(session, collection_name.c_str(), DEFAULT_FRAMEWORK_SCHEMA)); + ts = _timestamp_manager->get_next_ts(); + testutil_check(tracking->save(tracking_operation::CREATE, collection_name, 0, "", ts)); + } + debug_print(std::to_string(collection_count) + " collections created", DEBUG_TRACE); + + /* Open a cursor on each collection and use the configuration to insert key/value pairs. */ + testutil_check(_config->get_int(KEY_COUNT, key_count)); + testutil_check(_config->get_int(VALUE_SIZE, value_size)); + testutil_assert(value_size > 0); + testutil_check(_config->get_int(KEY_SIZE, key_size)); + testutil_assert(key_size > 0); + /* Keys must be unique. */ + testutil_assert(key_count <= pow(10, key_size)); + + for (const auto &it_collections : database.collections) { + collection_name = it_collections.first; + key_cpt = 0; + /* WiredTiger lets you open a cursor on a collection using the same pointer. When a + * session is closed, WiredTiger APIs close the cursors too. */ + testutil_check( + session->open_cursor(session, collection_name.c_str(), NULL, NULL, &cursor)); + for (size_t j = 0; j < key_count; ++j) { + /* Generation of a unique key. */ + generated_key = number_to_string(key_size, key_cpt); + ++key_cpt; + /* + * Generation of a random string value using the size defined in the test + * configuration. + */ + generated_value = + random_generator::random_generator::instance().generate_string(value_size); + ts = _timestamp_manager->get_next_ts(); + if (ts_enabled) + testutil_check(session->begin_transaction(session, "")); + testutil_check(insert(cursor, tracking, collection_name, generated_key.c_str(), + generated_value.c_str(), ts)); + if (ts_enabled) { + config = std::string(COMMIT_TS) + "=" + _timestamp_manager->decimal_to_hex(ts); + testutil_check(session->commit_transaction(session, config.c_str())); + } + /* Update the memory representation of the collections. */ + database.collections[collection_name].keys[generated_key].exists = true; + /* Values are not stored here. */ + database.collections[collection_name].values = nullptr; + } + } + debug_print("Populate stage done", DEBUG_TRACE); + } + + /* Basic read operation that walks a cursors across all collections. */ + virtual void + read_operation(thread_context &context, WT_SESSION *session) + { + WT_CURSOR *cursor; + std::vector<WT_CURSOR *> cursors; + + testutil_assert(session != nullptr); + /* Get a cursor for each collection in collection_names. */ + for (const auto &it : context.get_collection_names()) { + testutil_check(session->open_cursor(session, it.c_str(), NULL, NULL, &cursor)); + cursors.push_back(cursor); + } + + while (!cursors.empty() && context.is_running()) { + /* Walk each cursor. */ + for (const auto &it : cursors) { + if (it->next(it) != 0) + it->reset(it); + } + } + } + + /* + * Basic update operation that updates all the keys to a random value in each collection. + */ + virtual void + update_operation(thread_context &context, WT_SESSION *session) + { + WT_CURSOR *cursor; + wt_timestamp_t ts; + std::vector<WT_CURSOR *> cursors; + std::string collection_name; + std::vector<std::string> collection_names = context.get_collection_names(); + key_value_t generated_value, key; + int64_t cpt, value_size = context.get_value_size(); + + testutil_assert(session != nullptr); + /* Get a cursor for each collection in collection_names. */ + for (const auto &it : collection_names) { + testutil_check(session->open_cursor(session, it.c_str(), NULL, NULL, &cursor)); + cursors.push_back(cursor); + } + + cpt = 0; + /* Walk each cursor. */ + for (const auto &it : cursors) { + collection_name = collection_names[cpt]; + /* Walk each key. */ + for (keys_iterator_t iter_key = context.get_collection_keys_begin(collection_name); + iter_key != context.get_collection_keys_end(collection_name); ++iter_key) { + /* Do not process removed keys. */ + if (!iter_key->second.exists) + continue; + + /* Start a transaction if possible. */ + if (!context.is_in_transaction()) { + context.begin_transaction(session, ""); + ts = context.set_commit_timestamp(session); + } + generated_value = + random_generator::random_generator::instance().generate_string(value_size); + testutil_check(update(context.get_tracking(), it, collection_name, + iter_key->first.c_str(), generated_value.c_str(), ts)); + + /* Commit the current transaction if possible. */ + context.increment_operation_count(); + if (context.can_commit_transaction()) + context.commit_transaction(session, ""); + } + ++cpt; + } + + /* + * The update operations will be later on inside a loop that will be managed through + * throttle management. + */ + while (context.is_running()) + std::this_thread::sleep_for(std::chrono::seconds(1)); + + /* Make sure the last operation is committed now the work is finished. */ + if (context.is_in_transaction()) + context.commit_transaction(session, ""); + } + + private: + /* WiredTiger APIs wrappers for single operations. */ + template <typename K, typename V> + int + insert(WT_CURSOR *cursor, workload_tracking *tracking, const std::string &collection_name, + const K &key, const V &value, wt_timestamp_t ts) + { + int error_code; + + testutil_assert(cursor != nullptr); + cursor->set_key(cursor, key); + cursor->set_value(cursor, value); + error_code = cursor->insert(cursor); + + if (error_code == 0) { + debug_print("key/value inserted", DEBUG_TRACE); + error_code = + tracking->save(tracking_operation::INSERT, collection_name, key, value, ts); + } else + debug_print("key/value insertion failed", DEBUG_ERROR); + + return (error_code); + } + + template <typename K, typename V> + static int + update(workload_tracking *tracking, WT_CURSOR *cursor, const std::string &collection_name, + K key, V value, wt_timestamp_t ts) + { + int error_code; + + testutil_assert(tracking != nullptr); + testutil_assert(cursor != nullptr); + cursor->set_key(cursor, key); + cursor->set_value(cursor, value); + error_code = cursor->update(cursor); + + if (error_code == 0) { + debug_print("key/value update", DEBUG_TRACE); + error_code = + tracking->save(tracking_operation::UPDATE, collection_name, key, value, ts); + } else + debug_print("key/value update failed", DEBUG_ERROR); + + return (error_code); + } + + /* + * Convert a number to a string. If the resulting string is less than the given length, padding + * of '0' is added. + */ + static std::string + number_to_string(uint64_t size, uint64_t value) + { + std::string str, value_str = std::to_string(value); + testutil_assert(size >= value_str.size()); + uint64_t diff = size - value_str.size(); + std::string s(diff, '0'); + str = s.append(value_str); + return (str); + } +}; +} // namespace test_harness +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h index e81a8bfe47b..2ed58c54f40 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h @@ -104,7 +104,7 @@ class cache_limit_statistic : public statistic { debug_print(error_string, DEBUG_ERROR); testutil_assert(use_percent < limit); } else - debug_print("Usage: " + std::to_string(use_percent), DEBUG_TRACE); + debug_print("Cache usage: " + std::to_string(use_percent), DEBUG_TRACE); } private: diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h index e11d17ab51b..51c39958086 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h @@ -52,9 +52,11 @@ namespace test_harness { /* * The base class for a test, the standard usage pattern is to just call run(). */ -class test { +class test : public database_operation { public: test(const std::string &config, const std::string &name) + : _runtime_monitor(nullptr), _thread_manager(nullptr), _timestamp_manager(nullptr), + _workload_generator(nullptr), _workload_tracking(nullptr) { _configuration = new configuration(name, config); _runtime_monitor = new runtime_monitor(_configuration->get_subconfig(RUNTIME_MONITOR)); @@ -64,7 +66,7 @@ class test { OPERATION_TRACKING_TABLE_CONFIG, TABLE_OPERATION_TRACKING, SCHEMA_TRACKING_TABLE_CONFIG, TABLE_SCHEMA_TRACKING); _workload_generator = - new workload_generator(_configuration->get_subconfig(WORKLOAD_GENERATOR), + new workload_generator(_configuration->get_subconfig(WORKLOAD_GENERATOR), this, _timestamp_manager, _workload_tracking); _thread_manager = new thread_manager(); /* @@ -139,7 +141,7 @@ class test { if (_workload_tracking->is_enabled()) { workload_validation wv; is_success = wv.validate(_workload_tracking->get_operation_table_name(), - _workload_tracking->get_schema_table_name()); + _workload_tracking->get_schema_table_name(), _workload_generator->get_database()); } debug_print(is_success ? "SUCCESS" : "FAILED", DEBUG_INFO); diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_context.h index 61ee7e01a88..532f6bc669c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_context.h @@ -29,6 +29,7 @@ #ifndef THREAD_CONTEXT_H #define THREAD_CONTEXT_H +#include "database_model.h" #include "random_generator.h" #include "workload_tracking.h" @@ -48,13 +49,11 @@ enum class thread_operation { /* Container class for a thread and any data types it may need to interact with the database. */ class thread_context { public: - thread_context(timestamp_manager *timestamp_manager, workload_tracking *tracking, - std::vector<std::string> collection_names, thread_operation type, int64_t max_op, - int64_t min_op, int64_t value_size) - : _collection_names(collection_names), _current_op_count(0U), _in_txn(false), - _running(false), _min_op(min_op), _max_op(max_op), _max_op_count(0), - _timestamp_manager(timestamp_manager), _type(type), _tracking(tracking), - _value_size(value_size) + thread_context(timestamp_manager *timestamp_manager, workload_tracking *tracking, database &db, + thread_operation type, int64_t max_op, int64_t min_op, int64_t value_size) + : _database(db), _current_op_count(0U), _in_txn(false), _running(false), _min_op(min_op), + _max_op(max_op), _max_op_count(0), _timestamp_manager(timestamp_manager), _type(type), + _tracking(tracking), _value_size(value_size) { } @@ -64,10 +63,22 @@ class thread_context { _running = false; } - const std::vector<std::string> & + const std::vector<std::string> get_collection_names() const { - return (_collection_names); + return (_database.get_collection_names()); + } + + const keys_iterator_t + get_collection_keys_begin(const std::string &collection_name) const + { + return (_database.get_collection_keys_begin(collection_name)); + } + + const keys_iterator_t + get_collection_keys_end(const std::string &collection_name) const + { + return (_database.get_collection_keys_end(collection_name)); } thread_operation @@ -94,6 +105,12 @@ class thread_context { return (_running); } + bool + is_in_transaction() const + { + return (_in_txn); + } + void set_running(bool running) { @@ -105,7 +122,7 @@ class thread_context { { if (!_in_txn && _timestamp_manager->is_enabled()) { testutil_check( - session->begin_transaction(session, config.empty() ? NULL : config.c_str())); + session->begin_transaction(session, config.empty() ? nullptr : config.c_str())); /* This randomizes the number of operations to be executed in one transaction. */ _max_op_count = random_generator::instance().generate_integer(_min_op, _max_op); _current_op_count = 0; @@ -113,28 +130,36 @@ class thread_context { } } - /* Returns true if the current transaction has been committed. */ + /* + * The current transaction can be committed if: + * - The timestamp manager is enabled and + * - A transaction has started and + * - The thread is done working. This is useful when the test is ended and the thread has + * not reached the maximum number of operations per transaction or + * - The number of operations executed in the current transaction has exceeded the + * threshold. + */ bool - commit_transaction(WT_SESSION *session, const std::string &config) + can_commit_transaction() const { - if (!_timestamp_manager->is_enabled()) - return (true); + return (_timestamp_manager->is_enabled() && _in_txn && + (!_running || (_current_op_count > _max_op_count))); + } + void + commit_transaction(WT_SESSION *session, const std::string &config) + { /* A transaction cannot be committed if not started. */ testutil_assert(_in_txn); - /* The current transaction should be committed if: - * - The thread is done working. This is useful when the test is ended and the thread has - * not reached the maximum number of operations per transaction. - * - The number of operations executed in the current transaction has exceeded the - * threshold. - */ - if (!_running || (++_current_op_count > _max_op_count)) { - testutil_check( - session->commit_transaction(session, config.empty() ? nullptr : config.c_str())); - _in_txn = false; - } + testutil_check( + session->commit_transaction(session, config.empty() ? nullptr : config.c_str())); + _in_txn = false; + } - return (!_in_txn); + void + increment_operation_count(uint64_t inc = 1) + { + _current_op_count += inc; } /* @@ -157,7 +182,8 @@ class thread_context { } private: - const std::vector<std::string> _collection_names; + /* Representation of the collections and their key/value pairs in memory. */ + database _database; /* * _current_op_count is the current number of operations that have been executed in the current * transaction. diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h index 749f5c1d8f3..ba4616b4756 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h @@ -31,6 +31,7 @@ #include <thread> +#include "database_operation.h" #include "thread_context.h" namespace test_harness { @@ -56,10 +57,10 @@ class thread_manager { */ template <typename Callable> void - add_thread(thread_context *tc, Callable &&fct) + add_thread(thread_context *tc, database_operation *db_operation, Callable &&fct) { tc->set_running(true); - std::thread *t = new std::thread(fct, std::ref(*tc)); + std::thread *t = new std::thread(fct, std::ref(*tc), std::ref(*db_operation)); _workers.push_back(t); } diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h index 8a5940c7637..0a72eadc9cf 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h @@ -88,7 +88,7 @@ class timestamp_manager : public component { * Keep a time window between the stable and oldest ts less than the max defined in the * configuration. */ - testutil_assert(_stable_ts > _oldest_ts); + testutil_assert(_stable_ts >= _oldest_ts); if ((_stable_ts - _oldest_ts) > _oldest_lag) { _oldest_ts = _stable_ts - _oldest_lag; if (!config.empty()) diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h index f9445cd892a..d73a30c7bf0 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h @@ -33,6 +33,8 @@ #include <atomic> #include <map> +#include "database_model.h" +#include "database_operation.h" #include "random_generator.h" #include "workload_tracking.h" @@ -42,9 +44,10 @@ namespace test_harness { */ class workload_generator : public component { public: - workload_generator(configuration *configuration, timestamp_manager *timestamp_manager, - workload_tracking *tracking) - : component(configuration), _timestamp_manager(timestamp_manager), _tracking(tracking) + workload_generator(configuration *configuration, database_operation *db_operation, + timestamp_manager *timestamp_manager, workload_tracking *tracking) + : component(configuration), _database_operation(db_operation), + _timestamp_manager(timestamp_manager), _tracking(tracking) { } @@ -58,86 +61,20 @@ class workload_generator : public component { workload_generator(const workload_generator &) = delete; workload_generator &operator=(const workload_generator &) = delete; - /* - * Function that performs the following steps using the configuration that is defined by the - * test: - * - Create the working dir. - * - Open a connection. - * - Open a session. - * - Create n collections as per the configuration. - * - Open a cursor on each collection. - * - Insert m key/value pairs in each collection. Values are random strings which size is - * defined by the configuration. - */ - void - populate() - { - WT_CURSOR *cursor; - WT_SESSION *session; - wt_timestamp_t ts; - int64_t collection_count, key_count, value_size; - std::string collection_name, config, generated_value, home; - bool ts_enabled = _timestamp_manager->is_enabled(); - - cursor = nullptr; - collection_count = key_count = value_size = 0; - collection_name = ""; - - /* Get a session. */ - session = connection_manager::instance().create_session(); - /* Create n collections as per the configuration and store each collection name. */ - testutil_check(_config->get_int(COLLECTION_COUNT, collection_count)); - for (int i = 0; i < collection_count; ++i) { - collection_name = "table:collection" + std::to_string(i); - testutil_check(session->create(session, collection_name.c_str(), DEFAULT_TABLE_SCHEMA)); - ts = _timestamp_manager->get_next_ts(); - testutil_check(_tracking->save(tracking_operation::CREATE, collection_name, 0, "", ts)); - _collection_names.push_back(collection_name); - } - debug_print(std::to_string(collection_count) + " collections created", DEBUG_TRACE); - - /* Open a cursor on each collection and use the configuration to insert key/value pairs. */ - testutil_check(_config->get_int(KEY_COUNT, key_count)); - testutil_check(_config->get_int(VALUE_SIZE, value_size)); - testutil_assert(value_size >= 0); - for (const auto &collection_name : _collection_names) { - /* WiredTiger lets you open a cursor on a collection using the same pointer. When a - * session is closed, WiredTiger APIs close the cursors too. */ - testutil_check( - session->open_cursor(session, collection_name.c_str(), NULL, NULL, &cursor)); - for (size_t j = 0; j < key_count; ++j) { - /* - * Generation of a random string value using the size defined in the test - * configuration. - */ - generated_value = - random_generator::random_generator::instance().generate_string(value_size); - ts = _timestamp_manager->get_next_ts(); - if (ts_enabled) - testutil_check(session->begin_transaction(session, "")); - testutil_check(insert(cursor, collection_name, j + 1, generated_value.c_str(), ts)); - if (ts_enabled) { - config = std::string(COMMIT_TS) + "=" + _timestamp_manager->decimal_to_hex(ts); - testutil_check(session->commit_transaction(session, config.c_str())); - } - } - } - debug_print("Populate stage done", DEBUG_TRACE); - } - /* Do the work of the main part of the workload. */ void run() { configuration *sub_config; - int64_t read_threads, min_operation_per_transaction, max_operation_per_transaction, - value_size; + int64_t min_operation_per_transaction, max_operation_per_transaction, read_threads, + update_threads, value_size; /* Populate the database. */ - populate(); + _database_operation->populate(_database, _timestamp_manager, _config, _tracking); /* Retrieve useful parameters from the test configuration. */ testutil_check(_config->get_int(READ_THREADS, read_threads)); + testutil_check(_config->get_int(UPDATE_THREADS, update_threads)); sub_config = _config->get_subconfig(OPS_PER_TRANSACTION); testutil_check(sub_config->get_int(MIN, min_operation_per_transaction)); testutil_check(sub_config->get_int(MAX, max_operation_per_transaction)); @@ -149,27 +86,41 @@ class workload_generator : public component { /* Generate threads to execute read operations on the collections. */ for (int i = 0; i < read_threads; ++i) { - thread_context *tc = new thread_context(_timestamp_manager, _tracking, - _collection_names, thread_operation::READ, max_operation_per_transaction, - min_operation_per_transaction, value_size); + thread_context *tc = + new thread_context(_timestamp_manager, _tracking, _database, thread_operation::READ, + max_operation_per_transaction, min_operation_per_transaction, value_size); + _workers.push_back(tc); + _thread_manager.add_thread(tc, _database_operation, &execute_operation); + } + + /* Generate threads to execute update operations on the collections. */ + for (int i = 0; i < update_threads; ++i) { + thread_context *tc = + new thread_context(_timestamp_manager, _tracking, _database, thread_operation::UPDATE, + max_operation_per_transaction, min_operation_per_transaction, value_size); _workers.push_back(tc); - _thread_manager.add_thread(tc, &execute_operation); + _thread_manager.add_thread(tc, _database_operation, &execute_operation); } } void finish() { - for (const auto &it : _workers) { + for (const auto &it : _workers) it->finish(); - } _thread_manager.join(); debug_print("Workload generator: run stage done", DEBUG_TRACE); } + database & + get_database() + { + return _database; + } + /* Workload threaded operations. */ static void - execute_operation(thread_context &context) + execute_operation(thread_context &context, database_operation &db_operation) { WT_SESSION *session; @@ -177,7 +128,7 @@ class workload_generator : public component { switch (context.get_thread_operation()) { case thread_operation::READ: - read_operation(context, session); + db_operation.read_operation(context, session); break; case thread_operation::REMOVE: case thread_operation::INSERT: @@ -186,7 +137,7 @@ class workload_generator : public component { std::this_thread::sleep_for(std::chrono::seconds(1)); break; case thread_operation::UPDATE: - update_operation(context, session); + db_operation.update_operation(context, session); break; default: testutil_die(DEBUG_ABORT, "system: thread_operation is unknown : %d", @@ -195,134 +146,9 @@ class workload_generator : public component { } } - /* - * Basic update operation that currently update the same key with a random value in each - * collection. - */ - static void - update_operation(thread_context &context, WT_SESSION *session) - { - WT_CURSOR *cursor; - wt_timestamp_t ts; - std::vector<WT_CURSOR *> cursors; - std::vector<std::string> collection_names; - std::string generated_value; - bool has_committed = true; - int64_t cpt, value_size = context.get_value_size(); - - testutil_assert(session != nullptr); - /* Get a cursor for each collection in collection_names. */ - for (const auto &it : context.get_collection_names()) { - testutil_check(session->open_cursor(session, it.c_str(), NULL, NULL, &cursor)); - cursors.push_back(cursor); - collection_names.push_back(it); - } - - while (context.is_running()) { - /* Walk each cursor. */ - context.begin_transaction(session, ""); - ts = context.set_commit_timestamp(session); - cpt = 0; - for (const auto &it : cursors) { - generated_value = - random_generator::random_generator::instance().generate_string(value_size); - /* Key is hard coded for now. */ - testutil_check(update(context.get_tracking(), it, collection_names[cpt], 1, - generated_value.c_str(), ts)); - ++cpt; - } - has_committed = context.commit_transaction(session, ""); - } - - /* Make sure the last operation is committed now the work is finished. */ - if (!has_committed) - context.commit_transaction(session, ""); - } - - /* Basic read operation that walks a cursors across all collections. */ - static void - read_operation(thread_context &context, WT_SESSION *session) - { - WT_CURSOR *cursor; - std::vector<WT_CURSOR *> cursors; - - testutil_assert(session != nullptr); - /* Get a cursor for each collection in collection_names. */ - for (const auto &it : context.get_collection_names()) { - testutil_check(session->open_cursor(session, it.c_str(), NULL, NULL, &cursor)); - cursors.push_back(cursor); - } - - while (context.is_running()) { - /* Walk each cursor. */ - for (const auto &it : cursors) { - if (it->next(it) != 0) - it->reset(it); - } - } - } - - /* WiredTiger APIs wrappers for single operations. */ - template <typename K, typename V> - int - insert(WT_CURSOR *cursor, const std::string &collection_name, K key, V value, wt_timestamp_t ts) - { - int error_code; - - testutil_assert(cursor != nullptr); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); - error_code = cursor->insert(cursor); - - if (error_code == 0) { - debug_print("key/value inserted", DEBUG_TRACE); - error_code = - _tracking->save(tracking_operation::INSERT, collection_name, key, value, ts); - } else - debug_print("key/value insertion failed", DEBUG_ERROR); - - return (error_code); - } - - static int - search(WT_CURSOR *cursor) - { - testutil_assert(cursor != nullptr); - return (cursor->search(cursor)); - } - - static int - search_near(WT_CURSOR *cursor, int *exact) - { - testutil_assert(cursor != nullptr); - return (cursor->search_near(cursor, exact)); - } - - template <typename K, typename V> - static int - update(workload_tracking *tracking, WT_CURSOR *cursor, const std::string &collection_name, - K key, V value, wt_timestamp_t ts) - { - int error_code; - - testutil_assert(tracking != nullptr); - testutil_assert(cursor != nullptr); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); - error_code = cursor->update(cursor); - - if (error_code == 0) { - debug_print("key/value update", DEBUG_TRACE); - error_code = - tracking->save(tracking_operation::UPDATE, collection_name, key, value, ts); - } else - debug_print("key/value update failed", DEBUG_ERROR); - - return (error_code); - } - private: - std::vector<std::string> _collection_names; + database _database; + database_operation *_database_operation; thread_manager _thread_manager; timestamp_manager *_timestamp_manager; workload_tracking *_tracking; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_tracking.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_tracking.h index d1464e60970..241ccf341d7 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_tracking.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_tracking.h @@ -33,7 +33,7 @@ * Default schema for tracking operations on collections (key_format: Collection name / Key / * Timestamp, value_format: Operation type / Value) */ -#define OPERATION_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(Sii) +#define OPERATION_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(SSQ) #define OPERATION_TRACKING_VALUE_FORMAT WT_UNCHECKED_STRING(iS) #define OPERATION_TRACKING_TABLE_CONFIG \ "key_format=" OPERATION_TRACKING_KEY_FORMAT ",value_format=" OPERATION_TRACKING_VALUE_FORMAT @@ -42,7 +42,7 @@ * Default schema for tracking schema operations on collections (key_format: Collection name / * Timestamp, value_format: Operation type) */ -#define SCHEMA_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(Si) +#define SCHEMA_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(SQ) #define SCHEMA_TRACKING_VALUE_FORMAT WT_UNCHECKED_STRING(i) #define SCHEMA_TRACKING_TABLE_CONFIG \ "key_format=" SCHEMA_TRACKING_KEY_FORMAT ",value_format=" SCHEMA_TRACKING_VALUE_FORMAT diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_validation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_validation.h index 86ff567bcc2..a8b9940c3e9 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_validation.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_validation.h @@ -35,112 +35,114 @@ extern "C" { #include "wiredtiger.h" } +#include "database_model.h" + namespace test_harness { + /* * Class that can validate database state and collection data. */ class workload_validation { public: /* - * Validate the on disk data against what has been tracked during the test. The first step is to - * replay the tracked operations so a representation in memory of the collections is created. - * This representation is then compared to what is on disk. The second step is to go through - * what has been saved on disk and make sure the memory representation has the same data. + * Validate the on disk data against what has been tracked during the test. + * - The first step is to replay the tracked operations so a representation in memory of the + * collections is created. This representation is then compared to what is on disk. + * - The second step is to go through what has been saved on disk and make sure the memory + * representation has the same data. * operation_table_name is the collection that contains all the operations about the key/value * pairs in the different collections used during the test. schema_table_name is the collection * that contains all the operations about the creation or deletion of collections during the * test. */ bool - validate(const std::string &operation_table_name, const std::string &schema_table_name) + validate(const std::string &operation_table_name, const std::string &schema_table_name, + database &database) { WT_SESSION *session; std::string collection_name; - /* - * Representation in memory of the collections at the end of the test. The first level is a - * map that contains collection names as keys. The second level is another map that contains - * the different key/value pairs within a given collection. If a collection yields to a null - * map of key/value pairs, this means the collection should not be present on disk. If a - * value from a key/value pair is null, this means the key should not be present in the - * collection on disk. - */ - std::map<std::string, std::map<int, std::string *> *> collections; /* Existing collections after the test. */ - std::vector<std::string> created_collections; - bool is_valid; + std::vector<std::string> created_collections, deleted_collections; + bool is_valid = true; session = connection_manager::instance().create_session(); - /* Retrieve the created collections that need to be checked. */ + /* Retrieve the collections that were created and deleted during the test. */ collection_name = schema_table_name; - created_collections = parse_schema_tracking_table(session, collection_name); + parse_schema_tracking_table( + session, collection_name, created_collections, deleted_collections); - /* Allocate memory to the operations performed on the created collections. */ + /* Make sure they exist in memory. */ for (auto const &it : created_collections) { - std::map<int, std::string *> *map = new std::map<int, std::string *>(); - collections[it] = map; + if (database.collections.count(it) == 0) { + debug_print("Collection missing in memory: " + it, DEBUG_ERROR); + is_valid = false; + break; + } } - /* - * Build in memory the final state of each created collection according to the tracked - * operations. - */ - collection_name = operation_table_name; - for (auto const &active_collection : created_collections) - parse_operation_tracking_table( - session, collection_name, active_collection, collections); - - /* Check all tracked operations in memory against the database on disk. */ - is_valid = check_reference(session, collections); + if (!is_valid) + return (is_valid); - /* Check what has been saved on disk against what has been tracked. */ - if (is_valid) { - for (auto const &collection : created_collections) { - is_valid = check_disk_state(session, collection, collections); - if (!is_valid) { - debug_print( - "check_disk_state failed for collection " + collection, DEBUG_ERROR); - break; - } + /* Make sure they don't exist in memory nor on disk. */ + for (auto const &it : deleted_collections) { + if (database.collections.count(it) > 0) { + debug_print( + "Collection present in memory while it has been tracked as deleted: " + it, + DEBUG_ERROR); + is_valid = false; + break; } + if (!verify_collection_state(session, it, false)) { + debug_print( + "Collection present on disk while it has been tracked as deleted: " + it, + DEBUG_ERROR); + is_valid = false; + break; + } + } - } else - debug_print("check_reference failed!", DEBUG_ERROR); - - /* Clean the allocated memory. */ - clean_memory(collections); - - return (is_valid); - } - - /* Clean the memory used to represent the collections after the test. */ - void - clean_memory(std::map<std::string, std::map<int, std::string *> *> &collections) - { - for (auto &it_collections : collections) { - if (it_collections.second == nullptr) - continue; + for (auto const &collection_name : database.get_collection_names()) { + if (!is_valid) + break; - for (auto &it_operations : *it_collections.second) { - delete it_operations.second; - it_operations.second = nullptr; + /* Get the values associated to the different keys in the current collection. */ + parse_operation_tracking_table( + session, operation_table_name, collection_name, database); + /* Check all tracked operations in memory against the database on disk. */ + if (!check_reference(session, collection_name, database)) { + debug_print( + "check_reference failed for collection " + collection_name, DEBUG_ERROR); + is_valid = false; + } + /* Check what has been saved on disk against what has been tracked. */ + else if (!check_disk_state(session, collection_name, database)) { + debug_print( + "check_disk_state failed for collection " + collection_name, DEBUG_ERROR); + is_valid = false; } - delete it_collections.second; - it_collections.second = nullptr; + /* Clear memory. */ + delete database.collections[collection_name].values; + database.collections[collection_name].values = nullptr; } + + return (is_valid); } + private: /* + * Read the tracking table to retrieve the created and deleted collections during the test. * collection_name is the collection that contains the operations on the different collections * during the test. */ - const std::vector<std::string> - parse_schema_tracking_table(WT_SESSION *session, const std::string &collection_name) + void + parse_schema_tracking_table(WT_SESSION *session, const std::string &collection_name, + std::vector<std::string> &created_collections, std::vector<std::string> &deleted_collections) { WT_CURSOR *cursor; + wt_timestamp_t key_timestamp; const char *key_collection_name; - int key_timestamp, value_operation_type; - std::vector<std::string> created_collections; + int value_operation_type; testutil_check(session->open_cursor(session, collection_name.c_str(), NULL, NULL, &cursor)); @@ -154,16 +156,18 @@ class workload_validation { if (static_cast<tracking_operation>(value_operation_type) == tracking_operation::CREATE) { + deleted_collections.erase(std::remove(deleted_collections.begin(), + deleted_collections.end(), key_collection_name), + deleted_collections.end()); created_collections.push_back(key_collection_name); } else if (static_cast<tracking_operation>(value_operation_type) == tracking_operation::DELETE_COLLECTION) { created_collections.erase(std::remove(created_collections.begin(), created_collections.end(), key_collection_name), created_collections.end()); + deleted_collections.push_back(key_collection_name); } } - - return (created_collections); } /* @@ -174,32 +178,42 @@ class workload_validation { */ void parse_operation_tracking_table(WT_SESSION *session, const std::string &tracking_collection_name, - const std::string &collection_name, - std::map<std::string, std::map<int, std::string *> *> &collections) + const std::string &collection_name, database &database) { WT_CURSOR *cursor; - int error_code, exact, key, key_timestamp, value_operation_type; - const char *key_collection_name, *value; + wt_timestamp_t key_timestamp; + int exact, value_operation_type; + const char *key, *key_collection_name, *value; + std::vector<key_value_t> collection_keys; + std::string key_str; + + /* Retrieve all keys from the given collection. */ + for (auto const &it : database.collections.at(collection_name).keys) + collection_keys.push_back(it.first); + /* There must be at least a key. */ + testutil_assert(!collection_keys.empty()); + /* Sort keys. */ + std::sort(collection_keys.begin(), collection_keys.end()); + /* Use the first key as a parameter for search_near. */ + key_str = collection_keys[0]; testutil_check( session->open_cursor(session, tracking_collection_name.c_str(), NULL, NULL, &cursor)); - /* Our keys start at 0. */ - cursor->set_key(cursor, collection_name.c_str(), 0); - error_code = cursor->search_near(cursor, &exact); - + cursor->set_key(cursor, collection_name.c_str(), key_str.c_str()); + testutil_check(cursor->search_near(cursor, &exact)); /* - * As we don't support deletion, the searched collection is expected to be found. Since the - * timestamp which is part of the key is not provided, exact is expected to be > 0. + * Since the timestamp which is part of the key is not provided, exact is expected to be + * greater than 0. */ - testutil_check(exact < 1); + testutil_assert(exact >= 0); - while (error_code == 0) { + do { testutil_check(cursor->get_key(cursor, &key_collection_name, &key, &key_timestamp)); testutil_check(cursor->get_value(cursor, &value_operation_type, &value)); debug_print("Collection name is " + std::string(key_collection_name), DEBUG_TRACE); - debug_print("Key is " + std::to_string(key), DEBUG_TRACE); + debug_print("Key is " + std::string(key), DEBUG_TRACE); debug_print("Timestamp is " + std::to_string(key_timestamp), DEBUG_TRACE); debug_print("Operation type is " + std::to_string(value_operation_type), DEBUG_TRACE); debug_print("Value is " + std::string(value), DEBUG_TRACE); @@ -217,141 +231,138 @@ class workload_validation { /* * Operations are parsed from the oldest to the most recent one. It is safe to * assume the key has been inserted previously in an existing collection and can be - * deleted safely. + * safely deleted. */ - delete collections.at(key_collection_name)->at(key); - collections.at(key_collection_name)->at(key) = nullptr; + database.collections.at(key_collection_name).keys.at(std::string(key)).exists = + false; + delete database.collections.at(key_collection_name).values; + database.collections.at(key_collection_name).values = nullptr; break; case tracking_operation::INSERT: { /* Keys are unique, it is safe to assume the key has not been encountered before. */ - std::pair<int, std::string *> pair(key, new std::string(value)); - collections.at(key_collection_name)->insert(pair); + database.collections[key_collection_name].keys[std::string(key)].exists = true; + if (database.collections[key_collection_name].values == nullptr) { + database.collections[key_collection_name].values = + new std::map<key_value_t, value_t>(); + } + value_t v; + v.value = key_value_t(value); + std::pair<key_value_t, value_t> pair(key_value_t(key), v); + database.collections[key_collection_name].values->insert(pair); break; } - case tracking_operation::CREATE: - case tracking_operation::DELETE_COLLECTION: - testutil_die(DEBUG_ABORT, "Unexpected operation in the tracking table: %d", - static_cast<tracking_operation>(value_operation_type)); + case tracking_operation::UPDATE: + database.collections[key_collection_name].values->at(key).value = + key_value_t(value); + break; default: - testutil_die( - DEBUG_ABORT, "tracking operation is unknown : %d", value_operation_type); + testutil_die(DEBUG_ABORT, "Unexpected operation in the tracking table: %d", + value_operation_type); break; } - error_code = cursor->next(cursor); - } + } while (cursor->next(cursor) == 0); if (cursor->reset(cursor) != 0) debug_print("Cursor could not be reset !", DEBUG_ERROR); } /* - * Compare the tracked operations against what has been saved on disk. collections is the + * Compare the tracked operations against what has been saved on disk. database is the * representation in memory of the collections after the test according to the tracking table. */ bool check_reference( - WT_SESSION *session, std::map<std::string, std::map<int, std::string *> *> &collections) + WT_SESSION *session, const std::string &collection_name, const database &database) { + bool is_valid; + collection_t collection; + key_t key; + key_value_t key_str; + + /* Check the collection exists on disk. */ + is_valid = verify_collection_state(session, collection_name, true); - bool collection_exists, is_valid = true; - std::map<int, std::string *> *collection; - workload_validation wv; - std::string *value; - - for (const auto &it_collections : collections) { - /* Check the collection is in the correct state. */ - collection_exists = (it_collections.second != nullptr); - is_valid = wv.verify_database_state(session, it_collections.first, collection_exists); - - if (is_valid && collection_exists) { - collection = it_collections.second; - for (const auto &it_operations : *collection) { - value = (*collection)[it_operations.first]; - /* The key/value pair exists. */ - if (value != nullptr) - is_valid = (wv.is_key_present( - session, it_collections.first, it_operations.first) == true); - /* The key has been deleted. */ - else - is_valid = (wv.is_key_present( - session, it_collections.first, it_operations.first) == false); - - /* Check the associated value is valid. */ - if (is_valid && (value != nullptr)) { - is_valid = (wv.verify_value( - session, it_collections.first, it_operations.first, *value)); - } - - if (!is_valid) { - debug_print( - "check_reference failed for key " + std::to_string(it_operations.first), - DEBUG_ERROR); - break; - } + if (is_valid) { + collection = database.collections.at(collection_name); + /* Walk through each key/value pair of the current collection. */ + for (const auto &keys : collection.keys) { + key_str = keys.first; + key = keys.second; + /* The key/value pair exists. */ + if (key.exists) + is_valid = (is_key_present(session, collection_name, key_str.c_str()) == true); + /* The key has been deleted. */ + else + is_valid = (is_key_present(session, collection_name, key_str.c_str()) == false); + + /* Check the associated value is valid. */ + if (is_valid && key.exists) { + testutil_assert(collection.values != nullptr); + is_valid = verify_value(session, collection_name, key_str.c_str(), + collection.values->at(key_str).value); } - } - if (!is_valid) { - debug_print( - "check_reference failed for collection " + it_collections.first, DEBUG_ERROR); - break; + if (!is_valid) { + debug_print("check_reference failed for key " + key_str, DEBUG_ERROR); + break; + } } } + if (!is_valid) + debug_print("check_reference failed for collection " + collection_name, DEBUG_ERROR); + return (is_valid); } /* Check what is present on disk against what has been tracked. */ bool - check_disk_state(WT_SESSION *session, const std::string &collection_name, - std::map<std::string, std::map<int, std::string *> *> &collections) + check_disk_state( + WT_SESSION *session, const std::string &collection_name, const database &database) { WT_CURSOR *cursor; - int key; - const char *value; - bool is_valid; - std::string *value_str; - std::map<int, std::string *> *collection; + collection_t collection; + bool is_valid = true; + /* Key/value pairs on disk. */ + const char *key_on_disk, *value_on_disk; + key_value_t key_str, value_str; testutil_check(session->open_cursor(session, collection_name.c_str(), NULL, NULL, &cursor)); - /* Check the collection has been tracked and contains data. */ - is_valid = - ((collections.count(collection_name) > 0) && (collections[collection_name] != nullptr)); - - if (!is_valid) - debug_print( - "Collection " + collection_name + " has not been tracked or has been deleted", - DEBUG_ERROR); - else - collection = collections[collection_name]; + collection = database.collections.at(collection_name); /* Read the collection on disk. */ while (is_valid && (cursor->next(cursor) == 0)) { - testutil_check(cursor->get_key(cursor, &key)); - testutil_check(cursor->get_value(cursor, &value)); + testutil_check(cursor->get_key(cursor, &key_on_disk)); + testutil_check(cursor->get_value(cursor, &value_on_disk)); - debug_print("Key is " + std::to_string(key), DEBUG_TRACE); - debug_print("Value is " + std::string(value), DEBUG_TRACE); + key_str = std::string(key_on_disk); + + debug_print("Key on disk is " + key_str, DEBUG_TRACE); + debug_print("Value on disk is " + std::string(value_on_disk), DEBUG_TRACE); - if (collection->count(key) > 0) { - value_str = collection->at(key); + /* Check the key on disk has been saved in memory too. */ + if ((collection.keys.count(key_str) > 0) && collection.keys.at(key_str).exists) { + /* Memory should be allocated for values. */ + testutil_assert(collection.values != nullptr); + value_str = collection.values->at(key_str).value; /* * Check the key/value pair on disk matches the one in memory from the tracked * operations. */ - is_valid = (value_str != nullptr) && (*value_str == std::string(value)); + is_valid = (value_str == key_value_t(value_on_disk)); if (!is_valid) - debug_print(" Key/Value pair mismatch.\n Disk key: " + std::to_string(key) + - "\n Disk value: " + std ::string(value) + - "\n Tracking table key: " + std::to_string(key) + - "\n Tracking table value: " + (value_str == nullptr ? "NULL" : *value_str), + debug_print(" Key/Value pair mismatch.\n Disk key: " + key_str + + "\n Disk value: " + std ::string(value_on_disk) + + "\n Tracking table key: " + key_str + "\n Tracking table value exists: " + + std::to_string(collection.keys.at(key_str).exists) + + "\n Tracking table value: " + value_str, DEBUG_ERROR); } else { is_valid = false; debug_print( - "The key " + std::to_string(key) + " present on disk has not been tracked", + "The key " + std::string(key_on_disk) + " present on disk has not been tracked", DEBUG_ERROR); } } @@ -364,7 +375,7 @@ class workload_validation { * needs to be set to true if the collection is expected to be existing, false otherwise. */ bool - verify_database_state( + verify_collection_state( WT_SESSION *session, const std::string &collection_name, bool exists) const { WT_CURSOR *cursor; @@ -396,10 +407,8 @@ class workload_validation { testutil_check(cursor->search(cursor)); testutil_check(cursor->get_value(cursor, &value)); - return (value == expected_value); + return (key_value_t(value) == expected_value); } - - private: }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx new file mode 100644 index 00000000000..cc08d3d003a --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx @@ -0,0 +1,55 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_harness/test.h" + +/* + * Class that defines operations that do nothing as an example. + * This shows how database operations can be overriden and customized. + */ +class example_test : public test_harness::test { + public: + example_test(const std::string &config, const std::string &name) : test(config, name) {} + + virtual void + populate(test_harness::database &database, test_harness::timestamp_manager *_timestamp_manager, + test_harness::configuration *_config, test_harness::workload_tracking *tracking) + { + std::cout << "populate: nothing done." << std::endl; + } + virtual void + read_operation(test_harness::thread_context &context, WT_SESSION *session) + { + std::cout << "read_operation: nothing done." << std::endl; + } + virtual void + update_operation(test_harness::thread_context &context, WT_SESSION *session) + { + std::cout << "update_operation: nothing done." << std::endl; + } +}; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx index f9d5902b7ff..88932b96cde 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx @@ -33,6 +33,7 @@ #include "test_harness/debug_utils.h" #include "test_harness/test.h" +#include "example_test.cxx" #include "poc_test.cxx" std::string @@ -107,10 +108,12 @@ run_test(const std::string &test_name, const std::string &config_name = "") cfg_path = config_name; cfg = parse_configuration_from_file(cfg_path); - test_harness::debug_print("Configuration\t: " + cfg, DEBUG_INFO); + test_harness::debug_print("Configuration\t:" + cfg, DEBUG_INFO); if (test_name == "poc_test") poc_test(cfg, test_name).run(); + else if (test_name == "example_test") + example_test(cfg, test_name).run(); else { test_harness::debug_print("Test not found: " + test_name, DEBUG_ERROR); error_code = -1; @@ -127,7 +130,7 @@ main(int argc, char *argv[]) { std::string cfg, config_name, test_name; int64_t error_code = 0; - const std::vector<std::string> all_tests = {"poc_test"}; + const std::vector<std::string> all_tests = {"example_test", "poc_test"}; /* Parse args * -C : Configuration. Cannot be used with -f. diff --git a/src/third_party/wiredtiger/test/suite/hook_demo.py b/src/third_party/wiredtiger/test/suite/hook_demo.py new file mode 100755 index 00000000000..113c427c8b7 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/hook_demo.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# [TEST_TAGS] +# ignored_file +# [END_TAGS] + +# hook_demo.py +# Demonstration of hooks. Run via: +# python run.py --hook demo=N base01 +# +# These hooks are set up: +# - alter wiredtiger_open arguments (in a benign way) +# - report after wiredtiger_open is called. +# - notify on session.open_cursor +# - intercept the session.create call +# +# With N == 0, the session.create call reports its arguments and calls original session.create. +# with N == 1, it does an additional session.drop call (which should cause tests to fail); +# with N == 2, it does an additional session.create after the drop call (which should work). +# +# Note that notify hooks don't have to simply report, they can call other methods, +# set attributes on objects, etc. For example, one can save the open_cursor +# config string as an attribute on the cursor object, and examine it in another +# hooked method. +from __future__ import print_function + +import os, sys, wthooks +from wttest import WiredTigerTestCase + +# Print to /dev/tty for debugging, since anything extraneous to stdout/stderr will +# cause a test error. +def tty(s): + WiredTigerTestCase.tty(s) + +# These are the hook functions that are run when particular APIs are called. + +# Called to manipulate args for wiredtiger_open +def wiredtiger_open_args(ignored_self, args): + tty('>>> wiredtiger_open, adding cache_size') + args = list(args) # convert from a readonly tuple to a writeable list + args[-1] += ',,,cache_size=500M,,,' # modify the last arg + return args + +# Called to notify after successful wiredtiger_open +def wiredtiger_open_notify(ignored_self, ret, *args): + tty('>>> wiredtiger_open({}) returned {}'.format(args, ret)) + +# Called to notify after successful Session.open_cursor +def session_open_cursor_notify(self, ret, *args): + tty('>>> session.open_cursor({}) returned {}, session is {}'.format(args, ret, self)) + +# Called to replace Session.create +# We do different things (described above) as indicated by our command line argument. +def session_create_replace(arg, orig_session_create, session_self, uri, config): + tty('>>> session.create({},{}), session is {}'.format(uri, config, session_self)) + if arg == 0: + # Just do a regular create + return orig_session_create(session_self, uri, config) + elif arg == 1: + # Do a regular create, followed by a drop. This will cause test failures. + ret = orig_session_create(session_self, uri, config) + # We didn't replace drop, so we can call it as a method + tty('>>> session.drop({})'.format(uri)) + session_self.drop(uri) + return ret + elif arg == 2: + # Do a regular create, followed by a drop, then another create. Should work. + ret = orig_session_create(session_self, uri, config) + # We didn't replace drop, so we can call it as a method + tty('>>> session.drop({})'.format(uri)) + session_self.drop(uri) + tty('>>> session.create({},{})'.format(uri, config)) + orig_session_create(session_self, uri, config) + return ret + +# Every hook file must have one or more classes descended from WiredTigerHook +# This is where the hook functions are 'hooked' to API methods. +class DemoHookCreator(wthooks.WiredTigerHookCreator): + def __init__(self, arg=0): + # An argument may alter the test + if arg == None: + self.arg = 0 + else: + self.arg = int(arg) + + # We have an opportunity to filter the list of tests to be run. + # For this demo, we don't filter. + def filter_tests(self, tests): + print('Filtering: ' + str(tests)) + return tests + + def setup_hooks(self): + tty('>> SETUP HOOKS RUN') + orig_session_create = self.Session['create'] # gets original function + self.wiredtiger['wiredtiger_open'] = (wthooks.HOOK_ARGS, wiredtiger_open_args) + self.wiredtiger['wiredtiger_open'] = (wthooks.HOOK_NOTIFY, wiredtiger_open_notify) + self.Session['create'] = (wthooks.HOOK_REPLACE, lambda s, uri, config: + session_create_replace(self.arg, orig_session_create, s, uri, config)) + self.Session['open_cursor'] = (wthooks.HOOK_NOTIFY, session_open_cursor_notify) + +# Every hook file must have a top level initialize function, +# returning a list of WiredTigerHook objects. +def initialize(arg): + return [DemoHookCreator(arg)] diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index a5ae88fa966..8d74b84259d 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -119,8 +119,10 @@ Options:\n\ be run without executing any.\n\ -g | --gdb all subprocesses (like calls to wt) use gdb\n\ -h | --help show this message\n\ + | --hook name[=arg] set up hooks from hook_<name>.py, with optional arg\n\ -j N | --parallel N run all tests in parallel using N processes\n\ -l | --long run the entire test suite\n\ + | --noremove do not remove WT_TEST or -D target before run\n\ -p | --preserve preserve output files in WT_TEST/<testname>\n\ -r N | --random-sample N randomly sort scenarios to be run, then\n\ execute every Nth (2<=N<=1000) scenario.\n\ @@ -306,6 +308,7 @@ def error(exitval, prefix, msg): if __name__ == '__main__': # Turn numbers and ranges into test module names preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = ignoreStdout = False + removeAtStart = True asan = False parallel = 0 random_sample = 0 @@ -318,6 +321,7 @@ if __name__ == '__main__': verbose = 1 args = sys.argv[1:] testargs = [] + hook_names = [] while len(args) > 0: arg = args.pop(0) from unittest import defaultTestLoader as loader @@ -367,9 +371,18 @@ if __name__ == '__main__': if option == '-help' or option == 'h': usage() sys.exit(0) + if option == '-hook': + if len(args) == 0: + usage() + sys.exit(2) + hook_names.append(args.pop(0)) + continue if option == '-long' or option == 'l': longtest = True continue + if option == '-noremove': + removeAtStart = False + continue if option == '-random-sample' or option == 'r': if len(args) == 0: usage() @@ -519,11 +532,13 @@ if __name__ == '__main__': tests = unittest.TestSuite() from testscenarios.scenarios import generate_scenarios + import wthooks + hookmgr = wthooks.WiredTigerHookManager(hook_names) # All global variables should be set before any test classes are loaded. # That way, verbose printing can be done at the class definition level. - wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, lldbSub, - verbose, wt_builddir, dirarg, - longtest, ignoreStdout, seedw, seedz) + wttest.WiredTigerTestCase.globalSetup(preserve, removeAtStart, timestamp, gdbSub, lldbSub, + verbose, wt_builddir, dirarg, longtest, + ignoreStdout, seedw, seedz, hookmgr) # Without any tests listed as arguments, do discovery if len(testargs) == 0: @@ -542,6 +557,7 @@ if __name__ == '__main__': for arg in testargs: testsFromArg(tests, loader, arg, scenario) + tests = hookmgr.filter_tests(tests) # Shuffle the tests and create a new suite containing every Nth test from # the original suite if random_sample > 0: diff --git a/src/third_party/wiredtiger/test/suite/test_hs21.py b/src/third_party/wiredtiger/test/suite/test_hs21.py new file mode 100644 index 00000000000..e2c8885661f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs21.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import time, re +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wiredtiger import stat + +def timestamp_str(t): + return '%x' % t + +# test_hs21.py +# Test we don't lose any data when idle files with an active history are closed/sweeped. +# Files with active history, ie content newer than the oldest timestamp can be closed when idle. +# We want to ensure that when an active history file is idle closed we can continue reading the +# correct version of data and their base write generation hasn't changed (since we haven't +# restarted the system). +class test_hs21(wttest.WiredTigerTestCase): + # Configure handle sweeping to occur within a specific amount of time. + conn_config = 'file_manager=(close_handle_minimum=0,close_idle_time=2,close_scan_interval=1),' + \ + 'statistics=(all),operation_tracking=(enabled=false)' + session_config = 'isolation=snapshot' + file_name = 'test_hs21' + numfiles = 10 + nrows = 10000 + + def large_updates(self, uri, value, ds, nrows, commit_ts): + # Update a large number of records, we'll hang if the history store table isn't working. + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + for i in range(1, nrows + 1): + cursor[ds.key(i)] = value + session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + cursor.close() + + def check(self, session, check_value, uri, nrows, read_ts=-1): + # Validate we read an expected value (optionally at a given read timestamp). + if read_ts != -1: + session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + cursor = session.open_cursor(uri) + count = 0 + for k, v in cursor: + self.assertEqual(v, check_value) + count += 1 + if read_ts != -1: + session.rollback_transaction() + self.assertEqual(count, nrows) + cursor.close() + + def parse_run_write_gen(self, uri): + meta_cursor = self.session.open_cursor('metadata:') + config = meta_cursor[uri] + meta_cursor.close() + # The search string will look like: 'run_write_gen=<num>'. + # Just reverse the string and take the digits from the back until we hit '='. + write_gen = re.search('run_write_gen=\d+', config) + self.assertTrue(write_gen is not None) + write_gen_str = str() + for c in reversed(write_gen.group(0)): + if not c.isdigit(): + self.assertEqual(c, '=') + break + write_gen_str = c + write_gen_str + return int(write_gen_str) + + def test_hs(self): + active_files = [] + value1 = 'a' * 500 + value2 = 'd' * 500 + + # Set up 'numfiles' with 'numrows' entries. We want to create a number of files that + # contain active history (content newer than the oldest timestamp). + for f in range(self.numfiles): + table_uri = 'table:%s.%d' % (self.file_name, f) + file_uri = 'file:%s.%d.wt' % (self.file_name, f) + # Create a small table. + ds = SimpleDataSet( + self, table_uri, 0, key_format='S', value_format='S', config='log=(enabled=false)') + ds.populate() + # Checkpoint to ensure we write the files metadata checkpoint value. + self.session.checkpoint() + # Get the base write gen of the file so we can compare after the handles get closed. + base_write_gen = self.parse_run_write_gen(file_uri) + active_files.append((base_write_gen, ds)) + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + + ',stable_timestamp=' + timestamp_str(1)) + + # Perform a series of updates over our files at timestamp 2. This being data we can later assert + # to ensure the history store is working as intended. + for (_, ds) in active_files: + # Load data at timestamp 2. + self.large_updates(ds.uri, value1, ds, self.nrows // 2 , 2) + + # We want to create a long running read transaction in a seperate session which we will persist over the closing and + # re-opening of handles. We want to ensure the correct data gets read throughout this time period. + session_read = self.conn.open_session() + session_read.begin_transaction('read_timestamp=' + timestamp_str(2)) + # Check our inital set of updates are seen at the read timestamp. + for (_, ds) in active_files: + # Check that all updates at timestamp 2 are seen. + self.check(session_read, value1, ds.uri, self.nrows // 2) + + # Perform a series of updates over over files at a later timestamp. Checking the history store data is consistent + # with old and new timestamps. + for (_, ds) in active_files: + # Load more data with a later timestamp. + self.large_updates(ds.uri, value2, ds, self.nrows, 100) + # Check that the new updates are only seen after the update timestamp. + self.check(self.session, value1, ds.uri, self.nrows // 2, 2) + self.check(self.session, value2, ds.uri, self.nrows, 100) + + # Our sweep scan interval is every 1 second and the amount of idle time needed for a handle to be closed is 2 seconds. + # It should take roughly 3 seconds for the sweep server to close our file handles. Lets wait at least double + # that to be safe. + max = 6 + sleep = 0 + # After waiting for the sweep server to remove our idle handles, the only open + # handles that should be the metadata file, history store file and lock file. + final_numfiles = 3 + # Open the stats cursor to collect the dhandle sweep status. + stat_cursor = self.session.open_cursor('statistics:', None, None) + while sleep < max: + # We continue doing checkpoints which as a side effect runs the session handle sweep. This encouraging the idle + # handles get removed. + # Note, though checkpointing blocks sweeping, the checkpoint should be fast and not add too much extra time to the + # overall test time. + self.session.checkpoint() + sleep += 0.5 + time.sleep(0.5) + stat_cursor.reset() + curr_files_open = stat_cursor[stat.conn.file_open][2] + curr_dhandles_removed = stat_cursor[stat.conn.dh_sweep_remove][2] + curr_dhandle_sweep_closes = stat_cursor[stat.conn.dh_sweep_close][2] + + self.printVerbose(3, "==== loop " + str(sleep)) + self.printVerbose(3, "Number of files open: " + str(curr_files_open)) + self.printVerbose(3, "Number of connection sweep dhandles closed: " + str(curr_dhandle_sweep_closes)) + self.printVerbose(3, "Number of connection sweep dhandles removed from hashlist: " + str(curr_dhandles_removed)) + + # We've sweeped all the handles we can if we are left with the number of final dhandles + # that we expect to be always open. + if curr_files_open == final_numfiles and curr_dhandle_sweep_closes >= self.numfiles: + break + + stat_cursor.reset() + final_dhandle_sweep_closes = stat_cursor[stat.conn.dh_sweep_close][2] + stat_cursor.close() + # We want to assert our active history files have all been closed. + self.assertGreaterEqual(final_dhandle_sweep_closes, self.numfiles) + + # Using our long running read transaction, we want to now check the correct data can still be read after the + # handles have been closed. + for (_, ds) in active_files: + # Check that all updates at timestamp 2 are seen. + self.check(session_read, value1, ds.uri, self.nrows // 2) + session_read.rollback_transaction() + + # Perform a series of checks over our files to ensure that our transactions have been written + # before the dhandles were closed/sweeped. + # Also despite the dhandle is being re-opened, we don't expect the base write generation + # to have changed since we haven't actually restarted the system. + for idx, (initial_base_write_gen, ds) in enumerate(active_files): + # Check that the most recent transaction has the correct data. + self.check(self.session, value2, ds.uri, self.nrows, 100) + file_uri = 'file:%s.%d.wt' % (self.file_name, idx) + # Get the current base_write_gen and ensure it hasn't changed since being + # closed. + base_write_gen = self.parse_run_write_gen(file_uri) + self.assertEqual(initial_base_write_gen, base_write_gen) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py index b3bf62f03ef..da5b6d1ca91 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py @@ -110,6 +110,7 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): count += 1 session.commit_transaction() self.assertEqual(count, nrows) + cursor.close() # Test that rollback to stable clears the remove operation. class test_rollback_to_stable01(test_rollback_to_stable_base): diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py new file mode 100644 index 00000000000..68c2e8d0205 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch, os, shutil, time +from helper import simulate_crash_restart +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +def timestamp_str(t): + return '%x' % t + +# test_rollback_to_stable18.py +# Test the rollback to stable shouldn't skip any pages that don't have aggregated time window. +class test_rollback_to_stable18(test_rollback_to_stable_base): + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + prepare_values = [ + ('no_prepare', dict(prepare=False)), + ('prepare', dict(prepare=True)) + ] + + scenarios = make_scenarios(key_format_values, prepare_values) + + def conn_config(self): + config = 'cache_size=50MB,in_memory=true,statistics=(all),log=(enabled=false),eviction_dirty_trigger=5,eviction_updates_trigger=5' + return config + + def test_rollback_to_stable(self): + nrows = 10000 + + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + + # Create a table without logging. + uri = "table:rollback_to_stable18" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + ds.populate() + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + + ',stable_timestamp=' + timestamp_str(10)) + + value_a = "aaaaa" * 100 + + # Perform several updates. + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + + # Perform several removes. + self.large_removes(uri, ds, nrows, self.prepare, 30) + + # Verify data is visible and correct. + self.check(value_a, uri, nrows, 20) + self.check(None, uri, 0, 30) + + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + + # Search for the key so we position our cursor on the page that we want to evict. + evict_cursor.set_key(1) + evict_cursor.search() + evict_cursor.reset() + evict_cursor.close() + + # Pin stable and oldest to timestamp 30 if prepare otherwise 20. + if self.prepare: + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30) + + ',stable_timestamp=' + timestamp_str(30)) + else: + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20) + + ',stable_timestamp=' + timestamp_str(20)) + + # Perform rollback to stable. + self.conn.rollback_to_stable() + + # Verify data is not visible. + self.check(value_a, uri, nrows, 30) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + calls = stat_cursor[stat.conn.txn_rts][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + self.assertEqual(calls, 1) + self.assertEqual(upd_aborted, nrows) diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index aba6b8a81b2..d5eed1cbfe8 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import os, wiredtiger, wttest -StorageSource = wiredtiger.StorageSource # easy access to constants +FileSystem = wiredtiger.FileSystem # easy access to constants # test_tiered06.py # Test the local storage source. @@ -64,58 +64,58 @@ class test_tiered06(wttest.WiredTigerTestCase): local = self.get_local_storage_source() os.mkdir("objects") - location = local.ss_location_handle(session, - 'cluster="cluster1",bucket="./objects",auth_token="Secret"') + fs = local.ss_customize_file_system(session, "./objects", "cluster1-", "Secret", None) # The object doesn't exist yet. - self.assertFalse(local.ss_exist(session, location, 'foobar')) + self.assertFalse(fs.fs_exist(session, 'foobar')) - fh = local.ss_open_object(session, location, 'foobar', StorageSource.open_create) + fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_create) outbytes = ('MORE THAN ENOUGH DATA\n'*100000).encode() fh.fh_write(session, 0, outbytes) # The object doesn't even exist now. - self.assertFalse(local.ss_exist(session, location, 'foobar')) + self.assertFalse(fs.fs_exist(session, 'foobar')) # The object exists after close fh.close(session) - self.assertTrue(local.ss_exist(session, location, 'foobar')) + self.assertTrue(fs.fs_exist(session, 'foobar')) - fh = local.ss_open_object(session, location, 'foobar', StorageSource.open_readonly) + fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_readonly) inbytes = bytes(1000000) # An empty buffer with a million zero bytes. fh.fh_read(session, 0, inbytes) # read into the buffer self.assertEquals(outbytes[0:1000000], inbytes) - self.assertEquals(local.ss_size(session, location, 'foobar'), len(outbytes)) + self.assertEquals(fs.fs_size(session, 'foobar'), len(outbytes)) self.assertEquals(fh.fh_size(session), len(outbytes)) fh.close(session) # The fh_lock call doesn't do anything in the local store implementation. - fh = local.ss_open_object(session, location, 'foobar', StorageSource.open_readonly) + fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_readonly) fh.fh_lock(session, True) fh.fh_lock(session, False) fh.close(session) - self.assertEquals(local.ss_location_list(session, location, '', 0), ['foobar']) + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) # Make sure any new object is not in the list until it is closed. - fh = local.ss_open_object(session, location, 'zzz', StorageSource.open_create) - self.assertEquals(local.ss_location_list(session, location, '', 0), ['foobar']) + fh = fs.fs_open_file(session, 'zzz', FileSystem.open_file_type_data, FileSystem.open_create) + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) # Sync merely syncs to the local disk. fh.fh_sync(session) + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) fh.close(session) # zero length - self.assertEquals(sorted(local.ss_location_list(session, location, '', 0)), + self.assertEquals(sorted(fs.fs_directory_list(session, '', '')), ['foobar', 'zzz']) # See that we can remove objects. - local.ss_remove(session, location, 'zzz', 0) - self.assertEquals(local.ss_location_list(session, location, '', 0), ['foobar']) + fs.fs_remove(session, 'zzz', 0) + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) # Flushing doesn't do anything that's visible. - local.ss_flush(session, location, None, '') - self.assertEquals(local.ss_location_list(session, location, '', 0), ['foobar']) + local.ss_flush(session, fs, None, '') + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) - location.close(session) + fs.terminate(session) def test_local_write_read(self): # Write and read to a file non-sequentially. @@ -124,14 +124,13 @@ class test_tiered06(wttest.WiredTigerTestCase): local = self.get_local_storage_source() os.mkdir("objects") - location = local.ss_location_handle(session, - 'cluster="cluster1",bucket="./objects",auth_token="Secret"') + fs = local.ss_customize_file_system(session, "./objects", "cluster1-", "Secret", None) # We call these 4K chunks of data "blocks" for this test, but that doesn't # necessarily relate to WT block sizing. nblocks = 1000 block_size = 4096 - fh = local.ss_open_object(session, location, 'abc', StorageSource.open_create) + fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_create) # blocks filled with 'a', etc. a_block = ('a' * block_size).encode() @@ -153,7 +152,7 @@ class test_tiered06(wttest.WiredTigerTestCase): fh.close(session) in_block = bytes(block_size) - fh = local.ss_open_object(session, location, 'abc', StorageSource.open_readonly) + fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_readonly) # Do some spot checks, reading non-sequentially fh.fh_read(session, 500 * block_size, in_block) # divisible by 2, not 3 @@ -176,21 +175,21 @@ class test_tiered06(wttest.WiredTigerTestCase): self.assertEquals(in_block, a_block) fh.close(session) - def create_in_loc(self, loc, objname): + def create_with_fs(self, fs, fname): session = self.session - fh = self.local.ss_open_object(session, loc, objname, StorageSource.open_create) + fh = fs.fs_open_file(session, fname, FileSystem.open_file_type_data, FileSystem.open_create) fh.fh_write(session, 0, 'some stuff'.encode()) fh.close(session) - def check(self, loc, prefix, limit, expect): + def check(self, fs, prefix, expect): # We don't require any sorted output for location lists, # so we'll sort before comparing.' - got = sorted(self.local.ss_location_list(self.session, loc, prefix, limit)) + got = sorted(fs.fs_directory_list(self.session, '', prefix)) expect = sorted(expect) self.assertEquals(got, expect) def test_local_locations(self): - # Test using various buckets, clusters + # Test using various buckets, hosts session = self.session local = self.conn.get_storage_source('local_store') @@ -200,64 +199,60 @@ class test_tiered06(wttest.WiredTigerTestCase): # Any of the activity that happens in the various locations # should be independent. - location1 = local.ss_location_handle(session, - 'cluster="cluster1",bucket="./objects1",auth_token="k1"') - location2 = local.ss_location_handle(session, - 'cluster="cluster1",bucket="./objects2",auth_token="k2"') - location3 = local.ss_location_handle(session, - 'cluster="cluster2",bucket="./objects1",auth_token="k3"') - location4 = local.ss_location_handle(session, - 'cluster="cluster2",bucket="./objects2",auth_token="k4"') + fs1 = local.ss_customize_file_system(session, "./objects1", "cluster1-", "k1", None) + fs2 = local.ss_customize_file_system(session, "./objects2", "cluster1-", "k2", None) + fs3 = local.ss_customize_file_system(session, "./objects1", "cluster2-", "k3", None) + fs4 = local.ss_customize_file_system(session, "./objects2", "cluster2-", "k4", None) # Create files in the locations with some name overlap - self.create_in_loc(location1, 'alpaca') - self.create_in_loc(location2, 'bear') - self.create_in_loc(location3, 'crab') - self.create_in_loc(location4, 'deer') + self.create_with_fs(fs1, 'alpaca') + self.create_with_fs(fs2, 'bear') + self.create_with_fs(fs3, 'crab') + self.create_with_fs(fs4, 'deer') for a in ['beagle', 'bird', 'bison', 'bat']: - self.create_in_loc(location1, a) + self.create_with_fs(fs1, a) for a in ['bird', 'bison', 'bat', 'badger']: - self.create_in_loc(location2, a) + self.create_with_fs(fs2, a) for a in ['bison', 'bat', 'badger', 'baboon']: - self.create_in_loc(location3, a) + self.create_with_fs(fs3, a) for a in ['bat', 'badger', 'baboon', 'beagle']: - self.create_in_loc(location4, a) + self.create_with_fs(fs4, a) # Make sure we see the expected file names - self.check(location1, '', 0, ['alpaca', 'beagle', 'bird', 'bison', 'bat']) - self.check(location1, 'a', 0, ['alpaca']) - self.check(location1, 'b', 0, ['beagle', 'bird', 'bison', 'bat']) - self.check(location1, 'c', 0, []) - self.check(location1, 'd', 0, []) - - self.check(location2, '', 0, ['bear', 'bird', 'bison', 'bat', 'badger']) - self.check(location2, 'a', 0, []) - self.check(location2, 'b', 0, ['bear', 'bird', 'bison', 'bat', 'badger']) - self.check(location2, 'c', 0, []) - self.check(location2, 'd', 0, []) - - self.check(location3, '', 0, ['crab', 'bison', 'bat', 'badger', 'baboon']) - self.check(location3, 'a', 0, []) - self.check(location3, 'b', 0, ['bison', 'bat', 'badger', 'baboon']) - self.check(location3, 'c', 0, ['crab']) - self.check(location3, 'd', 0, []) - - self.check(location4, '', 0, ['deer', 'bat', 'badger', 'baboon', 'beagle']) - self.check(location4, 'a', 0, []) - self.check(location4, 'b', 0, ['bat', 'badger', 'baboon', 'beagle']) - self.check(location4, 'c', 0, []) - self.check(location4, 'd', 0, ['deer']) + self.check(fs1, '', ['alpaca', 'beagle', 'bird', 'bison', 'bat']) + self.check(fs1, 'a', ['alpaca']) + self.check(fs1, 'b', ['beagle', 'bird', 'bison', 'bat']) + self.check(fs1, 'c', []) + self.check(fs1, 'd', []) + + self.check(fs2, '', ['bear', 'bird', 'bison', 'bat', 'badger']) + self.check(fs2, 'a', []) + self.check(fs2, 'b', ['bear', 'bird', 'bison', 'bat', 'badger']) + self.check(fs2, 'c', []) + self.check(fs2, 'd', []) + + self.check(fs3, '', ['crab', 'bison', 'bat', 'badger', 'baboon']) + self.check(fs3, 'a', []) + self.check(fs3, 'b', ['bison', 'bat', 'badger', 'baboon']) + self.check(fs3, 'c', ['crab']) + self.check(fs3, 'd', []) + + self.check(fs4, '', ['deer', 'bat', 'badger', 'baboon', 'beagle']) + self.check(fs4, 'a', []) + self.check(fs4, 'b', ['bat', 'badger', 'baboon', 'beagle']) + self.check(fs4, 'c', []) + self.check(fs4, 'd', ['deer']) # Flushing doesn't do anything that's visible, but calling it still exercises code paths. # At some point, we'll have statistics we can check. # # For now, we can turn on the verbose config option for the local_store extension to verify. - local.ss_flush(session, location4, None, '') - local.ss_flush(session, location3, 'badger', '') - local.ss_flush(session, location3, 'c', '') # make sure we don't flush prefixes - local.ss_flush(session, location3, 'b', '') # or suffixes - local.ss_flush(session, location3, 'crab', '') - local.ss_flush(session, location3, 'crab', '') # should do nothing + local.ss_flush(session, fs4, None, '') + local.ss_flush(session, fs3, 'badger', '') + local.ss_flush(session, fs3, 'c', '') # make sure we don't flush prefixes + local.ss_flush(session, fs3, 'b', '') # or suffixes + local.ss_flush(session, fs3, 'crab', '') + local.ss_flush(session, fs3, 'crab', '') # should do nothing local.ss_flush(session, None, None, '') # flush everything else local.ss_flush(session, None, None, '') # should do nothing diff --git a/src/third_party/wiredtiger/test/suite/wthooks.py b/src/third_party/wiredtiger/test/suite/wthooks.py new file mode 100755 index 00000000000..56827350e29 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/wthooks.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# [TEST_TAGS] +# ignored_file +# [END_TAGS] +# +# WiredTigerHookManager +# Manage running of hooks +# +from __future__ import print_function + +import os, sys +from importlib import import_module +from abc import ABC, abstractmethod +import wiredtiger + +# Three kinds of hooks available: +HOOK_REPLACE = 1 # replace the call with the hook function +HOOK_NOTIFY = 2 # call the hook function after the function +HOOK_ARGS = 3 # transform the arg list before the call + +# Print to /dev/tty for debugging, since anything extraneous to stdout/stderr will +# cause a test error. +def tty(message): + from wttest import WiredTigerTestCase + WiredTigerTestCase.tty(message) + +################ +# Hooks Overview +# +# Here are some useful terms to know, with some commentary for each. +# +# API functions +# potentially any WiredTiger API functions that a hook creator wishes to modify (like +# Session.rename). In Python most everything is an object. Of course an instance of +# "Session" is an object, but also the "Session" class itself is an object. The Session.rename +# function is also an object (of a certain form that can be called). Also in Python, +# attributes on an object don't have to be "pre-declared", they can be created at any time. +# So it's easy to imagine assigning Session._rename_orig to be (the original value of) +# Session.rename, and then assigning Session.rename to be some other function object, that +# knows how to do something and then perhaps calls Session._rename_orig . This is the +# essence of the hook concept. +# +# Hook Creator: +# A way to attach a set of "behavior modifications" to various API functions. More precisely, +# a hook creator derives from WiredTigerHookCreator and sets up a number of "hook functions", +# that are actions that are done either just before, after, or instead of, an API function. +# A XxxxHookCreator lives in a hook_xxxx.py file. When a HookCreator is loaded, it may be +# given an optional argument. This argument comes from the original python command line. +# For example, "python run.py --hook abc" loads hook_abc.py (where it expects to find a hook). +# "python run.py --hook abc=123" loads hook_abc.py with an argument "123". +# +# Hook Function: +# One function that will be called before, after or instead of, an API function. A hook +# function will be bound to an API function. It is the job of the HookCreator to set up that +# binding. It is possible to have multiple hook functions bound to the same API function. +# A hook function that replaces an API function will have the same args as the function +# it replaces (but there is a trick to give it additional context if needed - +# see session_create_replace in hook_demo.py). + +# For every API function altered, there is one of these objects +# stashed in the <class>._<api_name>_hooks attribute. +class WiredTigerHookInfo(object): + def __init__(self): + self.arg_funcs = [] # The set of hook functions for manipulating arguments + self.notify_funcs = [] # The set of hook functions for manipulating arguments + # At the moment, we can only replace a method once. + # If needed, we can think about removing this restriction. + self.replace_func = None + +# hooked_function - +# A helper function for the hook manager. +def hooked_function(self, orig_func, hook_info_name, *args): + hook_info = getattr(self, hook_info_name) + + notifies = [] + replace_func = None + + # The three kinds of hooks are acted upon at different times. + # Before we call the function, we modify the args as indicated + # by hooks. Then we call the function, possibly with a replacement. + # Finally, we'll call any notify hooks. + # + # We only walk through the hook list once, and process the config + # hooks while we're doing that, and copy any other hooks needed. + for hook_func in hook_info.arg_funcs: + args = hook_func(self, args) + call_func = hook_info.replace_func + if call_func == None: + call_func = orig_func + if self == wiredtiger: + ret = call_func(*args) + else: + ret = call_func(self, *args) + for hook_func in hook_info.notify_funcs: + hook_func(ret, self, *args) + return ret + +# WiredTigerHookManager - +# The hook manager class. There is only one hook manager. It is responsible for finding all the +# HookCreators at the beginning of the run, and calling setup_hooks() for each one, to have it bind +# hook functions to API functions. The hook manager is initialized with a list of hook names. Each +# name is expanded, for example, "demo" causes the hook manager to load hook_demo.py, and to call +# the "initialize" global function in that file. We expect "initialize" to return a list of objects +# (hooks) derived from WiredTigerHook (class defined below). Generally, "initialize" returns a +# single object (setting up some number of "hook functions") but to allow flexibility for different +# sorts of packaging, we allow any number of hooks to be returned. +# +# A hook can set up any number of "hook functions". See hook_demo.py for a sample hook class. +class WiredTigerHookManager(object): + def __init__(self, hooknames = []): + self.hooks = [] + names_seen = [] + for name in hooknames: + # The hooks are indicated as "somename=arg" or simply "somename". + # hook_somename.py will be imported, and initialized with the arg. + # Names must be unique, as we stash some info into extra fields + # on the connection/session/cursor, these are named using the + # unique name of the hook. + if '=' in name: + name,arg = name.split('=', 1) + else: + arg = None + if name in names_seen: + raise Exception(name + ': hook name cannot be used multiple times') + names_seen.append(name) + + modname = 'hook_' + name + try: + imported = import_module(modname) + for hook in imported.initialize(arg): + hook._initialize(name, self) + self.hooks.append(hook) + except: + print('Cannot import hook: ' + name + ', check file ' + modname + '.py') + raise + for hook in self.hooks: + hook.setup_hooks() + + def add_hook(self, clazz, method_name, hook_type, hook_func): + if not hasattr(clazz, method_name): + raise Exception('Cannot find method ' + method_name + ' on class ' + str(clazz)) + + # We need to set up some extra attributes on the Connection class. + # Given that the method name is XXXX, and class is Connection, here's what we're doing: + # orig = wiredtiger.Connection.XXXX + # wiredtiger.Connection._XXXX_hooks = WiredTigerHookInfo() + # wiredtiger.Connection._XXXX_orig = wiredtiger.Connection.XXXX + # wiredtiger.Connection.XXXX = lambda self, *args: + # hooked_function(self, orig, '_XXXX_hooks', *args) + hook_info_name = '_' + method_name + '_hooks' + orig_name = '_' + method_name + '_orig' + if not hasattr(clazz, hook_info_name): + #tty('Setting up hook on ' + str(clazz) + '.' + method_name) + orig_func = getattr(clazz, method_name) + if orig_func == None: + raise Exception('method ' + method_name + ' hook setup: method does not exist') + setattr(clazz, hook_info_name, WiredTigerHookInfo()) + + # If we're using the wiredtiger module and not a class, we need a slightly different + # style of hooked_function, since there is no self. What would be the "self" argument + # is in fact the class. + if clazz == wiredtiger: + f = lambda *args: hooked_function(wiredtiger, orig_func, hook_info_name, *args) + else: + f = lambda self, *args: hooked_function(self, orig_func, hook_info_name, *args) + setattr(clazz, method_name, f) + setattr(clazz, orig_name, orig_func) + + # Now add to the list of hook functions + # If it's a replace hook, we only allow one of them for a given method name + hook_info = getattr(clazz, hook_info_name) + if hook_type == HOOK_ARGS: + hook_info.arg_funcs.append(hook_func) + elif hook_type == HOOK_NOTIFY: + hook_info.notify_funcs.append(hook_func) + elif hook_type == HOOK_REPLACE: + if hook_info.replace_func == None: + hook_info.replace_func = hook_func + else: + raise Exception('method ' + method_name + ' hook setup: trying to replace the same method with two hooks') + #tty('Setting up hooks list in ' + str(clazz) + '.' + hook_info_name) + + def get_function(self, clazz, method_name): + orig_name = '_' + method_name + '_orig' + if hasattr(clazz, orig_name): + orig_func = getattr(clazz, orig_name) + else: + orig_func = getattr(clazz, method_name) + return orig_func + + def filter_tests(self, tests): + for hook in self.hooks: + tests = hook.filter_tests(tests) + return tests + +class HookCreatorProxy(object): + def __init__(self, hookmgr, clazz): + self.hookmgr = hookmgr + self.clazz = clazz + + # Get the original function/method before any hooks applied + def __getitem__(self, name): + return self.hookmgr.get_function(self.clazz, name) + + # Get the original function/method before any hooks applied + def __setitem__(self, name, value): + try: + hooktype = int(value[0]) + fcn = value[1] + except: + raise ValueError('value must be (HOOK_xxxx, function)') + self.hookmgr.add_hook(self.clazz, name, hooktype, fcn) + +# Hooks must derive from this class +class WiredTigerHookCreator(ABC): + # This is called right after creation and should not be overridden. + def _initialize(self, name, hookmgr): + self.name = name + self.hookmgr = hookmgr + self.wiredtiger = HookCreatorProxy(self.hookmgr, wiredtiger) + self.Connection = HookCreatorProxy(self.hookmgr, wiredtiger.Connection) + self.Session = HookCreatorProxy(self.hookmgr, wiredtiger.Session) + self.Cursor = HookCreatorProxy(self.hookmgr, wiredtiger.Cursor) + + # default version of filter_tests, can be overridden + def filter_tests(self, tests): + return tests + + @abstractmethod + def setup_hooks(self): + """Set up all hooks using add_*_hook methods.""" + return diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index be38c1a748a..87db069fefe 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -43,7 +43,7 @@ except ImportError: from contextlib import contextmanager import errno, glob, os, re, shutil, sys, time, traceback -import wiredtiger, wtscenario +import wiredtiger, wtscenario, wthooks def shortenWithEllipsis(s, maxlen): if len(s) > maxlen: @@ -183,6 +183,7 @@ class ExtensionList(list): class WiredTigerTestCase(unittest.TestCase): _globalSetup = False _printOnceSeen = {} + _ttyDescriptor = None # set this early, to allow tty() to be called any time. # conn_config can be overridden to add to basic connection configuration. # Can be a string or a callable function or lambda expression. @@ -200,14 +201,15 @@ class WiredTigerTestCase(unittest.TestCase): conn_extensions = () @staticmethod - def globalSetup(preserveFiles = False, useTimestamp = False, + def globalSetup(preserveFiles = False, removeAtStart = True, useTimestamp = False, gdbSub = False, lldbSub = False, verbose = 1, builddir = None, dirarg = None, - longtest = False, ignoreStdout = False, seedw = 0, seedz = 0): + longtest = False, ignoreStdout = False, seedw = 0, seedz = 0, hookmgr = None): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg if useTimestamp: d += '.' + time.strftime('%Y%m%d-%H%M%S', time.localtime()) - shutil.rmtree(d, ignore_errors=True) + if removeAtStart: + shutil.rmtree(d, ignore_errors=True) os.makedirs(d) wtscenario.set_long_run(longtest) WiredTigerTestCase._parentTestdir = d @@ -224,9 +226,11 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._stderr = sys.stderr WiredTigerTestCase._concurrent = False WiredTigerTestCase._globalSetup = True - WiredTigerTestCase._ttyDescriptor = None WiredTigerTestCase._seeds = [521288629, 362436069] WiredTigerTestCase._randomseed = False + if hookmgr == None: + hookmgr = wthooks.WiredTigerHookManager() + WiredTigerTestCase._hookmgr = hookmgr if seedw != 0 and seedz != 0: WiredTigerTestCase._randomseed = True WiredTigerTestCase._seeds = [seedw, seedz] |