diff options
author | Luke Chen <luke.chen@mongodb.com> | 2021-02-23 16:48:05 +1100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-02-23 06:15:21 +0000 |
commit | f91e1e9716ec3758bdbbde9cdb81e5582049077d (patch) | |
tree | 06d7587c64fc64adc1b27ad8ec2399db3ce7e61c | |
parent | 39f500cd1d44d504fe1839cf18149a8849c3d2ef (diff) | |
download | mongo-f91e1e9716ec3758bdbbde9cdb81e5582049077d.tar.gz |
Import wiredtiger: ac48a5cfa32a9c8172fa87486d7521567ba4095d from branch mongodb-5.0
ref: 3f4d8ab737..ac48a5cfa3
for: 4.9.0
WT-7088 Define API for shared storage extensions
17 files changed, 1528 insertions, 61 deletions
diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index 7038cb8e4f7..d0707f82f7e 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -17,6 +17,7 @@ ext/encryptors/nop ext/encryptors/rotn ext/extractors/csv ext/test/fail_fs +ext/test/local_store . lang/python PYTHON diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index c42cf702495..ae5fdfb8f06 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1718,6 +1718,7 @@ methods = { 'WT_CONNECTION.add_data_source' : Method([]), 'WT_CONNECTION.add_encryptor' : Method([]), 'WT_CONNECTION.add_extractor' : Method([]), +'WT_CONNECTION.add_storage_source' : Method([]), 'WT_CONNECTION.close' : Method([ Config('leak_memory', 'false', r''' don't free memory during close''', diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 3e0f6af6581..98e39d86683 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -70,6 +70,8 @@ WT_SESSION_LOCKED_TURTLE WT_SINGLE_THREAD_CHECK_START WT_SINGLE_THREAD_CHECK_STOP WT_SIZE_CHECK +WT_SS_OPEN_CREATE +WT_SS_OPEN_READONLY WT_STATS_FIELD_TO_OFFSET WT_STATS_SLOT_ID WT_STAT_CONN_DATA_DECRV diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index c5b3fe73c20..7ecf175a20b 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1268,6 +1268,7 @@ spinlocks sprintf src srch +ss ssize startup statlog diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index dfbd32581e0..fb1519de91e 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -22,6 +22,7 @@ noinst_PROGRAMS = \ ex_pack \ ex_process \ ex_schema \ + ex_storage_source \ ex_smoke \ ex_stat \ ex_sync \ @@ -29,6 +30,7 @@ noinst_PROGRAMS = \ ex_encrypt_LDFLAGS = -rdynamic ex_file_system_LDFLAGS = -rdynamic +ex_storage_source_LDFLAGS = -rdynamic # The examples can be run with no arguments as simple smoke tests TESTS = $(noinst_PROGRAMS) diff --git a/src/third_party/wiredtiger/examples/c/ex_storage_source.c b/src/third_party/wiredtiger/examples/c/ex_storage_source.c new file mode 100644 index 00000000000..f0317dd09a4 --- /dev/null +++ b/src/third_party/wiredtiger/examples/c/ex_storage_source.c @@ -0,0 +1,1168 @@ +/*- + * Public Domain 2014-2020 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_storage_source.c + * demonstrates how to use the custom storage source interface + */ +#include <test_util.h> + +#ifdef __GNUC__ +#if __GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ > 0) +/* + * !!! + * GCC with -Wformat-truncation complains about calls to snprintf in this file. + * There's nothing wrong, this makes the warning go away. + */ +#pragma GCC diagnostic ignored "-Wformat-truncation" +#endif +#endif + +/* + * This example code uses pthread functions for portable locking, we ignore errors for simplicity. + */ +static void +allocate_storage_source_lock(pthread_rwlock_t *lockp) +{ + error_check(pthread_rwlock_init(lockp, NULL)); +} + +static void +destroy_storage_source_lock(pthread_rwlock_t *lockp) +{ + error_check(pthread_rwlock_destroy(lockp)); +} + +static void +lock_storage_source(pthread_rwlock_t *lockp) +{ + error_check(pthread_rwlock_wrlock(lockp)); +} + +static void +unlock_storage_source(pthread_rwlock_t *lockp) +{ + error_check(pthread_rwlock_unlock(lockp)); +} + +/* + * Example storage source implementation, using memory buffers to represent objects. + */ +typedef struct { + WT_STORAGE_SOURCE iface; + + /* + * WiredTiger performs schema and I/O operations in parallel, all storage sources and file + * handle access must be thread-safe. This example uses a single, global storage source lock for + * simplicity; real applications might require finer granularity, for example, a single lock for + * the storage source handle list and per-handle locks serializing I/O. + */ + pthread_rwlock_t lock; /* Lock */ + + int closed_object_count; + int opened_object_count; + int opened_unique_object_count; + int read_ops; + int write_ops; + + /* Queue of file handles */ + TAILQ_HEAD(demo_file_handle_qh, demo_file_handle) fileq; + + WT_EXTENSION_API *wtext; /* Extension functions */ + +} DEMO_STORAGE_SOURCE; + +typedef struct demo_file_handle { + WT_FILE_HANDLE iface; + + /* + * Add custom file handle fields after the interface. + */ + DEMO_STORAGE_SOURCE *demo_ss; /* Enclosing storage source */ + + TAILQ_ENTRY(demo_file_handle) q; /* Queue of handles */ + uint32_t ref; /* Reference count */ + + char *buf; /* In-memory contents */ + size_t bufsize; /* In-memory buffer size */ + + size_t size; /* Read/write data size */ +} DEMO_FILE_HANDLE; + +/* + * Extension initialization function. + */ +#ifdef _WIN32 +/* + * Explicitly export this function so it is visible when loading extensions. + */ +__declspec(dllexport) +#endif + int demo_storage_source_create(WT_CONNECTION *, WT_CONFIG_ARG *); + +/* + * Forward function declarations for storage source API implementation. + */ +static int demo_ss_exist( + WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, bool *); +static int demo_ss_location_handle( + WT_STORAGE_SOURCE *, WT_SESSION *, const char *, WT_LOCATION_HANDLE **); +static int demo_ss_location_handle_free(WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *); +static int demo_ss_location_list(WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, + const char *, uint32_t, char ***, uint32_t *); +static int demo_ss_location_list_free(WT_STORAGE_SOURCE *, WT_SESSION *, char **, uint32_t); +static int demo_ss_open(WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, + uint32_t, WT_FILE_HANDLE **); +static int demo_ss_remove( + WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, uint32_t); +static int demo_ss_size( + WT_STORAGE_SOURCE *, WT_SESSION *, WT_LOCATION_HANDLE *, const char *, wt_off_t *); +static int demo_ss_terminate(WT_STORAGE_SOURCE *, WT_SESSION *); + +/* + * Forward function declarations for file handle API implementation. + */ +static int demo_file_close(WT_FILE_HANDLE *, WT_SESSION *); +static int demo_file_lock(WT_FILE_HANDLE *, WT_SESSION *, bool); +static int demo_file_read(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, void *); +static int demo_file_size(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); +static int demo_file_sync(WT_FILE_HANDLE *, WT_SESSION *); +static int demo_file_truncate(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t); +static int demo_file_write(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, const void *); + +/* + * Forward function declarations for internal functions. + */ +static int demo_handle_remove(WT_SESSION *, DEMO_FILE_HANDLE *); +static DEMO_FILE_HANDLE *demo_handle_search( + WT_STORAGE_SOURCE *, WT_LOCATION_HANDLE *, const char *); + +#define DEMO_FILE_SIZE_INCREMENT 32768 + +/* + * Saved version of the storage source interface for direct testing. + */ +static WT_STORAGE_SOURCE *saved_storage_source; + +/* + * string_match -- + * Return if a string matches a byte string of len bytes. + */ +static bool +byte_string_match(const char *str, const char *bytes, size_t len) +{ + return (strncmp(str, bytes, len) == 0 && (str)[(len)] == '\0'); +} + +/* + * demo_storage_source_create -- + * Initialize the demo storage source. + */ +int +demo_storage_source_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) +{ + DEMO_STORAGE_SOURCE *demo_ss; + WT_CONFIG_ITEM k, v; + WT_CONFIG_PARSER *config_parser; + WT_EXTENSION_API *wtext; + WT_STORAGE_SOURCE *storage_source; + int ret = 0; + + wtext = conn->get_extension_api(conn); + + if ((demo_ss = calloc(1, sizeof(DEMO_STORAGE_SOURCE))) == NULL) { + (void)wtext->err_printf( + wtext, NULL, "demo_storage_source_create: %s", wtext->strerror(wtext, NULL, ENOMEM)); + return (ENOMEM); + } + demo_ss->wtext = wtext; + storage_source = (WT_STORAGE_SOURCE *)demo_ss; + + /* + * Applications may have their own configuration information to pass to the underlying + * filesystem implementation. See the main function for the setup of those configuration + * strings; here we parse configuration information as passed in by main, through WiredTiger. + */ + if ((ret = wtext->config_parser_open_arg(wtext, NULL, config, &config_parser)) != 0) { + (void)wtext->err_printf(wtext, NULL, "WT_EXTENSION_API.config_parser_open: config: %s", + wtext->strerror(wtext, NULL, ret)); + goto err; + } + + /* Step through our configuration values. */ + printf("Custom storage source configuration\n"); + while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { + if (byte_string_match("config_string", k.str, k.len)) { + printf( + "\t" + "key %.*s=\"%.*s\"\n", + (int)k.len, k.str, (int)v.len, v.str); + continue; + } + if (byte_string_match("config_value", k.str, k.len)) { + printf( + "\t" + "key %.*s=%" PRId64 "\n", + (int)k.len, k.str, v.val); + continue; + } + ret = EINVAL; + (void)wtext->err_printf(wtext, NULL, + "WT_CONFIG_PARSER.next: unexpected configuration " + "information: %.*s=%.*s: %s", + (int)k.len, k.str, (int)v.len, v.str, wtext->strerror(wtext, NULL, ret)); + goto err; + } + + /* Check for expected parser termination and close the parser. */ + if (ret != WT_NOTFOUND) { + (void)wtext->err_printf( + wtext, NULL, "WT_CONFIG_PARSER.next: config: %s", wtext->strerror(wtext, NULL, ret)); + goto err; + } + if ((ret = config_parser->close(config_parser)) != 0) { + (void)wtext->err_printf( + wtext, NULL, "WT_CONFIG_PARSER.close: config: %s", wtext->strerror(wtext, NULL, ret)); + goto err; + } + + allocate_storage_source_lock(&demo_ss->lock); + + /* Initialize the in-memory jump table. */ + storage_source->ss_exist = demo_ss_exist; + storage_source->ss_location_handle = demo_ss_location_handle; + storage_source->ss_location_handle_free = demo_ss_location_handle_free; + storage_source->ss_location_list = demo_ss_location_list; + storage_source->ss_location_list_free = demo_ss_location_list_free; + storage_source->ss_open_object = demo_ss_open; + storage_source->ss_remove = demo_ss_remove; + storage_source->ss_size = demo_ss_size; + storage_source->terminate = demo_ss_terminate; + + if ((ret = conn->add_storage_source(conn, "demo", storage_source, NULL)) != 0) { + (void)wtext->err_printf( + wtext, NULL, "WT_CONNECTION.set_storage_source: %s", wtext->strerror(wtext, NULL, ret)); + goto err; + } + + /* + * The WiredTiger API does not have a direct way to use the storage_source API. Save the + * structure so we can call it directly. + */ + saved_storage_source = storage_source; + return (0); + +err: + free(demo_ss); + /* An error installing the storage source is fatal. */ + exit(1); +} + +/* + * demo_ss_open -- + * fopen for our demo storage source. + */ +static int +demo_ss_open(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags, + WT_FILE_HANDLE **file_handlep) +{ + DEMO_FILE_HANDLE *demo_fh; + DEMO_STORAGE_SOURCE *demo_ss; + WT_EXTENSION_API *wtext; + WT_FILE_HANDLE *file_handle; + const char *location; + char *full_name; + size_t name_len; + int ret = 0; + + (void)flags; /* Unused */ + + *file_handlep = NULL; + + demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; + demo_fh = NULL; + wtext = demo_ss->wtext; + + lock_storage_source(&demo_ss->lock); + ++demo_ss->opened_object_count; + + /* + * First search the file queue, if we find it, assert there's only a single reference, we only + * support a single handle on any file. + */ + demo_fh = demo_handle_search(storage_source, location_handle, name); + if (demo_fh != NULL) { + if (demo_fh->ref != 0) { + (void)wtext->err_printf(wtext, session, "demo_ss_open: %s: file already open", name); + ret = EBUSY; + goto err; + } + + demo_fh->ref = 1; + *file_handlep = (WT_FILE_HANDLE *)demo_fh; + unlock_storage_source(&demo_ss->lock); + return (0); + } + + /* The file hasn't been opened before, create a new one. */ + if ((demo_fh = calloc(1, sizeof(DEMO_FILE_HANDLE))) == NULL) { + ret = ENOMEM; + goto err; + } + + /* Initialize private information. */ + demo_fh->demo_ss = demo_ss; + demo_fh->ref = 1; + if ((demo_fh->buf = calloc(1, DEMO_FILE_SIZE_INCREMENT)) == NULL) { + ret = ENOMEM; + goto err; + } + demo_fh->bufsize = DEMO_FILE_SIZE_INCREMENT; + demo_fh->size = 0; + + /* Construct the public name. */ + location = (const char *)location_handle; + name_len = strlen(location) + strlen(name) + 1; + full_name = calloc(1, name_len); + if (snprintf(full_name, name_len, "%s%s", location, name) != (ssize_t)(name_len - 1)) { + ret = ENOMEM; + goto err; + } + + /* Initialize public information. */ + file_handle = (WT_FILE_HANDLE *)demo_fh; + file_handle->name = full_name; + + /* + * Setup the function call table for our custom storage source. Set the function pointer to NULL + * where our implementation doesn't support the functionality. + */ + file_handle->close = demo_file_close; + file_handle->fh_advise = NULL; + file_handle->fh_extend = NULL; + file_handle->fh_extend_nolock = NULL; + file_handle->fh_lock = demo_file_lock; + file_handle->fh_map = NULL; + file_handle->fh_map_discard = NULL; + file_handle->fh_map_preload = NULL; + file_handle->fh_read = demo_file_read; + file_handle->fh_size = demo_file_size; + file_handle->fh_sync = demo_file_sync; + file_handle->fh_sync_nowait = NULL; + file_handle->fh_truncate = demo_file_truncate; + file_handle->fh_unmap = NULL; + file_handle->fh_write = demo_file_write; + + TAILQ_INSERT_HEAD(&demo_ss->fileq, demo_fh, q); + ++demo_ss->opened_unique_object_count; + + *file_handlep = file_handle; + + if (0) { +err: + free(demo_fh->buf); + free(demo_fh); + } + + unlock_storage_source(&demo_ss->lock); + return (ret); +} + +/* + * demo_ss_location_handle -- + * Return a location handle from a location string. + */ +static int +demo_ss_location_handle(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + const char *location_info, WT_LOCATION_HANDLE **location_handlep) +{ + size_t len; + char *p; + + (void)storage_source; /* Unused */ + (void)session; /* Unused */ + + /* + * Our "handle" is nothing more than the location string followed by a slash delimiter. We won't + * allow slashes in the location info parameter. + */ + if (strchr(location_info, '/') != NULL) + return (EINVAL); + len = strlen(location_info) + 2; + p = malloc(len); + if (snprintf(p, len, "%s/", location_info) != (ssize_t)(len - 1)) { + free(p); + return (ENOMEM); + } + *location_handlep = (WT_LOCATION_HANDLE *)p; + return (0); +} + +/* + * demo_ss_location_handle_free -- + * Free a location handle created by ss_location_handle. + */ +static int +demo_ss_location_handle_free( + WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, WT_LOCATION_HANDLE *location_handle) +{ + (void)storage_source; /* Unused */ + (void)session; /* Unused */ + + free(location_handle); + return (0); +} + +/* + * demo_ss_location_list -- + * Return a list of object names for the given location. + */ +static int +demo_ss_location_list(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_LOCATION_HANDLE *location_handle, const char *prefix, uint32_t limit, char ***dirlistp, + uint32_t *countp) +{ + DEMO_FILE_HANDLE *demo_fh; + DEMO_STORAGE_SOURCE *demo_ss; + size_t location_len, prefix_len; + uint32_t allocated, count; + int ret = 0; + const char *location; + char **entries, *name; + void *p; + + (void)session; /* Unused */ + + demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; + + *dirlistp = NULL; + *countp = 0; + + entries = NULL; + allocated = count = 0; + location = (const char *)location_handle; + location_len = strlen(location); + prefix_len = (prefix == NULL ? 0 : strlen(prefix)); + + lock_storage_source(&demo_ss->lock); + TAILQ_FOREACH (demo_fh, &demo_ss->fileq, q) { + name = demo_fh->iface.name; + if (strncmp(name, location, location_len) != 0) + continue; + name += location_len; + if (prefix != NULL && strncmp(name, prefix, prefix_len) != 0) + continue; + + /* + * Increase the list size in groups of 10, it doesn't matter if the list is a bit longer + * than necessary. + */ + if (count >= allocated) { + p = realloc(entries, (allocated + 10) * sizeof(*entries)); + if (p == NULL) { + ret = ENOMEM; + goto err; + } + + entries = p; + memset(entries + allocated * sizeof(*entries), 0, 10 * sizeof(*entries)); + allocated += 10; + } + entries[count++] = strdup(name); + if (limit > 0 && count >= limit) + break; + } + + *dirlistp = entries; + *countp = count; + +err: + unlock_storage_source(&demo_ss->lock); + if (ret == 0) + return (0); + + if (entries != NULL) { + while (count > 0) + free(entries[--count]); + free(entries); + } + + return (ret); +} + +/* + * demo_ss_location_list_free -- + * Free memory allocated by demo_ss_location_list. + */ +static int +demo_ss_location_list_free( + WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, char **dirlist, uint32_t count) +{ + (void)storage_source; + (void)session; + + if (dirlist != NULL) { + while (count > 0) + free(dirlist[--count]); + free(dirlist); + } + return (0); +} + +/* + * demo_ss_exist -- + * Return if the file exists. + */ +static int +demo_ss_exist(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_LOCATION_HANDLE *location_handle, const char *name, bool *existp) +{ + DEMO_STORAGE_SOURCE *demo_ss; + + (void)session; /* Unused */ + + demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; + + lock_storage_source(&demo_ss->lock); + *existp = demo_handle_search(storage_source, location_handle, name) != NULL; + unlock_storage_source(&demo_ss->lock); + + return (0); +} + +/* + * demo_ss_remove -- + * POSIX remove. + */ +static int +demo_ss_remove(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags) +{ + DEMO_STORAGE_SOURCE *demo_ss; + DEMO_FILE_HANDLE *demo_fh; + int ret = 0; + + (void)session; /* Unused */ + (void)flags; /* Unused */ + + demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; + + ret = ENOENT; + lock_storage_source(&demo_ss->lock); + if ((demo_fh = demo_handle_search(storage_source, location_handle, name)) != NULL) + ret = demo_handle_remove(session, demo_fh); + unlock_storage_source(&demo_ss->lock); + + return (ret); +} + +/* + * demo_ss_size -- + * Get the size of a file in bytes, by file name. + */ +static int +demo_ss_size(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_LOCATION_HANDLE *location_handle, const char *name, wt_off_t *sizep) +{ + DEMO_STORAGE_SOURCE *demo_ss; + DEMO_FILE_HANDLE *demo_fh; + int ret = 0; + + demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; + + ret = ENOENT; + lock_storage_source(&demo_ss->lock); + if ((demo_fh = demo_handle_search(storage_source, location_handle, name)) != NULL) + ret = demo_file_size((WT_FILE_HANDLE *)demo_fh, session, sizep); + unlock_storage_source(&demo_ss->lock); + + return (ret); +} + +/* + * demo_ss_terminate -- + * Discard any resources on termination. + */ +static int +demo_ss_terminate(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session) +{ + DEMO_FILE_HANDLE *demo_fh, *demo_fh_tmp; + DEMO_STORAGE_SOURCE *demo_ss; + int ret = 0, tret; + + demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; + + TAILQ_FOREACH_SAFE(demo_fh, &demo_ss->fileq, q, demo_fh_tmp) + if ((tret = demo_handle_remove(session, demo_fh)) != 0 && ret == 0) + ret = tret; + + printf("Custom storage source\n"); + printf("\t%d unique object opens\n", demo_ss->opened_unique_object_count); + printf("\t%d objects opened\n", demo_ss->opened_object_count); + printf("\t%d objects closed\n", demo_ss->closed_object_count); + printf("\t%d reads, %d writes\n", demo_ss->read_ops, demo_ss->write_ops); + + destroy_storage_source_lock(&demo_ss->lock); + free(demo_ss); + + return (ret); +} + +/* + * demo_file_close -- + * ANSI C close. + */ +static int +demo_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session) +{ + DEMO_FILE_HANDLE *demo_fh; + DEMO_STORAGE_SOURCE *demo_ss; + + (void)session; /* Unused */ + + demo_fh = (DEMO_FILE_HANDLE *)file_handle; + demo_ss = demo_fh->demo_ss; + + lock_storage_source(&demo_ss->lock); + if (--demo_fh->ref == 0) + ++demo_ss->closed_object_count; + unlock_storage_source(&demo_ss->lock); + + return (0); +} + +/* + * demo_file_lock -- + * Lock/unlock a file. + */ +static int +demo_file_lock(WT_FILE_HANDLE *file_handle, WT_SESSION *session, bool lock) +{ + /* Locks are always granted. */ + (void)file_handle; /* Unused */ + (void)session; /* Unused */ + (void)lock; /* Unused */ + return (0); +} + +/* + * demo_file_read -- + * POSIX pread. + */ +static int +demo_file_read( + WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, void *buf) +{ + DEMO_FILE_HANDLE *demo_fh; + DEMO_STORAGE_SOURCE *demo_ss; + WT_EXTENSION_API *wtext; + size_t off; + int ret = 0; + + demo_fh = (DEMO_FILE_HANDLE *)file_handle; + demo_ss = demo_fh->demo_ss; + wtext = demo_ss->wtext; + off = (size_t)offset; + + lock_storage_source(&demo_ss->lock); + ++demo_ss->read_ops; + if (off < demo_fh->size) { + if (len > demo_fh->size - off) + len = demo_fh->size - off; + memcpy(buf, (uint8_t *)demo_fh->buf + off, len); + } else + ret = EIO; /* EOF */ + unlock_storage_source(&demo_ss->lock); + if (ret == 0) + return (0); + + (void)wtext->err_printf(wtext, session, + "%s: handle-read: failed to read %zu bytes at offset %zu: %s", demo_fh->iface.name, len, off, + wtext->strerror(wtext, NULL, ret)); + return (ret); +} + +/* + * demo_file_size -- + * Get the size of a file in bytes, by file handle. + */ +static int +demo_file_size(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t *sizep) +{ + DEMO_FILE_HANDLE *demo_fh; + DEMO_STORAGE_SOURCE *demo_ss; + + (void)session; /* Unused */ + + demo_fh = (DEMO_FILE_HANDLE *)file_handle; + demo_ss = demo_fh->demo_ss; + + lock_storage_source(&demo_ss->lock); + *sizep = (wt_off_t)demo_fh->size; + unlock_storage_source(&demo_ss->lock); + return (0); +} + +/* + * demo_file_sync -- + * Ensure the content of the file is stable. This is a no-op in our memory backed storage + * source. + */ +static int +demo_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *session) +{ + (void)file_handle; /* Unused */ + (void)session; /* Unused */ + + return (0); +} + +/* + * demo_buffer_resize -- + * Resize the write buffer. + */ +static int +demo_buffer_resize(WT_SESSION *session, DEMO_FILE_HANDLE *demo_fh, wt_off_t offset) +{ + DEMO_STORAGE_SOURCE *demo_ss; + WT_EXTENSION_API *wtext; + size_t off; + void *p; + + demo_ss = demo_fh->demo_ss; + wtext = demo_ss->wtext; + off = (size_t)offset; + + /* Grow the buffer as necessary and clear any new space in the file. */ + if (demo_fh->bufsize >= off) + return (0); + + if ((p = realloc(demo_fh->buf, off)) == NULL) { + (void)wtext->err_printf(wtext, session, "%s: failed to resize buffer", demo_fh->iface.name, + wtext->strerror(wtext, NULL, ENOMEM)); + return (ENOMEM); + } + memset((uint8_t *)p + demo_fh->bufsize, 0, off - demo_fh->bufsize); + demo_fh->buf = p; + demo_fh->bufsize = off; + + return (0); +} + +/* + * demo_file_truncate -- + * POSIX ftruncate. + */ +static int +demo_file_truncate(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset) +{ + DEMO_FILE_HANDLE *demo_fh; + DEMO_STORAGE_SOURCE *demo_ss; + WT_EXTENSION_API *wtext; + + (void)file_handle; /* Unused */ + (void)session; /* Unused */ + (void)offset; /* Unused */ + + demo_fh = (DEMO_FILE_HANDLE *)file_handle; + demo_ss = demo_fh->demo_ss; + wtext = demo_ss->wtext; + + (void)wtext->err_printf(wtext, session, "%s: truncate not supported in storage source", + demo_fh->iface.name, wtext->strerror(wtext, NULL, ENOTSUP)); + return (ENOTSUP); +} + +/* + * demo_file_write -- + * POSIX pwrite. + */ +static int +demo_file_write( + WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, const void *buf) +{ + DEMO_FILE_HANDLE *demo_fh; + DEMO_STORAGE_SOURCE *demo_ss; + WT_EXTENSION_API *wtext; + size_t off; + int ret = 0; + + demo_fh = (DEMO_FILE_HANDLE *)file_handle; + demo_ss = demo_fh->demo_ss; + wtext = demo_ss->wtext; + off = (size_t)offset; + + lock_storage_source(&demo_ss->lock); + ++demo_ss->write_ops; + if ((ret = demo_buffer_resize( + session, demo_fh, offset + (wt_off_t)(len + DEMO_FILE_SIZE_INCREMENT))) == 0) { + memcpy((uint8_t *)demo_fh->buf + off, buf, len); + if (off + len > demo_fh->size) + demo_fh->size = off + len; + } + unlock_storage_source(&demo_ss->lock); + if (ret == 0) + return (0); + + (void)wtext->err_printf(wtext, session, + "%s: handle-write: failed to write %zu bytes at offset %zu: %s", demo_fh->iface.name, len, + off, wtext->strerror(wtext, NULL, ret)); + return (ret); +} + +/* + * demo_handle_remove -- + * Destroy an in-memory file handle. Should only happen on remove or shutdown. + */ +static int +demo_handle_remove(WT_SESSION *session, DEMO_FILE_HANDLE *demo_fh) +{ + DEMO_STORAGE_SOURCE *demo_ss; + WT_EXTENSION_API *wtext; + + demo_ss = demo_fh->demo_ss; + wtext = demo_ss->wtext; + + if (demo_fh->ref != 0) { + (void)wtext->err_printf(wtext, session, "demo_handle_remove: %s: file is currently open", + demo_fh->iface.name, wtext->strerror(wtext, NULL, EBUSY)); + return (EBUSY); + } + + TAILQ_REMOVE(&demo_ss->fileq, demo_fh, q); + + /* Clean up private information. */ + free(demo_fh->buf); + + /* Clean up public information. */ + free(demo_fh->iface.name); + + free(demo_fh); + + return (0); +} + +/* + * demo_handle_search -- + * Return a matching handle, if one exists. + */ +static DEMO_FILE_HANDLE * +demo_handle_search( + WT_STORAGE_SOURCE *storage_source, WT_LOCATION_HANDLE *location_handle, const char *name) +{ + DEMO_FILE_HANDLE *demo_fh; + DEMO_STORAGE_SOURCE *demo_ss; + size_t len; + char *location; + + demo_ss = (DEMO_STORAGE_SOURCE *)storage_source; + location = (char *)location_handle; + len = strlen(location); + + TAILQ_FOREACH (demo_fh, &demo_ss->fileq, q) + if (strncmp(demo_fh->iface.name, location, len) == 0 && + strcmp(&demo_fh->iface.name[len], name) == 0) + break; + return (demo_fh); +} + +static const char *home; + +static int +demo_test_create(WT_STORAGE_SOURCE *ss, WT_SESSION *session, WT_LOCATION_HANDLE *location, + const char *objname, const char *content) +{ + WT_FILE_HANDLE *fh; + const char *op; + size_t len; + int ret, t_ret; + + fh = NULL; + len = strlen(content) + 1; + op = "open"; + if ((ret = ss->ss_open_object(ss, session, location, objname, WT_SS_OPEN_CREATE, &fh)) != 0) + goto err; + op = "write"; + if ((ret = fh->fh_write(fh, session, 0, len, content)) != 0) + goto err; + +err: + if (fh != NULL && (t_ret = fh->close(fh, session)) != 0 && ret == 0) { + op = "close"; + ret = t_ret; + } + if (ret != 0) + fprintf(stderr, "demo failed during %s: %s\n", op, wiredtiger_strerror(ret)); + else + printf("demo succeeded create %s\n", objname); + + return (ret); +} + +static int +demo_test_read(WT_STORAGE_SOURCE *ss, WT_SESSION *session, WT_LOCATION_HANDLE *location, + const char *objname, const char *content) +{ + WT_FILE_HANDLE *fh; + char buf[100]; + const char *op; + size_t len; + wt_off_t size; + int ret, t_ret; + + fh = NULL; + len = strlen(content) + 1; + + /* Set the op string so that on error we know what failed. */ + op = "open"; + if ((ret = ss->ss_open_object(ss, session, location, objname, WT_SS_OPEN_READONLY, &fh)) != 0) + goto err; + op = "size"; + if ((ret = fh->fh_size(fh, session, &size)) != 0) + goto err; + op = "size-compare"; + if ((size_t)size != len || (size_t)size > sizeof(buf)) { + ret = EINVAL; + goto err; + } + op = "read"; + if ((ret = fh->fh_read(fh, session, 0, len, buf)) != 0) + goto err; + op = "read-compare"; + if (strncmp(buf, content, len) != 0) { + ret = EINVAL; + goto err; + } + +err: + if (fh != NULL && (t_ret = fh->close(fh, session)) != 0 && ret == 0) { + op = "close"; + ret = t_ret; + } + if (ret != 0) + fprintf(stderr, "demo failed during %s: %s\n", op, wiredtiger_strerror(ret)); + else + printf("demo succeeded read %s\n", objname); + + return (ret); +} + +static int +demo_test_list(WT_STORAGE_SOURCE *ss, WT_SESSION *session, WT_LOCATION_HANDLE *location, + const char *prefix, uint32_t limit, uint32_t expect) +{ + char **obj_list; + const char *op; + uint32_t i, obj_count; + int ret, t_ret; + + obj_list = NULL; + /* Set the op string so that on error we know what failed. */ + op = "location_list"; + if ((ret = ss->ss_location_list(ss, session, location, prefix, limit, &obj_list, &obj_count)) != + 0) + goto err; + op = "location_list count"; + if (obj_count != expect) { + ret = EINVAL; + goto err; + } + printf("list: %s:\n", (const char *)location); + for (i = 0; i < obj_count; i++) { + printf(" %s\n", obj_list[i]); + } + +err: + if (obj_list != NULL && + (t_ret = ss->ss_location_list_free(ss, session, obj_list, obj_count)) != 0 && ret == 0) { + op = "location_list_free"; + ret = t_ret; + } + if (ret != 0) + fprintf(stderr, "demo failed during %s: %s\n", op, wiredtiger_strerror(ret)); + else + printf("demo succeeded location_list %s\n", (const char *)location); + + return (ret); +} + +static int +demo_test_storage_source(WT_STORAGE_SOURCE *ss, WT_SESSION *session) +{ + WT_LOCATION_HANDLE *location1, *location2; + const char *op; + int ret, t_ret; + bool exist; + + location1 = location2 = NULL; + + /* Create two locations. Set the op string so that on error we know what failed. */ + op = "location_handle"; + if ((ret = ss->ss_location_handle(ss, session, "location-one", &location1)) != 0) + goto err; + if ((ret = ss->ss_location_handle(ss, session, "location-two", &location2)) != 0) + goto err; + + /* + * Create and existence checks. In location-one, create "A". In location-two, create "A", "B", + * "AA". We'll do simple lists of both locations, and a list of location-two with a prefix. + */ + op = "create/exist checks"; + if ((ret = demo_test_create(ss, session, location1, "A", "location-one-A")) != 0) + goto err; + + if ((ret = ss->ss_exist(ss, session, location1, "A", &exist)) != 0) + goto err; + if (!exist) { + fprintf(stderr, "Exist test failed for A\n"); + ret = EINVAL; + goto err; + } + if ((ret = ss->ss_exist(ss, session, location2, "A", &exist)) != 0) + goto err; + if (exist) { + fprintf(stderr, "Exist test failed for A in location2\n"); + ret = EINVAL; + goto err; + } + + if ((ret = demo_test_create(ss, session, location2, "A", "location-two-A")) != 0) + goto err; + if ((ret = demo_test_create(ss, session, location2, "B", "location-two-B")) != 0) + goto err; + if ((ret = demo_test_create(ss, session, location2, "AA", "location-two-AA")) != 0) + goto err; + + /* Make sure the objects contain the expected data. */ + op = "read checks"; + if ((ret = demo_test_read(ss, session, location1, "A", "location-one-A")) != 0) + goto err; + if ((ret = demo_test_read(ss, session, location2, "A", "location-two-A")) != 0) + goto err; + if ((ret = demo_test_read(ss, session, location2, "B", "location-two-B")) != 0) + goto err; + + /* + * List the locations. For location-one, we expect just one object. + */ + op = "list checks"; + if ((ret = demo_test_list(ss, session, location1, NULL, 0, 1)) != 0) + goto err; + + /* + * For location-two, we expect three objects. + */ + if ((ret = demo_test_list(ss, session, location2, NULL, 0, 3)) != 0) + goto err; + + /* + * If we limit the number of objects received to 2, we should only see 2. + */ + if ((ret = demo_test_list(ss, session, location2, NULL, 2, 2)) != 0) + goto err; + + /* + * With a prefix of "A", and no limit, we'll see two objects. + */ + if ((ret = demo_test_list(ss, session, location2, "A", 0, 2)) != 0) + goto err; + + /* + * With a prefix of "A", and a limit of one, we'll see just one object. + */ + if ((ret = demo_test_list(ss, session, location2, "A", 1, 1)) != 0) + goto err; + +err: + if (location1 != NULL && (t_ret = ss->ss_location_handle_free(ss, session, location1)) != 0 && + ret == 0) + ret = t_ret; + if (location2 != NULL && (t_ret = ss->ss_location_handle_free(ss, session, location2)) != 0 && + ret == 0) + ret = t_ret; + if (ret != 0) + fprintf(stderr, "demo failed during %s: %s\n", op, wiredtiger_strerror(ret)); + + return (ret); +} + +int +main(void) +{ + WT_CONNECTION *conn; + const char *open_config; + int ret = 0; + WT_SESSION *session; + + fprintf(stderr, "ex_storage_source: starting\n"); + /* + * Create a clean test directory for this run of the test program if the environment variable + * isn't already set (as is done by make check). + */ + if (getenv("WIREDTIGER_HOME") == NULL) { + home = "WT_HOME"; + ret = system("rm -rf WT_HOME && mkdir WT_HOME"); + } else + home = NULL; + + /*! [WT_STORAGE_SOURCE register] */ + /* + * Setup a configuration string that will load our custom storage source. Use the special local + * extension to indicate that the entry point is in the same executable. Finally, pass in two + * pieces of configuration information to our initialization function as the "config" value. + */ + open_config = + "create,log=(enabled=true),extensions=(local={entry=demo_storage_source_create," + "config={config_string=\"demo-storage-source\",config_value=37}})"; + /* Open a connection to the database, creating it if necessary. */ + if ((ret = wiredtiger_open(home, NULL, open_config, &conn)) != 0) { + fprintf(stderr, "Error connecting to %s: %s\n", home == NULL ? "." : home, + wiredtiger_strerror(ret)); + return (EXIT_FAILURE); + } + /*! [WT_STORAGE_SOURCE register] */ + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { + fprintf(stderr, "WT_CONNECTION.open_session: %s\n", wiredtiger_strerror(ret)); + return (EXIT_FAILURE); + } + /* + * At the moment, the infrastructure within WiredTiger that would use the storage source + * extension does not exist. So call the interface directly as a demonstration. + */ + if ((ret = demo_test_storage_source(saved_storage_source, session)) != 0) { + fprintf(stderr, "storage source test failed: %s\n", wiredtiger_strerror(ret)); + return (EXIT_FAILURE); + } + if ((ret = conn->close(conn, NULL)) != 0) { + fprintf(stderr, "Error closing connection to %s: %s\n", home == NULL ? "." : home, + wiredtiger_strerror(ret)); + return (EXIT_FAILURE); + } + + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 442e4497f19..f79b944384e 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "3f4d8ab73778407360edb6662dbe073897ebf5e1" + "commit": "ac48a5cfa32a9c8172fa87486d7521567ba4095d" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index d4552f9c711..4de8665089e 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -934,6 +934,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "", NULL, 0}, {"WT_CONNECTION.add_compressor", "", NULL, 0}, {"WT_CONNECTION.add_data_source", "", NULL, 0}, {"WT_CONNECTION.add_encryptor", "", NULL, 0}, {"WT_CONNECTION.add_extractor", "", NULL, 0}, + {"WT_CONNECTION.add_storage_source", "", NULL, 0}, {"WT_CONNECTION.close", "leak_memory=false,use_timestamp=true", confchk_WT_CONNECTION_close, 2}, {"WT_CONNECTION.debug_info", "cache=false,cursors=false,handles=false,log=false,sessions=false" diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 420494d985c..4687d85e309 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -664,6 +664,71 @@ __wt_conn_remove_extractor(WT_SESSION_IMPL *session) } /* + * __conn_add_storage_source -- + * WT_CONNECTION->add_storage_source method. + */ +static int +__conn_add_storage_source( + WT_CONNECTION *wt_conn, const char *name, WT_STORAGE_SOURCE *storage_source, const char *config) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_NAMED_STORAGE_SOURCE *nstorage_source; + WT_SESSION_IMPL *session; + + nstorage_source = NULL; + + conn = (WT_CONNECTION_IMPL *)wt_conn; + CONNECTION_API_CALL(conn, session, add_storage_source, config, cfg); + WT_UNUSED(cfg); + + WT_ERR(__wt_calloc_one(session, &nstorage_source)); + WT_ERR(__wt_strdup(session, name, &nstorage_source->name)); + nstorage_source->storage_source = storage_source; + + __wt_spin_lock(session, &conn->api_lock); + TAILQ_INSERT_TAIL(&conn->storagesrcqh, nstorage_source, q); + nstorage_source = NULL; + __wt_spin_unlock(session, &conn->api_lock); + +err: + if (nstorage_source != NULL) { + __wt_free(session, nstorage_source->name); + __wt_free(session, nstorage_source); + } + + API_END_RET_NOTFOUND_MAP(session, ret); +} + +/* + * __wt_conn_remove_storage_source -- + * Remove storage_source added by WT_CONNECTION->add_storage_source, only used internally. + */ +int +__wt_conn_remove_storage_source(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_NAMED_STORAGE_SOURCE *nstorage_source; + + conn = S2C(session); + + while ((nstorage_source = TAILQ_FIRST(&conn->storagesrcqh)) != NULL) { + /* Remove from the connection's list, free memory. */ + TAILQ_REMOVE(&conn->storagesrcqh, nstorage_source, q); + /* Call any termination method. */ + if (nstorage_source->storage_source->terminate != NULL) + WT_TRET(nstorage_source->storage_source->terminate( + nstorage_source->storage_source, (WT_SESSION *)session)); + + __wt_free(session, nstorage_source->name); + __wt_free(session, nstorage_source); + } + + return (ret); +} + +/* * __conn_get_extension_api -- * WT_CONNECTION.get_extension_api method. */ @@ -2313,7 +2378,8 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c __conn_get_home, __conn_configure_method, __conn_is_new, __conn_open_session, __conn_query_timestamp, __conn_set_timestamp, __conn_rollback_to_stable, __conn_load_extension, __conn_add_data_source, __conn_add_collator, __conn_add_compressor, - __conn_add_encryptor, __conn_add_extractor, __conn_set_file_system, __conn_get_extension_api}; + __conn_add_encryptor, __conn_add_extractor, __conn_set_file_system, __conn_add_storage_source, + __conn_get_extension_api}; static const WT_NAME_FLAG file_types[] = {{"checkpoint", WT_DIRECT_IO_CHECKPOINT}, {"data", WT_DIRECT_IO_DATA}, {"log", WT_DIRECT_IO_LOG}, {NULL, 0}}; diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 9668e8a26bc..d208f200ce1 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -19,14 +19,15 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) session = conn->default_session; - TAILQ_INIT(&conn->dhqh); /* Data handle list */ - TAILQ_INIT(&conn->dlhqh); /* Library list */ - TAILQ_INIT(&conn->dsrcqh); /* Data source list */ - TAILQ_INIT(&conn->fhqh); /* File list */ - TAILQ_INIT(&conn->collqh); /* Collator list */ - TAILQ_INIT(&conn->compqh); /* Compressor list */ - TAILQ_INIT(&conn->encryptqh); /* Encryptor list */ - TAILQ_INIT(&conn->extractorqh); /* Extractor list */ + TAILQ_INIT(&conn->dhqh); /* Data handle list */ + TAILQ_INIT(&conn->dlhqh); /* Library list */ + TAILQ_INIT(&conn->dsrcqh); /* Data source list */ + TAILQ_INIT(&conn->fhqh); /* File list */ + TAILQ_INIT(&conn->collqh); /* Collator list */ + TAILQ_INIT(&conn->compqh); /* Compressor list */ + TAILQ_INIT(&conn->encryptqh); /* Encryptor list */ + TAILQ_INIT(&conn->extractorqh); /* Extractor list */ + TAILQ_INIT(&conn->storagesrcqh); /* Storage source list */ TAILQ_INIT(&conn->lsmqh); /* WT_LSM_TREE list */ diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index a2722282a24..1b5880d1d40 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -126,6 +126,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) WT_TRET(__wt_conn_remove_data_source(session)); WT_TRET(__wt_conn_remove_encryptor(session)); WT_TRET(__wt_conn_remove_extractor(session)); + WT_TRET(__wt_conn_remove_storage_source(session)); /* Disconnect from shared cache - must be before cache destroy. */ WT_TRET(__wt_conn_cache_pool_destroy(session)); diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index 1a66e35fa74..80b1fc3b2f6 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -1591,6 +1591,7 @@ PREDEFINED = DOXYGEN \ __wt_lsn:=WT_LSN \ __wt_modify:=WT_MODIFY \ __wt_session:=WT_SESSION \ + __wt_storage_source:=WT_STORAGE_SOURCE \ __wt_txn_notify:=WT_TXN_NOTIFY \ WT_HANDLE_CLOSED(x):=x \ WT_HANDLE_NULLABLE(x):=x diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 8b87654bfb3..1ee86b9713e 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -56,55 +56,56 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_CONNECTION_add_data_source 2 #define WT_CONFIG_ENTRY_WT_CONNECTION_add_encryptor 3 #define WT_CONFIG_ENTRY_WT_CONNECTION_add_extractor 4 -#define WT_CONFIG_ENTRY_WT_CONNECTION_close 5 -#define WT_CONFIG_ENTRY_WT_CONNECTION_debug_info 6 -#define WT_CONFIG_ENTRY_WT_CONNECTION_load_extension 7 -#define WT_CONFIG_ENTRY_WT_CONNECTION_open_session 8 -#define WT_CONFIG_ENTRY_WT_CONNECTION_query_timestamp 9 -#define WT_CONFIG_ENTRY_WT_CONNECTION_reconfigure 10 -#define WT_CONFIG_ENTRY_WT_CONNECTION_rollback_to_stable 11 -#define WT_CONFIG_ENTRY_WT_CONNECTION_set_file_system 12 -#define WT_CONFIG_ENTRY_WT_CONNECTION_set_timestamp 13 -#define WT_CONFIG_ENTRY_WT_CURSOR_close 14 -#define WT_CONFIG_ENTRY_WT_CURSOR_reconfigure 15 -#define WT_CONFIG_ENTRY_WT_SESSION_alter 16 -#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 17 -#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 18 -#define WT_CONFIG_ENTRY_WT_SESSION_close 19 -#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 20 -#define WT_CONFIG_ENTRY_WT_SESSION_compact 21 -#define WT_CONFIG_ENTRY_WT_SESSION_create 22 -#define WT_CONFIG_ENTRY_WT_SESSION_drop 23 -#define WT_CONFIG_ENTRY_WT_SESSION_flush_tier 24 -#define WT_CONFIG_ENTRY_WT_SESSION_join 25 -#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 26 -#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 27 -#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 28 -#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 29 -#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 30 -#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 31 -#define WT_CONFIG_ENTRY_WT_SESSION_rename 32 -#define WT_CONFIG_ENTRY_WT_SESSION_reset 33 -#define WT_CONFIG_ENTRY_WT_SESSION_reset_snapshot 34 -#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 35 -#define WT_CONFIG_ENTRY_WT_SESSION_salvage 36 -#define WT_CONFIG_ENTRY_WT_SESSION_strerror 37 -#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 38 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 39 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 40 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 41 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 42 -#define WT_CONFIG_ENTRY_colgroup_meta 43 -#define WT_CONFIG_ENTRY_file_config 44 -#define WT_CONFIG_ENTRY_file_meta 45 -#define WT_CONFIG_ENTRY_index_meta 46 -#define WT_CONFIG_ENTRY_lsm_meta 47 -#define WT_CONFIG_ENTRY_table_meta 48 -#define WT_CONFIG_ENTRY_tiered_meta 49 -#define WT_CONFIG_ENTRY_wiredtiger_open 50 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 51 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 52 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 53 +#define WT_CONFIG_ENTRY_WT_CONNECTION_add_storage_source 5 +#define WT_CONFIG_ENTRY_WT_CONNECTION_close 6 +#define WT_CONFIG_ENTRY_WT_CONNECTION_debug_info 7 +#define WT_CONFIG_ENTRY_WT_CONNECTION_load_extension 8 +#define WT_CONFIG_ENTRY_WT_CONNECTION_open_session 9 +#define WT_CONFIG_ENTRY_WT_CONNECTION_query_timestamp 10 +#define WT_CONFIG_ENTRY_WT_CONNECTION_reconfigure 11 +#define WT_CONFIG_ENTRY_WT_CONNECTION_rollback_to_stable 12 +#define WT_CONFIG_ENTRY_WT_CONNECTION_set_file_system 13 +#define WT_CONFIG_ENTRY_WT_CONNECTION_set_timestamp 14 +#define WT_CONFIG_ENTRY_WT_CURSOR_close 15 +#define WT_CONFIG_ENTRY_WT_CURSOR_reconfigure 16 +#define WT_CONFIG_ENTRY_WT_SESSION_alter 17 +#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 18 +#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 19 +#define WT_CONFIG_ENTRY_WT_SESSION_close 20 +#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 21 +#define WT_CONFIG_ENTRY_WT_SESSION_compact 22 +#define WT_CONFIG_ENTRY_WT_SESSION_create 23 +#define WT_CONFIG_ENTRY_WT_SESSION_drop 24 +#define WT_CONFIG_ENTRY_WT_SESSION_flush_tier 25 +#define WT_CONFIG_ENTRY_WT_SESSION_join 26 +#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 27 +#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 28 +#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 29 +#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 30 +#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 31 +#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 32 +#define WT_CONFIG_ENTRY_WT_SESSION_rename 33 +#define WT_CONFIG_ENTRY_WT_SESSION_reset 34 +#define WT_CONFIG_ENTRY_WT_SESSION_reset_snapshot 35 +#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 36 +#define WT_CONFIG_ENTRY_WT_SESSION_salvage 37 +#define WT_CONFIG_ENTRY_WT_SESSION_strerror 38 +#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 39 +#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 40 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 41 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 42 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 43 +#define WT_CONFIG_ENTRY_colgroup_meta 44 +#define WT_CONFIG_ENTRY_file_config 45 +#define WT_CONFIG_ENTRY_file_meta 46 +#define WT_CONFIG_ENTRY_index_meta 47 +#define WT_CONFIG_ENTRY_lsm_meta 48 +#define WT_CONFIG_ENTRY_table_meta 49 +#define WT_CONFIG_ENTRY_tiered_meta 50 +#define WT_CONFIG_ENTRY_wiredtiger_open 51 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 52 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 53 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 54 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 1d89514f143..548504c4a1b 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -102,6 +102,17 @@ struct __wt_named_extractor { }; /* + * WT_NAMED_STORAGE_SOURCE -- + * A storage source list entry + */ +struct __wt_named_storage_source { + const char *name; /* Name of storage source */ + WT_STORAGE_SOURCE *storage_source; /* User supplied callbacks */ + /* Linked list of compressors */ + TAILQ_ENTRY(__wt_named_storage_source) q; +}; + +/* * WT_NAME_FLAG -- * Simple structure for name and flag configuration searches */ @@ -455,6 +466,9 @@ struct __wt_connection_impl { /* Locked: extractor list */ TAILQ_HEAD(__wt_extractor_qh, __wt_named_extractor) extractorqh; + /* Locked: storage source list */ + TAILQ_HEAD(__wt_storage_source_qh, __wt_named_storage_source) storagesrcqh; + void *lang_private; /* Language specific private storage */ /* If non-zero, all buffers used for I/O will be aligned to this. */ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index c7490d643b6..bf1505b2229 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -456,6 +456,8 @@ extern int __wt_conn_remove_encryptor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_remove_extractor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_remove_storage_source(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_statistics_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_connection_close(WT_CONNECTION_IMPL *conn) diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index efcde12d1db..d041c231680 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -78,6 +78,11 @@ struct __wt_file_system; typedef struct __wt_file_system WT_FILE_SYSTEM; struct __wt_item; typedef struct __wt_item WT_ITEM; struct __wt_modify; typedef struct __wt_modify WT_MODIFY; struct __wt_session; typedef struct __wt_session WT_SESSION; +#if !defined(SWIG) +#if !defined(DOXYGEN) +struct __wt_storage_source; typedef struct __wt_storage_source WT_STORAGE_SOURCE; +#endif +#endif #if defined(SWIGJAVA) #define WT_HANDLE_NULLABLE(typename) typename##_NULLABLE @@ -2560,6 +2565,27 @@ struct __wt_connection { int __F(set_file_system)( WT_CONNECTION *connection, WT_FILE_SYSTEM *fs, const char *config); +#if !defined(SWIG) +#if !defined(DOXYGEN) + /*! + * Add a storage source implementation. + * + * The application must first implement the WT_STORAGE_SOURCE + * interface and then register the implementation with WiredTiger: + * + * @snippet ex_storage_source.c WT_STORAGE_SOURCE register + * + * @param connection the connection handle + * @param name the name of the storage source implementation + * @param storage_source the populated storage source structure + * @configempty{WT_CONNECTION.add_storage_source, see dist/api_data.py} + * @errors + */ + int __F(add_storage_source)(WT_CONNECTION *connection, const char *name, + WT_STORAGE_SOURCE *storage_source, const char *config); +#endif +#endif + /*! * Return a reference to the WiredTiger extension functions. * @@ -4230,7 +4256,7 @@ typedef enum { * the implementation to protect any shared data. * * Applications register implementations with WiredTiger by calling - * WT_CONNECTION::add_file_system. See @ref custom_file_systems for more + * WT_CONNECTION::set_file_system. See @ref custom_file_systems for more * information. * * @snippet ex_file_system.c WT_FILE_SYSTEM register @@ -4336,7 +4362,6 @@ struct __wt_file_system { * @param file_system the WT_FILE_SYSTEM * @param session the current WiredTiger session * @param name the name of the file system object - * @param durable if the operation requires durability * @param flags 0 or ::WT_FS_DURABLE */ int (*fs_remove)(WT_FILE_SYSTEM *file_system, @@ -4672,6 +4697,184 @@ struct __wt_file_handle { }; #endif /* !defined(SWIG) */ +#if !defined(SWIG) +#if !defined(DOXYGEN) +/* This interface is not yet public. */ + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_SS_OPEN_CREATE 0x1u +#define WT_SS_OPEN_READONLY 0x2u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + +/*! + * A location handle, and its encoding is defined by each implementation + * of the WT_STORAGE_SOURCE interface. + */ +typedef struct __wt_location_handle WT_LOCATION_HANDLE; + +/*! + * The interface implemented by applications to provide a storage source + * implementation. This documentation refers to "object" and "bucket" + * to mean a "file-like object" and a "container of objects", respectively. + * + * <b>Thread safety:</b> WiredTiger may invoke methods on the WT_STORAGE_SOURCE + * interface from multiple threads concurrently. It is the responsibility of + * the implementation to protect any shared data. + * + * Applications register implementations with WiredTiger by calling + * WT_CONNECTION::add_storage_source. + * + * @snippet ex_storage_source.c WT_STORAGE_SOURCE register + */ +struct __wt_storage_source { + /*! + * Return a location handle from a location string. + * A location string may encode a bucket name, or the equivalent for this + * storage source, authorization information for that bucket, + * naming prefixes to be used for objects in that bucket, etc. + * + * @errors + * + * @param storage_source the WT_STORAGE_SOURCE + * @param session the current WiredTiger session + * @param location the location string + * @param[out] location_handle the allocated handle + */ + int (*ss_location_handle)(WT_STORAGE_SOURCE *storage_source, + WT_SESSION *session, const char *location, WT_LOCATION_HANDLE **location_handle); + + /*! + * Free a location handle created by WT_STORAGE_SOURCE::ss_location_handle. + * + * @errors + * + * @param storage_source the WT_STORAGE_SOURCE + * @param session the current WiredTiger session + * @param location_handle the handle to be freed. + */ + int (*ss_location_handle_free)(WT_STORAGE_SOURCE *storage_source, + WT_SESSION *session, WT_LOCATION_HANDLE *location_handle); + + /*! + * Return a list of object names for the given location. + * + * @errors + * + * @param storage_source the WT_STORAGE_SOURCE + * @param session the current WiredTiger session + * @param location_handle the location to list + * @param prefix if not NULL, only files with names matching the prefix + * are returned + * @param limit if not 0, limits the number of objects listed to this number. + * @param[out] object_list the method returns an allocated array of + * individually allocated strings, one for each object in the location. + * @param[out] countp the number of entries returned + */ + int (*ss_location_list)(WT_STORAGE_SOURCE *storage_source, + WT_SESSION *session, WT_LOCATION_HANDLE *location_handle, const char *prefix, + uint32_t limit, char ***object_list, uint32_t *countp); + + /*! + * Free memory allocated by WT_STORAGE_SOURCE::location_list. + * + * @errors + * + * @param storage_source the WT_STORAGE_SOURCE + * @param session the current WiredTiger session + * @param object_list array returned by WT_STORAGE_SOURCE::location_list + * @param count count returned by WT_STORAGE_SOURCE::location_list + */ + int (*ss_location_list_free)(WT_STORAGE_SOURCE *storage_source, + WT_SESSION *session, char **object_list, uint32_t count); + + /*! + * Return if the named object exists in the location. + * + * @errors + * + * @param storage_source the WT_STORAGE_SOURCE + * @param session the current WiredTiger session + * @param location_handle the location to search + * @param name the name of the object + * @param[out] existp If the named storage source object exists + */ + int (*ss_exist)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_LOCATION_HANDLE *location_handle, const char *name, bool *existp); + + /*! + * Open a handle for a named storage source object + * + * The method should return ENOENT if the object is not being created and + * does not exist. + * + * The method should return EACCES if the object cannot be opened given + * permissions by the location. + * + * @errors + * + * @param storage_source the WT_STORAGE_SOURCE + * @param session the current WiredTiger session + * @param location_handle the location where the object will be stored. + * @param name the name of the object within the location. + * @param flags flags indicating how to open the object, exactly one of + * ::WT_SS_OPEN_CREATE, ::WT_SS_OPEN_READONLY. + * @param[out] file_handlep the handle to the newly opened object. Storage + * source implementations must allocate memory for the handle and + * the WT_FILE_HANDLE::name field, and fill in the WT_FILE_HANDLE:: + * fields. Applications wanting to associate private information + * with the WT_FILE_HANDLE:: structure should declare and allocate + * their own structure as a superset of a WT_FILE_HANDLE:: structure. + */ + int (*ss_open_object)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags, + WT_FILE_HANDLE **file_handlep); + + /*! + * Remove a named storage source object + * + * This method is not required if storage source is configured readonly + * and should be set to NULL when not required by the storage source implementation. + * + * @errors + * + * @param storage_source the WT_STORAGE_SOURCE + * @param session the current WiredTiger session + * @param location_handle the location containing the object + * @param name the name of the storage source object + * @param flags must be 0 + */ + int (*ss_remove)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_LOCATION_HANDLE *location_handle, const char *name, uint32_t flags); + + /*! + * Return the size of a named storage source object + * + * @errors + * + * @param storage_source the WT_STORAGE_SOURCE + * @param session the current WiredTiger session + * @param location_handle the location containing the object + * @param name the name of the storage source object + * @param[out] sizep the size of the storage source object + */ + int (*ss_size)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, + WT_LOCATION_HANDLE *location_handle, const char *name, wt_off_t *sizep); + + /*! + * A callback performed when the storage source is closed and will no + * longer be accessed by the WiredTiger database. + * + * This method is not required and should be set to NULL when not + * required by the storage source implementation. + * + * The WT_STORAGE_SOURCE::terminate callback is intended to allow cleanup, + * the handle will not be subsequently accessed by WiredTiger. + */ + int (*terminate)(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session); +}; +#endif +#endif /* !defined(SWIG) */ + /*! * Entry point to an extension, called when the extension is loaded. * diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 91d36e90221..148c5cd0ed5 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -255,6 +255,8 @@ struct __wt_named_encryptor; typedef struct __wt_named_encryptor WT_NAMED_ENCRYPTOR; struct __wt_named_extractor; typedef struct __wt_named_extractor WT_NAMED_EXTRACTOR; +struct __wt_named_storage_source; +typedef struct __wt_named_storage_source WT_NAMED_STORAGE_SOURCE; struct __wt_optrack_header; typedef struct __wt_optrack_header WT_OPTRACK_HEADER; struct __wt_optrack_record; |