diff options
author | Michael Cahill <michael.cahill@mongodb.com> | 2016-11-14 14:09:19 +1100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-11-14 14:09:19 +1100 |
commit | 25f02285b804413361835d89caa4c195b93dcfdc (patch) | |
tree | 1b9eb63a8e84551a07243487c83b6b16764fdc8f | |
parent | 370154159bc7e3b06488cbdf7deb0585baf841b6 (diff) | |
download | mongo-25f02285b804413361835d89caa4c195b93dcfdc.tar.gz |
WT-2962 Allow configuration of builtin extensions. (#3137)
While in the area, fix sending "config={values}" to extensions: just the values should be passed in.
-rw-r--r-- | dist/api_data.py | 5 | ||||
-rw-r--r-- | examples/c/ex_file_system.c | 14 | ||||
-rw-r--r-- | ext/compressors/zlib/zlib_compress.c | 57 | ||||
-rw-r--r-- | ext/compressors/zstd/zstd_compress.c | 37 | ||||
-rw-r--r-- | ext/datasources/helium/helium.c | 10 | ||||
-rw-r--r-- | ext/encryptors/rotn/rotn_encrypt.c | 42 | ||||
-rw-r--r-- | src/config/config.c | 2 | ||||
-rw-r--r-- | src/config/config_def.c | 122 | ||||
-rw-r--r-- | src/config/config_ext.c | 77 | ||||
-rw-r--r-- | src/conn/conn_api.c | 79 | ||||
-rw-r--r-- | src/include/extern.h | 4 | ||||
-rw-r--r-- | src/include/wiredtiger.in | 5 | ||||
-rw-r--r-- | src/include/wiredtiger_ext.h | 36 | ||||
-rw-r--r-- | test/format/config.c | 2 |
14 files changed, 270 insertions, 222 deletions
diff --git a/dist/api_data.py b/dist/api_data.py index 7016be374a8..5a81e8dd080 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -657,6 +657,11 @@ wiredtiger_open_common =\ should be used (4KB on Linux systems when direct I/O is configured, zero elsewhere)''', min='-1', max='1MB'), + Config('builtin_extension_config', '', r''' + A structure where the keys are the names of builtin extensions and the + values are passed to WT_CONNECTION::load_extension as the \c config + parameter (for example, + <code>builtin_extension_config={zlib={compression_level=3}}</code>)'''), Config('checkpoint_sync', 'true', r''' flush files to stable storage when closing or writing checkpoints''', diff --git a/examples/c/ex_file_system.c b/examples/c/ex_file_system.c index 524a5d03f89..f51cad328c3 100644 --- a/examples/c/ex_file_system.c +++ b/examples/c/ex_file_system.c @@ -194,19 +194,9 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) * the underlying filesystem implementation. See the main function for * the setup of those configuration strings; here we parse configuration * information as passed in by main, through WiredTiger. - * - * Retrieve our configuration information, the "config" value. */ - if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) { - (void)wtext->err_printf(wtext, NULL, - "WT_EXTENSION_API.config_get: config: %s", - wtext->strerror(wtext, NULL, ret)); - goto err; - } - - /* Open a WiredTiger parser on the "config" value. */ - if ((ret = wtext->config_parser_open( - wtext, NULL, v.str, v.len, &config_parser)) != 0) { + if ((ret = wtext->config_parser_open_arg( + wtext, NULL, config, &config_parser)) != 0) { (void)wtext->err_printf(wtext, NULL, "WT_EXTENSION_API.config_parser_open: config: %s", wtext->strerror(wtext, NULL, ret)); diff --git a/ext/compressors/zlib/zlib_compress.c b/ext/compressors/zlib/zlib_compress.c index 3665ec48b9a..09a793646e7 100644 --- a/ext/compressors/zlib/zlib_compress.c +++ b/ext/compressors/zlib/zlib_compress.c @@ -483,8 +483,7 @@ static int zlib_init_config( WT_CONNECTION *connection, WT_CONFIG_ARG *config, int *zlib_levelp) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; int ret, zlib_level; @@ -497,49 +496,27 @@ zlib_init_config( * level; review the configuration. */ wt_api = connection->get_extension_api(connection); - if ((ret = - wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_get: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_parser_open: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) - if (strlen("compression_level") == k.len && - strncmp("compression_level", k.str, k.len) == 0) { - /* - * Between 0-9: level: see zlib manual. - */ - zlib_level = (int)v.val; - if (zlib_level < 0 || zlib_level > 9) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zlib configure: " - "unsupported compression level %d", - zlib_level); - return (EINVAL); - } - *zlib_levelp = zlib_level; - continue; + if ((ret = wt_api->config_get( + wt_api, NULL, config, "compression_level", &v)) == 0) { + /* + * Between 0-9: level: see zlib manual. + */ + zlib_level = (int)v.val; + if (zlib_level < 0 || zlib_level > 9) { + (void)wt_api->err_printf(wt_api, NULL, + "zlib_init_config: " + "unsupported compression level %d", + zlib_level); + return (EINVAL); } - if (ret != WT_NOTFOUND) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = config_parser->close(config_parser)) != 0) { + *zlib_levelp = zlib_level; + } else if (ret != WT_NOTFOUND) { (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.close: zlib configure: %s", + "zlib_init_config: %s", wt_api->strerror(wt_api, NULL, ret)); return (ret); } + return (0); } diff --git a/ext/compressors/zstd/zstd_compress.c b/ext/compressors/zstd/zstd_compress.c index a459b01d60a..ea8ec97602f 100644 --- a/ext/compressors/zstd/zstd_compress.c +++ b/ext/compressors/zstd/zstd_compress.c @@ -232,8 +232,7 @@ static int zstd_init_config( WT_CONNECTION *connection, WT_CONFIG_ARG *config, int *compression_levelp) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; int ret; @@ -246,38 +245,16 @@ zstd_init_config( * level; review the configuration. */ wt_api = connection->get_extension_api(connection); - if ((ret = - wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_get: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_parser_open: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) - if (strlen("compression_level") == k.len && - strncmp("compression_level", k.str, k.len) == 0) { - *compression_levelp = (int)v.val; - continue; - } - if (ret != WT_NOTFOUND) { + if ((ret = wt_api->config_get( + wt_api, NULL, config, "compression_level", &v)) == 0) + *compression_levelp = (int)v.val; + else if (ret != WT_NOTFOUND) { (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = config_parser->close(config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.close: zstd configure: %s", + "zstd_init_config: %s", wt_api->strerror(wt_api, NULL, ret)); return (ret); } + return (0); } diff --git a/ext/datasources/helium/helium.c b/ext/datasources/helium/helium.c index 473c569f0cc..dff86bd73ac 100644 --- a/ext/datasources/helium/helium.c +++ b/ext/datasources/helium/helium.c @@ -3380,15 +3380,9 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) goto err; ds->lockinit = 1; - /* Get the configuration string. */ - if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) - EMSG_ERR(wt_api, NULL, ret, - "WT_EXTENSION_API.config_get: config: %s", - wt_api->strerror(wt_api, NULL, ret)); - /* Step through the list of Helium sources, opening each one. */ - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) + if ((ret = wt_api->config_parser_open_arg( + wt_api, NULL, config, &config_parser)) != 0) EMSG_ERR(wt_api, NULL, ret, "WT_EXTENSION_API.config_parser_open: config: %s", wt_api->strerror(wt_api, NULL, ret)); diff --git a/ext/encryptors/rotn/rotn_encrypt.c b/ext/encryptors/rotn/rotn_encrypt.c index 559c8e6e33a..0b905a0540d 100644 --- a/ext/encryptors/rotn/rotn_encrypt.c +++ b/ext/encryptors/rotn/rotn_encrypt.c @@ -76,7 +76,7 @@ typedef struct { u_char *shift_forw; /* Encrypt shift data from secretkey */ u_char *shift_back; /* Decrypt shift data from secretkey */ size_t shift_len; /* Length of shift* byte arrays */ - int force_error; /* Force a decrypt error for testing */ + bool force_error; /* Force a decrypt error for testing */ } ROTN_ENCRYPTOR; /*! [WT_ENCRYPTOR initialization structure] */ @@ -429,43 +429,19 @@ rotn_terminate(WT_ENCRYPTOR *encryptor, WT_SESSION *session) static int rotn_configure(ROTN_ENCRYPTOR *rotn_encryptor, WT_CONFIG_ARG *config) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; /* Extension API */ - int ret, t_ret; + int ret; wt_api = rotn_encryptor->wt_api; /* Get the configuration string. */ - if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_EXTENSION_API.config_get")); - - /* Step through the list of configuration options. */ - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_EXTENSION_API.config_parser_open")); - - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { - if (strncmp("rotn_force_error", k.str, k.len) == 0 && - strlen("rotn_force_error") == k.len) { - rotn_encryptor->force_error = v.val == 0 ? 0 : 1; - continue; - } else { - if ((ret = config_parser->close(config_parser)) != 0) - return (rotn_error(rotn_encryptor, - NULL, ret, "WT_CONFIG_PARSER.close")); - return (rotn_error(rotn_encryptor, NULL, EINVAL, - "unknown config key")); - } - } - if ((t_ret = config_parser->close(config_parser)) != 0) - return (rotn_error(rotn_encryptor, NULL, t_ret, - "WT_CONFIG_PARSER.close")); - if (ret != WT_NOTFOUND) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_CONFIG_PARSER.next")); + if ((ret = wt_api->config_get( + wt_api, NULL, config, "rotn_force_error", &v)) == 0) + rotn_encryptor->force_error = v.val != 0; + else if (ret != WT_NOTFOUND) + return (rotn_error(rotn_encryptor, NULL, EINVAL, + "error parsing config")); return (0); } diff --git a/src/config/config.c b/src/config/config.c index 3416153d160..a47dfe76aec 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -711,7 +711,7 @@ __wt_config_getones(WT_SESSION_IMPL *session, /* * __wt_config_getones_none -- * Get the value for a given string key from a single config string. - * Treat "none" as empty. + * Treat "none" as empty. */ int __wt_config_getones_none(WT_SESSION_IMPL *session, diff --git a/src/config/config_def.c b/src/config/config_def.c index 540cf846e90..35fea16b1a5 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -659,6 +659,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -744,6 +745,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -830,6 +832,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -910,6 +913,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -1228,62 +1232,64 @@ static const WT_CONFIG_ENTRY config_entries[] = { }, { "wiredtiger_open", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,config_base=true,create=false," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,in_memory=false," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "config_base=true,create=false,direct_io=,encryption=(keyid=," + "name=,secretkey=),error_prefix=,eviction=(threads_max=1," + "threads_min=1),eviction_checkpoint_target=5," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95,exclusive=false," + "extensions=,file_extend=,file_manager=(close_handle_minimum=250," + "close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "in_memory=false,log=(archive=true,compressor=,enabled=false," + "file_max=100MB,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," + "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" + ",name=,quota=0,reserve=0,size=500MB),statistics=none," + "statistics_log=(json=false,on_close=false,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "write_through=", - confchk_wiredtiger_open, 39 + confchk_wiredtiger_open, 40 }, { "wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,config_base=true,create=false," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,in_memory=false," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "config_base=true,create=false,direct_io=,encryption=(keyid=," + "name=,secretkey=),error_prefix=,eviction=(threads_max=1," + "threads_min=1),eviction_checkpoint_target=5," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95,exclusive=false," + "extensions=,file_extend=,file_manager=(close_handle_minimum=250," + "close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "in_memory=false,log=(archive=true,compressor=,enabled=false," + "file_max=100MB,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," + "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" + ",name=,quota=0,reserve=0,size=500MB),statistics=none," + "statistics_log=(json=false,on_close=false,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 40 + confchk_wiredtiger_open_all, 41 }, { "wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,direct_io=,encryption=(keyid=,name=" - ",secretkey=),error_prefix=,eviction=(threads_max=1," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,log=(archive=true," - "compressor=,enabled=false,file_max=100MB,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=1,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" + ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," @@ -1291,20 +1297,20 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 34 + confchk_wiredtiger_open_basecfg, 35 }, { "wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,direct_io=,encryption=(keyid=,name=" - ",secretkey=),error_prefix=,eviction=(threads_max=1," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,log=(archive=true," - "compressor=,enabled=false,file_max=100MB,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=1,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" + ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," @@ -1312,7 +1318,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "write_through=", - confchk_wiredtiger_open_usercfg, 33 + confchk_wiredtiger_open_usercfg, 34 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/config/config_ext.c b/src/config/config_ext.c index 56c0018f8c3..88f1390843a 100644 --- a/src/config/config_ext.c +++ b/src/config/config_ext.c @@ -9,22 +9,9 @@ #include "wt_internal.h" /* - * __wt_ext_config_parser_open -- - * WT_EXTENSION_API->config_parser_open implementation - */ -int -__wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) -{ - WT_UNUSED(wt_ext); - return (wiredtiger_config_parser_open( - wt_session, config, len, config_parserp)); -} - -/* * __wt_ext_config_get -- * Given a NULL-terminated list of configuration strings, find the final - * value for a given string key (external API version). + * value for a given string key (external API version). */ int __wt_ext_config_get(WT_EXTENSION_API *wt_api, @@ -43,3 +30,65 @@ __wt_ext_config_get(WT_EXTENSION_API *wt_api, return (WT_NOTFOUND); return (__wt_config_gets(session, cfg, key, cval)); } + +/* + * __wt_ext_config_get_string -- + * Given a configuration string, find the value for a given string key + * (external API version). + */ +int +__wt_ext_config_get_string(WT_EXTENSION_API *wt_api, + WT_SESSION *wt_session, const char *config, const char *key, + WT_CONFIG_ITEM *cval) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *session; + + conn = (WT_CONNECTION_IMPL *)wt_api->conn; + if ((session = (WT_SESSION_IMPL *)wt_session) == NULL) + session = conn->default_session; + + return (__wt_config_getones(session, config, key, cval)); +} + +/* + * __wt_ext_config_parser_open -- + * WT_EXTENSION_API->config_parser_open implementation + */ +int +__wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) +{ + WT_UNUSED(wt_ext); + return (wiredtiger_config_parser_open( + wt_session, config, len, config_parserp)); +} + +/* + * __wt_ext_config_parser_open_arg -- + * WT_EXTENSION_API->config_parser_open_arg implementation + */ +int +__wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, + WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, + WT_CONFIG_PARSER **config_parserp) +{ + const char **cfg, *p; + size_t len; + + WT_UNUSED(wt_ext); + + /* Find the last non-NULL entry in the configuration stack. */ + if ((cfg = (const char **)cfg_arg) == NULL || *cfg == NULL) { + p = NULL; + len = 0; + } else { + while (cfg[1] != NULL) + ++cfg; + p = *cfg; + len = strlen(p); + } + + return (wiredtiger_config_parser_open( + wt_session, p, len, config_parserp)); +} diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 494735faaf0..dcb5fc74921 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -754,8 +754,11 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) conn->extension_api.scr_free = __wt_ext_scr_free; conn->extension_api.collator_config = ext_collator_config; conn->extension_api.collate = ext_collate; - conn->extension_api.config_parser_open = __wt_ext_config_parser_open; conn->extension_api.config_get = __wt_ext_config_get; + conn->extension_api.config_get_string = __wt_ext_config_get_string; + conn->extension_api.config_parser_open = __wt_ext_config_parser_open; + conn->extension_api.config_parser_open_arg = + __wt_ext_config_parser_open_arg; conn->extension_api.metadata_insert = __wt_ext_metadata_insert; conn->extension_api.metadata_remove = __wt_ext_metadata_remove; conn->extension_api.metadata_search = __wt_ext_metadata_search; @@ -787,40 +790,75 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) return (&conn->extension_api); } +/* + * __conn_builtin_init -- + * Initialize and configure a builtin extension. + */ +static int +__conn_builtin_init(WT_CONNECTION_IMPL *conn, const char *name, + int (*extension_init)(WT_CONNECTION *, WT_CONFIG_ARG *), + const char *cfg[]) +{ + WT_CONFIG_ITEM all_configs, cval; + WT_DECL_RET; + WT_SESSION_IMPL *session; + char *config; + const char *ext_cfg[] = { NULL, NULL }; + + session = conn->default_session; + + WT_RET(__wt_config_gets( + session, cfg, "builtin_extension_config", &all_configs)); + WT_CLEAR(cval); + WT_RET_NOTFOUND_OK(__wt_config_subgets( + session, &all_configs, name, &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &config)); + ext_cfg[0] = config; + + ret = extension_init(&conn->iface, (WT_CONFIG_ARG *)ext_cfg); + __wt_free(session, config); + + return (ret); +} + #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - extern int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - extern int snappy_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int snappy_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - extern int zlib_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int zlib_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - extern int zstd_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int zstd_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif /* - * __conn_load_default_extensions -- + * __conn_builtin_extensions -- * Load extensions that are enabled via --with-builtins */ static int -__conn_load_default_extensions(WT_CONNECTION_IMPL *conn) +__conn_builtin_extensions(WT_CONNECTION_IMPL *conn, const char *cfg[]) { - WT_UNUSED(conn); - #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - WT_RET(lz4_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "lz4", lz4_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - WT_RET(snappy_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "snappy", snappy_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - WT_RET(zlib_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "zlib", zlib_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - WT_RET(zstd_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "zstd", zstd_extension_init, cfg)); #endif + + /* Avoid warnings if no builtin extensions are configured. */ + WT_UNUSED(conn); + WT_UNUSED(cfg); + WT_UNUSED(__conn_builtin_init); + return (0); } @@ -837,10 +875,11 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, WT_DLH *dlh; int (*load)(WT_CONNECTION *, WT_CONFIG_ARG *); bool is_local; - const char *init_name, *terminate_name; + const char *ext_config, *init_name, *terminate_name; + const char *ext_cfg[2]; dlh = NULL; - init_name = terminate_name = NULL; + ext_config = init_name = terminate_name = NULL; is_local = strcmp(path, "local") == 0; /* Ensure that the load matches the phase of startup we are in. */ @@ -870,8 +909,14 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, WT_ERR( __wt_dlsym(session, dlh, terminate_name, false, &dlh->terminate)); + WT_CLEAR(cval); + WT_ERR_NOTFOUND_OK(__wt_config_gets(session, cfg, "config", &cval)); + WT_ERR(__wt_strndup(session, cval.str, cval.len, &ext_config)); + ext_cfg[0] = ext_config; + ext_cfg[1] = NULL; + /* Call the load function last, it simplifies error handling. */ - WT_ERR(load(&S2C(session)->iface, (WT_CONFIG_ARG *)cfg)); + WT_ERR(load(&S2C(session)->iface, (WT_CONFIG_ARG *)ext_cfg)); /* Link onto the environment's list of open libraries. */ __wt_spin_lock(session, &S2C(session)->api_lock); @@ -2353,7 +2398,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * everything else to be in place, and the extensions call back into the * library. */ - WT_ERR(__conn_load_default_extensions(conn)); + WT_ERR(__conn_builtin_extensions(conn, cfg)); WT_ERR(__conn_load_extensions(session, cfg, false)); /* diff --git a/src/include/extern.h b/src/include/extern.h index 63eaa5e39a4..ef2e9efa9fd 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -226,8 +226,10 @@ extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const c extern int __wt_conn_config_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); extern const WT_CONFIG_ENTRY *__wt_conn_config_match(const char *method); -extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_config_get(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_get_string(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *config, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_wiredtiger_error(int error); extern int __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_CONFIG_ITEM *cname, WT_CONFIG_ITEM *metadata, WT_COLLATOR **collatorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 296e510d8d4..8da46582924 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -2214,6 +2214,11 @@ struct __wt_connection { * I/O. The default value of -1 indicates a platform-specific alignment value * should be used (4KB on Linux systems when direct I/O is configured\, zero * elsewhere)., an integer between -1 and 1MB; default \c -1.} + * @config{builtin_extension_config, A structure where the keys are the names of + * builtin extensions and the values are passed to WT_CONNECTION::load_extension + * as the \c config parameter (for example\, + * <code>builtin_extension_config={zlib={compression_level=3}}</code>)., a + * string; default empty.} * @config{cache_overhead, assume the heap allocator overhead is the specified * percentage\, and adjust the cache usage by that amount (for example\, if * there is 10GB of data in cache\, a percentage of 10 means WiredTiger treats diff --git a/src/include/wiredtiger_ext.h b/src/include/wiredtiger_ext.h index 3d65cd1fc24..236d4e07e67 100644 --- a/src/include/wiredtiger_ext.h +++ b/src/include/wiredtiger_ext.h @@ -204,25 +204,47 @@ struct __wt_extension_api { WT_COLLATOR *collator, WT_ITEM *first, WT_ITEM *second, int *cmp); /*! - * @copydoc wiredtiger_config_parser_open + * Return the value of a configuration key. + * + * @param wt_api the extension handle + * @param session the session handle (or NULL if none available) + * @param config the configuration information passed to an application + * @param key configuration key string + * @param value the returned value + * @errors + * + * @snippet ex_data_source.c WT_EXTENSION config_get */ - int (*config_parser_open)(WT_EXTENSION_API *wt_api, WT_SESSION *session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + int (*config_get)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + WT_CONFIG_ARG *config, const char *key, WT_CONFIG_ITEM *value); /*! - * Return the value of a configuration string. + * Return the value of a configuration key from a string. * * @param wt_api the extension handle * @param session the session handle (or NULL if none available) + * @param config the configuration string * @param key configuration key string - * @param config the configuration information passed to an application * @param value the returned value * @errors * * @snippet ex_data_source.c WT_EXTENSION config_get */ - int (*config_get)(WT_EXTENSION_API *wt_api, WT_SESSION *session, - WT_CONFIG_ARG *config, const char *key, WT_CONFIG_ITEM *value); + int (*config_get_string)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + const char *config, const char *key, WT_CONFIG_ITEM *value); + + /*! + * @copydoc wiredtiger_config_parser_open + */ + int (*config_parser_open)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + + /*! + * @copydoc wiredtiger_config_parser_open + */ + int (*config_parser_open_arg)(WT_EXTENSION_API *wt_api, + WT_SESSION *session, WT_CONFIG_ARG *config, + WT_CONFIG_PARSER **config_parserp); /*! * Insert a row into the metadata if it does not already exist. diff --git a/test/format/config.c b/test/format/config.c index 839ff5058de..cf922b5db04 100644 --- a/test/format/config.c +++ b/test/format/config.c @@ -301,7 +301,7 @@ config_compression(const char *conf_name) break; #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - case 15: case 16 case 17: /* 15% zstd */ + case 15: case 16: case 17: /* 15% zstd */ cstr = "zstd"; break; #endif |