From 307e63aac7a2f89ea80a7e8a806ab19166697a6e Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 8 Jul 2016 00:42:41 -0400 Subject: WT-2711 Change statistics log configuration options (#2834) No longer support setting the statistics_log path in WT_CONNECTION::reconfigure. No longer support setting a custom name for statistics files, only allow a destination directory. Be more explicit about which logging configuration options are allowed in WT_CONNECTION::reconfigure. The aim of these changes is to avoid situations where applications that embed WiredTiger allow their users to overwrite unexpected files on a file system. This potentially requires an upgrade step for applications that were specifying a non-standard file name component for statistics log file names, it's not backward compatible. (cherry picked from commit 9cc5d0f4b18753c6cd5cfcfda11a4e3fa7505d9e) --- dist/api_data.py | 174 +++++++++++++--------- dist/s_string.ok | 4 + examples/c/ex_all.c | 27 ++-- examples/java/com/wiredtiger/examples/ex_all.java | 22 +-- src/config/config_def.c | 81 +++++----- src/conn/conn_ckpt.c | 56 ++++--- src/conn/conn_log.c | 52 +++++-- src/conn/conn_stat.c | 102 ++++++++----- src/docs/security.dox | 21 ++- src/docs/spell.ok | 3 + src/docs/statistics.dox | 31 ++-- src/docs/upgrading.dox | 12 +- src/include/connection.h | 1 + src/include/wiredtiger.in | 52 ++----- src/os_win/os_path.c | 26 +++- test/suite/test_reconfig01.py | 17 ++- test/suite/test_reconfig02.py | 33 ++-- test/suite/test_stat_log01.py | 16 +- 18 files changed, 436 insertions(+), 294 deletions(-) diff --git a/dist/api_data.py b/dist/api_data.py index 90b1c8378a2..0d995a3e2e2 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -420,40 +420,6 @@ connection_runtime_config = [ interval in seconds at which to check for files that are inactive and close them''', min=1, max=100000), ]), - Config('log', '', r''' - enable logging. Enabling logging uses three sessions from the - configured session_max''', - type='category', subconfig=[ - Config('archive', 'true', r''' - automatically archive unneeded log files''', - type='boolean'), - Config('compressor', 'none', r''' - configure a compressor for log records. Permitted values are - \c "none" or custom compression engine name created with - WT_CONNECTION::add_compressor. If WiredTiger has builtin support - for \c "snappy", \c "lz4" or \c "zlib" compression, these names - are also available. See @ref compression for more information'''), - Config('enabled', 'false', r''' - enable logging subsystem''', - type='boolean'), - Config('file_max', '100MB', r''' - the maximum size of log files''', - min='100KB', max='2GB'), - Config('path', '"."', r''' - the path to a directory into which the log files are written. - If the value is not an absolute path name, the files are created - relative to the database home'''), - Config('prealloc', 'true', r''' - pre-allocate log files.''', - type='boolean'), - Config('recover', 'on', r''' - run recovery or error if recovery needs to run after an - unclean shutdown.''', - choices=['error','on']), - Config('zero_fill', 'false', r''' - manually write zeroes into log files''', - type='boolean'), - ]), Config('lsm_manager', '', r''' configure database wide options for LSM tree management. The LSM manager is started automatically the first time an LSM tree is opened. @@ -525,38 +491,6 @@ connection_runtime_config = [ are logged using the \c statistics_log configuration. See @ref statistics for more information''', type='list', choices=['all', 'fast', 'none', 'clear']), - Config('statistics_log', '', r''' - log any statistics the database is configured to maintain, - to a file. See @ref statistics for more information. Enabling - the statistics log server uses a session from the configured - session_max''', - type='category', subconfig=[ - Config('json', 'false', r''' - encode statistics in JSON format''', - type='boolean'), - Config('on_close', 'false', r'''log statistics on database close''', - type='boolean'), - Config('path', '"WiredTigerStat.%d.%H"', r''' - the pathname to a file into which the log records are written, - may contain ISO C standard strftime conversion specifications. - If the value is not an absolute path name, the file is created - relative to the database home'''), - Config('sources', '', r''' - if non-empty, include statistics for the list of data source - URIs, if they are open at the time of the statistics logging. - The list may include URIs matching a single data source - ("table:mytable"), or a URI matching all data sources of a - particular type ("table:")''', - type='list'), - Config('timestamp', '"%b %d %H:%M:%S"', r''' - a timestamp prepended to each log record, may contain strftime - conversion specifications, when \c json is configured, defaults - to \c "%FT%Y.000Z"'''), - Config('wait', '0', r''' - seconds to wait between each write of the log records; setting - this value above 0 configures statistics logging''', - min='0', max='100000'), - ]), Config('verbose', '', r''' enable messages for various events. Only available if WiredTiger is configured with --enable-verbose. Options are given as a @@ -590,13 +524,113 @@ connection_runtime_config = [ 'write']), ] +# wiredtiger_open and WT_CONNECTION.reconfigure log configurations. +log_configuration_common = [ + Config('archive', 'true', r''' + automatically archive unneeded log files''', + type='boolean'), + Config('prealloc', 'true', r''' + pre-allocate log files.''', + type='boolean'), + Config('zero_fill', 'false', r''' + manually write zeroes into log files''', + type='boolean') +] +connection_reconfigure_log_configuration = [ + Config('log', '', r''' + enable logging. Enabling logging uses three sessions from the + configured session_max''', + type='category', subconfig= + log_configuration_common) +] +wiredtiger_open_log_configuration = [ + Config('log', '', r''' + enable logging. Enabling logging uses three sessions from the + configured session_max''', + type='category', subconfig= + log_configuration_common + [ + Config('enabled', 'false', r''' + enable logging subsystem''', + type='boolean'), + Config('compressor', 'none', r''' + configure a compressor for log records. Permitted values are + \c "none" or custom compression engine name created with + WT_CONNECTION::add_compressor. If WiredTiger has builtin support + for \c "snappy", \c "lz4" or \c "zlib" compression, these names + are also available. See @ref compression for more information'''), + Config('file_max', '100MB', r''' + the maximum size of log files''', + min='100KB', max='2GB'), + Config('path', '"."', r''' + the name of a directory into which log files are written. The + directory must already exist. If the value is not an absolute + path, the path is relative to the database home (see @ref + absolute_path for more information)'''), + Config('recover', 'on', r''' + run recovery or error if recovery needs to run after an + unclean shutdown''', + choices=['error','on']) + ]), +] + +# wiredtiger_open and WT_CONNECTION.reconfigure statistics log configurations. +statistics_log_configuration_common = [ + Config('json', 'false', r''' + encode statistics in JSON format''', + type='boolean'), + Config('on_close', 'false', r'''log statistics on database close''', + type='boolean'), + Config('sources', '', r''' + if non-empty, include statistics for the list of data source + URIs, if they are open at the time of the statistics logging. + The list may include URIs matching a single data source + ("table:mytable"), or a URI matching all data sources of a + particular type ("table:")''', + type='list'), + Config('timestamp', '"%b %d %H:%M:%S"', r''' + a timestamp prepended to each log record, may contain strftime + conversion specifications, when \c json is configured, defaults + to \c "%FT%Y.000Z"'''), + Config('wait', '0', r''' + seconds to wait between each write of the log records; setting + this value above 0 configures statistics logging''', + min='0', max='100000'), +] +connection_reconfigure_statistics_log_configuration = [ + Config('statistics_log', '', r''' + log any statistics the database is configured to maintain, + to a file. See @ref statistics for more information. Enabling + the statistics log server uses a session from the configured + session_max''', + type='category', subconfig= + statistics_log_configuration_common) +] +wiredtiger_open_statistics_log_configuration = [ + Config('statistics_log', '', r''' + log any statistics the database is configured to maintain, + to a file. See @ref statistics for more information. Enabling + the statistics log server uses a session from the configured + session_max''', + type='category', subconfig= + statistics_log_configuration_common + [ + Config('path', '"."', r''' + the name of a directory into which statistics files are written. + The directory must already exist. If the value is not an absolute + path, the path is relative to the database home (see @ref + absolute_path for more information)''') + ]) +] + session_config = [ Config('isolation', 'read-committed', r''' the default isolation level for operations in this session''', choices=['read-uncommitted', 'read-committed', 'snapshot']), ] -wiredtiger_open_common = connection_runtime_config + [ +wiredtiger_open_common =\ + connection_runtime_config +\ + wiredtiger_open_log_configuration +\ + wiredtiger_open_statistics_log_configuration + [ Config('buffer_alignment', '-1', r''' in-memory alignment (in bytes) for buffers used for I/O. The default value of -1 indicates a platform-specific alignment value @@ -1084,7 +1118,11 @@ methods = { don't free memory during close''', type='boolean'), ]), -'WT_CONNECTION.reconfigure' : Method(connection_runtime_config), +'WT_CONNECTION.reconfigure' : Method( + connection_reconfigure_log_configuration +\ + connection_reconfigure_statistics_log_configuration +\ + connection_runtime_config +), 'WT_CONNECTION.set_file_system' : Method([]), 'WT_CONNECTION.load_extension' : Method([ diff --git a/dist/s_string.ok b/dist/s_string.ok index 7966ff2cf2e..85ccf9f95c7 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -326,6 +326,7 @@ UID UIDs UINT ULINE +UNC URI URIs UTF @@ -557,6 +558,7 @@ dequeued der dereference desc +designator dest destSize dev @@ -1065,6 +1067,7 @@ tV tablename tcbench td +tempdir testutil th tid @@ -1091,6 +1094,7 @@ txn txnc txnid txnmin +txt typedef uB uS diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index dd807922c10..e8727df3f60 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1160,34 +1160,27 @@ main(void) if (ret == 0) (void)conn->close(conn, NULL); +#ifdef MIGHT_NOT_RUN + /* + * Don't run this code, statistics logging doesn't yet support tables. + */ /*! [Statistics logging with a table] */ ret = wiredtiger_open(home, NULL, "create, statistics_log=(" - "sources=(\"lsm:table1\",\"lsm:table2\"), wait=5)", + "sources=(\"table:table1\",\"table:table2\"), wait=5)", &conn); /*! [Statistics logging with a table] */ if (ret == 0) (void)conn->close(conn, NULL); - /*! [Statistics logging with all tables] */ - ret = wiredtiger_open(home, NULL, - "create, statistics_log=(sources=(\"lsm:\"), wait=5)", - &conn); - /*! [Statistics logging with all tables] */ - if (ret == 0) - (void)conn->close(conn, NULL); - -#ifdef MIGHT_NOT_RUN /* - * This example code gets run, and a non-existent log file path might - * cause the open to fail. The documentation requires code snippets, - * use #ifdef's to avoid running it. + * Don't run this code, statistics logging doesn't yet support indexes. */ - /*! [Statistics logging with path] */ + /*! [Statistics logging with a source type] */ ret = wiredtiger_open(home, NULL, - "create," - "statistics_log=(wait=120,path=/log/log.%m.%d.%y)", &conn); - /*! [Statistics logging with path] */ + "create, statistics_log=(sources=(\"index:\"), wait=5)", + &conn); + /*! [Statistics logging with a source type] */ if (ret == 0) (void)conn->close(conn, NULL); diff --git a/examples/java/com/wiredtiger/examples/ex_all.java b/examples/java/com/wiredtiger/examples/ex_all.java index 48e85c9fade..83a37e9a6a5 100644 --- a/examples/java/com/wiredtiger/examples/ex_all.java +++ b/examples/java/com/wiredtiger/examples/ex_all.java @@ -988,6 +988,10 @@ allExample() /*! [Statistics logging] */ conn.close(null); + if (false) { // MIGHT_NOT_RUN + /* + * Don't run this code, statistics logging doesn't yet support tables. + */ /*! [Statistics logging with a table] */ conn = wiredtiger.open(home, "create," + @@ -995,23 +999,13 @@ allExample() /*! [Statistics logging with a table] */ conn.close(null); - /*! [Statistics logging with all tables] */ - conn = wiredtiger.open(home, - "create,statistics_log=(sources=(\"table:\"))"); - /*! [Statistics logging with all tables] */ - conn.close(null); - - if (false) { // MIGHT_NOT_RUN /* - * This example code gets run, and a non-existent log file path might - * cause the open to fail. The documentation requires code snippets, - * use if (false) to avoid running it. + * Don't run this code, statistics logging doesn't yet support indexes. */ - /*! [Statistics logging with path] */ + /*! [Statistics logging with a source type] */ conn = wiredtiger.open(home, - "create," + - "statistics_log=(wait=120,path=/log/log.%m.%d.%y)"); - /*! [Statistics logging with path] */ + "create,statistics_log=(sources=(\"index:\"))"); + /*! [Statistics logging with a source type] */ conn.close(null); /* diff --git a/src/config/config_def.c b/src/config/config_def.c index 1b656c5a0aa..352464ee92a 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -67,16 +67,9 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK - confchk_wiredtiger_open_log_subconfigs[] = { + confchk_WT_CONNECTION_reconfigure_log_subconfigs[] = { { "archive", "boolean", NULL, NULL, NULL, 0 }, - { "compressor", "string", NULL, NULL, NULL, 0 }, - { "enabled", "boolean", NULL, NULL, NULL, 0 }, - { "file_max", "int", NULL, "min=100KB,max=2GB", NULL, 0 }, - { "path", "string", NULL, NULL, NULL, 0 }, { "prealloc", "boolean", NULL, NULL, NULL, 0 }, - { "recover", "string", - NULL, "choices=[\"error\",\"on\"]", - NULL, 0 }, { "zero_fill", "boolean", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -99,10 +92,9 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK - confchk_wiredtiger_open_statistics_log_subconfigs[] = { + confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs[] = { { "json", "boolean", NULL, NULL, NULL, 0 }, { "on_close", "boolean", NULL, NULL, NULL, 0 }, - { "path", "string", NULL, NULL, NULL, 0 }, { "sources", "list", NULL, NULL, NULL, 0 }, { "timestamp", "string", NULL, NULL, NULL, 0 }, { "wait", "int", NULL, "min=0,max=100000", NULL, 0 }, @@ -135,7 +127,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { confchk_wiredtiger_open_file_manager_subconfigs, 3 }, { "log", "category", NULL, NULL, - confchk_wiredtiger_open_log_subconfigs, 8 }, + confchk_WT_CONNECTION_reconfigure_log_subconfigs, 3 }, { "lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2 }, @@ -148,7 +140,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, + confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," @@ -607,6 +599,32 @@ static const WT_CONFIG_CHECK { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK + confchk_wiredtiger_open_log_subconfigs[] = { + { "archive", "boolean", NULL, NULL, NULL, 0 }, + { "compressor", "string", NULL, NULL, NULL, 0 }, + { "enabled", "boolean", NULL, NULL, NULL, 0 }, + { "file_max", "int", NULL, "min=100KB,max=2GB", NULL, 0 }, + { "path", "string", NULL, NULL, NULL, 0 }, + { "prealloc", "boolean", NULL, NULL, NULL, 0 }, + { "recover", "string", + NULL, "choices=[\"error\",\"on\"]", + NULL, 0 }, + { "zero_fill", "boolean", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + +static const WT_CONFIG_CHECK + confchk_wiredtiger_open_statistics_log_subconfigs[] = { + { "json", "boolean", NULL, NULL, NULL, 0 }, + { "on_close", "boolean", NULL, NULL, NULL, 0 }, + { "path", "string", NULL, NULL, NULL, 0 }, + { "sources", "list", NULL, NULL, NULL, 0 }, + { "timestamp", "string", NULL, NULL, NULL, 0 }, + { "wait", "int", NULL, "min=0,max=100000", NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_wiredtiger_open_transaction_sync_subconfigs[] = { { "enabled", "boolean", NULL, NULL, NULL, 0 }, @@ -975,12 +993,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "eviction=(threads_max=1,threads_min=1),eviction_dirty_target=80," "eviction_dirty_trigger=95,eviction_target=80,eviction_trigger=95" ",file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),log=(archive=,compressor=,enabled=0," - "file_max=100MB,path=\".\",prealloc=,recover=on,zero_fill=0)," + "close_scan_interval=10),log=(archive=,prealloc=,zero_fill=0)," "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," - "statistics=none,statistics_log=(json=0,on_close=0," - "path=\"WiredTigerStat.%d.%H\",sources=," + "statistics=none,statistics_log=(json=0,on_close=0,sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", confchk_WT_CONNECTION_reconfigure, 18 }, @@ -1181,11 +1197,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0" ",session_max=100,session_scratch_max=2MB," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," - "statistics=none,statistics_log=(json=0,on_close=0," - "path=\"WiredTigerStat.%d.%H\",sources=," - "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" - ",method=fsync),use_environment=,use_environment_priv=0,verbose=," - "write_through=", + "statistics=none,statistics_log=(json=0,on_close=0,path=\".\"," + "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=0,method=fsync),use_environment=," + "use_environment_priv=0,verbose=,write_through=", confchk_wiredtiger_open, 38 }, { "wiredtiger_open_all", @@ -1203,11 +1218,11 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0" ",session_max=100,session_scratch_max=2MB," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," - "statistics=none,statistics_log=(json=0,on_close=0," - "path=\"WiredTigerStat.%d.%H\",sources=," - "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" - ",method=fsync),use_environment=,use_environment_priv=0,verbose=," - "version=(major=0,minor=0),write_through=", + "statistics=none,statistics_log=(json=0,on_close=0,path=\".\"," + "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=0,method=fsync),use_environment=," + "use_environment_priv=0,verbose=,version=(major=0,minor=0)," + "write_through=", confchk_wiredtiger_open_all, 39 }, { "wiredtiger_open_basecfg", @@ -1224,10 +1239,9 @@ static const WT_CONFIG_ENTRY config_entries[] = { "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," - "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),verbose=," - "version=(major=0,minor=0),write_through=", + "statistics_log=(json=0,on_close=0,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" + ",method=fsync),verbose=,version=(major=0,minor=0),write_through=", confchk_wiredtiger_open_basecfg, 33 }, { "wiredtiger_open_usercfg", @@ -1244,10 +1258,9 @@ static const WT_CONFIG_ENTRY config_entries[] = { "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," - "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),verbose=," - "write_through=", + "statistics_log=(json=0,on_close=0,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" + ",method=fsync),verbose=,write_through=", confchk_wiredtiger_open_usercfg, 32 }, { NULL, NULL, NULL, 0 } diff --git a/src/conn/conn_ckpt.c b/src/conn/conn_ckpt.c index a23350a5e46..b1d619dc1e8 100644 --- a/src/conn/conn_ckpt.c +++ b/src/conn/conn_ckpt.c @@ -23,37 +23,16 @@ __ckpt_server_config(WT_SESSION_IMPL *session, const char **cfg, bool *startp) WT_DECL_RET; char *p; + *startp = false; + conn = S2C(session); - /* - * The checkpoint configuration requires a wait time and/or a log - * size -- if one is not set, we're not running at all. - * Checkpoints based on log size also require logging be enabled. - */ WT_RET(__wt_config_gets(session, cfg, "checkpoint.wait", &cval)); conn->ckpt_usecs = (uint64_t)cval.val * WT_MILLION; WT_RET(__wt_config_gets(session, cfg, "checkpoint.log_size", &cval)); conn->ckpt_logsize = (wt_off_t)cval.val; - /* Checkpoints are incompatible with in-memory configuration */ - if (conn->ckpt_usecs != 0 || conn->ckpt_logsize != 0) { - WT_RET(__wt_config_gets(session, cfg, "in_memory", &cval)); - if (cval.val != 0) - WT_RET_MSG(session, EINVAL, - "In memory configuration incompatible with " - "checkpoints"); - } - - __wt_log_written_reset(session); - if ((conn->ckpt_usecs == 0 && conn->ckpt_logsize == 0) || - (conn->ckpt_logsize && conn->ckpt_usecs == 0 && - !FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED))) { - *startp = false; - return (0); - } - *startp = true; - /* * The application can specify a checkpoint name, which we ignore if * it's our default. @@ -72,6 +51,26 @@ __ckpt_server_config(WT_SESSION_IMPL *session, const char **cfg, bool *startp) conn->ckpt_config = p; } + /* + * The checkpoint configuration requires a wait time and/or a log size, + * if neither is set, we're not running at all. Checkpoints based on log + * size also require logging be enabled. + */ + if (conn->ckpt_usecs != 0 || + (conn->ckpt_logsize != 0 && + FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED))) { + /* Checkpoints are incompatible with in-memory configuration */ + WT_ERR(__wt_config_gets(session, cfg, "in_memory", &cval)); + if (cval.val != 0) + WT_ERR_MSG(session, EINVAL, + "checkpoint configuration incompatible with " + "in-memory configuration"); + + __wt_log_written_reset(session); + + *startp = true; + } + err: __wt_scr_free(session, &tmp); return (ret); } @@ -179,7 +178,16 @@ __wt_checkpoint_server_create(WT_SESSION_IMPL *session, const char *cfg[]) conn = S2C(session); start = false; - /* If there is already a server running, shut it down. */ + /* + * Stop any server that is already running. This means that each time + * reconfigure is called we'll bounce the server even if there are no + * configuration changes. This makes our life easier as the underlying + * configuration routine doesn't have to worry about freeing objects + * in the connection structure (it's guaranteed to always start with a + * blank slate), and we don't have to worry about races where a running + * server is reading configuration information that we're updating, and + * it's not expected that reconfiguration will happen a lot. + */ if (conn->ckpt_session != NULL) WT_RET(__wt_checkpoint_server_destroy(session)); diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 1ae370ef2fa..18ed71e4688 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -51,6 +51,25 @@ __logmgr_config( WT_CONNECTION_IMPL *conn; bool enabled; + /* + * A note on reconfiguration: the standard "is this configuration string + * allowed" checks should fail if reconfiguration has invalid strings, + * for example, "log=(enabled)", or "statistics_log=(path=XXX)", because + * the connection reconfiguration method doesn't allow those strings. + * Additionally, the base configuration values during reconfiguration + * are the currently configured values (so we don't revert to default + * values when repeatedly reconfiguring), and configuration processing + * of a currently set value should not change the currently set value. + * + * In this code path, log server reconfiguration does not stop/restart + * the log server, so there's no point in re-evaluating configuration + * strings that cannot be reconfigured, risking bugs in configuration + * setup, and depending on evaluation of currently set values to always + * result in the currently set value. Skip tests for any configuration + * strings which don't make sense during reconfiguration, but don't + * worry about error reporting because it should never happen. + */ + conn = S2C(session); WT_RET(__wt_config_gets(session, cfg, "log.enabled", &cval)); @@ -62,6 +81,8 @@ __logmgr_config( * * If it is off and the user it turning it on, or it is on * and the user is turning it off, return an error. + * + * See above: should never happen. */ if (reconfig && ((enabled && !FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) || @@ -83,6 +104,8 @@ __logmgr_config( * Setup a log path and compression even if logging is disabled in case * we are going to print a log. Only do this on creation. Once a * compressor or log path are set they cannot be changed. + * + * See above: should never happen. */ if (!reconfig) { conn->log_compressor = NULL; @@ -95,6 +118,7 @@ __logmgr_config( WT_RET(__wt_strndup( session, cval.str, cval.len, &conn->log_path)); } + /* We are done if logging isn't enabled. */ if (!*runp) return (0); @@ -103,13 +127,14 @@ __logmgr_config( if (cval.val != 0) FLD_SET(conn->log_flags, WT_CONN_LOG_ARCHIVE); + /* + * The file size cannot be reconfigured. The amount of memory allocated + * to the log slots may be based on the log file size at creation and we + * don't want to re-allocate that memory while running. + * + * See above: should never happen. + */ if (!reconfig) { - /* - * Ignore if the user tries to change the file size. The - * amount of memory allocated to the log slots may be based - * on the log file size at creation and we don't want to - * re-allocate that memory while running. - */ WT_RET(__wt_config_gets(session, cfg, "log.file_max", &cval)); conn->log_file_max = (wt_off_t)cval.val; WT_STAT_FAST_CONN_SET(session, @@ -125,12 +150,17 @@ __logmgr_config( conn->log_prealloc = 1; /* - * Note that it is meaningless to reconfigure this value during - * runtime. It only matters on create before recovery runs. + * Note it's meaningless to reconfigure this value during runtime, it + * only matters on create before recovery runs. + * + * See above: should never happen. */ - WT_RET(__wt_config_gets_def(session, cfg, "log.recover", 0, &cval)); - if (cval.len != 0 && WT_STRING_MATCH("error", cval.str, cval.len)) - FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR); + if (!reconfig) { + WT_RET(__wt_config_gets_def( + session, cfg, "log.recover", 0, &cval)); + if (WT_STRING_MATCH("error", cval.str, cval.len)) + FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR); + } WT_RET(__wt_config_gets(session, cfg, "log.zero_fill", &cval)); if (cval.val != 0) { diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index 855ff57808e..5ca0f1855d9 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -73,20 +73,37 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) WT_CONFIG objectconf; WT_CONFIG_ITEM cval, k, v; WT_CONNECTION_IMPL *conn; + WT_DECL_ITEM(tmp); WT_DECL_RET; int cnt; char **sources; + /* + * A note on reconfiguration: the standard "is this configuration string + * allowed" checks should fail if reconfiguration has invalid strings, + * for example, "log=(enabled)", or "statistics_log=(path=XXX)", because + * the connection reconfiguration method doesn't allow those strings. + * Additionally, the base configuration values during reconfiguration + * are the currently configured values (so we don't revert to default + * values when repeatedly reconfiguring), and configuration processing + * of a currently set value should not change the currently set value. + * + * In this code path, a previous statistics log server reconfiguration + * may have stopped the server (and we're about to restart it). Because + * stopping the server discarded the configured information stored in + * the connection structure, we have to re-evaluate all configuration + * values, reconfiguration can't skip any of them. + */ + conn = S2C(session); sources = NULL; - WT_RET(__wt_config_gets(session, cfg, "statistics_log.wait", &cval)); /* Only start the server if wait time is non-zero */ + WT_RET(__wt_config_gets(session, cfg, "statistics_log.wait", &cval)); *runp = cval.val != 0; conn->stat_usecs = (uint64_t)cval.val * WT_MILLION; - WT_RET(__wt_config_gets( - session, cfg, "statistics_log.json", &cval)); + WT_RET(__wt_config_gets(session, cfg, "statistics_log.json", &cval)); if (cval.val != 0) FLD_SET(conn->stat_flags, WT_CONN_STAT_JSON); @@ -96,24 +113,30 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) FLD_SET(conn->stat_flags, WT_CONN_STAT_ON_CLOSE); /* - * Statistics logging configuration requires either a wait time or an - * on-close setting. - */ - if (!*runp && !FLD_ISSET(conn->stat_flags, WT_CONN_STAT_ON_CLOSE)) - return (0); - - /* - * If any statistics logging is done, this must not be a read-only - * connection. + * We don't allow the log path to be reconfigured for security reasons. + * (Applications passing input strings directly to reconfigure would + * expose themselves to a potential security problem, the utility of + * reconfiguring a statistics log path isn't worth the security risk.) + * + * See above for the details, but during reconfiguration we're loading + * the path value from the saved configuration information, and it's + * required during reconfiguration because we potentially stopped and + * are restarting, the server. */ - WT_RET(__wt_config_gets(session, cfg, "statistics_log.sources", &cval)); - WT_RET(__wt_config_subinit(session, &objectconf, &cval)); + WT_RET(__wt_config_gets(session, cfg, "statistics_log.path", &cval)); + WT_ERR(__wt_scr_alloc(session, 0, &tmp)); + WT_ERR(__wt_buf_fmt(session, + tmp, "%.*s/%s", (int)cval.len, cval.str, WT_STATLOG_FILENAME)); + WT_ERR(__wt_filename(session, tmp->data, &conn->stat_path)); + + WT_ERR(__wt_config_gets(session, cfg, "statistics_log.sources", &cval)); + WT_ERR(__wt_config_subinit(session, &objectconf, &cval)); for (cnt = 0; (ret = __wt_config_next(&objectconf, &k, &v)) == 0; ++cnt) ; - WT_RET_NOTFOUND_OK(ret); + WT_ERR_NOTFOUND_OK(ret); if (cnt != 0) { - WT_RET(__wt_calloc_def(session, cnt + 1, &sources)); - WT_RET(__wt_config_subinit(session, &objectconf, &cval)); + WT_ERR(__wt_calloc_def(session, cnt + 1, &sources)); + WT_ERR(__wt_config_subinit(session, &objectconf, &cval)); for (cnt = 0; (ret = __wt_config_next(&objectconf, &k, &v)) == 0; ++cnt) { /* @@ -138,29 +161,34 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) sources = NULL; } - WT_ERR(__wt_config_gets(session, cfg, "statistics_log.path", &cval)); - WT_ERR(__wt_nfilename(session, cval.str, cval.len, &conn->stat_path)); - /* * When using JSON format, use the same timestamp format as MongoDB by - * default. + * default. This requires caution: the user might have set the timestamp + * in a previous reconfigure call and we don't want to override that, so + * compare the retrieved value with the default value to decide if we + * should use the JSON default. + * + * (This still implies if the user explicitly sets the timestamp to the + * default value, then sets the JSON flag in a separate reconfigure + * call, or vice-versa, we will incorrectly switch to the JSON default + * timestamp. But there's no way to detect that, and this is all a low + * probability path.) */ - if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON)) { - ret = __wt_config_gets( - session, &cfg[1], "statistics_log.timestamp", &cval); - if (ret == WT_NOTFOUND) - WT_ERR(__wt_strdup( - session, "%FT%T.000Z", &conn->stat_format)); - WT_ERR_NOTFOUND_OK(ret); - } - if (conn->stat_format == NULL) { - WT_ERR(__wt_config_gets( - session, cfg, "statistics_log.timestamp", &cval)); +#define WT_TIMESTAMP_DEFAULT "%b %d %H:%M:%S" +#define WT_TIMESTAMP_JSON_DEFAULT "%FT%T.000Z" + WT_ERR(__wt_config_gets( + session, cfg, "statistics_log.timestamp", &cval)); + if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON) && + WT_STRING_MATCH(WT_TIMESTAMP_DEFAULT, cval.str, cval.len)) + WT_ERR(__wt_strdup( + session, WT_TIMESTAMP_JSON_DEFAULT, &conn->stat_format)); + else WT_ERR(__wt_strndup( session, cval.str, cval.len, &conn->stat_format)); - } err: __stat_sources_free(session, &sources); + __wt_scr_free(session, &tmp); + return (ret); } @@ -538,12 +566,16 @@ __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]) bool start; conn = S2C(session); - start = false; /* * Stop any server that is already running. This means that each time * reconfigure is called we'll bounce the server even if there are no - * configuration changes - but that makes our lives easier. + * configuration changes. This makes our life easier as the underlying + * configuration routine doesn't have to worry about freeing objects + * in the connection structure (it's guaranteed to always start with a + * blank slate), and we don't have to worry about races where a running + * server is reading configuration information that we're updating, and + * it's not expected that reconfiguration will happen a lot. */ if (conn->stat_session != NULL) WT_RET(__wt_statlog_destroy(session, false)); diff --git a/src/docs/security.dox b/src/docs/security.dox index 331f74d969b..82e13ae7ad3 100644 --- a/src/docs/security.dox +++ b/src/docs/security.dox @@ -2,10 +2,23 @@ @section directory_permissions Database directory permissions -All WiredTiger files are stored in the database home directory, and the -WiredTiger database directory should have its permissions set to ensure -database objects are not accessible to users without appropriate -permissions. See @ref home for more information. +By default, WiredTiger files are stored beneath the database home directory. +The WiredTiger database directory should have its permissions set to ensure +database objects are not accessible to users without appropriate permissions. +See @ref home for more information. + +@section absolute_path Absolute paths + +WiredTiger prepends the name of the database home to file names which +do not appear to be absolute paths. (The absolute path test is +simplistic, matching a leading slash character on POSIX systems or a +leading alphabetic character and colon on Windows.) No file path +sanitization or validation is done by WiredTiger, for example, file +paths may match universal naming conventions (UNC), or include \c "../" +(dot dot slash) components. + +Applications are responsible for validating user-supplied file paths as +necessary to prevent directory traversal attacks. @section file_permissions File permissions diff --git a/src/docs/spell.ok b/src/docs/spell.ok index 96fe04d7426..e08eb7d1447 100644 --- a/src/docs/spell.ok +++ b/src/docs/spell.ok @@ -80,6 +80,7 @@ SCons Seward's SiH TXT +UNC URIs WILLNEED WiredTiger @@ -368,6 +369,7 @@ php png posix pre +prepends primary's printf printlog @@ -411,6 +413,7 @@ runtime rwlock sHQ sHq +sanitization scalable scanf schemas diff --git a/src/docs/statistics.dox b/src/docs/statistics.dox index 0a29e351e4e..36ce2711dc5 100644 --- a/src/docs/statistics.dox +++ b/src/docs/statistics.dox @@ -90,11 +90,20 @@ cursor. @section statistics_log Statistics logging -WiredTiger will optionally log database statistics into a file when the +WiredTiger will optionally log database statistics into files when the the ::wiredtiger_open \c statistics_log configuration is set. -The resulting statistics can be displayed using the \c wtstats visualization -tool. For more information, see @ref_single wtstats. +The log files are named \c WiredTiger.%%d.%%H, where \c %%d is replaced +with the day of the month as a decimal number (01-31), and \c %%H +is replaced by the hour (24-hour clock) as a decimal number (00-23). +Each log file contains the statistics for the hour specified in its name. + +The location of the log files may be changed with the \c statistics_log.path +configuration string. + +The resulting statistics can be displayed and interactively examined +using the \c wtstats visualization tool. For more information, see +@ref_single wtstats. The following example logs statistics every 30 seconds: @@ -120,7 +129,7 @@ Statistics for all underlying data sources of a particular type may be included by adding a partial data source URI to the \c statistics_log configuration string: -@snippet ex_all.c Statistics logging with all tables +@snippet ex_all.c Statistics logging with a source type When database statistics are logged, the database home will be the first space-separated entry for each record in the log file. For example: @@ -151,23 +160,9 @@ currently open in the database, nor will any statistics requiring the traversal of a tree (as if the \c statistics_fast configuration string were set). -The location of the log files may be changed with the \c statistics_log.path -configuration string. The \c path value value may contain ISO C90 standard -strftime conversion specifications. WiredTiger will not create non-existent -directories in the path, they must exist before ::wiredtiger_open is called. - -The following example logs statistics into files named with the month, -day and year: - -@snippet ex_all.c Statistics logging with path - A Python script that parses the default logging output and uses the gnuplot, utility to generate Portable Network Graphics (PNG) format graphs is included in the WiredTiger distribution in the file \c tools/statlog.py. -@m_if{c} -To interactively examine statistics results, see @ref wtstats. -@m_endif - */ diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox index 5e824fee977..c5fbc0a86a2 100644 --- a/src/docs/upgrading.dox +++ b/src/docs/upgrading.dox @@ -1,5 +1,16 @@ /*! @page upgrading Upgrading WiredTiger applications +@section version_281 Upgrading to Version 2.8.1 +
+
Statistics logging path
+
+The statistics logging path configuration has been simplified to be only a +path to a directory, and the file name component of the path may no longer +be specified. Applications depending on the ability to set statistics log +file names will require modifications. +
+ +
@section version_280 Upgrading to Version 2.8.0
LSM metadata
@@ -55,7 +66,6 @@ The WiredTiger public API used to define a structure that could encapsulate log sequence numbers. That structure is no longer exposed publicly. -
@section version_270 Upgrading to Version 2.7.0 diff --git a/src/include/connection.h b/src/include/connection.h index 0e0c357279a..804498f6869 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -314,6 +314,7 @@ struct __wt_connection_impl { uint32_t evict_workers; /* Number of eviction workers */ WT_EVICT_WORKER *evict_workctx; /* Eviction worker context */ +#define WT_STATLOG_FILENAME "WiredTigerStat.%d.%H" WT_SESSION_IMPL *stat_session; /* Statistics log session */ wt_thread_t stat_tid; /* Statistics log thread */ bool stat_tid_set; /* Statistics log thread set */ diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index f578f4e6c08..daefe09044c 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1836,25 +1836,8 @@ struct __wt_connection { * configuration options defined below.} * @config{    archive, automatically archive * unneeded log files., a boolean flag; default \c true.} - * @config{    compressor, configure a compressor - * for log records. Permitted values are \c "none" or custom - * compression engine name created with WT_CONNECTION::add_compressor. - * If WiredTiger has builtin support for \c "snappy"\, \c "lz4" or \c - * "zlib" compression\, these names are also available. See @ref - * compression for more information., a string; default \c none.} - * @config{    enabled, enable logging subsystem., a - * boolean flag; default \c false.} - * @config{    file_max, the maximum size of log - * files., an integer between 100KB and 2GB; default \c 100MB.} - * @config{    path, the path to a directory into - * which the log files are written. If the value is not an absolute - * path name\, the files are created relative to the database home., a - * string; default \c ".".} - * @config{    prealloc, - * pre-allocate log files., a boolean flag; default \c true.} - * @config{    recover, run recovery or error if - * recovery needs to run after an unclean shutdown., a string\, chosen - * from the following options: \c "error"\, \c "on"; default \c on.} + * @config{    prealloc, pre-allocate log files., a + * boolean flag; default \c true.} * @config{    zero_fill, manually write zeroes into * log files., a boolean flag; default \c false.} * @config{ ),,} @@ -1914,11 +1897,6 @@ struct __wt_connection { * statistics in JSON format., a boolean flag; default \c false.} * @config{    on_close, log statistics on database * close., a boolean flag; default \c false.} - * @config{    path, the pathname to a file into - * which the log records are written\, may contain ISO C standard - * strftime conversion specifications. If the value is not an absolute - * path name\, the file is created relative to the database home., a - * string; default \c "WiredTigerStat.%d.%H".} * @config{    sources, if non-empty\, include * statistics for the list of data source URIs\, if they are open at the * time of the statistics logging. The list may include URIs matching a @@ -2343,9 +2321,10 @@ struct __wt_connection { * subsystem., a boolean flag; default \c false.} * @config{    file_max, the maximum size of log files., an * integer between 100KB and 2GB; default \c 100MB.} - * @config{    path, the path to a directory into which the - * log files are written. If the value is not an absolute path name\, the files - * are created relative to the database home., a string; default \c ".".} + * @config{    path, the name of a directory into which log + * files are written. The directory must already exist. If the value is not an + * absolute path\, the path is relative to the database home (see @ref + * absolute_path for more information)., a string; default \c ".".} * @config{    prealloc, pre-allocate log files., a boolean * flag; default \c true.} * @config{    recover, run recovery @@ -2415,16 +2394,15 @@ struct __wt_connection { * boolean flag; default \c false.} * @config{    on_close, * log statistics on database close., a boolean flag; default \c false.} - * @config{    path, the pathname to a file into which the - * log records are written\, may contain ISO C standard strftime conversion - * specifications. If the value is not an absolute path name\, the file is - * created relative to the database home., a string; default \c - * "WiredTigerStat.%d.%H".} - * @config{    sources, if - * non-empty\, include statistics for the list of data source URIs\, if they are - * open at the time of the statistics logging. The list may include URIs - * matching a single data source ("table:mytable")\, or a URI matching all data - * sources of a particular type ("table:")., a list of strings; default empty.} + * @config{    path, the name of a directory into which + * statistics files are written. The directory must already exist. If the + * value is not an absolute path\, the path is relative to the database home + * (see @ref absolute_path for more information)., a string; default \c ".".} + * @config{    sources, if non-empty\, include statistics + * for the list of data source URIs\, if they are open at the time of the + * statistics logging. The list may include URIs matching a single data source + * ("table:mytable")\, or a URI matching all data sources of a particular type + * ("table:")., a list of strings; default empty.} * @config{    timestamp, a timestamp prepended to each log * record\, may contain strftime conversion specifications\, when \c json is * configured\, defaults to \c "%FT%Y.000Z"., a string; default \c "%b %d diff --git a/src/os_win/os_path.c b/src/os_win/os_path.c index 220752ce7a1..74050600417 100644 --- a/src/os_win/os_path.c +++ b/src/os_win/os_path.c @@ -16,8 +16,30 @@ bool __wt_absolute_path(const char *path) { /* - * Check for a drive name (for example, "D:"), allow both forward and - * backward slashes. + * https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247 + * + * For Windows API functions that manipulate files, file names can often + * be relative to the current directory, while some APIs require a fully + * qualified path. A file name is relative to the current directory if + * it does not begin with one of the following: + * + * -- A UNC name of any format, which always start with two backslash + * characters ("\\"). + * -- A disk designator with a backslash, for example "C:\" or "d:\". + * -- A single backslash, for example, "\directory" or "\file.txt". This + * is also referred to as an absolute path. + * + * If a file name begins with only a disk designator but not the + * backslash after the colon, it is interpreted as a relative path to + * the current directory on the drive with the specified letter. Note + * that the current directory may or may not be the root directory + * depending on what it was set to during the most recent "change + * directory" operation on that disk. + * + * -- "C:tmp.txt" refers to a file named "tmp.txt" in the current + * directory on drive C. + * -- "C:tempdir\tmp.txt" refers to a file in a subdirectory to the + * current directory on drive C. */ if (strlen(path) >= 3 && __wt_isalpha(path[0]) && path[1] == ':') path += 2; diff --git a/test/suite/test_reconfig01.py b/test/suite/test_reconfig01.py index 876de1fe5af..419ff876dc2 100644 --- a/test/suite/test_reconfig01.py +++ b/test/suite/test_reconfig01.py @@ -95,19 +95,24 @@ class test_reconfig01(wttest.WiredTigerTestCase): self.conn.reconfigure("checkpoint=(wait=0,name=hi)") self.conn.reconfigure("checkpoint=(wait=5,name=hi)") - def test_reconfig_stat_log(self): + # Statistics logging: reconfigure the things we can reconfigure. + def test_reconfig_statistics_log_ok(self): self.conn.reconfigure("statistics=[all],statistics_log=(wait=0)") self.conn.reconfigure("statistics_log=(wait=0)") - self.conn.reconfigure("statistics_log=(wait=2)") + self.conn.reconfigure("statistics_log=(wait=2,json=true)") + self.conn.reconfigure("statistics_log=(wait=0)") + self.conn.reconfigure("statistics_log=(wait=2,on_close=true)") self.conn.reconfigure("statistics_log=(wait=0)") self.conn.reconfigure("statistics_log=(wait=2,sources=[lsm:])") self.conn.reconfigure("statistics_log=(wait=0)") self.conn.reconfigure("statistics_log=(wait=2,timestamp=\"t%b %d\")") self.conn.reconfigure("statistics_log=(wait=0)") - self.conn.reconfigure("statistics_log=(wait=2,path=\"wts.%d.%H\")") - self.conn.reconfigure("statistics_log=(wait=0)") - self.conn.reconfigure( - "statistics_log=(wait=2,sources=[lsm:],timestamp=\"%b\")") + + # Statistics logging: reconfigure the things we can't reconfigure. + def test_reconfig_statistics_log_fail(self): + msg = '/unknown configuration key/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.reconfigure("log=(path=foo)"), msg) def test_file_manager(self): self.conn.reconfigure("file_manager=(close_scan_interval=3)") diff --git a/test/suite/test_reconfig02.py b/test/suite/test_reconfig02.py index 85a9ceb2a34..9d9ac220aa7 100644 --- a/test/suite/test_reconfig02.py +++ b/test/suite/test_reconfig02.py @@ -41,24 +41,29 @@ class test_reconfig02(wttest.WiredTigerTestCase): self.conn_config = self.init_config return wttest.WiredTigerTestCase.setUpConnectionOpen(self, dir) - # Call reconfigure for zero filling a file. There is nothing - # we can actually look for to confirm it did anything. - # Also changing the log file size is a no-op, but should not fail. + # Logging: reconfigure the things we can reconfigure. def test_reconfig02_simple(self): + self.conn.reconfigure("log=(archive=false)") + self.conn.reconfigure("log=(prealloc=false)") + self.conn.reconfigure("log=(zero_fill=false)") + + self.conn.reconfigure("log=(archive=true)") + self.conn.reconfigure("log=(prealloc=true)") self.conn.reconfigure("log=(zero_fill=true)") - self.conn.reconfigure("log=(file_max=1MB)") - # Test that we get an error if we try to turn logging off. + # Logging: reconfigure the things we can't reconfigure. def test_reconfig02_disable(self): - msg = 'Invalid argument' - gotException = False - try: - self.conn.reconfigure("log=(enabled=false)") - except wiredtiger.WiredTigerError as e: - gotException = True - self.pr('got exception: ' + str(e)) - self.assertTrue(str(e).find(msg) >= 0) - self.assertTrue(gotException) + msg = '/unknown configuration key/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.reconfigure("log=(enabled=true)"), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.reconfigure("log=(compressor=foo)"), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.reconfigure("log=(file_max=1MB)"), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.reconfigure("log=(path=foo)"), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.reconfigure("log=(recovery=true)"), msg) # Logging starts on, but prealloc is off. Verify it is off. # Reconfigure it on and run again, making sure that log files diff --git a/test/suite/test_stat_log01.py b/test/suite/test_stat_log01.py index f6033d940c5..65ce80dfe7d 100644 --- a/test/suite/test_stat_log01.py +++ b/test/suite/test_stat_log01.py @@ -51,9 +51,10 @@ class test_stat_log01(wttest.WiredTigerTestCase): None, "create,statistics=(fast),statistics_log=(wait=1)") # Wait for the default interval, to ensure stats have been written. time.sleep(2) - self.check_stats_file("WiredTigerStat") + self.check_stats_file(".") def test_stats_log_name(self): + os.mkdir("foo") self.conn = self.wiredtiger_open( None, "create,statistics=(fast),statistics_log=(wait=1,path=foo)") # Wait for the default interval, to ensure stats have been written. @@ -66,21 +67,18 @@ class test_stat_log01(wttest.WiredTigerTestCase): # Wait for the default interval, to ensure stats have been written. time.sleep(2) self.close_conn() - self.check_stats_file("WiredTigerStat") + self.check_stats_file(".") def test_stats_log_on_close(self): self.conn = self.wiredtiger_open(None, "create,statistics=(fast),statistics_log=(on_close=true)") # Close the connection to ensure the statistics get generated. self.close_conn() - self.check_stats_file("WiredTigerStat") + self.check_stats_file(".") - def check_stats_file(self, filename): - if filename == "WiredTigerStat": - files = glob.glob(filename + '.[0-9]*') - self.assertTrue(files) - else: - self.assertTrue(os.path.isfile(filename)) + def check_stats_file(self, dir): + files = glob.glob(dir + '/' + 'WiredTigerStat.[0-9]*') + self.assertTrue(files) if __name__ == '__main__': wttest.run() -- cgit v1.2.1