diff options
-rw-r--r-- | build_win/filelist.win | 1 | ||||
-rw-r--r-- | dist/filelist | 1 | ||||
-rw-r--r-- | dist/s_string.ok | 2 | ||||
-rwxr-xr-x | dist/s_win | 1 | ||||
-rw-r--r-- | src/btree/bt_io.c | 4 | ||||
-rw-r--r-- | src/btree/bt_split.c | 11 | ||||
-rw-r--r-- | src/conn/conn_api.c | 88 | ||||
-rw-r--r-- | src/evict/evict_lru.c | 17 | ||||
-rw-r--r-- | src/evict/evict_page.c | 18 | ||||
-rw-r--r-- | src/include/cache.h | 3 | ||||
-rw-r--r-- | src/include/extern.h | 1 | ||||
-rw-r--r-- | src/include/wiredtiger.in | 7 | ||||
-rw-r--r-- | src/os_posix/os_getenv.c | 25 | ||||
-rw-r--r-- | src/os_win/os_getenv.c | 35 | ||||
-rw-r--r-- | src/schema/schema_list.c | 6 | ||||
-rw-r--r-- | src/session/session_api.c | 9 | ||||
-rw-r--r-- | test/format/config.h | 16 | ||||
-rw-r--r-- | test/format/format.h | 5 | ||||
-rw-r--r-- | test/format/ops.c | 12 | ||||
-rw-r--r-- | test/format/recover.sh | 31 | ||||
-rw-r--r-- | test/format/wts.c | 175 | ||||
-rw-r--r-- | test/suite/test_schema03.py | 16 |
22 files changed, 342 insertions, 142 deletions
diff --git a/build_win/filelist.win b/build_win/filelist.win index c7903c0a161..78ddcb14a59 100644 --- a/build_win/filelist.win +++ b/build_win/filelist.win @@ -113,6 +113,7 @@ src/os_win/os_filesize.c src/os_win/os_flock.c src/os_win/os_fsync.c src/os_win/os_ftruncate.c +src/os_win/os_getenv.c src/os_win/os_map.c src/os_win/os_mtx_cond.c src/os_win/os_once.c diff --git a/dist/filelist b/dist/filelist index 07469f49311..c2346caf1ea 100644 --- a/dist/filelist +++ b/dist/filelist @@ -109,6 +109,7 @@ src/os_posix/os_filesize.c src/os_posix/os_flock.c src/os_posix/os_fsync.c src/os_posix/os_ftruncate.c +src/os_posix/os_getenv.c src/os_posix/os_getline.c src/os_posix/os_getopt.c src/os_posix/os_map.c diff --git a/dist/s_string.ok b/dist/s_string.ok index aa5d65bcc8b..94f3ea6b6a5 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -138,6 +138,7 @@ GETTIMEOFDAY GIDs Gcc Geoff +GetEnvironmentVariableA GetFileAttributesEx GetFileSizeEx GetLastError @@ -639,6 +640,7 @@ func funcs gcc gdb +getenv getfiles getid getline diff --git a/dist/s_win b/dist/s_win index 7c5fecb08bc..187de91e498 100755 --- a/dist/s_win +++ b/dist/s_win @@ -44,6 +44,7 @@ win_filelist() -e 's;os_posix/os_flock.c;os_win/os_flock.c;' \ -e 's;os_posix/os_fsync.c;os_win/os_fsync.c;' \ -e 's;os_posix/os_ftruncate.c;os_win/os_ftruncate.c;' \ + -e 's;os_posix/os_getenv.c;os_win/os_getenv.c;' \ -e 's;os_posix/os_map.c;os_win/os_map.c;' \ -e 's;os_posix/os_mtx_cond.c;os_win/os_mtx_cond.c;' \ -e 's;os_posix/os_once.c;os_win/os_once.c;' \ diff --git a/src/btree/bt_io.c b/src/btree/bt_io.c index 8ab5a6a30c0..9d154311a8c 100644 --- a/src/btree/bt_io.c +++ b/src/btree/bt_io.c @@ -295,8 +295,8 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_STAT_FAST_CONN_INCR(session, cache_write); WT_STAT_FAST_DATA_INCR(session, cache_write); - WT_STAT_FAST_CONN_INCRV(session, cache_bytes_write, ip->size); - WT_STAT_FAST_DATA_INCRV(session, cache_bytes_write, ip->size); + WT_STAT_FAST_CONN_INCRV(session, cache_bytes_write, dsk->mem_size); + WT_STAT_FAST_DATA_INCRV(session, cache_bytes_write, dsk->mem_size); err: __wt_scr_free(&tmp); return (ret); diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index f1b96ad37f5..a14e82d980d 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -716,11 +716,10 @@ __split_multi_inmem( /* * We modified the page above, which will have set the first dirty * transaction to the last transaction current running. However, the - * updates we installed may be older than that. Take the oldest active - * transaction ID to make sure these updates are not skipped by a - * checkpoint. + * updates we installed may be older than that. Inherit the first + * dirty transaction from the original page. */ - page->modify->first_dirty_txn = S2C(session)->txn_global.oldest_id; + page->modify->first_dirty_txn = orig->modify->first_dirty_txn; err: /* Free any resources that may have been cached in the cursor. */ WT_TRET(__wt_btcur_close(&cbt)); @@ -1137,8 +1136,8 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) /* * We modified the page above, which will have set the first dirty * transaction to the last transaction current running. However, the - * updates we are moving may be older than that: inherit the original - * page's transaction ID. + * updates we installed may be older than that. Inherit the first + * dirty transaction from the original page. */ right->modify->first_dirty_txn = page->modify->first_dirty_txn; diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index cc88f848861..c878024edb4 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -519,12 +519,11 @@ err: if (nextractor != NULL) { * Given a configuration, configure the extractor. */ int -__wt_extractor_config(WT_SESSION_IMPL *session, const char *config, - WT_EXTRACTOR **extractorp, int *ownp) +__wt_extractor_config(WT_SESSION_IMPL *session, + const char *config, WT_EXTRACTOR **extractorp, int *ownp) { WT_CONNECTION_IMPL *conn; WT_CONFIG_ITEM cval; - WT_DECL_RET; WT_NAMED_EXTRACTOR *nextractor; *extractorp = NULL; @@ -532,35 +531,32 @@ __wt_extractor_config(WT_SESSION_IMPL *session, const char *config, conn = S2C(session); - if ((ret = - __wt_config_getones_none(session, config, "extractor", &cval)) != 0) - return (ret == WT_NOTFOUND || cval.len == 0 ? 0 : ret); - - if (cval.len > 0) { - TAILQ_FOREACH(nextractor, &conn->extractorqh, q) - if (WT_STRING_MATCH( - nextractor->name, cval.str, cval.len)) - break; + WT_RET_NOTFOUND_OK( + __wt_config_getones_none(session, config, "extractor", &cval)); + if (cval.len == 0) + return (0); - if (nextractor == NULL) - WT_RET_MSG(session, EINVAL, - "unknown extractor '%.*s'", - (int)cval.len, cval.str); - - if (nextractor->extractor->customize != NULL) { - WT_RET(__wt_config_getones(session, - config, "app_metadata", &cval)); - WT_RET(nextractor->extractor->customize( - nextractor->extractor, &session->iface, - session->dhandle->name, &cval, extractorp)); - } + TAILQ_FOREACH(nextractor, &conn->extractorqh, q) + if (WT_STRING_MATCH(nextractor->name, cval.str, cval.len)) + break; - if (*extractorp == NULL) - *extractorp = nextractor->extractor; - else - *ownp = 1; + if (nextractor == NULL) + WT_RET_MSG(session, EINVAL, + "unknown extractor '%.*s'", (int)cval.len, cval.str); + + if (nextractor->extractor->customize != NULL) { + WT_RET(__wt_config_getones(session, + config, "app_metadata", &cval)); + WT_RET(nextractor->extractor->customize( + nextractor->extractor, &session->iface, + session->dhandle->name, &cval, extractorp)); } + if (*extractorp == NULL) + *extractorp = nextractor->extractor; + else + *ownp = 1; + return (0); } @@ -1007,15 +1003,19 @@ static int __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf) { WT_CONFIG_ITEM cval; + WT_DECL_RET; const char *env_config; size_t len; - if ((env_config = getenv("WIREDTIGER_CONFIG")) == NULL) + ret = __wt_getenv(session, "WIREDTIGER_CONFIG", &env_config); + if (ret == WT_NOTFOUND) return (0); + WT_ERR(ret); + len = strlen(env_config); if (len == 0) - return (0); - WT_RET(__wt_buf_set(session, cbuf, env_config, len + 1)); + goto err; /* Free the memory. */ + WT_ERR(__wt_buf_set(session, cbuf, env_config, len + 1)); /* * Security stuff: @@ -1023,26 +1023,28 @@ __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf) * If the "use_environment_priv" configuration string is set, use the * environment variable if the process has appropriate privileges. */ - WT_RET(__wt_config_gets(session, cfg, "use_environment_priv", &cval)); + WT_ERR(__wt_config_gets(session, cfg, "use_environment_priv", &cval)); if (cval.val == 0 && __wt_has_priv()) - WT_RET_MSG(session, WT_ERROR, "%s", + WT_ERR_MSG(session, WT_ERROR, "%s", "WIREDTIGER_CONFIG environment variable set but process " "lacks privileges to use that environment variable"); /* Check any version. */ - WT_RET(__conn_config_check_version(session, env_config)); + WT_ERR(__conn_config_check_version(session, env_config)); /* Upgrade the configuration string. */ - WT_RET(__wt_config_upgrade(session, cbuf)); + WT_ERR(__wt_config_upgrade(session, cbuf)); /* Check the configuration information. */ - WT_RET(__wt_config_check(session, + WT_ERR(__wt_config_check(session, WT_CONFIG_REF(session, wiredtiger_open), env_config, 0)); /* Append it to the stack. */ - __conn_config_append(cfg, env_config); + __conn_config_append(cfg, cbuf->data); - return (0); +err: __wt_free(session, env_config); + + return (ret); } /* @@ -1052,17 +1054,19 @@ __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf) static int __conn_home(WT_SESSION_IMPL *session, const char *home, const char *cfg[]) { + WT_DECL_RET; WT_CONFIG_ITEM cval; /* If the application specifies a home directory, use it. */ if (home != NULL) goto copy; + ret = __wt_getenv(session, "WIREDTIGER_HOME", &S2C(session)->home); + if (ret == 0) + return (0); + /* If there's no WIREDTIGER_HOME environment variable, use ".". */ - if ((home = getenv("WIREDTIGER_HOME")) == NULL || strlen(home) == 0) { - home = "."; - goto copy; - } + home = "."; /* * Security stuff: diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index dbf3a71f222..69fa1f85c93 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -407,6 +407,15 @@ __evict_has_work(WT_SESSION_IMPL *session, uint32_t *flagsp) (cache->eviction_dirty_target * bytes_max) / 100) /* Ignore clean pages unless the cache is too large */ LF_SET(WT_EVICT_PASS_DIRTY); + else if (F_ISSET(cache, WT_EVICT_WOULD_BLOCK)) { + /* + * Evict pages with oldest generation (which would otherwise + * block application threads) set regardless of whether we have + * reached the eviction trigger. + */ + LF_SET(WT_EVICT_PASS_WOULD_BLOCK); + F_CLR(cache, WT_EVICT_WOULD_BLOCK); + } if (F_ISSET(cache, WT_EVICT_STUCK)) LF_SET(WT_EVICT_PASS_AGGRESSIVE); @@ -1076,6 +1085,14 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp, uint32_t flags) if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)) continue; + /* + * If we are only trickling out pages marked for definite + * eviction, skip anything that isn't marked. + */ + if (LF_ISSET(WT_EVICT_PASS_WOULD_BLOCK) && + page->read_gen != WT_READGEN_OLDEST) + continue; + /* Limit internal pages to 50% unless we get aggressive. */ if ((page->type == WT_PAGE_COL_INT || page->type == WT_PAGE_ROW_INT) && diff --git a/src/evict/evict_page.c b/src/evict/evict_page.c index f7cbb55dbe4..4e96898fd92 100644 --- a/src/evict/evict_page.c +++ b/src/evict/evict_page.c @@ -24,9 +24,10 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) WT_PAGE *page; WT_PAGE_MODIFY *mod; WT_TXN_STATE *txn_state; - int inmem_split, istree; + int forced_eviction, inmem_split, istree; page = ref->page; + forced_eviction = (page->read_gen == WT_READGEN_OLDEST); inmem_split = istree = 0; WT_RET(__wt_verbose(session, WT_VERB_EVICT, @@ -116,6 +117,12 @@ done: session->excl_next = 0; if (txn_state != NULL) txn_state->snap_min = WT_TXN_NONE; + if ((inmem_split || (forced_eviction && ret == EBUSY)) && + !F_ISSET(S2C(session)->cache, WT_EVICT_WOULD_BLOCK)) { + F_SET(S2C(session)->cache, WT_EVICT_WOULD_BLOCK); + WT_TRET(__wt_evict_server_wake(session)); + } + return (ret); } @@ -308,7 +315,6 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags; btree = S2BT(session); - page = ref->page; /* * Get exclusive access to the page if our caller doesn't have the tree @@ -327,6 +333,10 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, __wt_evict_list_clear_page(session, ref); } + /* Now that we have exclusive access, review the page. */ + page = ref->page; + mod = page->modify; + /* * Recurse through the page's subtree: this happens first because we * have to write pages in depth-first order, otherwise we'll dirty @@ -338,8 +348,6 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, WT_RET(ret); } - mod = page->modify; - /* * If the tree was deepened, there's a requirement that newly created * internal pages not be evicted until all threads are known to have @@ -449,7 +457,7 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, if (exclusive) LF_SET(WT_SKIP_UPDATE_ERR); else if (top && !WT_PAGE_IS_INTERNAL(page) && - page->memory_footprint > 10 * btree->maxleafpage) + page->read_gen == WT_READGEN_OLDEST) LF_SET(WT_SKIP_UPDATE_RESTORE); WT_RET(__wt_reconcile(session, ref, NULL, flags)); WT_ASSERT(session, diff --git a/src/include/cache.h b/src/include/cache.h index a6425e97837..cf688b3993f 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -19,6 +19,7 @@ #define WT_EVICT_PASS_AGGRESSIVE 0x01 #define WT_EVICT_PASS_ALL 0x02 #define WT_EVICT_PASS_DIRTY 0x04 +#define WT_EVICT_PASS_WOULD_BLOCK 0x08 /* * WT_EVICT_ENTRY -- @@ -33,7 +34,6 @@ struct __wt_evict_entry { * WT_EVICT_WORKER -- * Encapsulation of an eviction worker thread. */ - struct __wt_evict_worker { WT_SESSION_IMPL *session; u_int id; @@ -114,6 +114,7 @@ struct __wt_cache { #define WT_EVICT_ACTIVE 0x04 /* Eviction server is active */ #define WT_EVICT_CLEAR_WALKS 0x08 /* Clear eviction walks */ #define WT_EVICT_STUCK 0x10 /* Eviction server is stuck */ +#define WT_EVICT_WOULD_BLOCK 0x20 /* Pages that would block apps */ uint32_t flags; }; diff --git a/src/include/extern.h b/src/include/extern.h index bb7caa991d6..056058f3828 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -442,6 +442,7 @@ extern int __wt_directory_sync(WT_SESSION_IMPL *session, char *path); extern int __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh); extern int __wt_fsync_async(WT_SESSION_IMPL *session, WT_FH *fh); extern int __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t len); +extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp); extern int __wt_getline(WT_SESSION_IMPL *session, WT_ITEM *buf, FILE *fp); extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr); extern int __wt_mmap(WT_SESSION_IMPL *session, WT_FH *fh, void *mapp, size_t *lenp, void **mappingcookie); diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index c31b44e67b0..19ccc313b49 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -2613,7 +2613,8 @@ struct __wt_collator { const char *uri, WT_CONFIG_ITEM *appcfg, WT_COLLATOR **customp); /*! - * If non-NULL, a callback performed when the database is closed. + * If non-NULL a callback performed when the data source is closed + * for customized extractors otherwise when the database is closed. * * The WT_COLLATOR::terminate callback is intended to allow cleanup, * the handle will not be subsequently accessed by WiredTiger. @@ -3037,7 +3038,9 @@ struct __wt_extractor { const char *uri, WT_CONFIG_ITEM *appcfg, WT_EXTRACTOR **customp); /*! - * If non-NULL, a callback performed when the database is closed. + * If non-NULL a callback performed when the index or column group + * is closed for customized extractors otherwise when the database + * is closed. * * The WT_EXTRACTOR::terminate callback is intended to allow cleanup, * the handle will not be subsequently accessed by WiredTiger. diff --git a/src/os_posix/os_getenv.c b/src/os_posix/os_getenv.c new file mode 100644 index 00000000000..37fe2af8501 --- /dev/null +++ b/src/os_posix/os_getenv.c @@ -0,0 +1,25 @@ +/*- + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_getenv -- + * Get a non-NULL, greater than zero-length environment variable. + */ +int +__wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) +{ + const char *temp; + + *envp = NULL; + + if (((temp = getenv(variable)) != NULL) && strlen(temp) > 0) + return (__wt_strdup(session, temp, envp)); + + return (WT_NOTFOUND); +} diff --git a/src/os_win/os_getenv.c b/src/os_win/os_getenv.c new file mode 100644 index 00000000000..8cc94abb524 --- /dev/null +++ b/src/os_win/os_getenv.c @@ -0,0 +1,35 @@ +/*- + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_getenv -- + * Get a non-NULL, greater than zero-length environment variable. + */ +int +__wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) +{ + WT_DECL_RET; + DWORD size; + + *envp = NULL; + + size = GetEnvironmentVariableA(variable, NULL, 0); + if (size <= 1) + return (WT_NOTFOUND); + + WT_RET(__wt_calloc(session, 1, size, envp)); + + ret = GetEnvironmentVariableA(variable, *envp, size); + /* We expect the number of bytes not including nul terminator. */ + if ((ret + 1) != size) + WT_RET_MSG(session, __wt_errno(), + "GetEnvironmentVariableA failed: %s", variable); + + return (0); +} diff --git a/src/schema/schema_list.c b/src/schema/schema_list.c index 02556eb942c..1fbc52821a6 100644 --- a/src/schema/schema_list.c +++ b/src/schema/schema_list.c @@ -139,9 +139,13 @@ __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX *idx) WT_DECL_RET; /* If there is a custom extractor configured, terminate it. */ - if (idx->extractor != NULL && idx->extractor->terminate != NULL) + if (idx->extractor != NULL && + idx->extractor_owned && idx->extractor->terminate != NULL) { WT_TRET(idx->extractor->terminate( idx->extractor, &session->iface)); + idx->extractor = NULL; + idx->extractor_owned = 0; + } __wt_free(session, idx->name); __wt_free(session, idx->source); diff --git a/src/session/session_api.c b/src/session/session_api.c index 8f460dcc29f..0effc1a7248 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -547,9 +547,12 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, salvage, config, cfg); + /* Block out checkpoints to avoid spurious EBUSY errors. */ + __wt_spin_lock(session, &S2C(session)->checkpoint_lock); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_salvage, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_SALVAGE)); + __wt_spin_unlock(session, &S2C(session)->checkpoint_lock); err: API_END_RET_NOTFOUND_MAP(session, ret); } @@ -682,9 +685,12 @@ __session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, upgrade, config, cfg); + /* Block out checkpoints to avoid spurious EBUSY errors. */ + __wt_spin_lock(session, &S2C(session)->checkpoint_lock); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_upgrade, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_UPGRADE)); + __wt_spin_unlock(session, &S2C(session)->checkpoint_lock); err: API_END_RET_NOTFOUND_MAP(session, ret); } @@ -702,9 +708,12 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, verify, config, cfg); + /* Block out checkpoints to avoid spurious EBUSY errors. */ + __wt_spin_lock(session, &S2C(session)->checkpoint_lock); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_verify, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_VERIFY)); + __wt_spin_unlock(session, &S2C(session)->checkpoint_lock); err: API_END_RET_NOTFOUND_MAP(session, ret); } diff --git a/test/format/config.h b/test/format/config.h index 1b5d21d109b..2ab13620486 100644 --- a/test/format/config.h +++ b/test/format/config.h @@ -66,6 +66,10 @@ typedef struct { #define CONF_RAND(cp) MMRAND((cp)->min, (cp)->maxrand) static CONFIG c[] = { + { "abort", + "if timed run should drop core", /* 0% */ + C_BOOL, 0, 0, 0, &g.c_abort, NULL }, + { "auto_throttle", "if LSM inserts are throttled", /* 90% */ C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL }, @@ -196,6 +200,18 @@ static CONFIG c[] = { "if logging configured", /* 30% */ C_BOOL, 30, 0, 0, &g.c_logging, NULL }, + { "logging_archive", + "if log file archival configured", /* 50% */ + C_BOOL, 50, 0, 0, &g.c_logging_archive, NULL }, + + { "logging_prealloc", + "if log file pre-allocation configured", /* 50% */ + C_BOOL, 50, 0, 0, &g.c_logging_prealloc, NULL }, + + { "logging", + "if logging configured", /* 30% */ + C_BOOL, 30, 0, 0, &g.c_logging, NULL }, + { "lsm_worker_threads", "the number of LSM worker threads", 0x0, 3, 4, 20, &g.c_lsm_worker_threads, NULL }, diff --git a/test/format/format.h b/test/format/format.h index 902cea6cc5d..9290a93e5db 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -166,7 +166,8 @@ typedef struct { char *config_open; /* Command-line configuration */ - uint32_t c_auto_throttle; /* Config values */ + uint32_t c_abort; /* Config values */ + uint32_t c_auto_throttle; uint32_t c_backups; uint32_t c_bitcnt; uint32_t c_bloom; @@ -199,6 +200,8 @@ typedef struct { uint32_t c_leaf_page_max; uint32_t c_leak_memory; uint32_t c_logging; + uint32_t c_logging_archive; + uint32_t c_logging_prealloc; uint32_t c_lsm_worker_threads; uint32_t c_merge_max; uint32_t c_mmap; diff --git a/test/format/ops.c b/test/format/ops.c index bbaeabcc479..906d32fee82 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -139,10 +139,18 @@ wts_ops(void) break; } - /* Tell the thread if it's done. */ if (thread_ops == 0) { - if (fourths == 0) + /* + * Optionally drop core (for testing recovery), + * otherwise tell the thread it's done. + */ + if (fourths == 0) { + if (g.c_abort) { + static char *core = NULL; + *core = 0; + } tinfo[i].quit = 1; + } } else if (tinfo[i].ops >= thread_ops) tinfo[i].quit = 1; diff --git a/test/format/recover.sh b/test/format/recover.sh new file mode 100644 index 00000000000..22e60332711 --- /dev/null +++ b/test/format/recover.sh @@ -0,0 +1,31 @@ +#! /bin/sh + +# Timer: how many minutes format runs before aborting. +timer=2 + +# Runs: set to 0 to run infinitely. +runs=1000 + +# Config: additional test/format configuration +config= + +count=0 +while true; do + count=`expr $count + 1` + if test $runs -eq 0; then + echo "recovery test: $count" + else + if test $count -gt $runs; then + exit 0 + fi + echo "recovery test: $count of $runs" + fi + + ./t $config -q abort=1 logging=1 timer=$timer + + uri='file:wt' + if `wt -h RUNDIR list | egrep table > /dev/null`; then + uri='table:wt' + fi + wt -h RUNDIR verify $uri || exit 1 +done diff --git a/test/format/wts.c b/test/format/wts.c index 21e7806e982..243d1deb6d7 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -63,6 +63,9 @@ static WT_EVENT_HANDLER event_handler = { NULL /* Close handler. */ }; +#undef REMAIN +#define REMAIN(p, end) (size_t)((p) >= (end) ? 0 : (end) - (p)) + /* * wts_open -- * Open a connection to a WiredTiger database. @@ -72,58 +75,76 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) { WT_CONNECTION *conn; int ret; - const char *buffer_align, *progname; - char config[2048], evict_config[64]; + char config[4096], *end, *p; *connp = NULL; - /* Build the eviction worker thread config string, if needed. */ - evict_config[0] = '\0'; - if (g.c_evict_max != 0 && - snprintf(evict_config, sizeof(evict_config), - "eviction=(threads_max=%" PRIu32 "),", - g.c_evict_max) >= (int)sizeof(evict_config)) - die(ENOMEM, "eviction configuration buffer too small"); + p = config; + end = config + sizeof(config); + + p += snprintf(p, REMAIN(p, end), + "create,checkpoint_sync=false,cache_size=%" PRIu32 "MB", + g.c_cache); - /* - * Put configuration file configuration options second to last. Put - * command line configuration options at the end. Do this so they - * override the standard configuration. - */ #ifdef _WIN32 - progname = "t_format.exe"; - buffer_align = ""; + p += snprintf(p, REMAIN(p, end), ",error_prefix=\"t_format.exe\""); #else - progname = g.progname; - buffer_align = "buffer_alignment=512"; + p += snprintf(p, REMAIN(p, end), ",error_prefix=\"%s\"", g.progname); +#endif + + /* LSM configuration. */ + if (DATASOURCE("lsm")) + p += snprintf(p, REMAIN(p, end), + ",lsm_manager=(worker_thread_max=%" PRIu32 "),", + g.c_lsm_worker_threads); + + /* Eviction worker configuration. */ + if (g.c_evict_max != 0) + p += snprintf(p, REMAIN(p, end), + ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); + + /* Logging configuration. */ + if (g.c_logging) + p += snprintf(p, REMAIN(p, end), + ",log=(enabled=true,archive=%d,prealloc=%d)", + g.c_logging_archive ? 1 : 0, + g.c_logging_prealloc ? 1 : 0); + + /* Miscellaneous. */ +#ifndef _WIN32 + p += snprintf(p, REMAIN(p, end), ",buffer_alignment=512"); #endif - if (snprintf(config, sizeof(config), - "create," - "checkpoint_sync=false,cache_size=%" PRIu32 "MB," - "%s,lsm_manager=(worker_thread_max=%" PRIu32 - "),error_prefix=\"%s\"," - "%s,%s,%s,%s,%s" - "extensions=" - "[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"]," - "%s,%s", - g.c_cache, - buffer_align, - g.c_lsm_worker_threads, - progname, - g.c_data_extend ? "file_extend=(data=8MB)" : "", - g.c_logging ? "log=(enabled=true)" : "", - g.c_mmap ? "mmap=true" : "mmap=false", - g.c_statistics ? "statistics=(fast)" : "statistics=(none)", - evict_config, + + p += snprintf(p, REMAIN(p, end), ",mmap=%d", g.c_mmap ? 1 : 0); + + if (g.c_data_extend) + p += snprintf(p, REMAIN(p, end), ",file_extend=(data=8MB)"); + + p += snprintf(p, REMAIN(p, end), + ",statistics=(%s)", g.c_statistics ? "fast" : "none"); + + /* Extensions. */ + p += snprintf(p, REMAIN(p, end), + ",extensions=[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", g.c_reverse ? REVERSE_PATH : "", access(BZIP_PATH, R_OK) == 0 ? BZIP_PATH : "", access(LZO_PATH, R_OK) == 0 ? LZO_PATH : "", access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "", access(ZLIB_PATH, R_OK) == 0 ? ZLIB_PATH : "", - DATASOURCE("kvsbdb") ? KVS_BDB_PATH : "", - g.c_config_open == NULL ? "" : g.c_config_open, - g.config_open == NULL ? "" : g.config_open) >= (int)sizeof(config)) - die(ENOMEM, "configuration buffer too small"); + DATASOURCE("kvsbdb") ? KVS_BDB_PATH : ""); + + /* + * Put configuration file configuration options second to last. Put + * command line configuration options at the end. Do this so they + * override the standard configuration. + */ + if (g.c_config_open != NULL) + p += snprintf(p, REMAIN(p, end), ",%s", g.c_config_open); + if (g.config_open != NULL) + p += snprintf(p, REMAIN(p, end), ",%s", g.config_open); + + if (REMAIN(p, end) == 0) + die(ENOMEM, "wiredtiger_open configuration buffer too small"); /* * Direct I/O may not work with backups, doing copies through the buffer @@ -200,7 +221,7 @@ wts_create(void) if (maxleafpage > 512) maxleafpage >>= 1; } - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), "key_format=%s," "allocation_size=512,%s" "internal_page_max=%d,leaf_page_max=%d", @@ -214,43 +235,43 @@ wts_create(void) */ maxintlkey = MMRAND(maxintlpage / 50, maxintlpage / 40); if (maxintlkey > 20) - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",internal_key_max=%d", maxintlkey); maxleafkey = MMRAND(maxleafpage / 50, maxleafpage / 40); if (maxleafkey > 20) - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",leaf_key_max=%d", maxleafkey); maxleafvalue = MMRAND(maxleafpage * 10, maxleafpage / 40); if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",leaf_value_max=%d", maxleafvalue); switch (g.type) { case FIX: - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",value_format=%" PRIu32 "t", g.c_bitcnt); break; case ROW: if (g.c_huffman_key) - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",huffman_key=english"); if (g.c_prefix_compression) - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",prefix_compression_min=%" PRIu32, g.c_prefix_compression_min); else - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",prefix_compression=false"); if (g.c_reverse) - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",collator=reverse"); /* FALLTHROUGH */ case VAR: if (g.c_huffman_value) - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",huffman_value=english"); if (g.c_dictionary) - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",dictionary=%d", MMRAND(123, 517)); break; } @@ -258,14 +279,13 @@ wts_create(void) /* Configure checksums. */ switch (g.c_checksum_flag) { case CHECKSUM_OFF: - p += snprintf(p, (size_t)(end - p), ",checksum=\"off\""); + p += snprintf(p, REMAIN(p, end), ",checksum=\"off\""); break; case CHECKSUM_ON: - p += snprintf(p, (size_t)(end - p), ",checksum=\"on\""); + p += snprintf(p, REMAIN(p, end), ",checksum=\"on\""); break; case CHECKSUM_UNCOMPRESSED: - p += snprintf( - p, (size_t)(end - p), ",checksum=\"uncompressed\""); + p += snprintf(p, REMAIN(p, end), ",checksum=\"uncompressed\""); break; } @@ -274,57 +294,55 @@ wts_create(void) case COMPRESS_NONE: break; case COMPRESS_BZIP: - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",block_compressor=\"bzip2\""); break; case COMPRESS_BZIP_RAW: - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",block_compressor=\"bzip2-raw-test\""); break; case COMPRESS_LZO: - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",block_compressor=\"LZO1B-6\""); break; case COMPRESS_SNAPPY: - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",block_compressor=\"snappy\""); break; case COMPRESS_ZLIB: - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",block_compressor=\"zlib\""); break; case COMPRESS_ZLIB_NO_RAW: - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",block_compressor=\"zlib-noraw\""); break; } /* Configure Btree internal key truncation. */ - p += snprintf( - p, (size_t)(end - p), ",internal_key_truncate=%s", + p += snprintf(p, REMAIN(p, end), ",internal_key_truncate=%s", g.c_internal_key_truncation ? "true" : "false"); /* Configure Btree page key gap. */ - p += snprintf(p, (size_t)(end - p), ",key_gap=%" PRIu32, g.c_key_gap); + p += snprintf(p, REMAIN(p, end), ",key_gap=%" PRIu32, g.c_key_gap); /* Configure Btree split page percentage. */ - p += snprintf(p, (size_t)(end - p), - ",split_pct=%" PRIu32, g.c_split_pct); + p += snprintf(p, REMAIN(p, end), ",split_pct=%" PRIu32, g.c_split_pct); /* Configure LSM and data-sources. */ if (DATASOURCE("helium")) - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",type=helium,helium_o_compress=%d,helium_o_truncate=1", g.c_compression_flag == COMPRESS_NONE ? 0 : 1); if (DATASOURCE("kvsbdb")) - p += snprintf(p, (size_t)(end - p), ",type=kvsbdb"); + p += snprintf(p, REMAIN(p, end), ",type=kvsbdb"); if (DATASOURCE("lsm")) { - p += snprintf(p, (size_t)(end - p), ",type=lsm,lsm=("); - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), ",type=lsm,lsm=("); + p += snprintf(p, REMAIN(p, end), "auto_throttle=%s,", g.c_auto_throttle ? "true" : "false"); - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); /* * We can't set bloom_oldest without bloom, and we want to test @@ -332,19 +350,22 @@ wts_create(void) */ if (g.c_bloom_oldest) g.c_bloom = 1; - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), "bloom=%s,", g.c_bloom ? "true" : "false"); - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), "bloom_bit_count=%" PRIu32 ",", g.c_bloom_bit_count); - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), "bloom_hash_count=%" PRIu32 ",", g.c_bloom_hash_count); - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), "bloom_oldest=%s,", g.c_bloom_oldest ? "true" : "false"); - p += snprintf(p, (size_t)(end - p), + p += snprintf(p, REMAIN(p, end), "merge_max=%" PRIu32 ",", g.c_merge_max); - p += snprintf(p, (size_t)(end - p), ",)"); + p += snprintf(p, REMAIN(p, end), ",)"); } + if (REMAIN(p, end) == 0) + die(ENOMEM, "WT_SESSION.create configuration buffer too small"); + /* * Create the underlying store. */ diff --git a/test/suite/test_schema03.py b/test/suite/test_schema03.py index 48d8e6b2986..e650d8b6964 100644 --- a/test/suite/test_schema03.py +++ b/test/suite/test_schema03.py @@ -25,10 +25,17 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import resource +import os import suite_random import wiredtiger, wtscenario, wttest +try: + # Windows does not getrlimit/setrlimit so we must catch the resource + # module load + import resource +except: + None + # test_schema03.py # Bigger, more 'randomly generated' schemas and data. # This test is complex. If it fails, rerun with modified values for @@ -190,7 +197,7 @@ class test_schema03(wttest.WiredTigerTestCase): created in various orders as much as the API allows. On some runs the connection will be closed and reopened at a particular point to test that the schemas (and data) are saved and read correctly. - + The test is run multiple times, using scenarios. The test always follows these steps: - table: create tables @@ -277,6 +284,9 @@ class test_schema03(wttest.WiredTigerTestCase): # This test requires a large number of open files. # Increase our resource limits before we start def setUp(self): + if os.name == "nt": + self.skipTest('Unix specific test skipped on Windows') + self.origFileLimit = resource.getrlimit(resource.RLIMIT_NOFILE) newlimit = (self.OPEN_FILE_LIMIT, self.origFileLimit[1]) if newlimit[0] > newlimit[1]: @@ -289,7 +299,7 @@ class test_schema03(wttest.WiredTigerTestCase): 'create,cache_size=100m,session_max=1000') self.pr(`conn`) return conn - + def tearDown(self): super(test_schema03, self).tearDown() resource.setrlimit(resource.RLIMIT_NOFILE, self.origFileLimit) |