summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--build_win/filelist.win1
-rw-r--r--dist/filelist1
-rw-r--r--dist/s_string.ok2
-rwxr-xr-xdist/s_win1
-rw-r--r--src/btree/bt_io.c4
-rw-r--r--src/btree/bt_split.c11
-rw-r--r--src/conn/conn_api.c88
-rw-r--r--src/evict/evict_lru.c17
-rw-r--r--src/evict/evict_page.c18
-rw-r--r--src/include/cache.h3
-rw-r--r--src/include/extern.h1
-rw-r--r--src/include/wiredtiger.in7
-rw-r--r--src/os_posix/os_getenv.c25
-rw-r--r--src/os_win/os_getenv.c35
-rw-r--r--src/schema/schema_list.c6
-rw-r--r--src/session/session_api.c9
-rw-r--r--test/format/config.h16
-rw-r--r--test/format/format.h5
-rw-r--r--test/format/ops.c12
-rw-r--r--test/format/recover.sh31
-rw-r--r--test/format/wts.c175
-rw-r--r--test/suite/test_schema03.py16
22 files changed, 342 insertions, 142 deletions
diff --git a/build_win/filelist.win b/build_win/filelist.win
index c7903c0a161..78ddcb14a59 100644
--- a/build_win/filelist.win
+++ b/build_win/filelist.win
@@ -113,6 +113,7 @@ src/os_win/os_filesize.c
src/os_win/os_flock.c
src/os_win/os_fsync.c
src/os_win/os_ftruncate.c
+src/os_win/os_getenv.c
src/os_win/os_map.c
src/os_win/os_mtx_cond.c
src/os_win/os_once.c
diff --git a/dist/filelist b/dist/filelist
index 07469f49311..c2346caf1ea 100644
--- a/dist/filelist
+++ b/dist/filelist
@@ -109,6 +109,7 @@ src/os_posix/os_filesize.c
src/os_posix/os_flock.c
src/os_posix/os_fsync.c
src/os_posix/os_ftruncate.c
+src/os_posix/os_getenv.c
src/os_posix/os_getline.c
src/os_posix/os_getopt.c
src/os_posix/os_map.c
diff --git a/dist/s_string.ok b/dist/s_string.ok
index aa5d65bcc8b..94f3ea6b6a5 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -138,6 +138,7 @@ GETTIMEOFDAY
GIDs
Gcc
Geoff
+GetEnvironmentVariableA
GetFileAttributesEx
GetFileSizeEx
GetLastError
@@ -639,6 +640,7 @@ func
funcs
gcc
gdb
+getenv
getfiles
getid
getline
diff --git a/dist/s_win b/dist/s_win
index 7c5fecb08bc..187de91e498 100755
--- a/dist/s_win
+++ b/dist/s_win
@@ -44,6 +44,7 @@ win_filelist()
-e 's;os_posix/os_flock.c;os_win/os_flock.c;' \
-e 's;os_posix/os_fsync.c;os_win/os_fsync.c;' \
-e 's;os_posix/os_ftruncate.c;os_win/os_ftruncate.c;' \
+ -e 's;os_posix/os_getenv.c;os_win/os_getenv.c;' \
-e 's;os_posix/os_map.c;os_win/os_map.c;' \
-e 's;os_posix/os_mtx_cond.c;os_win/os_mtx_cond.c;' \
-e 's;os_posix/os_once.c;os_win/os_once.c;' \
diff --git a/src/btree/bt_io.c b/src/btree/bt_io.c
index 8ab5a6a30c0..9d154311a8c 100644
--- a/src/btree/bt_io.c
+++ b/src/btree/bt_io.c
@@ -295,8 +295,8 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf,
WT_STAT_FAST_CONN_INCR(session, cache_write);
WT_STAT_FAST_DATA_INCR(session, cache_write);
- WT_STAT_FAST_CONN_INCRV(session, cache_bytes_write, ip->size);
- WT_STAT_FAST_DATA_INCRV(session, cache_bytes_write, ip->size);
+ WT_STAT_FAST_CONN_INCRV(session, cache_bytes_write, dsk->mem_size);
+ WT_STAT_FAST_DATA_INCRV(session, cache_bytes_write, dsk->mem_size);
err: __wt_scr_free(&tmp);
return (ret);
diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c
index f1b96ad37f5..a14e82d980d 100644
--- a/src/btree/bt_split.c
+++ b/src/btree/bt_split.c
@@ -716,11 +716,10 @@ __split_multi_inmem(
/*
* We modified the page above, which will have set the first dirty
* transaction to the last transaction current running. However, the
- * updates we installed may be older than that. Take the oldest active
- * transaction ID to make sure these updates are not skipped by a
- * checkpoint.
+ * updates we installed may be older than that. Inherit the first
+ * dirty transaction from the original page.
*/
- page->modify->first_dirty_txn = S2C(session)->txn_global.oldest_id;
+ page->modify->first_dirty_txn = orig->modify->first_dirty_txn;
err: /* Free any resources that may have been cached in the cursor. */
WT_TRET(__wt_btcur_close(&cbt));
@@ -1137,8 +1136,8 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp)
/*
* We modified the page above, which will have set the first dirty
* transaction to the last transaction current running. However, the
- * updates we are moving may be older than that: inherit the original
- * page's transaction ID.
+ * updates we installed may be older than that. Inherit the first
+ * dirty transaction from the original page.
*/
right->modify->first_dirty_txn = page->modify->first_dirty_txn;
diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c
index cc88f848861..c878024edb4 100644
--- a/src/conn/conn_api.c
+++ b/src/conn/conn_api.c
@@ -519,12 +519,11 @@ err: if (nextractor != NULL) {
* Given a configuration, configure the extractor.
*/
int
-__wt_extractor_config(WT_SESSION_IMPL *session, const char *config,
- WT_EXTRACTOR **extractorp, int *ownp)
+__wt_extractor_config(WT_SESSION_IMPL *session,
+ const char *config, WT_EXTRACTOR **extractorp, int *ownp)
{
WT_CONNECTION_IMPL *conn;
WT_CONFIG_ITEM cval;
- WT_DECL_RET;
WT_NAMED_EXTRACTOR *nextractor;
*extractorp = NULL;
@@ -532,35 +531,32 @@ __wt_extractor_config(WT_SESSION_IMPL *session, const char *config,
conn = S2C(session);
- if ((ret =
- __wt_config_getones_none(session, config, "extractor", &cval)) != 0)
- return (ret == WT_NOTFOUND || cval.len == 0 ? 0 : ret);
-
- if (cval.len > 0) {
- TAILQ_FOREACH(nextractor, &conn->extractorqh, q)
- if (WT_STRING_MATCH(
- nextractor->name, cval.str, cval.len))
- break;
+ WT_RET_NOTFOUND_OK(
+ __wt_config_getones_none(session, config, "extractor", &cval));
+ if (cval.len == 0)
+ return (0);
- if (nextractor == NULL)
- WT_RET_MSG(session, EINVAL,
- "unknown extractor '%.*s'",
- (int)cval.len, cval.str);
-
- if (nextractor->extractor->customize != NULL) {
- WT_RET(__wt_config_getones(session,
- config, "app_metadata", &cval));
- WT_RET(nextractor->extractor->customize(
- nextractor->extractor, &session->iface,
- session->dhandle->name, &cval, extractorp));
- }
+ TAILQ_FOREACH(nextractor, &conn->extractorqh, q)
+ if (WT_STRING_MATCH(nextractor->name, cval.str, cval.len))
+ break;
- if (*extractorp == NULL)
- *extractorp = nextractor->extractor;
- else
- *ownp = 1;
+ if (nextractor == NULL)
+ WT_RET_MSG(session, EINVAL,
+ "unknown extractor '%.*s'", (int)cval.len, cval.str);
+
+ if (nextractor->extractor->customize != NULL) {
+ WT_RET(__wt_config_getones(session,
+ config, "app_metadata", &cval));
+ WT_RET(nextractor->extractor->customize(
+ nextractor->extractor, &session->iface,
+ session->dhandle->name, &cval, extractorp));
}
+ if (*extractorp == NULL)
+ *extractorp = nextractor->extractor;
+ else
+ *ownp = 1;
+
return (0);
}
@@ -1007,15 +1003,19 @@ static int
__conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf)
{
WT_CONFIG_ITEM cval;
+ WT_DECL_RET;
const char *env_config;
size_t len;
- if ((env_config = getenv("WIREDTIGER_CONFIG")) == NULL)
+ ret = __wt_getenv(session, "WIREDTIGER_CONFIG", &env_config);
+ if (ret == WT_NOTFOUND)
return (0);
+ WT_ERR(ret);
+
len = strlen(env_config);
if (len == 0)
- return (0);
- WT_RET(__wt_buf_set(session, cbuf, env_config, len + 1));
+ goto err; /* Free the memory. */
+ WT_ERR(__wt_buf_set(session, cbuf, env_config, len + 1));
/*
* Security stuff:
@@ -1023,26 +1023,28 @@ __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf)
* If the "use_environment_priv" configuration string is set, use the
* environment variable if the process has appropriate privileges.
*/
- WT_RET(__wt_config_gets(session, cfg, "use_environment_priv", &cval));
+ WT_ERR(__wt_config_gets(session, cfg, "use_environment_priv", &cval));
if (cval.val == 0 && __wt_has_priv())
- WT_RET_MSG(session, WT_ERROR, "%s",
+ WT_ERR_MSG(session, WT_ERROR, "%s",
"WIREDTIGER_CONFIG environment variable set but process "
"lacks privileges to use that environment variable");
/* Check any version. */
- WT_RET(__conn_config_check_version(session, env_config));
+ WT_ERR(__conn_config_check_version(session, env_config));
/* Upgrade the configuration string. */
- WT_RET(__wt_config_upgrade(session, cbuf));
+ WT_ERR(__wt_config_upgrade(session, cbuf));
/* Check the configuration information. */
- WT_RET(__wt_config_check(session,
+ WT_ERR(__wt_config_check(session,
WT_CONFIG_REF(session, wiredtiger_open), env_config, 0));
/* Append it to the stack. */
- __conn_config_append(cfg, env_config);
+ __conn_config_append(cfg, cbuf->data);
- return (0);
+err: __wt_free(session, env_config);
+
+ return (ret);
}
/*
@@ -1052,17 +1054,19 @@ __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf)
static int
__conn_home(WT_SESSION_IMPL *session, const char *home, const char *cfg[])
{
+ WT_DECL_RET;
WT_CONFIG_ITEM cval;
/* If the application specifies a home directory, use it. */
if (home != NULL)
goto copy;
+ ret = __wt_getenv(session, "WIREDTIGER_HOME", &S2C(session)->home);
+ if (ret == 0)
+ return (0);
+
/* If there's no WIREDTIGER_HOME environment variable, use ".". */
- if ((home = getenv("WIREDTIGER_HOME")) == NULL || strlen(home) == 0) {
- home = ".";
- goto copy;
- }
+ home = ".";
/*
* Security stuff:
diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c
index dbf3a71f222..69fa1f85c93 100644
--- a/src/evict/evict_lru.c
+++ b/src/evict/evict_lru.c
@@ -407,6 +407,15 @@ __evict_has_work(WT_SESSION_IMPL *session, uint32_t *flagsp)
(cache->eviction_dirty_target * bytes_max) / 100)
/* Ignore clean pages unless the cache is too large */
LF_SET(WT_EVICT_PASS_DIRTY);
+ else if (F_ISSET(cache, WT_EVICT_WOULD_BLOCK)) {
+ /*
+ * Evict pages with oldest generation (which would otherwise
+ * block application threads) set regardless of whether we have
+ * reached the eviction trigger.
+ */
+ LF_SET(WT_EVICT_PASS_WOULD_BLOCK);
+ F_CLR(cache, WT_EVICT_WOULD_BLOCK);
+ }
if (F_ISSET(cache, WT_EVICT_STUCK))
LF_SET(WT_EVICT_PASS_AGGRESSIVE);
@@ -1076,6 +1085,14 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp, uint32_t flags)
if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU))
continue;
+ /*
+ * If we are only trickling out pages marked for definite
+ * eviction, skip anything that isn't marked.
+ */
+ if (LF_ISSET(WT_EVICT_PASS_WOULD_BLOCK) &&
+ page->read_gen != WT_READGEN_OLDEST)
+ continue;
+
/* Limit internal pages to 50% unless we get aggressive. */
if ((page->type == WT_PAGE_COL_INT ||
page->type == WT_PAGE_ROW_INT) &&
diff --git a/src/evict/evict_page.c b/src/evict/evict_page.c
index f7cbb55dbe4..4e96898fd92 100644
--- a/src/evict/evict_page.c
+++ b/src/evict/evict_page.c
@@ -24,9 +24,10 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive)
WT_PAGE *page;
WT_PAGE_MODIFY *mod;
WT_TXN_STATE *txn_state;
- int inmem_split, istree;
+ int forced_eviction, inmem_split, istree;
page = ref->page;
+ forced_eviction = (page->read_gen == WT_READGEN_OLDEST);
inmem_split = istree = 0;
WT_RET(__wt_verbose(session, WT_VERB_EVICT,
@@ -116,6 +117,12 @@ done: session->excl_next = 0;
if (txn_state != NULL)
txn_state->snap_min = WT_TXN_NONE;
+ if ((inmem_split || (forced_eviction && ret == EBUSY)) &&
+ !F_ISSET(S2C(session)->cache, WT_EVICT_WOULD_BLOCK)) {
+ F_SET(S2C(session)->cache, WT_EVICT_WOULD_BLOCK);
+ WT_TRET(__wt_evict_server_wake(session));
+ }
+
return (ret);
}
@@ -308,7 +315,6 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref,
uint32_t flags;
btree = S2BT(session);
- page = ref->page;
/*
* Get exclusive access to the page if our caller doesn't have the tree
@@ -327,6 +333,10 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref,
__wt_evict_list_clear_page(session, ref);
}
+ /* Now that we have exclusive access, review the page. */
+ page = ref->page;
+ mod = page->modify;
+
/*
* Recurse through the page's subtree: this happens first because we
* have to write pages in depth-first order, otherwise we'll dirty
@@ -338,8 +348,6 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref,
WT_RET(ret);
}
- mod = page->modify;
-
/*
* If the tree was deepened, there's a requirement that newly created
* internal pages not be evicted until all threads are known to have
@@ -449,7 +457,7 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref,
if (exclusive)
LF_SET(WT_SKIP_UPDATE_ERR);
else if (top && !WT_PAGE_IS_INTERNAL(page) &&
- page->memory_footprint > 10 * btree->maxleafpage)
+ page->read_gen == WT_READGEN_OLDEST)
LF_SET(WT_SKIP_UPDATE_RESTORE);
WT_RET(__wt_reconcile(session, ref, NULL, flags));
WT_ASSERT(session,
diff --git a/src/include/cache.h b/src/include/cache.h
index a6425e97837..cf688b3993f 100644
--- a/src/include/cache.h
+++ b/src/include/cache.h
@@ -19,6 +19,7 @@
#define WT_EVICT_PASS_AGGRESSIVE 0x01
#define WT_EVICT_PASS_ALL 0x02
#define WT_EVICT_PASS_DIRTY 0x04
+#define WT_EVICT_PASS_WOULD_BLOCK 0x08
/*
* WT_EVICT_ENTRY --
@@ -33,7 +34,6 @@ struct __wt_evict_entry {
* WT_EVICT_WORKER --
* Encapsulation of an eviction worker thread.
*/
-
struct __wt_evict_worker {
WT_SESSION_IMPL *session;
u_int id;
@@ -114,6 +114,7 @@ struct __wt_cache {
#define WT_EVICT_ACTIVE 0x04 /* Eviction server is active */
#define WT_EVICT_CLEAR_WALKS 0x08 /* Clear eviction walks */
#define WT_EVICT_STUCK 0x10 /* Eviction server is stuck */
+#define WT_EVICT_WOULD_BLOCK 0x20 /* Pages that would block apps */
uint32_t flags;
};
diff --git a/src/include/extern.h b/src/include/extern.h
index bb7caa991d6..056058f3828 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -442,6 +442,7 @@ extern int __wt_directory_sync(WT_SESSION_IMPL *session, char *path);
extern int __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh);
extern int __wt_fsync_async(WT_SESSION_IMPL *session, WT_FH *fh);
extern int __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t len);
+extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp);
extern int __wt_getline(WT_SESSION_IMPL *session, WT_ITEM *buf, FILE *fp);
extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr);
extern int __wt_mmap(WT_SESSION_IMPL *session, WT_FH *fh, void *mapp, size_t *lenp, void **mappingcookie);
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index c31b44e67b0..19ccc313b49 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -2613,7 +2613,8 @@ struct __wt_collator {
const char *uri, WT_CONFIG_ITEM *appcfg, WT_COLLATOR **customp);
/*!
- * If non-NULL, a callback performed when the database is closed.
+ * If non-NULL a callback performed when the data source is closed
+ * for customized extractors otherwise when the database is closed.
*
* The WT_COLLATOR::terminate callback is intended to allow cleanup,
* the handle will not be subsequently accessed by WiredTiger.
@@ -3037,7 +3038,9 @@ struct __wt_extractor {
const char *uri, WT_CONFIG_ITEM *appcfg, WT_EXTRACTOR **customp);
/*!
- * If non-NULL, a callback performed when the database is closed.
+ * If non-NULL a callback performed when the index or column group
+ * is closed for customized extractors otherwise when the database
+ * is closed.
*
* The WT_EXTRACTOR::terminate callback is intended to allow cleanup,
* the handle will not be subsequently accessed by WiredTiger.
diff --git a/src/os_posix/os_getenv.c b/src/os_posix/os_getenv.c
new file mode 100644
index 00000000000..37fe2af8501
--- /dev/null
+++ b/src/os_posix/os_getenv.c
@@ -0,0 +1,25 @@
+/*-
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ * All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+#include "wt_internal.h"
+
+/*
+ * __wt_getenv --
+ * Get a non-NULL, greater than zero-length environment variable.
+ */
+int
+__wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp)
+{
+ const char *temp;
+
+ *envp = NULL;
+
+ if (((temp = getenv(variable)) != NULL) && strlen(temp) > 0)
+ return (__wt_strdup(session, temp, envp));
+
+ return (WT_NOTFOUND);
+}
diff --git a/src/os_win/os_getenv.c b/src/os_win/os_getenv.c
new file mode 100644
index 00000000000..8cc94abb524
--- /dev/null
+++ b/src/os_win/os_getenv.c
@@ -0,0 +1,35 @@
+/*-
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ * All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+#include "wt_internal.h"
+
+/*
+ * __wt_getenv --
+ * Get a non-NULL, greater than zero-length environment variable.
+ */
+int
+__wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp)
+{
+ WT_DECL_RET;
+ DWORD size;
+
+ *envp = NULL;
+
+ size = GetEnvironmentVariableA(variable, NULL, 0);
+ if (size <= 1)
+ return (WT_NOTFOUND);
+
+ WT_RET(__wt_calloc(session, 1, size, envp));
+
+ ret = GetEnvironmentVariableA(variable, *envp, size);
+ /* We expect the number of bytes not including nul terminator. */
+ if ((ret + 1) != size)
+ WT_RET_MSG(session, __wt_errno(),
+ "GetEnvironmentVariableA failed: %s", variable);
+
+ return (0);
+}
diff --git a/src/schema/schema_list.c b/src/schema/schema_list.c
index 02556eb942c..1fbc52821a6 100644
--- a/src/schema/schema_list.c
+++ b/src/schema/schema_list.c
@@ -139,9 +139,13 @@ __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX *idx)
WT_DECL_RET;
/* If there is a custom extractor configured, terminate it. */
- if (idx->extractor != NULL && idx->extractor->terminate != NULL)
+ if (idx->extractor != NULL &&
+ idx->extractor_owned && idx->extractor->terminate != NULL) {
WT_TRET(idx->extractor->terminate(
idx->extractor, &session->iface));
+ idx->extractor = NULL;
+ idx->extractor_owned = 0;
+ }
__wt_free(session, idx->name);
__wt_free(session, idx->source);
diff --git a/src/session/session_api.c b/src/session/session_api.c
index 8f460dcc29f..0effc1a7248 100644
--- a/src/session/session_api.c
+++ b/src/session/session_api.c
@@ -547,9 +547,12 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config)
session = (WT_SESSION_IMPL *)wt_session;
SESSION_API_CALL(session, salvage, config, cfg);
+ /* Block out checkpoints to avoid spurious EBUSY errors. */
+ __wt_spin_lock(session, &S2C(session)->checkpoint_lock);
WT_WITH_SCHEMA_LOCK(session,
ret = __wt_schema_worker(session, uri, __wt_salvage,
NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_SALVAGE));
+ __wt_spin_unlock(session, &S2C(session)->checkpoint_lock);
err: API_END_RET_NOTFOUND_MAP(session, ret);
}
@@ -682,9 +685,12 @@ __session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config)
session = (WT_SESSION_IMPL *)wt_session;
SESSION_API_CALL(session, upgrade, config, cfg);
+ /* Block out checkpoints to avoid spurious EBUSY errors. */
+ __wt_spin_lock(session, &S2C(session)->checkpoint_lock);
WT_WITH_SCHEMA_LOCK(session,
ret = __wt_schema_worker(session, uri, __wt_upgrade,
NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_UPGRADE));
+ __wt_spin_unlock(session, &S2C(session)->checkpoint_lock);
err: API_END_RET_NOTFOUND_MAP(session, ret);
}
@@ -702,9 +708,12 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config)
session = (WT_SESSION_IMPL *)wt_session;
SESSION_API_CALL(session, verify, config, cfg);
+ /* Block out checkpoints to avoid spurious EBUSY errors. */
+ __wt_spin_lock(session, &S2C(session)->checkpoint_lock);
WT_WITH_SCHEMA_LOCK(session,
ret = __wt_schema_worker(session, uri, __wt_verify,
NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_VERIFY));
+ __wt_spin_unlock(session, &S2C(session)->checkpoint_lock);
err: API_END_RET_NOTFOUND_MAP(session, ret);
}
diff --git a/test/format/config.h b/test/format/config.h
index 1b5d21d109b..2ab13620486 100644
--- a/test/format/config.h
+++ b/test/format/config.h
@@ -66,6 +66,10 @@ typedef struct {
#define CONF_RAND(cp) MMRAND((cp)->min, (cp)->maxrand)
static CONFIG c[] = {
+ { "abort",
+ "if timed run should drop core", /* 0% */
+ C_BOOL, 0, 0, 0, &g.c_abort, NULL },
+
{ "auto_throttle",
"if LSM inserts are throttled", /* 90% */
C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL },
@@ -196,6 +200,18 @@ static CONFIG c[] = {
"if logging configured", /* 30% */
C_BOOL, 30, 0, 0, &g.c_logging, NULL },
+ { "logging_archive",
+ "if log file archival configured", /* 50% */
+ C_BOOL, 50, 0, 0, &g.c_logging_archive, NULL },
+
+ { "logging_prealloc",
+ "if log file pre-allocation configured", /* 50% */
+ C_BOOL, 50, 0, 0, &g.c_logging_prealloc, NULL },
+
+ { "logging",
+ "if logging configured", /* 30% */
+ C_BOOL, 30, 0, 0, &g.c_logging, NULL },
+
{ "lsm_worker_threads",
"the number of LSM worker threads",
0x0, 3, 4, 20, &g.c_lsm_worker_threads, NULL },
diff --git a/test/format/format.h b/test/format/format.h
index 902cea6cc5d..9290a93e5db 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -166,7 +166,8 @@ typedef struct {
char *config_open; /* Command-line configuration */
- uint32_t c_auto_throttle; /* Config values */
+ uint32_t c_abort; /* Config values */
+ uint32_t c_auto_throttle;
uint32_t c_backups;
uint32_t c_bitcnt;
uint32_t c_bloom;
@@ -199,6 +200,8 @@ typedef struct {
uint32_t c_leaf_page_max;
uint32_t c_leak_memory;
uint32_t c_logging;
+ uint32_t c_logging_archive;
+ uint32_t c_logging_prealloc;
uint32_t c_lsm_worker_threads;
uint32_t c_merge_max;
uint32_t c_mmap;
diff --git a/test/format/ops.c b/test/format/ops.c
index bbaeabcc479..906d32fee82 100644
--- a/test/format/ops.c
+++ b/test/format/ops.c
@@ -139,10 +139,18 @@ wts_ops(void)
break;
}
- /* Tell the thread if it's done. */
if (thread_ops == 0) {
- if (fourths == 0)
+ /*
+ * Optionally drop core (for testing recovery),
+ * otherwise tell the thread it's done.
+ */
+ if (fourths == 0) {
+ if (g.c_abort) {
+ static char *core = NULL;
+ *core = 0;
+ }
tinfo[i].quit = 1;
+ }
} else
if (tinfo[i].ops >= thread_ops)
tinfo[i].quit = 1;
diff --git a/test/format/recover.sh b/test/format/recover.sh
new file mode 100644
index 00000000000..22e60332711
--- /dev/null
+++ b/test/format/recover.sh
@@ -0,0 +1,31 @@
+#! /bin/sh
+
+# Timer: how many minutes format runs before aborting.
+timer=2
+
+# Runs: set to 0 to run infinitely.
+runs=1000
+
+# Config: additional test/format configuration
+config=
+
+count=0
+while true; do
+ count=`expr $count + 1`
+ if test $runs -eq 0; then
+ echo "recovery test: $count"
+ else
+ if test $count -gt $runs; then
+ exit 0
+ fi
+ echo "recovery test: $count of $runs"
+ fi
+
+ ./t $config -q abort=1 logging=1 timer=$timer
+
+ uri='file:wt'
+ if `wt -h RUNDIR list | egrep table > /dev/null`; then
+ uri='table:wt'
+ fi
+ wt -h RUNDIR verify $uri || exit 1
+done
diff --git a/test/format/wts.c b/test/format/wts.c
index 21e7806e982..243d1deb6d7 100644
--- a/test/format/wts.c
+++ b/test/format/wts.c
@@ -63,6 +63,9 @@ static WT_EVENT_HANDLER event_handler = {
NULL /* Close handler. */
};
+#undef REMAIN
+#define REMAIN(p, end) (size_t)((p) >= (end) ? 0 : (end) - (p))
+
/*
* wts_open --
* Open a connection to a WiredTiger database.
@@ -72,58 +75,76 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp)
{
WT_CONNECTION *conn;
int ret;
- const char *buffer_align, *progname;
- char config[2048], evict_config[64];
+ char config[4096], *end, *p;
*connp = NULL;
- /* Build the eviction worker thread config string, if needed. */
- evict_config[0] = '\0';
- if (g.c_evict_max != 0 &&
- snprintf(evict_config, sizeof(evict_config),
- "eviction=(threads_max=%" PRIu32 "),",
- g.c_evict_max) >= (int)sizeof(evict_config))
- die(ENOMEM, "eviction configuration buffer too small");
+ p = config;
+ end = config + sizeof(config);
+
+ p += snprintf(p, REMAIN(p, end),
+ "create,checkpoint_sync=false,cache_size=%" PRIu32 "MB",
+ g.c_cache);
- /*
- * Put configuration file configuration options second to last. Put
- * command line configuration options at the end. Do this so they
- * override the standard configuration.
- */
#ifdef _WIN32
- progname = "t_format.exe";
- buffer_align = "";
+ p += snprintf(p, REMAIN(p, end), ",error_prefix=\"t_format.exe\"");
#else
- progname = g.progname;
- buffer_align = "buffer_alignment=512";
+ p += snprintf(p, REMAIN(p, end), ",error_prefix=\"%s\"", g.progname);
+#endif
+
+ /* LSM configuration. */
+ if (DATASOURCE("lsm"))
+ p += snprintf(p, REMAIN(p, end),
+ ",lsm_manager=(worker_thread_max=%" PRIu32 "),",
+ g.c_lsm_worker_threads);
+
+ /* Eviction worker configuration. */
+ if (g.c_evict_max != 0)
+ p += snprintf(p, REMAIN(p, end),
+ ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max);
+
+ /* Logging configuration. */
+ if (g.c_logging)
+ p += snprintf(p, REMAIN(p, end),
+ ",log=(enabled=true,archive=%d,prealloc=%d)",
+ g.c_logging_archive ? 1 : 0,
+ g.c_logging_prealloc ? 1 : 0);
+
+ /* Miscellaneous. */
+#ifndef _WIN32
+ p += snprintf(p, REMAIN(p, end), ",buffer_alignment=512");
#endif
- if (snprintf(config, sizeof(config),
- "create,"
- "checkpoint_sync=false,cache_size=%" PRIu32 "MB,"
- "%s,lsm_manager=(worker_thread_max=%" PRIu32
- "),error_prefix=\"%s\","
- "%s,%s,%s,%s,%s"
- "extensions="
- "[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],"
- "%s,%s",
- g.c_cache,
- buffer_align,
- g.c_lsm_worker_threads,
- progname,
- g.c_data_extend ? "file_extend=(data=8MB)" : "",
- g.c_logging ? "log=(enabled=true)" : "",
- g.c_mmap ? "mmap=true" : "mmap=false",
- g.c_statistics ? "statistics=(fast)" : "statistics=(none)",
- evict_config,
+
+ p += snprintf(p, REMAIN(p, end), ",mmap=%d", g.c_mmap ? 1 : 0);
+
+ if (g.c_data_extend)
+ p += snprintf(p, REMAIN(p, end), ",file_extend=(data=8MB)");
+
+ p += snprintf(p, REMAIN(p, end),
+ ",statistics=(%s)", g.c_statistics ? "fast" : "none");
+
+ /* Extensions. */
+ p += snprintf(p, REMAIN(p, end),
+ ",extensions=[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],",
g.c_reverse ? REVERSE_PATH : "",
access(BZIP_PATH, R_OK) == 0 ? BZIP_PATH : "",
access(LZO_PATH, R_OK) == 0 ? LZO_PATH : "",
access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "",
access(ZLIB_PATH, R_OK) == 0 ? ZLIB_PATH : "",
- DATASOURCE("kvsbdb") ? KVS_BDB_PATH : "",
- g.c_config_open == NULL ? "" : g.c_config_open,
- g.config_open == NULL ? "" : g.config_open) >= (int)sizeof(config))
- die(ENOMEM, "configuration buffer too small");
+ DATASOURCE("kvsbdb") ? KVS_BDB_PATH : "");
+
+ /*
+ * Put configuration file configuration options second to last. Put
+ * command line configuration options at the end. Do this so they
+ * override the standard configuration.
+ */
+ if (g.c_config_open != NULL)
+ p += snprintf(p, REMAIN(p, end), ",%s", g.c_config_open);
+ if (g.config_open != NULL)
+ p += snprintf(p, REMAIN(p, end), ",%s", g.config_open);
+
+ if (REMAIN(p, end) == 0)
+ die(ENOMEM, "wiredtiger_open configuration buffer too small");
/*
* Direct I/O may not work with backups, doing copies through the buffer
@@ -200,7 +221,7 @@ wts_create(void)
if (maxleafpage > 512)
maxleafpage >>= 1;
}
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
"key_format=%s,"
"allocation_size=512,%s"
"internal_page_max=%d,leaf_page_max=%d",
@@ -214,43 +235,43 @@ wts_create(void)
*/
maxintlkey = MMRAND(maxintlpage / 50, maxintlpage / 40);
if (maxintlkey > 20)
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",internal_key_max=%d", maxintlkey);
maxleafkey = MMRAND(maxleafpage / 50, maxleafpage / 40);
if (maxleafkey > 20)
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",leaf_key_max=%d", maxleafkey);
maxleafvalue = MMRAND(maxleafpage * 10, maxleafpage / 40);
if (maxleafvalue > 40 && maxleafvalue < 100 * 1024)
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",leaf_value_max=%d", maxleafvalue);
switch (g.type) {
case FIX:
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",value_format=%" PRIu32 "t", g.c_bitcnt);
break;
case ROW:
if (g.c_huffman_key)
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",huffman_key=english");
if (g.c_prefix_compression)
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",prefix_compression_min=%" PRIu32,
g.c_prefix_compression_min);
else
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",prefix_compression=false");
if (g.c_reverse)
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",collator=reverse");
/* FALLTHROUGH */
case VAR:
if (g.c_huffman_value)
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",huffman_value=english");
if (g.c_dictionary)
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",dictionary=%d", MMRAND(123, 517));
break;
}
@@ -258,14 +279,13 @@ wts_create(void)
/* Configure checksums. */
switch (g.c_checksum_flag) {
case CHECKSUM_OFF:
- p += snprintf(p, (size_t)(end - p), ",checksum=\"off\"");
+ p += snprintf(p, REMAIN(p, end), ",checksum=\"off\"");
break;
case CHECKSUM_ON:
- p += snprintf(p, (size_t)(end - p), ",checksum=\"on\"");
+ p += snprintf(p, REMAIN(p, end), ",checksum=\"on\"");
break;
case CHECKSUM_UNCOMPRESSED:
- p += snprintf(
- p, (size_t)(end - p), ",checksum=\"uncompressed\"");
+ p += snprintf(p, REMAIN(p, end), ",checksum=\"uncompressed\"");
break;
}
@@ -274,57 +294,55 @@ wts_create(void)
case COMPRESS_NONE:
break;
case COMPRESS_BZIP:
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",block_compressor=\"bzip2\"");
break;
case COMPRESS_BZIP_RAW:
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",block_compressor=\"bzip2-raw-test\"");
break;
case COMPRESS_LZO:
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",block_compressor=\"LZO1B-6\"");
break;
case COMPRESS_SNAPPY:
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",block_compressor=\"snappy\"");
break;
case COMPRESS_ZLIB:
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",block_compressor=\"zlib\"");
break;
case COMPRESS_ZLIB_NO_RAW:
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",block_compressor=\"zlib-noraw\"");
break;
}
/* Configure Btree internal key truncation. */
- p += snprintf(
- p, (size_t)(end - p), ",internal_key_truncate=%s",
+ p += snprintf(p, REMAIN(p, end), ",internal_key_truncate=%s",
g.c_internal_key_truncation ? "true" : "false");
/* Configure Btree page key gap. */
- p += snprintf(p, (size_t)(end - p), ",key_gap=%" PRIu32, g.c_key_gap);
+ p += snprintf(p, REMAIN(p, end), ",key_gap=%" PRIu32, g.c_key_gap);
/* Configure Btree split page percentage. */
- p += snprintf(p, (size_t)(end - p),
- ",split_pct=%" PRIu32, g.c_split_pct);
+ p += snprintf(p, REMAIN(p, end), ",split_pct=%" PRIu32, g.c_split_pct);
/* Configure LSM and data-sources. */
if (DATASOURCE("helium"))
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
",type=helium,helium_o_compress=%d,helium_o_truncate=1",
g.c_compression_flag == COMPRESS_NONE ? 0 : 1);
if (DATASOURCE("kvsbdb"))
- p += snprintf(p, (size_t)(end - p), ",type=kvsbdb");
+ p += snprintf(p, REMAIN(p, end), ",type=kvsbdb");
if (DATASOURCE("lsm")) {
- p += snprintf(p, (size_t)(end - p), ",type=lsm,lsm=(");
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end), ",type=lsm,lsm=(");
+ p += snprintf(p, REMAIN(p, end),
"auto_throttle=%s,", g.c_auto_throttle ? "true" : "false");
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
"chunk_size=%" PRIu32 "MB,", g.c_chunk_size);
/*
* We can't set bloom_oldest without bloom, and we want to test
@@ -332,19 +350,22 @@ wts_create(void)
*/
if (g.c_bloom_oldest)
g.c_bloom = 1;
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
"bloom=%s,", g.c_bloom ? "true" : "false");
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
"bloom_bit_count=%" PRIu32 ",", g.c_bloom_bit_count);
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
"bloom_hash_count=%" PRIu32 ",", g.c_bloom_hash_count);
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
"bloom_oldest=%s,", g.c_bloom_oldest ? "true" : "false");
- p += snprintf(p, (size_t)(end - p),
+ p += snprintf(p, REMAIN(p, end),
"merge_max=%" PRIu32 ",", g.c_merge_max);
- p += snprintf(p, (size_t)(end - p), ",)");
+ p += snprintf(p, REMAIN(p, end), ",)");
}
+ if (REMAIN(p, end) == 0)
+ die(ENOMEM, "WT_SESSION.create configuration buffer too small");
+
/*
* Create the underlying store.
*/
diff --git a/test/suite/test_schema03.py b/test/suite/test_schema03.py
index 48d8e6b2986..e650d8b6964 100644
--- a/test/suite/test_schema03.py
+++ b/test/suite/test_schema03.py
@@ -25,10 +25,17 @@
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
-import resource
+import os
import suite_random
import wiredtiger, wtscenario, wttest
+try:
+ # Windows does not getrlimit/setrlimit so we must catch the resource
+ # module load
+ import resource
+except:
+ None
+
# test_schema03.py
# Bigger, more 'randomly generated' schemas and data.
# This test is complex. If it fails, rerun with modified values for
@@ -190,7 +197,7 @@ class test_schema03(wttest.WiredTigerTestCase):
created in various orders as much as the API allows. On some runs
the connection will be closed and reopened at a particular point
to test that the schemas (and data) are saved and read correctly.
-
+
The test is run multiple times, using scenarios.
The test always follows these steps:
- table: create tables
@@ -277,6 +284,9 @@ class test_schema03(wttest.WiredTigerTestCase):
# This test requires a large number of open files.
# Increase our resource limits before we start
def setUp(self):
+ if os.name == "nt":
+ self.skipTest('Unix specific test skipped on Windows')
+
self.origFileLimit = resource.getrlimit(resource.RLIMIT_NOFILE)
newlimit = (self.OPEN_FILE_LIMIT, self.origFileLimit[1])
if newlimit[0] > newlimit[1]:
@@ -289,7 +299,7 @@ class test_schema03(wttest.WiredTigerTestCase):
'create,cache_size=100m,session_max=1000')
self.pr(`conn`)
return conn
-
+
def tearDown(self):
super(test_schema03, self).tearDown()
resource.setrlimit(resource.RLIMIT_NOFILE, self.origFileLimit)