summaryrefslogtreecommitdiff
path: root/src/third_party
diff options
context:
space:
mode:
authorLuke Chen <luke.chen@mongodb.com>2018-08-13 15:46:37 +1000
committerLuke Chen <luke.chen@mongodb.com>2018-08-13 15:46:37 +1000
commitec6f2a54d8b901e95e556d53647944be8ce441d5 (patch)
treea75c521f1918cf0343acf2ced1669567e2a7b854 /src/third_party
parent1c52af393ebc9040fbc300293835f68f827304ce (diff)
downloadmongo-ec6f2a54d8b901e95e556d53647944be8ce441d5.tar.gz
Import wiredtiger: c91b80412603f283532e267893f9238dd4a5ec0f from branch mongodb-4.2
ref: 65531924e3..c91b804126 for: 4.1.2 WT-3856 Create a test that runs recovery to different points of time with schema operations WT-4026 Add implementation for existing file extension configuration API WT-4193 test/format snapshot-isolation search mismatch WT-4194 Improve fairness of eviction with multiple tables WT-4207 Coverity #1394567: null pointer dereference WT-4210 schema abort child process failing prematurely WT-4215 Allow recovery of backup without salvage WT-4229 Lint WT-4234 Remove documentation mention of legacy tool statlog.py WT-4235 Fix workgen tracking of table state across workloads WT-4242 New log file extension Python test failure
Diffstat (limited to 'src/third_party')
-rw-r--r--src/third_party/wiredtiger/.gitignore1
-rw-r--r--src/third_party/wiredtiger/bench/workgen/workgen.cxx7
-rw-r--r--src/third_party/wiredtiger/dist/api_data.py9
-rw-r--r--src/third_party/wiredtiger/import.data2
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_cursor.c20
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_read.c4
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_walk.c4
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_api.c22
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_log.c10
-rw-r--r--src/third_party/wiredtiger/src/docs/statistics.dox5
-rw-r--r--src/third_party/wiredtiger/src/docs/upgrading.dox16
-rw-r--r--src/third_party/wiredtiger/src/evict/evict_lru.c33
-rw-r--r--src/third_party/wiredtiger/src/include/btree.h2
-rw-r--r--src/third_party/wiredtiger/src/include/btree.i4
-rw-r--r--src/third_party/wiredtiger/src/include/cache.h1
-rw-r--r--src/third_party/wiredtiger/src/include/config.h1
-rw-r--r--src/third_party/wiredtiger/src/include/error.h2
-rw-r--r--src/third_party/wiredtiger/src/include/extern.h10
-rw-r--r--src/third_party/wiredtiger/src/include/log.h6
-rw-r--r--src/third_party/wiredtiger/src/include/misc.h2
-rw-r--r--src/third_party/wiredtiger/src/include/session.h4
-rw-r--r--src/third_party/wiredtiger/src/include/wiredtiger.in34
-rw-r--r--src/third_party/wiredtiger/src/log/log.c71
-rw-r--r--src/third_party/wiredtiger/src/lsm/lsm_tree.c1
-rw-r--r--src/third_party/wiredtiger/src/reconcile/rec_write.c2
-rw-r--r--src/third_party/wiredtiger/src/support/err.c16
-rw-r--r--src/third_party/wiredtiger/src/support/hazard.c8
-rw-r--r--src/third_party/wiredtiger/src/support/scratch.c9
-rw-r--r--src/third_party/wiredtiger/test/csuite/schema_abort/main.c6
-rw-r--r--src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c7
-rw-r--r--src/third_party/wiredtiger/test/format/ops.c16
-rw-r--r--src/third_party/wiredtiger/test/suite/test_config07.py103
-rw-r--r--src/third_party/wiredtiger/test/suite/test_cursor12.py10
-rw-r--r--src/third_party/wiredtiger/test/suite/test_las01.py2
-rw-r--r--src/third_party/wiredtiger/test/suite/test_schema08.py189
35 files changed, 521 insertions, 118 deletions
diff --git a/src/third_party/wiredtiger/.gitignore b/src/third_party/wiredtiger/.gitignore
index 4a9e098a17f..b57a37aad83 100644
--- a/src/third_party/wiredtiger/.gitignore
+++ b/src/third_party/wiredtiger/.gitignore
@@ -104,6 +104,7 @@ _wiredtiger.pyd
**/test/checkpoint/t
**/test/csuite/test_random_abort
**/test/csuite/test_rwlock
+**/test/csuite/test_schema_abort
**/test/csuite/test_scope
**/test/csuite/test_timestamp_abort
**/test/csuite/test_truncated_log
diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx
index 74393b13681..9ae63682f9c 100644
--- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx
+++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx
@@ -254,12 +254,11 @@ ContextInternal::~ContextInternal() {
}
int ContextInternal::create_all() {
- if (_runtime_alloced != _tint_last) {
+ if (_runtime_alloced < _tint_last) {
// The array references are 1-based, we'll waste one entry.
TableRuntime *new_table_runtime = new TableRuntime[_tint_last + 1];
- memcpy(new_table_runtime, _table_runtime, sizeof(uint64_t) * _runtime_alloced);
- memset(&new_table_runtime[_runtime_alloced], 0,
- sizeof(uint64_t) * (_tint_last - _runtime_alloced + 1));
+ for (int i = 0; i < _runtime_alloced; i++)
+ new_table_runtime[i + 1] = _table_runtime[i + 1];
delete _table_runtime;
_table_runtime = new_table_runtime;
_runtime_alloced = _tint_last;
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index 66825754a6a..36f78816b84 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -708,7 +708,8 @@ wiredtiger_open_log_configuration = [
information'''),
Config('file_max', '100MB', r'''
the maximum size of log files''',
- min='100KB', max='2GB'),
+ min='100KB', # !!! Must match WT_LOG_FILE_MIN
+ max='2GB'), # !!! Must match WT_LOG_FILE_MAX
Config('path', '"."', r'''
the name of a directory into which log files are written. The
directory must already exist. If the value is not an absolute path,
@@ -862,7 +863,11 @@ wiredtiger_open_common =\
file extension configuration. If set, extend files of the set
type in allocations of the set size, instead of a block at a
time as each new block is written. For example,
- <code>file_extend=(data=16MB)</code>''',
+ <code>file_extend=(data=16MB)</code>. If set to 0, disable the file
+ extension for the set type. For log files, the allowed range is
+ between 100KB and 2GB; values larger than the configured maximum log
+ size and the default config would extend log files in allocations of
+ the maximum log file size.''',
type='list', choices=['data', 'log']),
Config('hazard_max', '1000', r'''
maximum number of simultaneous hazard pointers per session
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 67a45c86ec2..bf83ce694d9 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -1,5 +1,5 @@
{
- "commit": "65531924e397aeb55f6835b73e4c199ea378ec97",
+ "commit": "c91b80412603f283532e267893f9238dd4a5ec0f",
"github": "wiredtiger/wiredtiger.git",
"vendor": "wiredtiger",
"branch": "mongodb-4.2"
diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c
index b6172966eae..75d868eed08 100644
--- a/src/third_party/wiredtiger/src/btree/bt_cursor.c
+++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c
@@ -1414,10 +1414,6 @@ __wt_btcur_modify(WT_CURSOR_BTREE *cbt, WT_MODIFY *entries, int nentries)
/* Save the cursor state. */
__cursor_state_save(cursor, &state);
- if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED)
- WT_ERR_MSG(session, ENOTSUP,
- "not supported in read-uncommitted transactions");
-
/*
* Get the current value and apply the modification to it, for a few
* reasons: first, we set the updated value so the application can
@@ -1428,7 +1424,23 @@ __wt_btcur_modify(WT_CURSOR_BTREE *cbt, WT_MODIFY *entries, int nentries)
* trouble if we attempt to modify a value that doesn't exist. For the
* fifth reason, verify we're not in a read-uncommitted transaction,
* that implies a value that might disappear out from under us.
+ *
+ * Also, an application might read a value outside of a transaction and
+ * then call modify. For that to work, the read must be part of the
+ * transaction that performs the update for correctness, otherwise we
+ * could race with another thread and end up modifying the wrong value.
+ * A clever application could get this right (imagine threads that only
+ * updated non-overlapping, fixed-length byte strings), but it's unsafe
+ * because it will work most of the time and the failure is unlikely to
+ * be detected. Require explicit transactions for modify operations.
*/
+ if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED)
+ WT_ERR_MSG(session, ENOTSUP,
+ "not supported in read-uncommitted transactions");
+ if (F_ISSET(&session->txn, WT_TXN_AUTOCOMMIT))
+ WT_ERR_MSG(session, ENOTSUP,
+ "not supported in implicit transactions");
+
if (!F_ISSET(cursor, WT_CURSTD_KEY_INT) ||
!F_ISSET(cursor, WT_CURSTD_VALUE_INT))
WT_ERR(__wt_btcur_search(cbt));
diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c
index 58853bff6ac..7f3391984b2 100644
--- a/src/third_party/wiredtiger/src/btree/bt_read.c
+++ b/src/third_party/wiredtiger/src/btree/bt_read.c
@@ -553,7 +553,7 @@ err: /*
int
__wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags
#ifdef HAVE_DIAGNOSTIC
- , const char *file, int line
+ , const char *func, int line
#endif
)
{
@@ -683,7 +683,7 @@ read: /*
*/
#ifdef HAVE_DIAGNOSTIC
WT_RET(
- __wt_hazard_set(session, ref, &busy, file, line));
+ __wt_hazard_set(session, ref, &busy, func, line));
#else
WT_RET(__wt_hazard_set(session, ref, &busy));
#endif
diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c
index 6434142824e..3dde08b43c1 100644
--- a/src/third_party/wiredtiger/src/btree/bt_walk.c
+++ b/src/third_party/wiredtiger/src/btree/bt_walk.c
@@ -443,8 +443,8 @@ restart: /*
goto done;
}
- WT_ASSERT(session, ret == WT_NOTFOUND);
- WT_ERR_NOTFOUND_OK(ret);
+ WT_ASSERT(session, ret == WT_NOTFOUND);
+ WT_ERR_NOTFOUND_OK(ret);
__wt_spin_backoff(&yield_count, &sleep_usecs);
}
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index 47ab622a7f4..3d06bac02c5 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -2626,6 +2626,12 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
#endif
WT_ERR(__wt_config_gets(session, cfg, "file_extend", &cval));
+ /*
+ * If the log extend length is not set use the default of the configured
+ * maximum log file size. That size is not known until it is initialized
+ * as part of the log server initialization.
+ */
+ conn->log_extend_len = WT_CONFIG_UNSET;
for (ft = file_types; ft->name != NULL; ft++) {
ret = __wt_config_subgets(session, &cval, ft->name, &sval);
if (ret == 0) {
@@ -2634,7 +2640,21 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
conn->data_extend_len = sval.val;
break;
case WT_DIRECT_IO_LOG:
- conn->log_extend_len = sval.val;
+ /*
+ * When using "file_extend=(log=)", the val
+ * returned is 1. Unset the log extend length
+ * in that case to use the default.
+ */
+ if (sval.val == 1)
+ conn->log_extend_len = WT_CONFIG_UNSET;
+ else if (sval.val == 0 ||
+ (sval.val >= WT_LOG_FILE_MIN &&
+ sval.val <= WT_LOG_FILE_MAX))
+ conn->log_extend_len = sval.val;
+ else
+ WT_ERR_MSG(session, EINVAL,
+ "invalid log extend length: %"
+ PRId64, sval.val);
break;
}
} else
diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c
index d42789c809e..9f4af1aba88 100644
--- a/src/third_party/wiredtiger/src/conn/conn_log.c
+++ b/src/third_party/wiredtiger/src/conn/conn_log.c
@@ -277,6 +277,15 @@ __logmgr_config(
if (!reconfig) {
WT_RET(__wt_config_gets(session, cfg, "log.file_max", &cval));
conn->log_file_max = (wt_off_t)cval.val;
+ /*
+ * With the default log file extend configuration or if the log
+ * file extension size is larger than the configured maximum log
+ * file size, set the log file extension size to the configured
+ * maximum log file size.
+ */
+ if (conn->log_extend_len == WT_CONFIG_UNSET ||
+ conn->log_extend_len > conn->log_file_max)
+ conn->log_extend_len = conn->log_file_max;
WT_STAT_CONN_SET(session, log_max_filesize, conn->log_file_max);
}
@@ -890,7 +899,6 @@ __log_wrlsn_server(void *arg)
__wt_log_wrlsn(session, NULL);
if (0) {
err: WT_PANIC_MSG(session, ret, "log wrlsn server error");
-
}
return (WT_THREAD_RET_VALUE);
}
diff --git a/src/third_party/wiredtiger/src/docs/statistics.dox b/src/third_party/wiredtiger/src/docs/statistics.dox
index 26c4b66fa40..19b7b17257b 100644
--- a/src/third_party/wiredtiger/src/docs/statistics.dox
+++ b/src/third_party/wiredtiger/src/docs/statistics.dox
@@ -156,9 +156,4 @@ currently open in the database, nor will any statistics requiring the
traversal of a tree (as if the \c statistics_fast configuration string
were set).
-A Python script that parses the default logging output and uses the
-<a href="http://www.gnuplot.info/">gnuplot</a>, utility to generate
-Portable Network Graphics (PNG) format graphs is included in the
-WiredTiger distribution in the file \c tools/statlog.py.
-
*/
diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox
index 7e89d23230f..0d8e5e1b428 100644
--- a/src/third_party/wiredtiger/src/docs/upgrading.dox
+++ b/src/third_party/wiredtiger/src/docs/upgrading.dox
@@ -1,4 +1,20 @@
/*! @page upgrading Upgrading WiredTiger applications
+
+</dl><hr>
+@section version_311 Upgrading to Version 3.1.1
+<dl>
+
+<dt>WT_CURSOR::modify transaction requirements</dt>
+<dd>
+In previous releases of WiredTiger, it was possible to use implicit
+transactions in combination with WT_CURSOR::modify operations. This
+requires applications be extraordinarily careful to avoid multiple
+threads which are changing the same values racing with each other. In
+the 3.1.1 release, WT_CURSOR::modify operations must be performed in an
+explicit transaction, and will fail if that's not the case.
+</dd>
+
+</dl><hr>
@section version_310 Upgrading to Version 3.1.0
<dl>
diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c
index 13b5dfc4c8d..b072c47a9ba 100644
--- a/src/third_party/wiredtiger/src/evict/evict_lru.c
+++ b/src/third_party/wiredtiger/src/evict/evict_lru.c
@@ -318,7 +318,7 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread)
WT_ERR(__evict_lru_pages(session, false));
if (0) {
-err: WT_PANIC_MSG(session, ret, "cache eviction thread error");
+err: WT_PANIC_RET(session, ret, "cache eviction thread error");
}
return (ret);
}
@@ -357,7 +357,7 @@ __wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread)
WT_VERB_EVICTSERVER, "%s", "cache eviction thread exiting");
if (0) {
-err: WT_PANIC_MSG(session, ret, "cache eviction thread error");
+err: WT_PANIC_RET(session, ret, "cache eviction thread error");
}
return (ret);
}
@@ -810,10 +810,8 @@ __evict_clear_walk(WT_SESSION_IMPL *session)
cache = S2C(session)->cache;
WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_PASS));
- if (session->dhandle == cache->walk_tree) {
+ if (session->dhandle == cache->walk_tree)
cache->walk_tree = NULL;
- cache->walk_target = 0;
- }
if ((ref = btree->evict_ref) == NULL)
return (0);
@@ -1414,10 +1412,8 @@ retry: while (slot < max_entries) {
*/
if ((dhandle = cache->walk_tree) != NULL)
cache->walk_tree = NULL;
- else {
+ else
dhandle = TAILQ_FIRST(&conn->dhqh);
- cache->walk_target = 0;
- }
} else {
if (incr) {
WT_ASSERT(session, dhandle->session_inuse > 0);
@@ -1427,7 +1423,6 @@ retry: while (slot < max_entries) {
cache->walk_tree = NULL;
}
dhandle = TAILQ_NEXT(dhandle, q);
- cache->walk_target = 0;
}
/* If we reach the end of the list, we're done. */
@@ -1601,8 +1596,7 @@ __evict_push_candidate(WT_SESSION_IMPL *session,
* Calculate how many pages to queue for a given tree.
*/
static uint32_t
-__evict_walk_target(
- WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_entries)
+__evict_walk_target(WT_SESSION_IMPL *session, u_int max_entries)
{
WT_CACHE *cache;
uint64_t btree_inuse, bytes_per_slot, cache_inuse;
@@ -1611,7 +1605,7 @@ __evict_walk_target(
cache = S2C(session)->cache;
target_pages_clean = target_pages_dirty = 0;
- total_slots = max_entries - queue->evict_entries;
+ total_slots = max_entries;
/*
* The number of times we should fill the queue by the end of
@@ -1717,14 +1711,13 @@ __evict_walk_tree(WT_SESSION_IMPL *session,
*/
start = queue->evict_queue + *slotp;
remaining_slots = max_entries - *slotp;
- if (cache->walk_target != 0) {
- WT_ASSERT(session, cache->walk_progress <= cache->walk_target);
- target_pages = cache->walk_target - cache->walk_progress;
- } else {
- target_pages = cache->walk_target =
- __evict_walk_target(session, queue, max_entries);
- cache->walk_progress = 0;
+ if (btree->evict_walk_progress >= btree->evict_walk_target) {
+ btree->evict_walk_target =
+ __evict_walk_target(session, max_entries);
+ btree->evict_walk_progress = 0;
}
+ target_pages = WT_MIN(btree->evict_walk_target / QUEUE_FILLS_PER_PASS,
+ btree->evict_walk_target - btree->evict_walk_progress);
if (target_pages > remaining_slots)
target_pages = remaining_slots;
@@ -2017,7 +2010,7 @@ fast: /* If the page can't be evicted, give up. */
continue;
++evict;
++pages_queued;
- ++cache->walk_progress;
+ ++btree->evict_walk_progress;
__wt_verbose(session, WT_VERB_EVICTSERVER,
"select: %p, size %" WT_SIZET_FMT,
diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h
index 2a5be782b06..93e1d8e44ea 100644
--- a/src/third_party/wiredtiger/src/include/btree.h
+++ b/src/third_party/wiredtiger/src/include/btree.h
@@ -192,6 +192,8 @@ struct __wt_btree {
*/
WT_REF *evict_ref; /* Eviction thread's location */
uint64_t evict_priority; /* Relative priority of cached pages */
+ uint32_t evict_walk_progress;/* Eviction walk progress */
+ uint32_t evict_walk_target; /* Eviction walk target */
u_int evict_walk_period; /* Skip this many LRU walks */
u_int evict_walk_saved; /* Saved walk skips for checkpoints */
u_int evict_walk_skips; /* Number of walks skipped */
diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i
index ed3480a40d0..86159e971db 100644
--- a/src/third_party/wiredtiger/src/include/btree.i
+++ b/src/third_party/wiredtiger/src/include/btree.i
@@ -1668,7 +1668,7 @@ static inline int
__wt_page_swap_func(
WT_SESSION_IMPL *session, WT_REF *held, WT_REF *want, uint32_t flags
#ifdef HAVE_DIAGNOSTIC
- , const char *file, int line
+ , const char *func, int line
#endif
)
{
@@ -1691,7 +1691,7 @@ __wt_page_swap_func(
/* Get the wanted page. */
ret = __wt_page_in_func(session, want, flags
#ifdef HAVE_DIAGNOSTIC
- , file, line
+ , func, line
#endif
);
diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h
index 7d07e6dfd98..8afedb30832 100644
--- a/src/third_party/wiredtiger/src/include/cache.h
+++ b/src/third_party/wiredtiger/src/include/cache.h
@@ -149,7 +149,6 @@ struct __wt_cache {
WT_SPINLOCK evict_pass_lock; /* Eviction pass lock */
WT_SESSION_IMPL *walk_session; /* Eviction pass session */
WT_DATA_HANDLE *walk_tree; /* LRU walk current tree */
- uint32_t walk_progress, walk_target;/* Progress in current tree */
WT_SPINLOCK evict_queue_lock; /* Eviction current queue lock */
WT_EVICT_QUEUE evict_queues[WT_EVICT_QUEUE_MAX];
diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h
index 06847117b7d..3ae4fe2aaea 100644
--- a/src/third_party/wiredtiger/src/include/config.h
+++ b/src/third_party/wiredtiger/src/include/config.h
@@ -50,6 +50,7 @@ struct __wt_config_parser_impl {
"", 0, 0, WT_CONFIG_ITEM_NUM \
}
+#define WT_CONFIG_UNSET -1
/*
* DO NOT EDIT: automatically built by dist/api_config.py.
* configuration section: BEGIN
diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h
index 6f28dfae2c8..1d2d21617a6 100644
--- a/src/third_party/wiredtiger/src/include/error.h
+++ b/src/third_party/wiredtiger/src/include/error.h
@@ -98,7 +98,7 @@
/* Called on unexpected code path: locate the failure. */
#define __wt_illegal_value(session, v) \
- __wt_illegal_value_func(session, (uintmax_t)v, __func__, __LINE__)
+ __wt_illegal_value_func(session, (uintmax_t)(v), __func__, __LINE__)
/* Return and branch-to-err-label cases for switch statements. */
#define WT_ILLEGAL_VALUE(session, v) \
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index 34cee52df99..bebc3ad7bc1 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -163,7 +163,7 @@ extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBU
extern int
__wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags
#ifdef HAVE_DIAGNOSTIC
- , const char *file, int line
+ , const char *func, int line
#endif
);
extern int __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
@@ -675,8 +675,8 @@ extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_
extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep);
extern void __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler);
-extern void __wt_err_func(WT_SESSION_IMPL *session, int error, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
-extern void __wt_errx_func(WT_SESSION_IMPL *session, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
+extern void __wt_err_func(WT_SESSION_IMPL *session, int error, const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
+extern void __wt_errx_func(WT_SESSION_IMPL *session, const char *func, int line, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
extern int __wt_set_return_func(WT_SESSION_IMPL *session, const char*func, int line, int err) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_ext_err_printf(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern void __wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((cold));
@@ -708,7 +708,7 @@ extern uint64_t __wt_hash_fnv64(const void *string, size_t len);
extern int
__wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp
#ifdef HAVE_DIAGNOSTIC
- , const char *file, int line
+ , const char *func, int line
#endif
);
extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
@@ -757,7 +757,7 @@ extern const char *__wt_buf_set_size(WT_SESSION_IMPL *session, uint64_t size, bo
extern int
__wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp
#ifdef HAVE_DIAGNOSTIC
- , const char *file, int line
+ , const char *func, int line
#endif
)
WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h
index 2613808e74e..d3e36d37da9 100644
--- a/src/third_party/wiredtiger/src/include/log.h
+++ b/src/third_party/wiredtiger/src/include/log.h
@@ -87,6 +87,12 @@ union __wt_lsn {
#define WT_LOGC_KEY_FORMAT WT_UNCHECKED_STRING(III)
#define WT_LOGC_VALUE_FORMAT WT_UNCHECKED_STRING(qIIIuu)
+/*
+ * Size range for the log files.
+ */
+#define WT_LOG_FILE_MAX ((int64_t)2 * WT_GIGABYTE)
+#define WT_LOG_FILE_MIN (100 * WT_KILOBYTE)
+
#define WT_LOG_SKIP_HEADER(data) \
((const uint8_t *)(data) + offsetof(WT_LOG_RECORD, record))
#define WT_LOG_REC_SIZE(size) \
diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h
index 1180c641222..d7540c333e7 100644
--- a/src/third_party/wiredtiger/src/include/misc.h
+++ b/src/third_party/wiredtiger/src/include/misc.h
@@ -352,7 +352,7 @@ union __wt_rand_state {
* buffer. If not concatenating, clear the size so we don't use \
* any existing contents. \
*/ \
- if (!concatenate) \
+ if (!(concatenate)) \
(buf)->size = 0; \
for (;;) { \
WT_ASSERT(session, (buf)->memsize >= (buf)->size); \
diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h
index cbf572f9a23..10ff7bd48dc 100644
--- a/src/third_party/wiredtiger/src/include/session.h
+++ b/src/third_party/wiredtiger/src/include/session.h
@@ -25,7 +25,7 @@ struct __wt_data_handle_cache {
struct __wt_hazard {
WT_REF *ref; /* Page reference */
#ifdef HAVE_DIAGNOSTIC
- const char *file; /* File/line where hazard acquired */
+ const char *func; /* Function/line hazard acquired */
int line;
#endif
};
@@ -120,7 +120,7 @@ struct __wt_session_impl {
* to applications, create a parallel structure instead.
*/
struct __wt_scratch_track {
- const char *file; /* Allocating file, line */
+ const char *func; /* Allocating function, line */
int line;
} *scratch_track;
#endif
diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in
index c80aaef53fb..99adacb0a1b 100644
--- a/src/third_party/wiredtiger/src/include/wiredtiger.in
+++ b/src/third_party/wiredtiger/src/include/wiredtiger.in
@@ -495,10 +495,22 @@ struct __wt_cursor {
* \c S), or raw byte arrays accessed using a WT_ITEM structure (value
* format type \c u).
*
- * Calling the WT_CURSOR::modify method outside of snapshot isolation
- * can lead to unexpected results. While \c read-committed isolation
- * is supported with the WT_CURSOR::modify method, \c read-uncommitted
- * isolation is not.
+ * The WT_CURSOR::modify method can only be called from within an
+ * explicit transaction configured at a higher isolation level than
+ * \c read-uncommitted. Using \c read-committed isolation is allowed,
+ * but requires caution: reading a value, re-positioning the cursor
+ * and then modifying the value based on the initial read could lead
+ * to unexpected results. Using \c snapshot isolation is recommended.
+ *
+ * The WT_CURSOR::modify method stores a change record in cache and
+ * writes a change record to the log instead of the usual complete
+ * values. Note that WT_CURSOR::modify is generally slower than the
+ * WT_CURSOR::update method, and can result in slower reads because
+ * the complete value must be assembled during retrieval. The
+ * WT_CURSOR::modify method is intended for applications modifying
+ * large records where there is cache or I/O pressure, that is,
+ * applications that will benefit when data updates require less cache
+ * and they write less logging information.
*
* @snippet ex_all.c Modify an existing record
*
@@ -510,12 +522,6 @@ struct __wt_cursor {
* (as it partially depends on the underlying file configuration), but
* is always a small number of bytes less than 4GB.
*
- * The WT_CURSOR::modify method stores a change record in cache and
- * writes a change record to the log, instead of the usual complete
- * value. This can reduce cache and logging requirements, but may result
- * in slower reads because the complete value must be assembled during
- * retrieval.
- *
* @param cursor the cursor handle
* @param entries an array of modification data structures
* @param nentries the number of modification data structures
@@ -2870,8 +2876,12 @@ struct __wt_connection {
* @config{file_extend, file extension configuration. If set\, extend files of
* the set type in allocations of the set size\, instead of a block at a time as
* each new block is written. For example\,
- * <code>file_extend=(data=16MB)</code>., a list\, with values chosen from the
- * following options: \c "data"\, \c "log"; default empty.}
+ * <code>file_extend=(data=16MB)</code>. If set to 0\, disable the file
+ * extension for the set type. For log files\, the allowed range is between
+ * 100KB and 2GB; values larger than the configured maximum log size and the
+ * default config would extend log files in allocations of the maximum log file
+ * size., a list\, with values chosen from the following options: \c "data"\, \c
+ * "log"; default empty.}
* @config{file_manager = (, control how file handles are managed., a set of
* related configuration options defined below.}
* @config{&nbsp;&nbsp;&nbsp;&nbsp;close_handle_minimum, number of handles open
diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c
index d8cff7afa10..c1d7b06ad6d 100644
--- a/src/third_party/wiredtiger/src/log/log.c
+++ b/src/third_party/wiredtiger/src/log/log.c
@@ -208,7 +208,7 @@ __log_fs_write(WT_SESSION_IMPL *session,
WT_RET(__wt_log_force_sync(session, &slot->slot_release_lsn));
}
if ((ret = __wt_write(session, slot->slot_fh, offset, len, buf)) != 0)
- WT_PANIC_MSG(session, ret,
+ WT_PANIC_RET(session, ret,
"%s: fatal log failure", slot->slot_fh->name);
return (ret);
}
@@ -674,11 +674,15 @@ __log_prealloc(WT_SESSION_IMPL *session, WT_FH *fh)
return (__log_zero(session, fh,
WT_LOG_END_HEADER, conn->log_file_max));
+ /* If configured to not extend the file, we're done. */
+ if (conn->log_extend_len == 0)
+ return (0);
+
/*
* We have exclusive access to the log file and there are no other
* writes happening concurrently, so there are no locking issues.
*/
- ret = __wt_fextend(session, fh, conn->log_file_max);
+ ret = __wt_fextend(session, fh, conn->log_extend_len);
return (ret == EBUSY || ret == ENOTSUP ? 0 : ret);
}
@@ -1862,7 +1866,7 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size,
WT_LOG *log;
WT_LOG_RECORD *logrec;
wt_off_t off, remainder;
- size_t buf_left, bufsz, rdlen;
+ size_t allocsize, buf_left, bufsz, rdlen;
char *buf, *p, *zerobuf;
bool corrupt;
@@ -1898,6 +1902,7 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size,
remainder -= (wt_off_t)rdlen, off += (wt_off_t)rdlen) {
rdlen = WT_MIN(bufsz, (size_t)remainder);
WT_ERR(__wt_read(session, fh, off, rdlen, buf));
+ allocsize = (log == NULL ? WT_LOG_ALIGN : log->allocsize);
if (memcmp(buf, zerobuf, rdlen) != 0) {
/*
* Find where the next log record starts after the
@@ -1905,7 +1910,7 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size,
*/
for (p = buf, buf_left = rdlen; buf_left > 0;
buf_left -= rdlen, p += rdlen) {
- rdlen = WT_MIN(log->allocsize, buf_left);
+ rdlen = WT_MIN(allocsize, buf_left);
if (memcmp(p, zerobuf, rdlen) != 0)
break;
}
@@ -1915,14 +1920,21 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size,
* present in the buffer, we either have a valid header
* or corruption. Verify the header of this record to
* determine whether it is just a hole or corruption.
+ *
+ * We don't bother making this check for backup copies,
+ * as records may have their beginning zeroed, hence
+ * the part after a hole may in fact be the middle of
+ * the record.
*/
- logrec = (WT_LOG_RECORD *)p;
- if (buf_left >= sizeof(WT_LOG_RECORD)) {
- off += p - buf;
- WT_ERR(__log_record_verify(session, fh,
- (uint32_t)off, logrec, &corrupt));
- if (corrupt)
- *error_offset = off;
+ if (!F_ISSET(conn, WT_CONN_WAS_BACKUP)) {
+ logrec = (WT_LOG_RECORD *)p;
+ if (buf_left >= sizeof(WT_LOG_RECORD)) {
+ off += p - buf;
+ WT_ERR(__log_record_verify(session, fh,
+ (uint32_t)off, logrec, &corrupt));
+ if (corrupt)
+ *error_offset = off;
+ }
}
*hole = true;
break;
@@ -1945,12 +1957,6 @@ err: __wt_free(session, buf);
* padded with zeroes before writing). The only way we have any certainty
* is if the last byte is non-zero, when that happens, we know that
* the write cannot be partial.
- *
- * When we have a checksum mismatch, it is important to know that whether
- * it may be 1) the result of a partial write or 2) the result of
- * corruption. The former can happen in normal operations, and we
- * will silently truncate the log when it occurs. The latter will
- * result in an error during recovery, and requires salvage to fix.
*/
static bool
__log_check_partial_write(WT_SESSION_IMPL *session, WT_ITEM *buf,
@@ -2480,7 +2486,30 @@ advance:
*/
if (log != NULL)
log->trunc_lsn = rd_lsn;
- /* Make a check to see if it may be a partial write. */
+ /*
+ * If the user asked for a specific LSN and it is not
+ * a valid LSN, return WT_NOTFOUND.
+ */
+ if (LF_ISSET(WT_LOGSCAN_ONE))
+ ret = WT_NOTFOUND;
+
+ /*
+ * When we have a checksum mismatch, we determine
+ * whether it may be the result of:
+ * 1) some expected corruption that can occur during
+ * backups
+ * 2) a partial write that can naturally occur when
+ * an application crashes
+ * 3) some other corruption
+ *
+ * As the first and second happen commonly in normal
+ * operations, we will silently truncate the log when
+ * it occurs. The third will result in an error during
+ * recovery, and requires salvage to fix.
+ */
+ if (F_ISSET(conn, WT_CONN_WAS_BACKUP))
+ break;
+
if (!__log_check_partial_write(session, buf, reclen)) {
/*
* It's not a partial write, and we have a bad
@@ -2505,12 +2534,6 @@ advance:
log_fh->name, "", rd_lsn.l.offset));
}
}
- /*
- * If the user asked for a specific LSN and it is not
- * a valid LSN, return WT_NOTFOUND.
- */
- if (LF_ISSET(WT_LOGSCAN_ONE))
- ret = WT_NOTFOUND;
break;
}
__wt_log_record_byteswap(logrec);
diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c
index 03feafe3c8c..3dd7222630f 100644
--- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c
+++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c
@@ -407,6 +407,7 @@ __lsm_tree_find(WT_SESSION_IMPL *session,
{
WT_LSM_TREE *lsm_tree;
+ *treep = NULL;
WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST));
/* See if the tree is already open. */
diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c
index a38b2ed0f02..145009be4df 100644
--- a/src/third_party/wiredtiger/src/reconcile/rec_write.c
+++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c
@@ -595,8 +595,6 @@ static int
__rec_write_check_complete(
WT_SESSION_IMPL *session, WT_RECONCILE *r, int tret, bool *lookaside_retryp)
{
- WT_UNUSED(session);
-
/*
* Tests in this function are lookaside tests and tests to decide if
* rewriting a page in memory is worth doing. In-memory configurations
diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c
index 1c6cd8e50ae..33dc1a0a0d4 100644
--- a/src/third_party/wiredtiger/src/support/err.c
+++ b/src/third_party/wiredtiger/src/support/err.c
@@ -174,7 +174,7 @@ __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler)
*/
static int
__eventv(WT_SESSION_IMPL *session, bool msg_event, int error,
- const char *func_name, int line_number, const char *fmt, va_list ap)
+ const char *func, int line, const char *fmt, va_list ap)
WT_GCC_FUNC_ATTRIBUTE((cold))
{
struct timespec ts;
@@ -231,8 +231,8 @@ __eventv(WT_SESSION_IMPL *session, bool msg_event, int error,
WT_ERROR_APPEND(p, remain, ", %s", prefix);
WT_ERROR_APPEND(p, remain, ": ");
- if (func_name != NULL)
- WT_ERROR_APPEND(p, remain, "%s, %d: ", func_name, line_number);
+ if (func != NULL)
+ WT_ERROR_APPEND(p, remain, "%s, %d: ", func, line);
WT_ERROR_APPEND_AP(p, remain, fmt, ap);
@@ -314,7 +314,7 @@ err: if (fprintf(stderr,
*/
void
__wt_err_func(WT_SESSION_IMPL *session,
- int error, const char *func_name, int line_number, const char *fmt, ...)
+ int error, const char *func, int line, const char *fmt, ...)
WT_GCC_FUNC_ATTRIBUTE((cold))
WT_GCC_FUNC_ATTRIBUTE((format (printf, 5, 6)))
WT_GCC_FUNC_ATTRIBUTE((visibility("default")))
@@ -326,8 +326,7 @@ __wt_err_func(WT_SESSION_IMPL *session,
* an error value to return.
*/
va_start(ap, fmt);
- WT_IGNORE_RET(__eventv(session,
- false, error, func_name, line_number, fmt, ap));
+ WT_IGNORE_RET(__eventv(session, false, error, func, line, fmt, ap));
va_end(ap);
}
@@ -337,7 +336,7 @@ __wt_err_func(WT_SESSION_IMPL *session,
*/
void
__wt_errx_func(WT_SESSION_IMPL *session,
- const char *func_name, int line_number, const char *fmt, ...)
+ const char *func, int line, const char *fmt, ...)
WT_GCC_FUNC_ATTRIBUTE((cold))
WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5)))
WT_GCC_FUNC_ATTRIBUTE((visibility("default")))
@@ -349,8 +348,7 @@ __wt_errx_func(WT_SESSION_IMPL *session,
* an error value to return.
*/
va_start(ap, fmt);
- WT_IGNORE_RET(__eventv(session,
- false, 0, func_name, line_number, fmt, ap));
+ WT_IGNORE_RET(__eventv(session, false, 0, func, line, fmt, ap));
va_end(ap);
}
diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c
index 4116638e31c..eb65c00741c 100644
--- a/src/third_party/wiredtiger/src/support/hazard.c
+++ b/src/third_party/wiredtiger/src/support/hazard.c
@@ -67,7 +67,7 @@ hazard_grow(WT_SESSION_IMPL *session)
int
__wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp
#ifdef HAVE_DIAGNOSTIC
- , const char *file, int line
+ , const char *func, int line
#endif
)
{
@@ -146,7 +146,7 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp
*/
hp->ref = ref;
#ifdef HAVE_DIAGNOSTIC
- hp->file = file;
+ hp->func = func;
hp->line = line;
#endif
/* Publish the hazard pointer before reading page's state. */
@@ -419,7 +419,7 @@ __wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor)
}
__wt_errx(session,
"hazard pointer reference to discarded object: (%p: %s, line %d)",
- (void *)hp->ref, hp->file, hp->line);
+ (void *)hp->ref, hp->func, hp->line);
return (false);
}
@@ -438,6 +438,6 @@ __hazard_dump(WT_SESSION_IMPL *session)
__wt_errx(session,
"session %p: hazard pointer %p: %s, line %d",
(void *)session,
- (void *)hp->ref, hp->file, hp->line);
+ (void *)hp->ref, hp->func, hp->line);
}
#endif
diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c
index 21c83146a31..a0f7de3179f 100644
--- a/src/third_party/wiredtiger/src/support/scratch.c
+++ b/src/third_party/wiredtiger/src/support/scratch.c
@@ -242,7 +242,7 @@ __wt_buf_set_size(
int
__wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp
#ifdef HAVE_DIAGNOSTIC
- , const char *file, int line
+ , const char *func, int line
#endif
)
WT_GCC_FUNC_ATTRIBUTE((visibility("default")))
@@ -330,15 +330,14 @@ __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp
F_SET(*best, WT_ITEM_INUSE);
#ifdef HAVE_DIAGNOSTIC
- session->scratch_track[best - session->scratch].file = file;
+ session->scratch_track[best - session->scratch].func = func;
session->scratch_track[best - session->scratch].line = line;
#endif
*scratchp = *best;
return (0);
-err: WT_RET_MSG(session, ret,
- "session unable to allocate a scratch buffer");
+err: WT_RET_MSG(session, ret, "session unable to allocate a scratch buffer");
}
/*
@@ -361,7 +360,7 @@ __wt_scr_discard(WT_SESSION_IMPL *session)
"scratch buffer allocated and never discarded"
": %s: %d",
session->
- scratch_track[bufp - session->scratch].file,
+ scratch_track[bufp - session->scratch].func,
session->
scratch_track[bufp - session->scratch].line
);
diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c
index 735faa417c3..ff0b74edcad 100644
--- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c
+++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c
@@ -537,7 +537,6 @@ thread_ckpt_run(void *arg)
testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
first_ckpt = true;
ts = 0;
- __wt_epoch(NULL, &start);
for (i = 0; ;++i) {
sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL;
sleep(sleep_time);
@@ -555,9 +554,12 @@ thread_ckpt_run(void *arg)
/*
* Create the checkpoint file so that the parent process knows
* at least one checkpoint has finished and can start its
- * timer.
+ * timer. Start the timer for stable after the first checkpoint
+ * completes because a slow I/O lag during the checkpoint can
+ * cause a false positive for a timeout.
*/
if (first_ckpt) {
+ __wt_epoch(NULL, &start);
testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL);
first_ckpt = false;
testutil_checksys(fclose(fp) != 0);
diff --git a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c
index 1efe22c1816..2dd0f86db20 100644
--- a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c
@@ -101,8 +101,7 @@ main(int argc, char *argv[])
"key_format=Q,value_format=u,"
"leaf_item_max=64M,leaf_page_max=32k,memory_page_max=1M"));
- testutil_check(
- session->open_cursor(session, uri, NULL, NULL, &c));
+ testutil_check(session->open_cursor(session, uri, NULL, NULL, &c));
/* Value is initialized with 'v' and has not significance to it. */
large_doc = dmalloc(DATASIZE);
@@ -137,6 +136,8 @@ main(int argc, char *argv[])
while (++j < MODIFY_COUNT) {
for (i = 0; i < NUM_DOCS; i++) {
/* Position the cursor. */
+ testutil_check(
+ session2->begin_transaction(session2, NULL));
c->set_key(c, i);
modify_entry.data.data =
"abcdefghijklmnopqrstuvwxyz";
@@ -146,6 +147,8 @@ main(int argc, char *argv[])
(void)alarm(1);
testutil_check(c->modify(c, &modify_entry, 1));
(void)alarm(0);
+ testutil_check(
+ session2->commit_transaction(session2, NULL));
}
/*
* Modify operations are done similar to append sequence.
diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c
index 705594a2156..f92f438a4f1 100644
--- a/src/third_party/wiredtiger/test/format/ops.c
+++ b/src/third_party/wiredtiger/test/format/ops.c
@@ -353,8 +353,15 @@ snap_check(WT_CURSOR *cursor,
testutil_assert(start->keyno != 0);
}
- /* Check for subsequent changes to this record. */
+ /*
+ * Check for subsequent changes to this record. If we find a
+ * read, don't treat it was a subsequent change, that way we
+ * verify the results of the change as well as the results of
+ * the read.
+ */
for (p = start + 1; p < stop; ++p) {
+ if (p->op == READ)
+ continue;
if (p->keyno == start->keyno)
break;
@@ -872,10 +879,11 @@ ops(void *arg)
break;
case MODIFY:
/*
- * Change modify into update if in a read-uncommitted
- * transaction, modify isn't supported in that case.
+ * Change modify into update if not in a transaction
+ * or in a read-uncommitted transaction, modify isn't
+ * supported in those cases.
*/
- if (iso_config == ISOLATION_READ_UNCOMMITTED)
+ if (!intxn || iso_config == ISOLATION_READ_UNCOMMITTED)
goto update_instead_of_chosen_op;
++tinfo->update;
diff --git a/src/third_party/wiredtiger/test/suite/test_config07.py b/src/third_party/wiredtiger/test/suite/test_config07.py
new file mode 100644
index 00000000000..edf37577d4b
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_config07.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2018 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import fnmatch, os, time
+import wiredtiger, wttest
+from wtscenario import make_scenarios
+
+# test_config07.py
+# Test that log files extend as configured and as documented.
+class test_config07(wttest.WiredTigerTestCase):
+ uri = "table:test"
+ entries = 100000
+ K = 1024
+ log_size = K * K
+
+ extend_len = [
+ ('default', dict(log_extend_len='()', expected_log_size = log_size)),
+ ('empty', dict(log_extend_len='(log=)', expected_log_size = log_size)),
+ ('disable', dict(log_extend_len='(log=0)', expected_log_size = 128)),
+ ('100K', dict(log_extend_len='(log=100K)', expected_log_size = 100 * K)),
+ ('too_small', dict(log_extend_len='(log=20K)', expected_log_size = None)),
+ ('too_large', dict(log_extend_len='(log=20G)', expected_log_size = None)),
+ ('small_in_allowed range', dict(log_extend_len='(log=200K)',
+ expected_log_size = 200 * K)),
+ ('large_in_allowed_range', dict(log_extend_len='(log=900K)',
+ expected_log_size = 900 * K)),
+ ('larger_than_log_file_size', dict(log_extend_len='(log=20M)',
+ expected_log_size = log_size)),
+ ('with_data_file_extend_conf', dict(log_extend_len='(log=100K,data=16M)',
+ expected_log_size = 100 * K)),
+ ]
+
+ scenarios = make_scenarios(extend_len)
+
+ def populate(self):
+ cur = self.session.open_cursor(self.uri, None, None)
+ for i in range(0, self.entries):
+ cur[i] = i
+ cur.close()
+
+ def checkLogFileSize(self, size):
+ # Wait for a log file to be preallocated. Avoid timing problems, but
+ # assert that a file is created within 1 minute.
+ for i in range(1,60):
+ logs = fnmatch.filter(os.listdir('.'), "*Prep*")
+ if logs:
+ f = logs[-1]
+ file_size = os.stat(f).st_size
+ self.assertEqual(size, file_size)
+ break
+ time.sleep(1)
+ self.assertTrue(logs)
+
+ def test_log_extend(self):
+ self.conn.close()
+ msg = '/invalid log extend length/'
+
+ config = 'log=(enabled,file_max=1M),file_extend=' + self.log_extend_len
+ configarg = 'create,statistics=(fast)' + ',' + config
+
+ # Expect an error when an invalid log extend size is provided.
+ if self.expected_log_size is None:
+ self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+ lambda: self.wiredtiger_open('.', configarg), msg)
+ return
+
+ self.conn = self.wiredtiger_open('.', configarg)
+ self.session = self.conn.open_session(None)
+
+ # Create a table, insert data in it to trigger log file writes.
+ self.session.create(self.uri, 'key_format=i,value_format=i')
+ self.populate()
+ self.session.checkpoint()
+
+ self.checkLogFileSize(self.expected_log_size)
+
+if __name__ == '__main__':
+ wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_cursor12.py b/src/third_party/wiredtiger/test/suite/test_cursor12.py
index 50204274b94..96bf48ef83c 100644
--- a/src/third_party/wiredtiger/test/suite/test_cursor12.py
+++ b/src/third_party/wiredtiger/test/suite/test_cursor12.py
@@ -194,12 +194,14 @@ class test_cursor12(wttest.WiredTigerTestCase):
self.assertEquals(c.update(), 0)
c.reset()
+ self.session.begin_transaction()
c.set_key(ds.key(row))
mods = []
for j in i['mods']:
mod = wiredtiger.Modify(j[0], j[1], j[2])
mods.append(mod)
self.assertEquals(c.modify(mods), 0)
+ self.session.commit_transaction()
c.reset()
c.set_key(ds.key(row))
@@ -288,6 +290,7 @@ class test_cursor12(wttest.WiredTigerTestCase):
ds.populate()
c = self.session.open_cursor(self.uri, None)
+ self.session.begin_transaction()
c.set_key(ds.key(10))
orig = 'abcdefghijklmnopqrstuvwxyz'
c.set_value(orig)
@@ -299,6 +302,7 @@ class test_cursor12(wttest.WiredTigerTestCase):
mod = wiredtiger.Modify(new, 10, 5)
mods.append(mod)
self.assertEquals(c.modify(mods), 0)
+ self.session.commit_transaction()
c.set_key(ds.key(10))
self.assertEquals(c.search(), 0)
@@ -314,12 +318,14 @@ class test_cursor12(wttest.WiredTigerTestCase):
c.set_key(ds.key(10))
self.assertEquals(c.remove(), 0)
+ self.session.begin_transaction()
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(10))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
+ self.session.commit_transaction()
# Check that modify returns not-found when an insert is not yet committed
# and after it's aborted.
@@ -347,6 +353,7 @@ class test_cursor12(wttest.WiredTigerTestCase):
# Test that another transaction cannot modify our uncommitted record.
xs = self.conn.open_session()
xc = xs.open_cursor(self.uri, None)
+ xs.begin_transaction()
xc.set_key(ds.key(30))
xc.set_value(ds.value(30))
mods = []
@@ -354,16 +361,19 @@ class test_cursor12(wttest.WiredTigerTestCase):
mods.append(mod)
xc.set_key(ds.key(30))
self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND)
+ xs.rollback_transaction()
# Rollback our transaction.
self.session.rollback_transaction()
# Test that we can't modify our aborted insert.
+ self.session.begin_transaction()
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
+ self.session.rollback_transaction()
if __name__ == '__main__':
wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_las01.py b/src/third_party/wiredtiger/test/suite/test_las01.py
index 8b6132f8418..f47df3be9ac 100644
--- a/src/third_party/wiredtiger/test/suite/test_las01.py
+++ b/src/third_party/wiredtiger/test/suite/test_las01.py
@@ -139,8 +139,10 @@ class test_las01(wttest.WiredTigerTestCase):
session2 = self.conn.open_session()
session2.begin_transaction('isolation=snapshot')
# Apply two modify operations - replacing the first two items with 'A'
+ self.session.begin_transaction()
self.large_modifies(self.session, uri, 0, ds, nrows)
self.large_modifies(self.session, uri, 1, ds, nrows)
+ self.session.commit_transaction()
# Check to see the value after recovery
self.durable_check(bigvalue3, uri, ds, nrows)
session2.rollback_transaction()
diff --git a/src/third_party/wiredtiger/test/suite/test_schema08.py b/src/third_party/wiredtiger/test/suite/test_schema08.py
new file mode 100644
index 00000000000..e7b44219ba3
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_schema08.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2018 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import fnmatch, os, shutil, sys
+from suite_subprocess import suite_subprocess
+import wiredtiger, wttest
+from wtscenario import make_scenarios
+
+# test_schema08.py
+# Test schema operations on recovery.
+# Test all schema operations alter, create, drop, rename.
+# After doing the operation, create a backup copy of the directory,
+# walk the log recording each LSN, truncate the backup copy of the
+# log walking backward from the LSNs and then run recovery.
+class test_schema08(wttest.WiredTigerTestCase, suite_subprocess):
+ # We want to copy, truncate and run recovery so keep the log
+ # file small and don't pre-allocate any. We expect a small log.
+ conn_config = 'log=(enabled,archive=false,file_max=100k,prealloc=false)'
+ types = [
+ ('file', dict(uri='file:', use_cg=False, use_index=False)),
+ ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)),
+ ('table-cg', dict(uri='table:', use_cg=True, use_index=False)),
+ ('table-index', dict(uri='table:', use_cg=False, use_index=True)),
+ ('table-simple', dict(uri='table:', use_cg=False, use_index=False)),
+ ]
+ ops = [
+ ('none', dict(schema_ops='none')),
+ ('alter', dict(schema_ops='alter')),
+ ('drop', dict(schema_ops='drop')),
+ ('rename', dict(schema_ops='rename')),
+ ]
+ ckpt = [
+ ('no_ckpt', dict(ckpt=False)),
+ ('with_ckpt', dict(ckpt=True)),
+ ]
+ scenarios = make_scenarios(types, ops, ckpt)
+ count = 0
+ lsns = []
+ backup_pfx = "BACKUP."
+
+ def do_alter(self, uri, suburi):
+ alter_param = 'cache_resident=true'
+ self.session.alter(uri, alter_param)
+ if suburi != None:
+ self.session.alter(suburi, alter_param)
+
+ def do_ops(self, uri, suburi):
+ if (self.schema_ops == 'none'):
+ return
+ if (self.schema_ops == 'alter'):
+ self.do_alter(uri, suburi)
+ elif (self.schema_ops == 'drop'):
+ self.session.drop(uri, None)
+ elif (self.schema_ops == 'rename'):
+ newuri = self.uri + "new-table"
+ self.session.rename(uri, newuri, None)
+
+ # Count actual log records in the log. Log cursors walk the individual
+ # operations of a transaction as well as the entire record. Skip counting
+ # any individual commit operations and only count entire records.
+ def find_logrecs(self):
+ self.count = 0
+ self.session.log_flush('sync=on')
+ c = self.session.open_cursor('log:', None, None)
+ self.lsns.append(0)
+ while c.next() == 0:
+ # lsn.file, lsn.offset, opcount
+ keys = c.get_key()
+ # We don't expect to need more than one log file. We only store
+ # the offsets in a list so assert lsn.file is 1.
+ self.assertTrue(keys[0] == 1)
+
+ # Only count whole records, which is when opcount is zero.
+ # If opcount is not zero it is an operation of a commit.
+ # Skip LSN 128, that is a system record and its existence
+ # is assumed within the system.
+ if keys[2] == 0 and keys[1] != 128:
+ self.count += 1
+ self.lsns.append(keys[1])
+ c.close()
+ self.pr("Find " + str(self.count) + " logrecs LSNS: ")
+ self.pr(str(self.lsns))
+
+ def make_backups(self):
+ # With the connection still open, copy files to the new directory.
+ # Make an initial copy as well as a copy for each LSN we save.
+ # Truncate the log to the appropriate offset as we make each copy.
+ olddir = "."
+ log1 = 'WiredTigerLog.0000000001'
+ for lsn in self.lsns:
+ newdir = self.backup_pfx + str(lsn)
+ shutil.rmtree(newdir, ignore_errors=True)
+ os.mkdir(newdir)
+ for fname in os.listdir(olddir):
+ fullname = os.path.join(olddir, fname)
+ # Skip lock file on Windows since it is locked
+ if os.path.isfile(fullname) and \
+ "WiredTiger.lock" not in fullname and \
+ "Tmplog" not in fullname and \
+ "Preplog" not in fullname:
+ shutil.copy(fullname, newdir)
+ # Truncate the file to the LSN offset.
+ # NOTE: This removes the record at that offset
+ # resulting in recovery running to just before
+ # that record.
+ if lsn != 0:
+ logf = os.path.join(newdir + '/' + log1)
+ f = open(logf, "r+")
+ f.truncate(lsn)
+ f.close()
+ # print "New size " + logf + ": " + str(os.path.getsize(logf))
+
+ def run_recovery(self, uri, suburi):
+ # With the connection still open, copy files to the new directory.
+ # Make an initial copy as well as a copy for each LSN we save.
+ # Truncate the log to the appropriate offset as we make each copy.
+ olddir = "."
+ for lsn in self.lsns:
+ newdir = self.backup_pfx + str(lsn)
+ outfile = newdir + '.txt'
+ self.runWt(['-R', '-h', newdir, 'list', '-v'], outfilename=outfile)
+
+ # Test that creating and dropping tables does not write individual
+ # log records.
+ def test_schema08_create(self):
+ self.count = 0
+ self.lsns = []
+ uri = self.uri + 'table0'
+ create_params = 'key_format=i,value_format=S,'
+
+ cgparam = ''
+ suburi = None
+ if self.use_cg or self.use_index:
+ cgparam = 'columns=(k,v),'
+ if self.use_cg:
+ cgparam += 'colgroups=(g0),'
+
+ # Create main table.
+ self.session.create(uri, create_params + cgparam)
+
+ # Checkpoint after the main table creation if wanted.
+ if self.ckpt:
+ self.session.checkpoint()
+
+ # Add in column group or index tables.
+ if self.use_cg:
+ # Create.
+ cgparam = 'columns=(v),'
+ suburi = 'colgroup:table0:g0'
+ self.session.create(suburi, cgparam)
+
+ if self.use_index:
+ # Create.
+ suburi = 'index:table0:i0'
+ self.session.create(suburi, cgparam)
+
+ self.do_ops(uri, suburi)
+ self.find_logrecs()
+ # print "Found " + str(self.count) + " log records"
+ self.make_backups()
+ self.run_recovery(uri, suburi)
+
+if __name__ == '__main__':
+ wttest.run()