From 437e37763a3f49b328dadd1281ce976a73e4ef61 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Tue, 28 Nov 2017 18:09:21 +1100 Subject: Import wiredtiger: 780a2fb15d6c0acb9f5fd942c90d0f0774a37628 from branch mongodb-3.6 ref: 7a7d6ffec4..780a2fb15d for: 3.6.0-rc7 WT-3607 Compact skips all in-cache blocks with associated disk images WT-3658 Fixed-sized strings can be stored without a trailing NUL WT-3761 Review when cursor operations force evict pages WT-3762 Allow the oldest_timestamp to be moved backwards. --- src/third_party/wiredtiger/dist/api_data.py | 4 ++++ src/third_party/wiredtiger/dist/s_string.ok | 1 + src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/btree/bt_compact.c | 2 +- src/third_party/wiredtiger/src/config/config_def.c | 6 ++++-- src/third_party/wiredtiger/src/docs/schema.dox | 5 ++++- src/third_party/wiredtiger/src/include/btree.i | 22 ++++++++++++-------- src/third_party/wiredtiger/src/include/extern.h | 2 +- src/third_party/wiredtiger/src/include/misc.i | 14 +++++++++++++ src/third_party/wiredtiger/src/include/packing.i | 19 ++++++++++------- .../wiredtiger/src/include/wiredtiger.in | 3 +++ src/third_party/wiredtiger/src/txn/txn.c | 2 +- src/third_party/wiredtiger/src/txn/txn_timestamp.c | 24 +++++++++++++++------- .../wiredtiger/test/suite/test_timestamp09.py | 10 +++++++++ 14 files changed, 86 insertions(+), 30 deletions(-) diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index a8b1a30a333..68a8f4908d8 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1303,6 +1303,10 @@ methods = { where the application is rolling back locally committed transactions. The supplied value should not be older than the current oldest and stable timestamps. See @ref transaction_timestamps'''), + Config('force', 'false', r''' + set timestamps even if they violate normal ordering requirements. + For example allow the \c oldest_timestamp to move backwards''', + type='boolean'), Config('oldest_timestamp', '', r''' future commits and queries will be no earlier than the specified timestamp. Supplied values must be monotonically increasing, any diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 75f237f39fb..1a252fdd615 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1165,6 +1165,7 @@ strget strlen strncpy strndup +strnlen strtok strtoll strtouq diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 546c9967ece..6a71011e82e 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "7a7d6ffec48afeef4e1d7b23bceb4015986f47b9", + "commit": "780a2fb15d6c0acb9f5fd942c90d0f0774a37628", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.6" diff --git a/src/third_party/wiredtiger/src/btree/bt_compact.c b/src/third_party/wiredtiger/src/btree/bt_compact.c index cbc19df65fa..b3e23a8251c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_compact.c +++ b/src/third_party/wiredtiger/src/btree/bt_compact.c @@ -58,7 +58,7 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) if (mod->rec_result == WT_PM_REC_MULTIBLOCK) for (multi = mod->mod_multi, i = 0; i < mod->mod_multi_entries; ++multi, ++i) { - if (multi->disk_image != NULL) + if (multi->addr.addr == NULL) continue; if ((ret = bm->compact_page_skip(bm, session, multi->addr.addr, multi->addr.size, skipp)) != 0) diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index f0e1dc1f701..14234badcfa 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -191,6 +191,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { static const WT_CONFIG_CHECK confchk_WT_CONNECTION_set_timestamp[] = { { "commit_timestamp", "string", NULL, NULL, NULL, 0 }, + { "force", "boolean", NULL, NULL, NULL, 0 }, { "oldest_timestamp", "string", NULL, NULL, NULL, 0 }, { "stable_timestamp", "string", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } @@ -1194,8 +1195,9 @@ static const WT_CONFIG_ENTRY config_entries[] = { NULL, 0 }, { "WT_CONNECTION.set_timestamp", - "commit_timestamp=,oldest_timestamp=,stable_timestamp=", - confchk_WT_CONNECTION_set_timestamp, 3 + "commit_timestamp=,force=false,oldest_timestamp=," + "stable_timestamp=", + confchk_WT_CONNECTION_set_timestamp, 4 }, { "WT_CURSOR.close", "", diff --git a/src/third_party/wiredtiger/src/docs/schema.dox b/src/third_party/wiredtiger/src/docs/schema.dox index 65ad7f6919c..4427111ba2a 100644 --- a/src/third_party/wiredtiger/src/docs/schema.dox +++ b/src/third_party/wiredtiger/src/docs/schema.dox @@ -94,7 +94,10 @@ a size, that indicates the number of bytes to store; the default is a length of 1 byte. The \c 'S' type is encoded as a C language string terminated by a -NUL character. +NUL character. When preceded by a size, that indicates the maximum number of +bytes the string can store. In a string with characters less than the +specified size, the remaining bytes are NUL padded. If the supplied string +is longer than the specified size, it will be stored without a trailing NUL. @m_if{java} Because of this, the associated Java String may not contain the NUL character. @m_endif diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 35cb868ac26..19b300908b1 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1425,15 +1425,19 @@ __wt_page_release(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) * tree, then perform a general check if eviction will be possible. */ page = ref->page; - if (!WT_READGEN_EVICT_SOON(page->read_gen) || - LF_ISSET(WT_READ_NO_SPLIT) || - btree->evict_disabled > 0 || - !__wt_page_can_evict(session, ref, &inmem_split) || - (F_ISSET(session, WT_SESSION_NO_RECONCILE) && !inmem_split)) - return (__wt_hazard_clear(session, ref)); - - WT_RET_BUSY_OK(__wt_page_release_evict(session, ref)); - return (0); + if (WT_READGEN_EVICT_SOON(page->read_gen) && + btree->evict_disabled == 0 && + __wt_page_can_evict(session, ref, &inmem_split)) { + if ((LF_ISSET(WT_READ_NO_SPLIT) || (!inmem_split && + F_ISSET(session, WT_SESSION_NO_RECONCILE)))) + __wt_page_evict_urgent(session, ref); + else { + WT_RET_BUSY_OK(__wt_page_release_evict(session, ref)); + return (0); + } + } + + return (__wt_hazard_clear(session, ref)); } /* diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 17267861717..ce9e1e57a47 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -824,7 +824,7 @@ extern int __wt_timestamp_to_hex_string( WT_SESSION_IMPL *session, char *hex_tim extern void __wt_verbose_timestamp(WT_SESSION_IMPL *session, const wt_timestamp_t *ts, const char *msg); extern int __wt_txn_parse_timestamp(WT_SESSION_IMPL *session, const char *name, wt_timestamp_t *timestamp, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_global_query_timestamp( WT_SESSION_IMPL *session, char *hex_timestamp, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_timestamp_validate(WT_SESSION_IMPL *session, const char *name, wt_timestamp_t *ts, WT_CONFIG_ITEM *cval, bool cmp_oldest, bool cmp_stable, bool cmp_commit) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index dbb921f0946..bedd3121037 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -40,6 +40,20 @@ __wt_strdup(WT_SESSION_IMPL *session, const char *str, void *retp) session, str, (str == NULL) ? 0 : strlen(str), retp)); } +/* + * __wt_strnlen -- + * Determine the length of a fixed-size string + */ +static inline size_t +__wt_strnlen(const char *s, size_t maxlen) +{ + size_t i; + + for (i = 0; i < maxlen && *s != '\0'; i++, s++) + ; + return (i); +} + /* * __wt_snprintf -- * snprintf convenience function, ignoring the returned size. diff --git a/src/third_party/wiredtiger/src/include/packing.i b/src/third_party/wiredtiger/src/include/packing.i index e1cf158c660..d5181738fbd 100644 --- a/src/third_party/wiredtiger/src/include/packing.i +++ b/src/third_party/wiredtiger/src/include/packing.i @@ -342,15 +342,20 @@ __pack_write( *pp += pv->size; break; case 'S': - s = strlen(pv->u.s); + /* + * When preceded by a size, that indicates the maximum number + * of bytes the string can store, this does not include the + * terminating NUL character. In a string with characters + * less than the specified size, the remaining bytes are + * NULL padded. + */ if (pv->havesize) { - if (pv->size < s) { - s = pv->size; - pad = 0; - } else - pad = pv->size - s; - } else + s = __wt_strnlen(pv->u.s, pv->size); + pad = (s < pv->size) ? pv->size - s : 0; + } else { + s = strlen(pv->u.s); pad = 1; + } WT_SIZE_CHECK_PACK(s + pad, maxlen); if (s > 0) memcpy(*pp, pv->u.s, s); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index c24cffe2371..d4c1fa956c3 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2341,6 +2341,9 @@ struct __wt_connection { * supplied value should not be older than the current oldest and stable * timestamps. See @ref transaction_timestamps., a string; default * empty.} + * @config{force, set timestamps even if they violate normal ordering + * requirements. For example allow the \c oldest_timestamp to move + * backwards., a boolean flag; default \c false.} * @config{oldest_timestamp, future commits and queries will be no * earlier than the specified timestamp. Supplied values must be * monotonically increasing\, any attempt to set the value to older than diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index c461e5f8e45..fb61b037723 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -287,7 +287,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) #ifdef HAVE_TIMESTAMPS /* Try to move the pinned timestamp forward. */ if (strict) - WT_RET(__wt_txn_update_pinned_timestamp(session)); + WT_RET(__wt_txn_update_pinned_timestamp(session, false)); #endif /* diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 5a39a6d84dc..6735946fbb3 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -292,7 +292,7 @@ __wt_txn_global_query_timestamp( * maintained for current or future readers). */ int -__wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session) +__wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force) { WT_DECL_RET; WT_TXN_GLOBAL *txn_global; @@ -321,7 +321,7 @@ __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session) } else __wt_timestamp_set(&pinned_timestamp, &active_timestamp); - if (txn_global->has_pinned_timestamp) { + if (txn_global->has_pinned_timestamp && !force) { WT_WITH_TIMESTAMP_READLOCK(session, &txn_global->rwlock, __wt_timestamp_set( &last_pinned_timestamp, &txn_global->pinned_timestamp)); @@ -332,7 +332,7 @@ __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session) } __wt_writelock(session, &txn_global->rwlock); - if (!txn_global->has_pinned_timestamp || __wt_timestamp_cmp( + if (!txn_global->has_pinned_timestamp || force || __wt_timestamp_cmp( &txn_global->pinned_timestamp, &pinned_timestamp) < 0) { __wt_timestamp_set( &txn_global->pinned_timestamp, &pinned_timestamp); @@ -377,11 +377,14 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) #ifdef HAVE_TIMESTAMPS { + WT_CONFIG_ITEM cval; WT_TXN_GLOBAL *txn_global; wt_timestamp_t commit_ts, oldest_ts, stable_ts; wt_timestamp_t last_oldest_ts, last_stable_ts; + bool force; txn_global = &S2C(session)->txn_global; + /* * Parsing will initialize the timestamp to zero even if * it is not configured. @@ -393,6 +396,13 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_txn_parse_timestamp( session, "stable", &stable_ts, &stable_cval)); + WT_RET(__wt_config_gets_def(session, + cfg, "force", 0, &cval)); + force = cval.val != 0; + + if (force) + goto set; + __wt_readlock(session, &txn_global->rwlock); __wt_timestamp_set(&last_oldest_ts, &txn_global->oldest_timestamp); @@ -460,7 +470,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) if (!has_commit && !has_oldest && !has_stable) return (0); - __wt_writelock(session, &txn_global->rwlock); +set: __wt_writelock(session, &txn_global->rwlock); /* * This method can be called from multiple threads, check that we are * moving the global timestamps forwards. @@ -479,7 +489,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) } if (has_oldest && (!txn_global->has_oldest_timestamp || - __wt_timestamp_cmp( + force || __wt_timestamp_cmp( &oldest_ts, &txn_global->oldest_timestamp) > 0)) { __wt_timestamp_set(&txn_global->oldest_timestamp, &oldest_ts); txn_global->has_oldest_timestamp = true; @@ -489,7 +499,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) } if (has_stable && (!txn_global->has_stable_timestamp || - __wt_timestamp_cmp( + force || __wt_timestamp_cmp( &stable_ts, &txn_global->stable_timestamp) > 0)) { __wt_timestamp_set(&txn_global->stable_timestamp, &stable_ts); txn_global->has_stable_timestamp = true; @@ -500,7 +510,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) __wt_writeunlock(session, &txn_global->rwlock); if (has_oldest || has_stable) - WT_RET(__wt_txn_update_pinned_timestamp(session)); + WT_RET(__wt_txn_update_pinned_timestamp(session, force)); } return (0); #else diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp09.py b/src/third_party/wiredtiger/test/suite/test_timestamp09.py index b79521329e7..0e3a6b8dab3 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp09.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp09.py @@ -186,5 +186,15 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(c[8], 8) self.session.commit_transaction() + # We can move the oldest timestamp backwards with "force" + self.conn.set_timestamp( + 'oldest_timestamp=' + timestamp_str(5) + ',force') + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.begin_transaction('read_timestamp=' + + timestamp_str(4)), + '/older than oldest timestamp/') + self.session.begin_transaction('read_timestamp=' + timestamp_str(6)) + self.session.commit_transaction() + if __name__ == '__main__': wttest.run() -- cgit v1.2.1