diff options
author | Luke Chen <luke.chen@mongodb.com> | 2018-10-04 13:58:18 +1000 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2018-10-04 13:58:18 +1000 |
commit | e80b323e722f4e3e55b660d680ddc160434d8f04 (patch) | |
tree | 4ed1eef1c18d7733316665570dc75b90ca78c23d | |
parent | bf4a5f8242579de57cfeacef57ac9a77e94e80d3 (diff) | |
download | mongo-e80b323e722f4e3e55b660d680ddc160434d8f04.tar.gz |
Import wiredtiger: e7d742daa2d2500cd94a7061f754a1d0c4aa963c from branch mongodb-4.2
ref: 9ee7cc6956..e7d742daa2
for: 4.1.4
WT-3995 Enhance timestamp abort to accept more number of threads
WT-4164 Ensure test/format configures a reasonably sized cache
WT-4224 Add statistics for prepared transactions
WT-4315 In rollback_to_stable, only check timestamp order if enforced
WT-4328 Use an internal session handle for schema operations in a txn
WT-4342 Set session max on all configuration strings in timestamp_abort test
WT-4347 Limit the threads spawned by timestamp_abort with default config
WT-4348 Create all tables before spawning threads in random_directio test
22 files changed, 327 insertions, 169 deletions
diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index c452efa9cc6..13b1675550e 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -526,6 +526,9 @@ connection_stats = [ TxnStat('txn_prepare_active', 'prepared transactions currently active'), TxnStat('txn_prepare_commit', 'prepared transactions committed'), TxnStat('txn_prepare_rollback', 'prepared transactions rolled back'), + TxnStat('txn_prepared_updates_count', 'Number of prepared updates'), + TxnStat('txn_prepared_updates_lookaside_inserts', 'Number of prepared updates added to cache overflow'), + TxnStat('txn_prepared_updates_resolved', 'Number of prepared updates resolved'), TxnStat('txn_query_ts', 'query timestamp calls'), TxnStat('txn_read_queue_empty', 'read timestamp queue insert to empty'), TxnStat('txn_read_queue_head', 'read timestamp queue inserts to head'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 12145d03db3..553ff5a7b55 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "9ee7cc6956dda425f3522f6d32ec55c67e9c49f8", + "commit": "e7d742daa2d2500cd94a7061f754a1d0c4aa963c", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 47b84ad7a25..3ffc63d94ba 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -542,11 +542,13 @@ __debug_dsk_cell(WT_DBG *ds, const WT_PAGE_HEADER *dsk) * Pretty-print information about a page. */ static char * -__debug_tree_shape_info(WT_PAGE *page, char *buf, size_t len) +__debug_tree_shape_info(WT_REF *ref, char *buf, size_t len) { + WT_PAGE *page; uint64_t v; const char *unit; + page = ref->page; v = page->memory_footprint; if (v > WT_GIGABYTE) { @@ -564,7 +566,7 @@ __debug_tree_shape_info(WT_PAGE *page, char *buf, size_t len) (void)__wt_snprintf(buf, len, "(%p, %" PRIu64 "%s, evict gen %" PRIu64 ", create gen %" PRIu64 ")", - (void *)page, v, unit, + (void *)ref, v, unit, page->evict_pass_gen, page->cache_create_gen); return (buf); } @@ -574,27 +576,27 @@ __debug_tree_shape_info(WT_PAGE *page, char *buf, size_t len) * Dump information about the current page and descend. */ static int -__debug_tree_shape_worker(WT_DBG *ds, WT_PAGE *page, int level) +__debug_tree_shape_worker(WT_DBG *ds, WT_REF *ref, int level) { - WT_REF *ref; + WT_REF *walk; WT_SESSION_IMPL *session; char buf[128]; session = ds->session; - if (WT_PAGE_IS_INTERNAL(page)) { + if (WT_PAGE_IS_INTERNAL(ref->page)) { WT_RET(ds->f(ds, "%*s" "I" "%d %s\n", level * 3, " ", level, - __debug_tree_shape_info(page, buf, sizeof(buf)))); - WT_INTL_FOREACH_BEGIN(session, page, ref) { - if (ref->state == WT_REF_MEM) + __debug_tree_shape_info(ref, buf, sizeof(buf)))); + WT_INTL_FOREACH_BEGIN(session, ref->page, walk) { + if (walk->state == WT_REF_MEM) WT_RET(__debug_tree_shape_worker( - ds, ref->page, level + 1)); + ds, walk, level + 1)); } WT_INTL_FOREACH_END; } else WT_RET(ds->f(ds, "%*s" "L" " %s\n", level * 3, " ", - __debug_tree_shape_info(page, buf, sizeof(buf)))); + __debug_tree_shape_info(ref, buf, sizeof(buf)))); return (0); } @@ -604,7 +606,7 @@ __debug_tree_shape_worker(WT_DBG *ds, WT_PAGE *page, int level) */ int __wt_debug_tree_shape( - WT_SESSION_IMPL *session, WT_PAGE *page, const char *ofile) + WT_SESSION_IMPL *session, WT_REF *ref, const char *ofile) { WT_DBG *ds, _ds; WT_DECL_RET; @@ -614,12 +616,12 @@ __wt_debug_tree_shape( ds = &_ds; WT_RET(__debug_config(session, ds, ofile)); - /* A NULL page starts at the top of the tree -- it's a convenience. */ - if (page == NULL) - page = S2BT(session)->root.page; + /* A NULL WT_REF starts at the top of the tree -- it's a convenience. */ + if (ref == NULL) + ref = &S2BT(session)->root; WT_WITH_PAGE_INDEX(session, - ret = __debug_tree_shape_worker(ds, page, 1)); + ret = __debug_tree_shape_worker(ds, ref, 1)); WT_TRET(__debug_wrapup(ds)); return (ret); diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index cf28027d8b7..f99bb0bbd9d 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -622,7 +622,7 @@ __wt_las_insert_block(WT_CURSOR *cursor, WT_SESSION_IMPL *session; WT_TXN_ISOLATION saved_isolation; WT_UPDATE *upd; - uint64_t insert_cnt; + uint64_t insert_cnt, prepared_insert_cnt; uint64_t las_counter, las_pageid; uint32_t btree_id, i, slot; uint8_t *p; @@ -632,7 +632,7 @@ __wt_las_insert_block(WT_CURSOR *cursor, conn = S2C(session); WT_CLEAR(las_timestamp); WT_CLEAR(las_value); - insert_cnt = 0; + insert_cnt = prepared_insert_cnt = 0; btree_id = btree->id; local_txn = false; @@ -763,6 +763,8 @@ __wt_las_insert_block(WT_CURSOR *cursor, */ WT_ERR(cursor->update(cursor)); ++insert_cnt; + if (upd->prepare_state == WT_PREPARE_INPROGRESS) + ++prepared_insert_cnt; } while ((upd = upd->next) != NULL); } @@ -774,9 +776,13 @@ err: /* Resolve the transaction. */ WT_TRET(__wt_txn_rollback(session, NULL)); /* Adjust the entry count. */ - if (ret == 0) + if (ret == 0) { (void)__wt_atomic_add64( &conn->cache->las_insert_count, insert_cnt); + WT_STAT_CONN_INCRV(session, + txn_prepared_updates_lookaside_inserts, + prepared_insert_cnt); + } } __las_restore_isolation(session, saved_isolation); diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index b05e3ba2556..05a2a2f3f52 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -123,7 +123,7 @@ extern int __wt_debug_addr(WT_SESSION_IMPL *session, const uint8_t *addr, size_t extern int __wt_debug_offset_blind(WT_SESSION_IMPL *session, wt_off_t offset, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_offset(WT_SESSION_IMPL *session, wt_off_t offset, uint32_t size, uint32_t checksum, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_disk(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_debug_tree_shape(WT_SESSION_IMPL *session, WT_PAGE *page, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_tree_shape(WT_SESSION_IMPL *session, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_tree_all(void *session_arg, WT_BTREE *btree, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_tree(void *session_arg, WT_BTREE *btree, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_page(void *session_arg, WT_BTREE *btree, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -644,6 +644,8 @@ extern int __wt_range_truncate(WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DE extern int __wt_schema_range_truncate(WT_SESSION_IMPL *session, WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const char *name); +extern int __wt_schema_internal_session(WT_SESSION_IMPL *session, WT_SESSION_IMPL **int_sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_session_release(WT_SESSION_IMPL *session, WT_SESSION_IMPL *int_session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_str_name_check(WT_SESSION_IMPL *session, const char *str) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_name_check(WT_SESSION_IMPL *session, const char *str, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_exclusive_handle_operation(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[], uint32_t open_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 1ae4e56be03..6407b5fabfe 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -624,6 +624,9 @@ struct __wt_connection_stats { int64_t page_sleep; int64_t page_del_rollback_blocked; int64_t child_modify_blocked_page; + int64_t txn_prepared_updates_count; + int64_t txn_prepared_updates_lookaside_inserts; + int64_t txn_prepared_updates_resolved; int64_t txn_commit_queue_walked; int64_t txn_commit_queue_empty; int64_t txn_commit_queue_head; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 093c7aa11b7..0b7e9ae1aa2 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -364,6 +364,7 @@ __wt_txn_resolve_prepared_op( #endif op->u.op_upd = upd; + WT_STAT_CONN_INCR(session, txn_prepared_updates_resolved); for (; upd != NULL; upd = upd->next) { if (upd->txnid != txn->id) diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 333c74b5e80..2fb4af02900 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -5652,123 +5652,129 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1309 /*! thread-yield: page reconciliation yielded due to child modification */ #define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1310 +/*! transaction: Number of prepared updates */ +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1311 +/*! transaction: Number of prepared updates added to cache overflow */ +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_LOOKASIDE_INSERTS 1312 +/*! transaction: Number of prepared updates resolved */ +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_RESOLVED 1313 /*! transaction: commit timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_WALKED 1311 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_WALKED 1314 /*! transaction: commit timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_EMPTY 1312 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_EMPTY 1315 /*! transaction: commit timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_HEAD 1313 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_HEAD 1316 /*! transaction: commit timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_INSERTS 1314 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_INSERTS 1317 /*! transaction: commit timestamp queue length */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_LEN 1315 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_LEN 1318 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1316 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1319 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1317 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1320 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1318 +#define WT_STAT_CONN_TXN_PREPARE 1321 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1319 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1322 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1320 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1323 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1321 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1324 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1322 +#define WT_STAT_CONN_TXN_QUERY_TS 1325 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1323 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1326 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1324 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1327 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1325 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1328 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1326 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1329 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1327 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1330 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1328 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1331 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1329 +#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1332 /*! transaction: rollback to stable updates removed from cache overflow */ -#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1330 +#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1333 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1331 +#define WT_STAT_CONN_TXN_SET_TS 1334 /*! transaction: set timestamp commit calls */ -#define WT_STAT_CONN_TXN_SET_TS_COMMIT 1332 +#define WT_STAT_CONN_TXN_SET_TS_COMMIT 1335 /*! transaction: set timestamp commit updates */ -#define WT_STAT_CONN_TXN_SET_TS_COMMIT_UPD 1333 +#define WT_STAT_CONN_TXN_SET_TS_COMMIT_UPD 1336 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1334 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1337 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1335 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1338 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1336 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1339 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1337 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1340 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1338 +#define WT_STAT_CONN_TXN_BEGIN 1341 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1339 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1342 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1340 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1343 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1341 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1344 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1342 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1345 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1343 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1346 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1344 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1347 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1345 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1348 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1346 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1349 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1347 +#define WT_STAT_CONN_TXN_CHECKPOINT 1350 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1348 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1351 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1349 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1352 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1350 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1353 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1351 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1354 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1352 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1355 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1353 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1356 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1354 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1357 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1355 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1358 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1356 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1359 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1357 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1360 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1358 +#define WT_STAT_CONN_TXN_SYNC 1361 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1359 +#define WT_STAT_CONN_TXN_COMMIT 1362 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1360 +#define WT_STAT_CONN_TXN_ROLLBACK 1363 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1361 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1364 /*! * @} diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index aba708f0e0b..6e1fa71f584 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -250,9 +250,12 @@ __wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) { WT_DECL_RET; + WT_SESSION_IMPL *int_session; - WT_RET(__wt_meta_track_on(session)); - ret = __schema_alter(session, uri, newcfg); - WT_TRET(__wt_meta_track_off(session, true, ret != 0)); + WT_RET(__wt_schema_internal_session(session, &int_session)); + WT_ERR(__wt_meta_track_on(int_session)); + ret = __schema_alter(int_session, uri, newcfg); + WT_TRET(__wt_meta_track_off(int_session, true, ret != 0)); +err: WT_TRET(__wt_schema_session_release(session, int_session)); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index 13298f56e9e..e26ae00802c 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -677,11 +677,11 @@ __create_data_source(WT_SESSION_IMPL *session, } /* - * __wt_schema_create -- + * __schema_create -- * Process a WT_SESSION::create operation for all supported types. */ -int -__wt_schema_create( +static int +__schema_create( WT_SESSION_IMPL *session, const char *uri, const char *config) { WT_CONFIG_ITEM cval; @@ -721,3 +721,20 @@ __wt_schema_create( return (ret); } + +/* + * __wt_schema_create -- + * Process a WT_SESSION::create operation for all supported types. + */ +int +__wt_schema_create( + WT_SESSION_IMPL *session, const char *uri, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *int_session; + + WT_RET(__wt_schema_internal_session(session, &int_session)); + ret = __schema_create(int_session, uri, config); + WT_TRET(__wt_schema_session_release(session, int_session)); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index 64438f1e0c7..22793a9c6dd 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -182,11 +182,11 @@ err: if (table != NULL && !tracked) } /* - * __wt_schema_drop -- + * __schema_drop -- * Process a WT_SESSION::drop operation for all supported types. */ -int -__wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +static int +__schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) { WT_CONFIG_ITEM cval; WT_DATA_SOURCE *dsrc; @@ -230,3 +230,19 @@ __wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) return (ret); } + +/* + * __wt_schema_drop -- + * Process a WT_SESSION::drop operation for all supported types. + */ +int +__wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_DECL_RET; + WT_SESSION_IMPL *int_session; + + WT_RET(__wt_schema_internal_session(session, &int_session)); + ret = __schema_drop(int_session, uri, cfg); + WT_TRET(__wt_schema_session_release(session, int_session)); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index 128f0ff3f86..0c44d8caf9b 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -256,11 +256,11 @@ err: if (!tracked) } /* - * __wt_schema_rename -- + * __schema_rename -- * WT_SESSION::rename. */ -int -__wt_schema_rename(WT_SESSION_IMPL *session, +static int +__schema_rename(WT_SESSION_IMPL *session, const char *uri, const char *newuri, const char *cfg[]) { WT_DATA_SOURCE *dsrc; @@ -299,3 +299,20 @@ __wt_schema_rename(WT_SESSION_IMPL *session, /* If we didn't find a metadata entry, map that error to ENOENT. */ return (ret == WT_NOTFOUND ? ENOENT : ret); } + +/* + * __wt_schema_rename -- + * WT_SESSION::rename. + */ +int +__wt_schema_rename(WT_SESSION_IMPL *session, + const char *uri, const char *newuri, const char *cfg[]) +{ + WT_DECL_RET; + WT_SESSION_IMPL *int_session; + + WT_RET(__wt_schema_internal_session(session, &int_session)); + ret = __schema_rename(int_session, uri, newuri, cfg); + WT_TRET(__wt_schema_session_release(session, int_session)); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/schema/schema_util.c b/src/third_party/wiredtiger/src/schema/schema_util.c index a281ec3fe12..7951f7ab8d7 100644 --- a/src/third_party/wiredtiger/src/schema/schema_util.c +++ b/src/third_party/wiredtiger/src/schema/schema_util.c @@ -63,6 +63,48 @@ __wt_schema_get_source(WT_SESSION_IMPL *session, const char *name) } /* + * __wt_schema_internal_session -- + * Create and return an internal schema session if necessary. + */ +int +__wt_schema_internal_session( + WT_SESSION_IMPL *session, WT_SESSION_IMPL **int_sessionp) +{ + /* + * Open an internal session if a transaction is running so that the + * schema operations are not logged and buffered with any log records + * in the transaction. The new session inherits its flags from the + * original. + */ + *int_sessionp = session; + if (F_ISSET(&session->txn, WT_TXN_RUNNING)) { + /* We should not have a schema txn running now. */ + WT_ASSERT(session, !F_ISSET(session, WT_SESSION_SCHEMA_TXN)); + WT_RET(__wt_open_internal_session(S2C(session), "schema", + true, session->flags, int_sessionp)); + } + return (0); +} + +/* + * __wt_schema_session_release -- + * Release an internal schema session if needed. + */ +int +__wt_schema_session_release( + WT_SESSION_IMPL *session, WT_SESSION_IMPL *int_session) +{ + WT_SESSION *wt_session; + + if (session != int_session) { + wt_session = &int_session->iface; + WT_RET(wt_session->close(wt_session, NULL)); + } + + return (0); +} + +/* * __wt_str_name_check -- * Disallow any use of the WiredTiger name space. */ diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 61300dfeab9..a0417b3952c 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -1058,6 +1058,9 @@ static const char * const __stats_connection_desc[] = { "thread-yield: page acquire time sleeping (usecs)", "thread-yield: page delete rollback time sleeping for state change (usecs)", "thread-yield: page reconciliation yielded due to child modification", + "transaction: Number of prepared updates", + "transaction: Number of prepared updates added to cache overflow", + "transaction: Number of prepared updates resolved", "transaction: commit timestamp queue entries walked", "transaction: commit timestamp queue insert to empty", "transaction: commit timestamp queue inserts to head", @@ -1462,6 +1465,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->page_sleep = 0; stats->page_del_rollback_blocked = 0; stats->child_modify_blocked_page = 0; + stats->txn_prepared_updates_count = 0; + stats->txn_prepared_updates_lookaside_inserts = 0; + stats->txn_prepared_updates_resolved = 0; stats->txn_commit_queue_walked = 0; stats->txn_commit_queue_empty = 0; stats->txn_commit_queue_head = 0; @@ -1999,6 +2005,12 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, page_del_rollback_blocked); to->child_modify_blocked_page += WT_STAT_READ(from, child_modify_blocked_page); + to->txn_prepared_updates_count += + WT_STAT_READ(from, txn_prepared_updates_count); + to->txn_prepared_updates_lookaside_inserts += + WT_STAT_READ(from, txn_prepared_updates_lookaside_inserts); + to->txn_prepared_updates_resolved += + WT_STAT_READ(from, txn_prepared_updates_resolved); to->txn_commit_queue_walked += WT_STAT_READ(from, txn_commit_queue_walked); to->txn_commit_queue_empty += diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index ee7e5707096..1add958e226 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -1003,7 +1003,8 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) WT_ASSERT(session, S2C(session)->cache->las_fileid == 0 || !F_ISSET(op->btree, WT_BTREE_LOOKASIDE)); - /* Metadata updates are never prepared. */ + /* Metadata updates should never be prepared. */ + WT_ASSERT(session, !WT_IS_METADATA(op->btree->dhandle)); if (WT_IS_METADATA(op->btree->dhandle)) continue; @@ -1035,6 +1036,7 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) #ifdef HAVE_LONG_RUNNING_PREPARE op->u.op_upd = NULL; #endif + WT_STAT_CONN_INCR(session, txn_prepared_updates_count); break; case WT_TXN_OP_REF_DELETE: __wt_timestamp_set( @@ -1101,7 +1103,8 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) WT_ASSERT(session, S2C(session)->cache->las_fileid == 0 || !F_ISSET(op->btree, WT_BTREE_LOOKASIDE)); - /* Metadata updates are never rolled back. */ + /* Metadata updates should never be rolled back. */ + WT_ASSERT(session, !WT_IS_METADATA(op->btree->dhandle)); if (WT_IS_METADATA(op->btree->dhandle)) continue; diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 23d7cc67728..aa4512b80b1 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -104,10 +104,6 @@ __txn_abort_newer_update(WT_SESSION_IMPL *session, WT_UPDATE *first_upd, wt_timestamp_t *rollback_timestamp) { WT_UPDATE *upd; - bool skip_zero_timestamps; - - skip_zero_timestamps = !FLD_ISSET(S2BT(session)->assert_flags, - WT_ASSERT_COMMIT_TS_ALWAYS | WT_ASSERT_COMMIT_TS_KEYS); for (upd = first_upd; upd != NULL; upd = upd->next) { /* @@ -116,23 +112,30 @@ __txn_abort_newer_update(WT_SESSION_IMPL *session, * strict timestamp checking, assert that all more recent * updates were also rolled back. */ - if (upd->txnid == WT_TXN_ABORTED && upd == first_upd) - first_upd = upd->next; - else if (__wt_timestamp_iszero(&upd->timestamp)) { - if (skip_zero_timestamps && upd == first_upd) + if (upd->txnid == WT_TXN_ABORTED || + __wt_timestamp_iszero(&upd->timestamp)) { + if (upd == first_upd) first_upd = upd->next; } else if (__wt_timestamp_cmp( rollback_timestamp, &upd->timestamp) < 0) { - upd->txnid = WT_TXN_ABORTED; - WT_STAT_CONN_INCR(session, txn_rollback_upd_aborted); - __wt_timestamp_set_zero(&upd->timestamp); - /* * If any updates are aborted, all newer updates * better be aborted as well. + * + * Timestamp ordering relies on the validations at + * the time of commit. Thus if the table is not + * configured for key consistency check, the + * the timestamps could be out of order here. */ - WT_ASSERT(session, upd == first_upd); + WT_ASSERT(session, + !FLD_ISSET(S2BT(session)->assert_flags, + WT_ASSERT_COMMIT_TS_KEYS) || + upd == first_upd); first_upd = upd->next; + + upd->txnid = WT_TXN_ABORTED; + WT_STAT_CONN_INCR(session, txn_rollback_upd_aborted); + __wt_timestamp_set_zero(&upd->timestamp); } } } diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/main.c b/src/third_party/wiredtiger/test/csuite/random_directio/main.c index 513985d1913..b2831b5d2ea 100644 --- a/src/third_party/wiredtiger/test/csuite/random_directio/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_directio/main.c @@ -27,12 +27,12 @@ */ /* - * This test simulates system crashes. It uses direct IO, and currently + * This test simulates system crashes. It uses direct I/O, and currently * runs only on Linux. * * Our strategy is to run a subordinate 'writer' process that creates/modifies * data, including schema modifications. Every N seconds, asynchronously, we - * send a stop signal to the writer and then copy (with direct IO) the entire + * send a stop signal to the writer and then copy (with direct I/O) the entire * contents of its database home to a new saved location where we can run and * verify the recovered home. Then we send a continue signal. We repeat this: * @@ -41,7 +41,7 @@ * which allows the writer to make continuing progress, while the main * process is verifying what's on disk. * - * By using stop signal to suspend the process and copying with direct IO, + * By using stop signal to suspend the process and copying with direct I/O, * we are roughly simulating a system crash, by seeing what's actually on * disk (not in file system buffer cache) at the moment that the copy is * made. It's not quite as harsh as a system crash, as suspending does not @@ -76,12 +76,7 @@ static char home[1024]; /* Program working dir */ -/* - * These two names for the URI and file system must be maintained in tandem. - */ static const char * const uri_main = "table:main"; -static const char * const fs_main = "main.wt"; - static const char * const uri_rev = "table:rev"; /* @@ -570,8 +565,14 @@ again: ret = schema_operation(session, td->id, i, op, td->flags); if (ret == EBUSY) { - testutil_check(session->rollback_transaction( - session, NULL)); + /* + * Only rollback if integrated and we have + * an active transaction. + */ + if (F_ISSET(td, SCHEMA_INTEGRATED)) + testutil_check( + session->rollback_transaction( + session, NULL)); sleep(1); goto again; } @@ -588,17 +589,47 @@ again: } /* + * create_db -- + * Creates the database and tables so they are fully ready to be + * accessed by subordinate threads, and copied/recovered. + */ +static void +create_db(const char *method) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + char envconf[512]; + + testutil_check(__wt_snprintf(envconf, sizeof(envconf), + ENV_CONFIG, method)); + + testutil_check(wiredtiger_open(home, NULL, envconf, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create( + session, uri_main, "key_format=S,value_format=S")); + testutil_check(session->create( + session, uri_rev, "key_format=S,value_format=S")); + /* + * Checkpoint to help ensure that everything gets out to disk, + * so any direct I/O copy will have at least have tables that + * can be opened. + */ + testutil_check(session->checkpoint(session, NULL)); + testutil_check(session->close(session, NULL)); + testutil_check(conn->close(conn, NULL)); +} + +/* * fill_db -- - * The child process creates the database and table, and then creates - * worker threads to add data until it is killed by the parent. + * The child process creates worker threads to add data until it is + * killed by the parent. */ static void fill_db(uint32_t, uint32_t, const char *, uint32_t) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void - fill_db(uint32_t nth, uint32_t datasize, const char *method, uint32_t flags) +fill_db(uint32_t nth, uint32_t datasize, const char *method, uint32_t flags) { WT_CONNECTION *conn; - WT_SESSION *session; WT_THREAD_DATA *td; wt_thread_t *thr; uint32_t i; @@ -612,17 +643,6 @@ static void ENV_CONFIG, method)); testutil_check(wiredtiger_open(".", NULL, envconf, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->create( - session, uri_main, "key_format=S,value_format=S")); - testutil_check(session->create( - session, uri_rev, "key_format=S,value_format=S")); - /* - * Checkpoint to help ensure that at least the main tables - * can be opened after recovery. - */ - testutil_check(session->checkpoint(session, NULL)); - testutil_check(session->close(session, NULL)); datasize += 1; /* Add an extra byte for string termination */ printf("Create %" PRIu32 " writer threads\n", nth); @@ -831,7 +851,7 @@ check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) "%s.SAVE", home)); /* - * We make a copy of the directory (possibly using direct IO) + * We make a copy of the directory (possibly using direct I/O) * for recovery and checking, and an identical copy that * keeps the state of all files before recovery starts. */ @@ -1071,7 +1091,6 @@ int main(int argc, char *argv[]) { struct sigaction sa; - struct stat sb; WT_RAND_STATE rnd; pid_t pid; size_t size; @@ -1238,6 +1257,7 @@ main(int argc, char *argv[]) printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); + create_db(method); if (!populate_only) { /* * Fork a child to insert as many items. We will @@ -1259,14 +1279,8 @@ main(int argc, char *argv[]) /* parent */ /* * Sleep for the configured amount of time before killing - * the child. Start the timeout from the time we notice that - * the table has been created. That allows the test to run - * correctly on really slow machines. + * the child. */ - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s/%s", home, fs_main)); - while (stat(buf, &sb) != 0 || sb.st_size < 4096) - testutil_sleep_wait(1, pid); testutil_sleep_wait(timeout, pid); /* diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 5aec69cf034..f5f34fe3505 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -58,7 +58,8 @@ static char home[1024]; /* Program working dir */ */ #define INVALID_KEY UINT64_MAX #define MAX_CKPT_INVL 5 /* Maximum interval between checkpoints */ -#define MAX_TH 12 +#define MAX_DEFAULT_TH 12 +#define MAX_TH 200 /* Maximum configurable threads */ #define MAX_TIME 40 #define MAX_VAL 1024 #define MIN_TH 5 @@ -67,6 +68,7 @@ static char home[1024]; /* Program working dir */ #define PREPARE_YIELD (PREPARE_FREQ * 10) #define RECORDS_FILE "records-%" PRIu32 #define STABLE_PERIOD 100 +#define SESSION_MAX MAX_TH + 3 /* Include program worker threads */ static const char * table_pfx = "table"; static const char * const uri_local = "local"; @@ -81,10 +83,10 @@ static volatile uint64_t th_ts[MAX_TH]; #define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" #define ENV_CONFIG_DEF \ - "create,log=(archive=false,file_max=10M,enabled)" + "create,log=(archive=false,file_max=10M,enabled),session_max=%" PRIu32 #define ENV_CONFIG_TXNSYNC \ - "create,log=(archive=false,file_max=10M,enabled)," \ - "transaction_sync=(enabled,method=none)" + "create,log=(archive=false,file_max=10M,enabled)," \ + "transaction_sync=(enabled,method=none),session_max=%" PRIu32 #define ENV_CONFIG_REC "log=(archive=false,recover=on)" typedef struct { @@ -445,9 +447,11 @@ run_workload(uint32_t nth) if (chdir(home) != 0) testutil_die(errno, "Child chdir: %s", home); if (inmem) - strcpy(envconf, ENV_CONFIG_DEF); + (void)__wt_snprintf(envconf, sizeof(envconf), + ENV_CONFIG_DEF, SESSION_MAX); else - strcpy(envconf, ENV_CONFIG_TXNSYNC); + (void)__wt_snprintf(envconf, sizeof(envconf), + ENV_CONFIG_TXNSYNC, SESSION_MAX); if (compat) strcat(envconf, ENV_CONFIG_COMPAT); @@ -628,6 +632,12 @@ main(int argc, char *argv[]) case 'T': rand_th = false; nth = (uint32_t)atoi(__wt_optarg); + if (nth > MAX_TH) { + fprintf(stderr, + "Number of threads is larger than the" + " maximum %" PRId32 "\n", MAX_TH); + return (EXIT_FAILURE); + } break; case 't': rand_time = false; @@ -666,7 +676,7 @@ main(int argc, char *argv[]) timeout = MIN_TIME; } if (rand_th) { - nth = __wt_random(&rnd) % MAX_TH; + nth = __wt_random(&rnd) % MAX_DEFAULT_TH; if (nth < MIN_TH) nth = MIN_TH; } diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 33ef6864b64..9a32a96423d 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -260,12 +260,13 @@ config_setup(void) static void config_cache(void) { - uint32_t max_dirty_bytes, required; + uint32_t required; /* Page sizes are powers-of-two for bad historic reasons. */ g.intl_page_max = 1U << g.c_intl_page_max; g.leaf_page_max = 1U << g.c_leaf_page_max; + /* Check if a minimum cache size has been specified. */ if (config_is_perm("cache")) { if (config_is_perm("cache_minimum") && g.c_cache_minimum != 0 && g.c_cache < g.c_cache_minimum) @@ -276,35 +277,24 @@ config_cache(void) return; } - /* Check if a minimum cache size has been specified. */ - if (g.c_cache_minimum != 0 && g.c_cache < g.c_cache_minimum) - g.c_cache = g.c_cache_minimum; - - /* Ensure there is at least 1MB of cache per thread. */ - if (g.c_cache < g.c_threads) - g.c_cache = g.c_threads; + g.c_cache = WT_MAX(g.c_cache, g.c_cache_minimum); /* * Maximum internal/leaf page size sanity. * * Ensure we can service at least one operation per-thread concurrently * without filling the cache with pinned pages, that is, every thread - * consuming an internal page and a leaf page. Page-size configurations - * control on-disk sizes and in-memory pages are often larger than their - * disk counterparts, so it's hard to translate from one to the other. - * Use a size-adjustment multiplier as an estimate. + * consuming an internal page and a leaf page (or a pair of leaf pages + * for cursor movements). * - * Assuming all of those pages are dirty, don't let the maximum dirty - * bytes exceed 40% of the cache (the default eviction trigger is 20%). + * Maximum memory pages are in units of MB. + * + * This code is what dramatically increases the cache size when there + * are lots of threads, it grows the cache to several megabytes per + * thread. */ -#define SIZE_ADJUSTMENT 3 - for (;;) { - max_dirty_bytes = ((g.c_cache * WT_MEGABYTE) / 10) * 4; - if (SIZE_ADJUSTMENT * g.c_threads * - (g.intl_page_max + g.leaf_page_max) <= max_dirty_bytes) - break; - ++g.c_cache; - } + g.c_cache = WT_MAX(g.c_cache, + 2 * g.c_threads * g.c_memory_page_max); /* * Ensure cache size sanity for LSM runs. An LSM tree open requires 3 diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 70e8165e97d..a64a808d968 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -250,6 +250,10 @@ static CONFIG c[] = { "the number of LSM worker threads", 0x0, 3, 4, 20, &g.c_lsm_worker_threads, NULL }, + { "memory_page_max", + "maximum size of in-memory pages", + 0x0, 1, 10, 128, &g.c_memory_page_max, NULL }, + { "merge_max", "the maximum number of chunks to include in a merge operation", 0x0, 4, 20, 100, &g.c_merge_max, NULL }, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index a83aa0d2dcb..8c4132d137e 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -65,11 +65,11 @@ #define LZO_PATH ".libs/lzo_compress.so" #undef M -#define M(v) ((v) * 1000000) /* Million */ +#define M(v) ((v) * WT_MILLION) /* Million */ #undef KILOBYTE -#define KILOBYTE(v) ((v) * 1024) +#define KILOBYTE(v) ((v) * WT_KILOBYTE) #undef MEGABYTE -#define MEGABYTE(v) ((v) * 1048576) +#define MEGABYTE(v) ((v) * WT_MEGABYTE) #define WT_NAME "wt" /* Object name */ @@ -201,6 +201,7 @@ typedef struct { uint32_t c_logging_prealloc; uint32_t c_long_running_txn; uint32_t c_lsm_worker_threads; + uint32_t c_memory_page_max; uint32_t c_merge_max; uint32_t c_mmap; uint32_t c_modify_pct; diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 30b910435d4..f331282ad85 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -367,12 +367,15 @@ wts_init(void) max = sizeof(config); CONFIG_APPEND(p, - "key_format=%s," - "allocation_size=512,%s" - "internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32, + "key_format=%s" + ",allocation_size=512" + ",%s" + ",internal_page_max=%" PRIu32 + ",leaf_page_max=%" PRIu32 + ",memory_page_max=%" PRIu32, (g.type == ROW) ? "u" : "r", - g.c_firstfit ? "block_allocation=first," : "", - g.intl_page_max, g.leaf_page_max); + g.c_firstfit ? "block_allocation=first" : "", + g.intl_page_max, g.leaf_page_max, MEGABYTE(g.c_memory_page_max)); /* * Configure the maximum key/value sizes, but leave it as the default |