diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2020-05-13 16:20:56 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-05-13 06:37:42 +0000 |
| commit | 736bf1bca376a50dbd038098baf0a2f6ace29db7 (patch) | |
| tree | 0c2fa5977198486c1ba6a16848a35864adc20c3f | |
| parent | 32673beb9193176de717feb7cd8d37c6b580f32d (diff) | |
| download | mongo-736bf1bca376a50dbd038098baf0a2f6ace29db7.tar.gz | |
Import wiredtiger: 63b37d1861f396e09a76b69f9b786740709a6f8c from branch mongodb-4.4
ref: bdff12c233..63b37d1861
for: 4.5.1
WT-4414 Only flush dirty data during checkpoints if logging is disabled
WT-5543 Failed to open cursor on table that had writing failure on close
WT-6075 Re-enable csuite-wt2323-join-visibility-test
WT-6080 Add statistics to count frequency of saving validity window in the data store
WT-6118 Fix missing checkpoint in backup
WT-6136 Record incremental extent lists before merging them with earlier checkpoints
31 files changed, 903 insertions, 338 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 6bef5f96b93..3543aac5e5a 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -983,6 +983,16 @@ wiredtiger_open_common =\ size and the default config would extend log files in allocations of the maximum log file size.''', type='list', choices=['data', 'log']), + Config('file_close_sync', 'true', r''' + control whether to flush modified files to storage independent + of a global checkpoint when closing file handles to acquire exclusive + access to a table. If set to false, and logging is disabled, API calls that + require exclusive access to tables will return EBUSY if there have been + changes made to the table since the last global checkpoint. When logging + is enabled, the value for <code>file_close_sync</code> has no effect, and, + modified file is always flushed to storage when closing file handles to + acquire exclusive access to the table''', + type='boolean'), Config('hazard_max', '1000', r''' maximum number of simultaneous hazard pointers per session handle''', diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 181a3c29847..471ee528981 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -528,6 +528,9 @@ connection_stats = [ RecStat('rec_page_delete_fast', 'fast-path pages deleted'), RecStat('rec_pages', 'page reconciliation calls'), RecStat('rec_pages_eviction', 'page reconciliation calls for eviction'), + RecStat('rec_pages_with_prepare', 'page reconciliation calls that resulted in values with prepared transaction metadata'), + RecStat('rec_pages_with_ts', 'page reconciliation calls that resulted in values with timestamps'), + RecStat('rec_pages_with_txn', 'page reconciliation calls that resulted in values with transaction ids'), RecStat('rec_split_stashed_bytes', 'split bytes currently awaiting free', 'no_clear,no_scale,size'), RecStat('rec_split_stashed_objects', 'split objects currently awaiting free', 'no_clear,no_scale'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index e64af3c37d8..68ce2c05a34 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "bdff12c2331ab0478a22309a6d35519d2e2ca441" + "commit": "63b37d1861f396e09a76b69f9b786740709a6f8c" } diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 7480ddfca2b..2f97613368a 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -343,6 +343,119 @@ __ckpt_verify(WT_SESSION_IMPL *session, WT_CKPT *ckptbase) #endif /* + * __ckpt_add_blkmod_entry -- + * Add an offset/length entry to the bitstring based on granularity. + */ +static int +__ckpt_add_blkmod_entry( + WT_SESSION_IMPL *session, WT_BLOCK_MODS *blk_mod, wt_off_t offset, wt_off_t len) +{ + uint64_t end_bit, start_bit; + uint32_t end_buf_bytes, end_rdup_bits, end_rdup_bytes; + + WT_ASSERT(session, blk_mod->granularity != 0); + /* + * Figure out how the starting and ending bits based on the granularity and our offset and + * length. + */ + start_bit = (uint64_t)offset / blk_mod->granularity; + end_bit = (uint64_t)(offset + len - 1) / blk_mod->granularity; + WT_ASSERT(session, end_bit < UINT32_MAX); + /* We want to grow the bitmap by 64 bits, or 8 bytes at a time. */ + end_rdup_bits = WT_MAX(__wt_rduppo2((uint32_t)end_bit, 64), WT_BLOCK_MODS_LIST_MIN); + end_rdup_bytes = end_rdup_bits >> 3; + end_buf_bytes = (uint32_t)blk_mod->nbits >> 3; + /* + * We are doing a lot of shifting. Make sure that the number of bytes we end up with is a + * multiple of eight. We guarantee that in the rounding up call, but also make sure that the + * constant stays a multiple of eight. + */ + WT_ASSERT(session, end_rdup_bytes % 8 == 0); + if (end_rdup_bytes > end_buf_bytes) { + /* If we don't have enough, extend the buffer. */ + if (blk_mod->nbits == 0) { + WT_RET(__wt_buf_initsize(session, &blk_mod->bitstring, end_rdup_bytes)); + memset(blk_mod->bitstring.mem, 0, end_rdup_bytes); + } else { + WT_RET( + __wt_buf_set(session, &blk_mod->bitstring, blk_mod->bitstring.data, end_rdup_bytes)); + memset( + (uint8_t *)blk_mod->bitstring.mem + end_buf_bytes, 0, end_rdup_bytes - end_buf_bytes); + } + blk_mod->nbits = end_rdup_bits; + } + /* Set all the bits needed to record this offset/length pair. */ + __bit_nset(blk_mod->bitstring.mem, start_bit, end_bit); + return (0); +} + +/* + * __ckpt_add_blk_mods_alloc -- + * Add the checkpoint's allocated blocks to all valid incremental backup source identifiers. + */ +static int +__ckpt_add_blk_mods_alloc(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_BLOCK_CKPT *ci) +{ + WT_BLOCK_MODS *blk_mod; + WT_CKPT *ckpt; + WT_EXT *ext; + u_int i; + + WT_CKPT_FOREACH (ckptbase, ckpt) { + if (F_ISSET(ckpt, WT_CKPT_ADD)) + break; + } + /* If this is not the live checkpoint or we don't care about incremental blocks, we're done. */ + if (ckpt == NULL || !F_ISSET(ckpt, WT_CKPT_BLOCK_MODS)) + return (0); + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk_mod = &ckpt->backup_blocks[i]; + /* If there is no information at this entry, we're done. */ + if (!F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) + continue; + + WT_EXT_FOREACH (ext, ci->alloc.off) { + WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ext->off, ext->size)); + } + } + return (0); +} + +/* + * __ckpt_add_blk_mods_ext -- + * Add a set of extent blocks to all valid incremental backup source identifiers. + */ +static int +__ckpt_add_blk_mods_ext(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_BLOCK_CKPT *ci) +{ + WT_BLOCK_MODS *blk_mod; + WT_CKPT *ckpt; + u_int i; + + WT_CKPT_FOREACH (ckptbase, ckpt) { + if (F_ISSET(ckpt, WT_CKPT_ADD)) + break; + } + /* If this is not the live checkpoint or we don't care about incremental blocks, we're done. */ + if (ckpt == NULL || !F_ISSET(ckpt, WT_CKPT_BLOCK_MODS)) + return (0); + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk_mod = &ckpt->backup_blocks[i]; + /* If there is no information at this entry, we're done. */ + if (!F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) + continue; + + if (ci->alloc.offset != WT_BLOCK_INVALID_OFFSET) + WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->alloc.offset, ci->alloc.size)); + if (ci->discard.offset != WT_BLOCK_INVALID_OFFSET) + WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->discard.offset, ci->discard.size)); + if (ci->avail.offset != WT_BLOCK_INVALID_OFFSET) + WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->avail.offset, ci->avail.size)); + } + return (0); +} + +/* * __ckpt_process -- * Process the list of checkpoints. */ @@ -475,6 +588,12 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) ckpt_size += ci->alloc.bytes; ckpt_size -= ci->discard.bytes; + /* + * Record the checkpoint's allocated blocks. Do so before skipping any processing and before + * possibly merging in blocks from any previous checkpoint. + */ + WT_ERR(__ckpt_add_blk_mods_alloc(session, ckptbase, ci)); + /* Skip the additional processing if we aren't deleting checkpoints. */ if (!deleting) goto live_update; @@ -581,11 +700,9 @@ live_update: if (F_ISSET(ckpt, WT_CKPT_ADD)) { /* * !!! - * Our caller wants the final checkpoint size. Setting - * the size here violates layering, but the alternative - * is a call for the btree layer to crack the checkpoint - * cookie into its components, and that's a fair amount - * of work. + * Our caller wants the final checkpoint size. Setting the size here violates layering, + * but the alternative is a call for the btree layer to crack the checkpoint cookie into + * its components, and that's a fair amount of work. */ ckpt->size = ckpt_size; @@ -654,83 +771,6 @@ err: } /* - * __ckpt_add_blkmod_entry -- - * Add an offset/length entry to the bitstring based on granularity. - */ -static int -__ckpt_add_blkmod_entry( - WT_SESSION_IMPL *session, WT_BLOCK_MODS *blk_mod, wt_off_t offset, wt_off_t len) -{ - uint64_t end_bit, start_bit; - uint32_t end_buf_bytes, end_rdup_bits, end_rdup_bytes; - - WT_ASSERT(session, blk_mod->granularity != 0); - /* - * Figure out how the starting and ending bits based on the granularity and our offset and - * length. - */ - start_bit = (uint64_t)offset / blk_mod->granularity; - end_bit = (uint64_t)(offset + len - 1) / blk_mod->granularity; - WT_ASSERT(session, end_bit < UINT32_MAX); - /* We want to grow the bitmap by 64 bits, or 8 bytes at a time. */ - end_rdup_bits = WT_MAX(__wt_rduppo2((uint32_t)end_bit, 64), WT_BLOCK_MODS_LIST_MIN); - end_rdup_bytes = end_rdup_bits >> 3; - end_buf_bytes = (uint32_t)blk_mod->nbits >> 3; - /* - * We are doing a lot of shifting. Make sure that the number of bytes we end up with is a - * multiple of eight. We guarantee that in the rounding up call, but also make sure that the - * constant stays a multiple of eight. - */ - WT_ASSERT(session, end_rdup_bytes % 8 == 0); - if (end_rdup_bytes > end_buf_bytes) { - /* If we don't have enough, extend the buffer. */ - if (blk_mod->nbits == 0) { - WT_RET(__wt_buf_initsize(session, &blk_mod->bitstring, end_rdup_bytes)); - memset(blk_mod->bitstring.mem, 0, end_rdup_bytes); - } else { - WT_RET( - __wt_buf_set(session, &blk_mod->bitstring, blk_mod->bitstring.data, end_rdup_bytes)); - memset( - (uint8_t *)blk_mod->bitstring.mem + end_buf_bytes, 0, end_rdup_bytes - end_buf_bytes); - } - blk_mod->nbits = end_rdup_bits; - } - /* Set all the bits needed to record this offset/length pair. */ - __bit_nset(blk_mod->bitstring.mem, start_bit, end_bit); - return (0); -} - -/* - * __ckpt_add_blk_mods -- - * Add the blocks to all valid incremental backup source identifiers. - */ -static int -__ckpt_add_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci) -{ - WT_BLOCK_MODS *blk_mod; - WT_EXT *ext; - u_int i; - - for (i = 0; i < WT_BLKINCR_MAX; ++i) { - blk_mod = &ckpt->backup_blocks[i]; - /* If there is no information at this entry, we're done. */ - if (!F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) - continue; - - WT_EXT_FOREACH (ext, ci->alloc.off) - WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ext->off, ext->size)); - - if (ci->alloc.offset != WT_BLOCK_INVALID_OFFSET) - WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->alloc.offset, ci->alloc.size)); - if (ci->discard.offset != WT_BLOCK_INVALID_OFFSET) - WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->discard.offset, ci->discard.size)); - if (ci->avail.offset != WT_BLOCK_INVALID_OFFSET) - WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->avail.offset, ci->avail.size)); - } - return (0); -} - -/* * __ckpt_update -- * Update a checkpoint. */ @@ -752,9 +792,8 @@ __ckpt_update( WT_RET(__wt_block_extlist_check(session, &ci->alloc, &ci->discard)); #endif /* - * Write the checkpoint's alloc and discard extent lists. After each write, remove any allocated - * blocks from the system's allocation list, checkpoint extent blocks don't appear on any extent - * lists. + * Write the checkpoint's alloc and discard extent lists. Note these blocks never appear on the + * system's allocation list, checkpoint extent blocks don't appear on any extent lists. */ WT_RET(__wt_block_extlist_write(session, block, &ci->alloc, NULL)); WT_RET(__wt_block_extlist_write(session, block, &ci->discard, NULL)); @@ -803,29 +842,34 @@ __ckpt_update( } /* - * If this is the live system, we need to record the list of blocks written for this checkpoint - * (including the blocks we allocated to write the extent lists). + * Record the blocks allocated to write the extent lists. We must record blocks in the live + * system's extent lists, as those blocks are a necessary part of the checkpoint a hot backup + * might recover. Update blocks in extent lists used to rewrite other checkpoints (for example, + * an intermediate checkpoint rewritten because a checkpoint was rolled into it), even though + * it's not necessary: those blocks aren't the last checkpoint in the file and so aren't + * included in a recoverable checkpoint, they don't matter on a hot backup target until they're + * allocated and used in the context of a live system. Regardless, they shouldn't materially + * affect how much data we're writing, and it keeps things more consistent on the target to + * update them. (Ignore the live system's ckpt_avail list here. The blocks on that list were + * written into the final avail extent list which will be copied to the hot backup, and that's + * all that matters.) */ - if (F_ISSET(ckpt, WT_CKPT_BLOCK_MODS)) - WT_RET(__ckpt_add_blk_mods(session, ckpt, ci)); + WT_RET(__ckpt_add_blk_mods_ext(session, ckptbase, ci)); /* * Set the file size for the live system. * * !!! - * We do NOT set the file size when re-writing checkpoints because we - * want to test the checkpoint's blocks against a reasonable maximum - * file size during verification. This is bad: imagine a checkpoint - * appearing early in the file, re-written, and then the checkpoint - * requires blocks at the end of the file, blocks after the listed file - * size. If the application opens that checkpoint for writing - * (discarding subsequent checkpoints), we would truncate the file to - * the early chunk, discarding the re-written checkpoint information. - * The alternative, updating the file size has its own problems, in - * that case we'd work correctly, but we'd lose all of the blocks - * between the original checkpoint and the re-written checkpoint. - * Currently, there's no API to roll-forward intermediate checkpoints, - * if there ever is, this will need to be fixed. + * We do NOT set the file size when re-writing checkpoints because we want to test the + * checkpoint's blocks against a reasonable maximum file size during verification. This is bad: + * imagine a checkpoint appearing early in the file, re-written, and then the checkpoint + * requires blocks at the end of the file, blocks after the listed file size. If the application + * opens that checkpoint for writing (discarding subsequent checkpoints), we would truncate the + * file to the early chunk, discarding the re-written checkpoint information. The alternative, + * updating the file size has its own problems, in that case we'd work correctly, but we'd lose + * all of the blocks between the original checkpoint and the re-written checkpoint. Currently, + * there's no API to roll-forward intermediate checkpoints, if there ever is, this will need to + * be fixed. */ if (is_live) ci->file_size = block->size; diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index a7cae12139a..a29bcbe25d0 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -549,6 +549,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"eviction_target", "int", NULL, "min=10,max=10TB", NULL, 0}, {"eviction_trigger", "int", NULL, "min=10,max=10TB", NULL, 0}, {"exclusive", "boolean", NULL, NULL, NULL, 0}, {"extensions", "list", NULL, NULL, NULL, 0}, + {"file_close_sync", "boolean", NULL, NULL, NULL, 0}, {"file_extend", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {"file_manager", "category", NULL, NULL, confchk_wiredtiger_open_file_manager_subconfigs, 3}, {"hazard_max", "int", NULL, "min=15", NULL, 0}, @@ -621,6 +622,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"eviction_target", "int", NULL, "min=10,max=10TB", NULL, 0}, {"eviction_trigger", "int", NULL, "min=10,max=10TB", NULL, 0}, {"exclusive", "boolean", NULL, NULL, NULL, 0}, {"extensions", "list", NULL, NULL, NULL, 0}, + {"file_close_sync", "boolean", NULL, NULL, NULL, 0}, {"file_extend", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {"file_manager", "category", NULL, NULL, confchk_wiredtiger_open_file_manager_subconfigs, 3}, {"hazard_max", "int", NULL, "min=15", NULL, 0}, @@ -691,7 +693,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"eviction_dirty_trigger", "int", NULL, "min=1,max=10TB", NULL, 0}, {"eviction_target", "int", NULL, "min=10,max=10TB", NULL, 0}, {"eviction_trigger", "int", NULL, "min=10,max=10TB", NULL, 0}, - {"extensions", "list", NULL, NULL, NULL, 0}, + {"extensions", "list", NULL, NULL, NULL, 0}, {"file_close_sync", "boolean", NULL, NULL, NULL, 0}, {"file_extend", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {"file_manager", "category", NULL, NULL, confchk_wiredtiger_open_file_manager_subconfigs, 3}, {"hazard_max", "int", NULL, "min=15", NULL, 0}, @@ -759,7 +761,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"eviction_dirty_trigger", "int", NULL, "min=1,max=10TB", NULL, 0}, {"eviction_target", "int", NULL, "min=10,max=10TB", NULL, 0}, {"eviction_trigger", "int", NULL, "min=10,max=10TB", NULL, 0}, - {"extensions", "list", NULL, NULL, NULL, 0}, + {"extensions", "list", NULL, NULL, NULL, 0}, {"file_close_sync", "boolean", NULL, NULL, NULL, 0}, {"file_extend", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {"file_manager", "category", NULL, NULL, confchk_wiredtiger_open_file_manager_subconfigs, 3}, {"hazard_max", "int", NULL, "min=15", NULL, 0}, @@ -1009,7 +1011,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "eviction=(threads_max=8,threads_min=1)," "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," + ",exclusive=false,extensions=,file_close_sync=true,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),hazard_max=1000," "history_store=(file_max=0),in_memory=false,io_capacity=(total=0)" @@ -1026,7 +1028,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),use_environment=true,use_environment_priv=false," "verbose=,verify_metadata=false,write_through=", - confchk_wiredtiger_open, 53}, + confchk_wiredtiger_open, 54}, {"wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1041,7 +1043,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "eviction=(threads_max=8,threads_min=1)," "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," + ",exclusive=false,extensions=,file_close_sync=true,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),hazard_max=1000," "history_store=(file_max=0),in_memory=false,io_capacity=(total=0)" @@ -1059,7 +1061,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "method=fsync),use_environment=true,use_environment_priv=false," "verbose=,verify_metadata=false,version=(major=0,minor=0)," "write_through=", - confchk_wiredtiger_open_all, 54}, + confchk_wiredtiger_open_all, 55}, {"wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1073,8 +1075,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "eviction=(threads_max=8,threads_min=1)," "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" - ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + ",extensions=,file_close_sync=true,file_extend=," + "file_manager=(close_handle_minimum=250,close_idle_time=30," + "close_scan_interval=10),hazard_max=1000," "history_store=(file_max=0),io_capacity=(total=0)," "log=(archive=true,compressor=,enabled=false,file_max=100MB," "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," @@ -1089,7 +1092,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),verbose=,verify_metadata=false,version=(major=0," "minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 48}, + confchk_wiredtiger_open_basecfg, 49}, {"wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1103,8 +1106,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "eviction=(threads_max=8,threads_min=1)," "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" - ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + ",extensions=,file_close_sync=true,file_extend=," + "file_manager=(close_handle_minimum=250,close_idle_time=30," + "close_scan_interval=10),hazard_max=1000," "history_store=(file_max=0),io_capacity=(total=0)," "log=(archive=true,compressor=,enabled=false,file_max=100MB," "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," @@ -1118,7 +1122,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),verbose=,verify_metadata=false,write_through=", - confchk_wiredtiger_open_usercfg, 47}, + confchk_wiredtiger_open_usercfg, 48}, {NULL, NULL, NULL, 0}}; int diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 82639177844..7369cbad72b 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2546,6 +2546,10 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c if (cval.val) F_SET(conn, WT_CONN_CKPT_SYNC); + WT_ERR(__wt_config_gets(session, cfg, "file_close_sync", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_FILE_CLOSE_SYNC); + WT_ERR(__wt_config_gets(session, cfg, "file_extend", &cval)); /* * If the log extend length is not set use the default of the configured maximum log file size. diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 076e64c73ce..7ad449ed492 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -39,7 +39,7 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) */ conn->default_session = session; - __wt_seconds(session, &conn->ckpt_finish_secs); + __wt_seconds(session, &conn->ckpt_most_recent); /* * Publish: there must be a barrier to ensure the connection structure fields are set before diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 4011a62b0d5..01ffa7a9699 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -311,7 +311,7 @@ __backup_add_id(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval) for (i = 0; i < WT_BLKINCR_MAX; ++i) { blk = &conn->incr_backups[i]; __wt_verbose(session, WT_VERB_BACKUP, "blk[%u] flags 0x%" PRIx64, i, blk->flags); - /* If it isn't use, we can use it. */ + /* If it isn't already in use, we can use it. */ if (!F_ISSET(blk, WT_BLKINCR_INUSE)) break; } diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index 427ca83b124..fad797a842e 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -60,7 +60,8 @@ __cell_check_value_validity(WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw) * Pack the validity window for a value. */ static inline void -__cell_pack_value_validity(WT_SESSION_IMPL *session, uint8_t **pp, WT_TIME_WINDOW *tw) +__cell_pack_value_validity( + WT_SESSION_IMPL *session, WT_RECONCILE *r, uint8_t **pp, WT_TIME_WINDOW *tw) { uint8_t flags, *flagsp; @@ -112,8 +113,15 @@ __cell_pack_value_validity(WT_SESSION_IMPL *session, uint8_t **pp, WT_TIME_WINDO LF_SET(WT_CELL_TS_DURABLE_STOP); } } - if (tw->prepare) + if (LF_ISSET( + WT_CELL_TS_START | WT_CELL_TS_DURABLE_START | WT_CELL_TS_STOP | WT_CELL_TS_DURABLE_STOP)) + r->rec_page_cell_with_ts = true; + if (LF_ISSET(WT_CELL_TXN_START | WT_CELL_TXN_STOP)) + r->rec_page_cell_with_txn_id = true; + if (tw->prepare) { LF_SET(WT_CELL_PREPARE); + r->rec_page_cell_with_prepared_txn = true; + } *flagsp = flags; } @@ -276,8 +284,8 @@ __wt_cell_pack_addr(WT_SESSION_IMPL *session, WT_CELL *cell, u_int cell_type, ui * Set a value item's WT_CELL contents. */ static inline size_t -__wt_cell_pack_value( - WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, size_t size) +__wt_cell_pack_value(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw, + uint64_t rle, size_t size) { uint8_t byte, *p; bool validity; @@ -286,7 +294,7 @@ __wt_cell_pack_value( p = cell->__chunk; *p = '\0'; - __cell_pack_value_validity(session, &p, tw); + __cell_pack_value_validity(session, r, &p, tw); /* * Short data cells without a validity window or run-length encoding have 6 bits of data length @@ -408,8 +416,8 @@ __wt_cell_pack_value_match( * Write a copy value cell. */ static inline size_t -__wt_cell_pack_copy( - WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, uint64_t v) +__wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw, + uint64_t rle, uint64_t v) { uint8_t *p; @@ -417,7 +425,7 @@ __wt_cell_pack_copy( p = cell->__chunk; *p = '\0'; - __cell_pack_value_validity(session, &p, tw); + __cell_pack_value_validity(session, r, &p, tw); if (rle < 2) cell->__chunk[0] |= WT_CELL_VALUE_COPY; /* Type */ @@ -437,7 +445,8 @@ __wt_cell_pack_copy( * Write a deleted value cell. */ static inline size_t -__wt_cell_pack_del(WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle) +__wt_cell_pack_del( + WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle) { uint8_t *p; @@ -446,7 +455,7 @@ __wt_cell_pack_del(WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, *p = '\0'; /* FIXME-WT-6124: we should set the time window prepare value. */ - __cell_pack_value_validity(session, &p, tw); + __cell_pack_value_validity(session, r, &p, tw); if (rle < 2) cell->__chunk[0] |= WT_CELL_DEL; /* Type */ @@ -532,8 +541,8 @@ __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size) * Pack an overflow cell. */ static inline size_t -__wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_CELL *cell, uint8_t type, WT_TIME_WINDOW *tw, - uint64_t rle, size_t size) +__wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, uint8_t type, + WT_TIME_WINDOW *tw, uint64_t rle, size_t size) { uint8_t *p; @@ -549,7 +558,7 @@ __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_CELL *cell, uint8_t type, WT_TI break; case WT_CELL_VALUE_OVFL: case WT_CELL_VALUE_OVFL_RM: - __cell_pack_value_validity(session, &p, tw); + __cell_pack_value_validity(session, r, &p, tw); break; } diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 547feaa54a3..9c0ebf552be 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -152,7 +152,7 @@ struct __wt_named_extractor { */ #define WT_CONN_HOTBACKUP_START(conn) \ do { \ - (conn)->hot_backup_start = (conn)->ckpt_finish_secs; \ + (conn)->hot_backup_start = (conn)->ckpt_most_recent; \ (conn)->hot_backup_list = NULL; \ } while (0) @@ -276,7 +276,7 @@ struct __wt_connection_impl { wt_thread_t ckpt_tid; /* Checkpoint thread */ bool ckpt_tid_set; /* Checkpoint thread set */ WT_CONDVAR *ckpt_cond; /* Checkpoint wait mutex */ - uint64_t ckpt_finish_secs; /* Clock value of last completed checkpoint */ + uint64_t ckpt_most_recent; /* Clock value of most recent checkpoint */ #define WT_CKPT_LOGSIZE(conn) ((conn)->ckpt_logsize != 0) wt_off_t ckpt_logsize; /* Checkpoint log size period */ bool ckpt_signalled; /* Checkpoint signalled */ @@ -527,25 +527,26 @@ struct __wt_connection_impl { #define WT_CONN_DEBUG_REALLOC_EXACT 0x00000200u #define WT_CONN_DEBUG_SLOW_CKPT 0x00000400u #define WT_CONN_EVICTION_RUN 0x00000800u -#define WT_CONN_HS_OPEN 0x00001000u -#define WT_CONN_INCR_BACKUP 0x00002000u -#define WT_CONN_IN_MEMORY 0x00004000u -#define WT_CONN_LEAK_MEMORY 0x00008000u -#define WT_CONN_LSM_MERGE 0x00010000u -#define WT_CONN_OPTRACK 0x00020000u -#define WT_CONN_PANIC 0x00040000u -#define WT_CONN_READONLY 0x00080000u -#define WT_CONN_RECONFIGURING 0x00100000u -#define WT_CONN_RECOVERING 0x00200000u -#define WT_CONN_SALVAGE 0x00400000u -#define WT_CONN_SERVER_ASYNC 0x00800000u -#define WT_CONN_SERVER_CAPACITY 0x01000000u -#define WT_CONN_SERVER_CHECKPOINT 0x02000000u -#define WT_CONN_SERVER_LOG 0x04000000u -#define WT_CONN_SERVER_LSM 0x08000000u -#define WT_CONN_SERVER_STATISTICS 0x10000000u -#define WT_CONN_SERVER_SWEEP 0x20000000u -#define WT_CONN_WAS_BACKUP 0x40000000u +#define WT_CONN_FILE_CLOSE_SYNC 0x00001000u +#define WT_CONN_HS_OPEN 0x00002000u +#define WT_CONN_INCR_BACKUP 0x00004000u +#define WT_CONN_IN_MEMORY 0x00008000u +#define WT_CONN_LEAK_MEMORY 0x00010000u +#define WT_CONN_LSM_MERGE 0x00020000u +#define WT_CONN_OPTRACK 0x00040000u +#define WT_CONN_PANIC 0x00080000u +#define WT_CONN_READONLY 0x00100000u +#define WT_CONN_RECONFIGURING 0x00200000u +#define WT_CONN_RECOVERING 0x00400000u +#define WT_CONN_SALVAGE 0x00800000u +#define WT_CONN_SERVER_ASYNC 0x01000000u +#define WT_CONN_SERVER_CAPACITY 0x02000000u +#define WT_CONN_SERVER_CHECKPOINT 0x04000000u +#define WT_CONN_SERVER_LOG 0x08000000u +#define WT_CONN_SERVER_LSM 0x10000000u +#define WT_CONN_SERVER_STATISTICS 0x20000000u +#define WT_CONN_SERVER_SWEEP 0x40000000u +#define WT_CONN_WAS_BACKUP 0x80000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 81e6a7c81aa..abaf0575fee 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -2051,17 +2051,18 @@ static inline int __wt_write(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offse static inline size_t __wt_cell_pack_addr(WT_SESSION_IMPL *session, WT_CELL *cell, u_int cell_type, uint64_t recno, WT_TIME_AGGREGATE *ta, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline size_t __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_CELL *cell, +static inline size_t __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, uint64_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline size_t __wt_cell_pack_del(WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, - uint64_t rle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline size_t __wt_cell_pack_del(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, + WT_TIME_WINDOW *tw, uint64_t rle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline size_t __wt_cell_pack_int_key(WT_CELL *cell, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline size_t __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline size_t __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_CELL *cell, uint8_t type, - WT_TIME_WINDOW *tw, uint64_t rle, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline size_t __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_CELL *cell, +static inline size_t __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, + uint8_t type, WT_TIME_WINDOW *tw, uint64_t rle, size_t size) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline size_t __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline size_t __wt_cell_total_len(void *unpack_arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/reconcile.h b/src/third_party/wiredtiger/src/include/reconcile.h index a8d0c205aad..c31a92d0edd 100644 --- a/src/third_party/wiredtiger/src/include/reconcile.h +++ b/src/third_party/wiredtiger/src/include/reconcile.h @@ -224,6 +224,14 @@ struct __wt_reconcile { * violation and fragile, we need a better solution. */ WT_CURSOR_BTREE update_modify_cbt; + + /* + * Variables to track reconciled pages containing cells with time window values and prepared + * transactions. + */ + bool rec_page_cell_with_ts; + bool rec_page_cell_with_txn_id; + bool rec_page_cell_with_prepared_txn; }; typedef struct { diff --git a/src/third_party/wiredtiger/src/include/reconcile.i b/src/third_party/wiredtiger/src/include/reconcile.i index 8da14bc93ad..39b4c683334 100644 --- a/src/third_party/wiredtiger/src/include/reconcile.i +++ b/src/third_party/wiredtiger/src/include/reconcile.i @@ -219,7 +219,7 @@ __wt_rec_cell_build_val(WT_SESSION_IMPL *session, WT_RECONCILE *r, const void *d if (tw->prepare) WT_STAT_DATA_INCR(session, rec_prepare_value); - val->cell_len = __wt_cell_pack_value(session, &val->cell, tw, rle, val->buf.size); + val->cell_len = __wt_cell_pack_value(session, r, &val->cell, tw, rle, val->buf.size); val->len = val->cell_len + val->buf.size; return (0); @@ -267,7 +267,7 @@ __wt_rec_dict_replace( * offset from the beginning of the page. */ offset = (uint64_t)WT_PTRDIFF(r->first_free, (uint8_t *)r->cur_ptr->image.mem + dp->offset); - val->len = val->cell_len = __wt_cell_pack_copy(session, &val->cell, tw, rle, offset); + val->len = val->cell_len = __wt_cell_pack_copy(session, r, &val->cell, tw, rle, offset); val->buf.data = NULL; val->buf.size = 0; } diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 59936321cbc..2b646a640e7 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -613,6 +613,9 @@ struct __wt_connection_stats { int64_t rec_page_delete_fast; int64_t rec_pages; int64_t rec_pages_eviction; + int64_t rec_pages_with_prepare; + int64_t rec_pages_with_ts; + int64_t rec_pages_with_txn; int64_t rec_page_delete; int64_t rec_split_stashed_bytes; int64_t rec_split_stashed_objects; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 74fbe62d15b..4171b48d614 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2893,6 +2893,13 @@ struct __wt_connection { * specified to a library extension are passed to WT_CONNECTION::load_extension as the \c config * parameter (for example\, <code>extensions=(/path/ext.so={entry=my_entry})</code>)., a list of * strings; default empty.} + * @config{file_close_sync, control whether to flush modified files to storage independent of a + * global checkpoint when closing file handles to acquire exclusive access to a table. If set to + * false\, and logging is disabled\, API calls that require exclusive access to tables will return + * EBUSY if there have been changes made to the table since the last global checkpoint. When + * logging is enabled\, the value for <code>file_close_sync</code> has no effect\, and\, modified + * file is always flushed to storage when closing file handles to acquire exclusive access to the + * table., a boolean flag; default \c true.} * @config{file_extend, file extension configuration. If set\, extend files of the set type in * allocations of the set size\, instead of a block at a time as each new block is written. For * example\, <code>file_extend=(data=16MB)</code>. If set to 0\, disable the file extension for the @@ -5600,241 +5607,256 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_REC_PAGES 1316 /*! reconciliation: page reconciliation calls for eviction */ #define WT_STAT_CONN_REC_PAGES_EVICTION 1317 +/*! + * reconciliation: page reconciliation calls that resulted in values with + * prepared transaction metadata + */ +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1318 +/*! + * reconciliation: page reconciliation calls that resulted in values with + * timestamps + */ +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1319 +/*! + * reconciliation: page reconciliation calls that resulted in values with + * transaction ids + */ +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1320 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1318 +#define WT_STAT_CONN_REC_PAGE_DELETE 1321 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1319 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1322 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1320 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1323 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1321 +#define WT_STAT_CONN_SESSION_OPEN 1324 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1322 +#define WT_STAT_CONN_SESSION_QUERY_TS 1325 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1323 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1326 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1324 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1327 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1325 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1328 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1326 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1329 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1327 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1330 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1328 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1331 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1329 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1332 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1330 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1333 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1331 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1334 /*! session: table import failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1332 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1335 /*! session: table import successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1333 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1336 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1334 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1337 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1335 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1338 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1336 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1339 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1337 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1340 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1338 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1341 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1339 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1342 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1340 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1343 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1341 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1344 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1342 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1345 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1343 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1346 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1344 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1347 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1345 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1348 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1346 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1349 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1347 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1350 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1348 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1351 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1349 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1352 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1350 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1353 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1351 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1354 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1352 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1355 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1353 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1356 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1354 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1357 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1355 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1358 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1356 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1359 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1357 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1360 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1358 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1361 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1359 +#define WT_STAT_CONN_PAGE_SLEEP 1362 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1360 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1363 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1361 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1364 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1362 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1365 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1363 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1366 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1364 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1367 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1365 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1368 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1366 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1369 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1367 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1370 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1368 +#define WT_STAT_CONN_TXN_PREPARE 1371 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1369 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1372 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1370 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1373 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1371 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1374 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1372 +#define WT_STAT_CONN_TXN_QUERY_TS 1375 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1373 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1376 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1374 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1377 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1375 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1378 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1376 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1379 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1377 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1380 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1378 +#define WT_STAT_CONN_TXN_RTS 1381 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1379 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1382 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1380 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1383 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1381 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1384 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1382 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1385 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1383 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1386 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1384 +#define WT_STAT_CONN_TXN_SET_TS 1387 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1385 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1388 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1386 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1389 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1387 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1390 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1388 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1391 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1389 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1392 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1390 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1393 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1391 +#define WT_STAT_CONN_TXN_BEGIN 1394 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1392 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1395 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1393 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1396 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1394 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1397 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1395 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1398 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1396 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1399 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1397 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1400 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1398 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1401 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1399 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1402 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1400 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1403 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1401 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1404 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1402 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1405 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1403 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1406 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1404 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1407 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1405 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1408 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1406 +#define WT_STAT_CONN_TXN_CHECKPOINT 1409 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1407 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1410 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1408 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1411 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1409 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1412 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1410 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1413 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1411 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1414 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1412 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1415 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1413 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1416 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1414 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1417 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1415 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1418 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1416 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1419 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1417 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1420 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1418 +#define WT_STAT_CONN_TXN_SYNC 1421 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1419 +#define WT_STAT_CONN_TXN_COMMIT 1422 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1420 +#define WT_STAT_CONN_TXN_ROLLBACK 1423 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1421 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1424 /*! * @} diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 9742d16a9ef..194efb7e8d3 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -461,8 +461,10 @@ __wt_meta_ckptlist_get( WT_CKPT *ckpt, *ckptbase; WT_CONFIG ckptconf; WT_CONFIG_ITEM k, v; + WT_CONNECTION_IMPL *conn; WT_DECL_RET; size_t allocated, slot; + uint64_t most_recent; char *config; *ckptbasep = NULL; @@ -470,6 +472,7 @@ __wt_meta_ckptlist_get( ckptbase = NULL; allocated = slot = 0; config = NULL; + conn = S2C(session); /* Retrieve the metadata information for the file. */ WT_RET(__wt_metadata_search(session, fname, &config)); @@ -511,6 +514,17 @@ __wt_meta_ckptlist_get( ckpt->order = (slot == 0) ? 1 : ckptbase[slot - 1].order + 1; __wt_seconds(session, &ckpt->sec); /* + * Update time value for most recent checkpoint, not letting it move backwards. It is + * possible to race here, so use atomic CAS. This code relies on the fact that anyone we + * race with will only increase (never decrease) the most recent checkpoint time value. + */ + for (;;) { + WT_ORDERED_READ(most_recent, conn->ckpt_most_recent); + if (ckpt->sec <= most_recent || + __wt_atomic_cas64(&conn->ckpt_most_recent, most_recent, ckpt->sec)) + break; + } + /* * Load most recent checkpoint backup blocks to this checkpoint. */ WT_ERR(__ckpt_load_blk_mods(session, config, ckpt)); @@ -522,7 +536,7 @@ __wt_meta_ckptlist_get( * the checkpoint's modified blocks from the block manager. */ F_SET(ckpt, WT_CKPT_ADD); - if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) { + if (F_ISSET(conn, WT_CONN_INCR_BACKUP)) { F_SET(ckpt, WT_CKPT_BLOCK_MODS); WT_ERR(__ckpt_valid_blk_mods(session, ckpt)); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index c49a3f11bfa..613f83f3f55 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -118,7 +118,7 @@ __wt_bulk_insert_var(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool delet val = &r->v; if (deleted) { - val->cell_len = __wt_cell_pack_del(session, &val->cell, &tw, cbulk->rle); + val->cell_len = __wt_cell_pack_del(session, r, &val->cell, &tw, cbulk->rle); val->buf.data = NULL; val->buf.size = 0; val->len = val->cell_len; @@ -526,13 +526,13 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_SALVAGE_COOKI } if (deleted) { - val->cell_len = __wt_cell_pack_del(session, &val->cell, tw, rle); + val->cell_len = __wt_cell_pack_del(session, r, &val->cell, tw, rle); val->buf.data = NULL; val->buf.size = 0; val->len = val->cell_len; } else if (overflow_type) { val->cell_len = - __wt_cell_pack_ovfl(session, &val->cell, WT_CELL_VALUE_OVFL, tw, rle, value->size); + __wt_cell_pack_ovfl(session, r, &val->cell, WT_CELL_VALUE_OVFL, tw, rle, value->size); val->buf.data = value->data; val->buf.size = value->size; val->len = val->cell_len + value->size; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index 66331a663b5..ccde8b402d7 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -797,8 +797,8 @@ __wt_rec_row_leaf( val->buf.size = vpack->size; /* Rebuild the cell. */ - val->cell_len = - __wt_cell_pack_ovfl(session, &val->cell, vpack->raw, &tw, 0, val->buf.size); + val->cell_len = __wt_cell_pack_ovfl( + session, r, &val->cell, vpack->raw, &tw, 0, val->buf.size); val->len = val->cell_len + val->buf.size; } else WT_ERR(__rec_cell_repack(session, btree, r, vpack, &tw)); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 4dd14173c5c..bdecb82b226 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -210,6 +210,14 @@ __reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, u WT_STAT_CONN_INCR(session, cache_write_restore); WT_STAT_DATA_INCR(session, cache_write_restore); } + if (!WT_IS_HS(btree)) { + if (r->rec_page_cell_with_txn_id) + WT_STAT_CONN_INCR(session, rec_pages_with_txn); + if (r->rec_page_cell_with_ts) + WT_STAT_CONN_INCR(session, rec_pages_with_ts); + if (r->rec_page_cell_with_prepared_txn) + WT_STAT_CONN_INCR(session, rec_pages_with_prepare); + } if (r->multi_next > btree->rec_multiblock_max) btree->rec_multiblock_max = r->multi_next; @@ -596,6 +604,11 @@ __rec_init(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags, WT_SALVAGE_COO r->update_modify_cbt.iface.value_format = btree->value_format; r->update_modify_cbt.upd_value = &r->update_modify_cbt._upd_value; + /* Clear stats related data. */ + r->rec_page_cell_with_ts = false; + r->rec_page_cell_with_txn_id = false; + r->rec_page_cell_with_prepared_txn = false; + /* * If we allocated the reconciliation structure and there was an error, clean up. If our caller * passed in a structure, they own it. @@ -2304,7 +2317,7 @@ __wt_rec_cell_build_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REC_KV *k WT_ERR(__wt_buf_set(session, &kv->buf, addr, size)); /* Build the cell and return. */ - kv->cell_len = __wt_cell_pack_ovfl(session, &kv->cell, type, tw, rle, kv->buf.size); + kv->cell_len = __wt_cell_pack_ovfl(session, r, &kv->cell, type, tw, rle, kv->buf.size); kv->len = kv->cell_len + kv->buf.size; err: diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 751a86deb53..b8e154e110b 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -839,8 +839,12 @@ static const char *const __stats_connection_desc[] = { "perf: operation write latency histogram (bucket 4) - 1000-9999us", "perf: operation write latency histogram (bucket 5) - 10000us+", "reconciliation: fast-path pages deleted", "reconciliation: page reconciliation calls", - "reconciliation: page reconciliation calls for eviction", "reconciliation: pages deleted", - "reconciliation: split bytes currently awaiting free", + "reconciliation: page reconciliation calls for eviction", + "reconciliation: page reconciliation calls that resulted in values with prepared transaction " + "metadata", + "reconciliation: page reconciliation calls that resulted in values with timestamps", + "reconciliation: page reconciliation calls that resulted in values with transaction ids", + "reconciliation: pages deleted", "reconciliation: split bytes currently awaiting free", "reconciliation: split objects currently awaiting free", "session: open session count", "session: session query timestamp calls", "session: table alter failed calls", "session: table alter successful calls", "session: table alter unchanged and skipped", @@ -1274,6 +1278,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->rec_page_delete_fast = 0; stats->rec_pages = 0; stats->rec_pages_eviction = 0; + stats->rec_pages_with_prepare = 0; + stats->rec_pages_with_ts = 0; + stats->rec_pages_with_txn = 0; stats->rec_page_delete = 0; /* not clearing rec_split_stashed_bytes */ /* not clearing rec_split_stashed_objects */ @@ -1735,6 +1742,9 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); to->rec_pages += WT_STAT_READ(from, rec_pages); to->rec_pages_eviction += WT_STAT_READ(from, rec_pages_eviction); + to->rec_pages_with_prepare += WT_STAT_READ(from, rec_pages_with_prepare); + to->rec_pages_with_ts += WT_STAT_READ(from, rec_pages_with_ts); + to->rec_pages_with_txn += WT_STAT_READ(from, rec_pages_with_txn); to->rec_page_delete += WT_STAT_READ(from, rec_page_delete); to->rec_split_stashed_bytes += WT_STAT_READ(from, rec_split_stashed_bytes); to->rec_split_stashed_objects += WT_STAT_READ(from, rec_split_stashed_objects); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 2297f957e5f..1dd95932105 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -745,8 +745,8 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN_GLOBAL *txn_global; WT_TXN_ISOLATION saved_isolation; wt_timestamp_t ckpt_tmp_ts; - uint64_t finish_secs, hs_ckpt_duration_usecs, time_start_hs, time_stop_hs; - uint64_t fsync_duration_usecs, generation, time_start_fsync, time_stop_fsync; + uint64_t fsync_duration_usecs, generation, hs_ckpt_duration_usecs; + uint64_t time_start_fsync, time_stop_fsync, time_start_hs, time_stop_hs; u_int i; bool can_skip, failed, full, idle, logging, tracking, use_timestamp; void *saved_meta_next; @@ -985,16 +985,6 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) conn->txn_global.last_ckpt_timestamp = conn->txn_global.recovery_timestamp; } else conn->txn_global.last_ckpt_timestamp = WT_TS_NONE; - - /* - * Save clock value marking end of checkpoint processing. If a hot backup starts before the - * next checkpoint, we will need to keep all checkpoints up to this clock value until the - * backup completes. - */ - __wt_seconds(session, &finish_secs); - /* Be defensive: time is only monotonic per session */ - if (finish_secs > conn->ckpt_finish_secs) - conn->ckpt_finish_secs = finish_secs; } err: @@ -1259,27 +1249,20 @@ __checkpoint_lock_dirty_tree_int(WT_SESSION_IMPL *session, bool is_checkpoint, b continue; is_wt_ckpt = WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT); -/* - * If there is a hot backup, don't delete any WiredTiger checkpoint that could possibly have been - * created before the backup started. Fail if trying to delete any other named checkpoint. - */ -#ifdef DISABLED_CODE - if (conn->hot_backup_start != 0 && ckpt->sec <= conn->hot_backup_start) { -#else /* - * N.B. Despite the comment above, dropping checkpoints during backup can corrupt the - * backup. For now we retain all WiredTiger checkpoints. + * If there is a hot backup, don't delete any WiredTiger checkpoint that could possibly have + * been created before the backup started. Fail if trying to delete any other named + * checkpoint. */ - if (conn->hot_backup_start != 0) { -#endif + if (conn->hot_backup_start != 0 && ckpt->sec <= conn->hot_backup_start) { if (is_wt_ckpt) { F_CLR(ckpt, WT_CKPT_DELETE); continue; } WT_RET_MSG(session, EBUSY, "checkpoint %s blocked by hot backup: it would " - "delete an existing checkpoint, and checkpoints " - "cannot be deleted during a hot backup", + "delete an existing named checkpoint, and such " + "checkpoints cannot be deleted during a hot backup", ckpt->name); } /* @@ -1816,16 +1799,17 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) { WT_BTREE *btree; WT_DECL_RET; - bool bulk, need_tracking; + bool bulk, metadata, need_tracking; btree = S2BT(session); bulk = F_ISSET(btree, WT_BTREE_BULK); + metadata = WT_IS_METADATA(session->dhandle); /* * We've done the final checkpoint before the final close, subsequent writes to normal objects * are wasted effort. Discard the objects to validate exit accounting. */ - if (final && !WT_IS_METADATA(session->dhandle)) + if (final && !metadata) return (__wt_evict_file(session, WT_SYNC_DISCARD)); /* @@ -1839,11 +1823,13 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) } /* - * Don't flush data from trees when there is a stable timestamp set: that can lead to files that - * are inconsistent on disk after a crash. + * Don't flush data from modified trees independent of system-wide checkpoint when either there + * is a stable timestamp set or the connection is configured to disallow such operation. + * Flushing trees can lead to files that are inconsistent on disk after a crash. */ - if (btree->modified && !bulk && S2C(session)->txn_global.has_stable_timestamp && - !__wt_btree_immediately_durable(session)) + if (btree->modified && !bulk && !__wt_btree_immediately_durable(session) && + (S2C(session)->txn_global.has_stable_timestamp || + (!F_ISSET(S2C(session), WT_CONN_FILE_CLOSE_SYNC) && !metadata && !final))) return (__wt_set_return(session, EBUSY)); /* diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index fdfb42ffc3f..1aa06aa2094 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -57,8 +57,7 @@ noinst_PROGRAMS += test_wt2246_col_append test_wt2323_join_visibility_SOURCES = wt2323_join_visibility/main.c noinst_PROGRAMS += test_wt2323_join_visibility -# Temporarily disabled -# all_TESTS += test_wt2323_join_visibility +all_TESTS += test_wt2323_join_visibility test_wt2535_insert_race_SOURCES = wt2535_insert_race/main.c noinst_PROGRAMS += test_wt2535_insert_race diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c index edf0a9b851e..95bc274b0f8 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c @@ -242,8 +242,13 @@ thread_insert(void *arg) */ key = (int)(__wt_random(&rnd) % N_RECORDS); maincur->set_key(maincur, key); +/* FIXME-WT-6180: disable lower isolation levels. */ +#if 0 if (sharedopts->remove) testutil_check(session->begin_transaction(session, "isolation=snapshot")); +#else + testutil_check(session->begin_transaction(session, "isolation=snapshot")); +#endif if (sharedopts->remove && __wt_random(&rnd) % 5 == 0 && maincur->search(maincur) == 0) { /* * Another thread can be removing at the same time. @@ -278,12 +283,20 @@ thread_insert(void *arg) else if (ret == WT_ROLLBACK) threadargs->rollbacks++; } +/* FIXME-WT-6180: disable lower isolation levels. */ +#if 0 if (sharedopts->remove) { if (ret == WT_ROLLBACK) testutil_check(session->rollback_transaction(session, NULL)); else testutil_check(session->commit_transaction(session, NULL)); } +#else + if (ret == WT_ROLLBACK) + testutil_check(session->rollback_transaction(session, NULL)); + else + testutil_check(session->commit_transaction(session, NULL)); +#endif if (i % 1000 == 0 && i != 0) { if (i % 10000 == 0) fprintf(stderr, "*"); diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 7d81b1c2b80..55ef711399a 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -1111,21 +1111,20 @@ tasks: # ${test_env_vars|} $(pwd)/test/csuite/test_wt2246_col_append 2>&1 - # Temporarily disabled - # - name: csuite-wt2323-join-visibility-test - # tags: ["pull_request"] - # depends_on: - # - name: compile - # commands: - # - func: "fetch artifacts" - # - command: shell.exec - # params: - # working_dir: "wiredtiger/build_posix" - # script: | - # set -o errexit - # set -o verbose + - name: csuite-wt2323-join-visibility-test + tags: ["pull_request"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + script: | + set -o errexit + set -o verbose - # ${test_env_vars|} $(pwd)/test/csuite/test_wt2323_join_visibility 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2323_join_visibility 2>&1 - name: csuite-wt2535-insert-race-test tags: ["pull_request"] diff --git a/src/third_party/wiredtiger/test/suite/test_backup01.py b/src/third_party/wiredtiger/test/suite/test_backup01.py index 2494945dba2..f2745710545 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup01.py +++ b/src/third_party/wiredtiger/test/suite/test_backup01.py @@ -189,18 +189,16 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.session.checkpoint("name=three,drop=(two)"), msg) self.session.checkpoint() - # Confirm that a named checkpoint created after a backup cursor can be dropped. # Need to pause a couple seconds; checkpoints that are assigned the same timestamp as # the backup will be pinned, even if they occur after the backup starts. - time.sleep(2) - # N.B. This test is temporarily disabled because deleting checkpoints during backup - # can corrupt the backup. - #self.session.checkpoint("name=four") - #self.session.checkpoint("drop=(four)") - #self.assertRaises(wiredtiger.WiredTigerError, - # lambda: self.session.open_cursor( - # self.objs[0][0], None, "checkpoint=four")) + + # Confirm that a named checkpoint created after a backup cursor can be dropped. + self.session.checkpoint("name=four") + self.session.checkpoint("drop=(four)") + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.open_cursor( + self.objs[0][0], None, "checkpoint=four")) # Confirm that after closing the backup cursor the original named checkpoint can # be deleted. diff --git a/src/third_party/wiredtiger/test/suite/test_backup14.py b/src/third_party/wiredtiger/test/suite/test_backup14.py index 7a2ec4f427f..c312020bcef 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup14.py +++ b/src/third_party/wiredtiger/test/suite/test_backup14.py @@ -175,9 +175,9 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): shutil.copy(copy_from, copy_to) else: self.pr('Range copy file ' + newfile + ' offset ' + str(offset) + ' len ' + str(size)) - write_from = newfile + read_from = newfile write_to = self.home_incr + '.' + str(self.counter) + '/' + newfile - rfp = open(write_from, "r+b") + rfp = open(read_from, "r+b") wfp = open(write_to, "w+b") rfp.seek(offset, 0) wfp.seek(offset, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_backup15.py b/src/third_party/wiredtiger/test/suite/test_backup15.py new file mode 100644 index 00000000000..08ddf177ea6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup15.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +import os, shutil +from helper import compare_files +from suite_subprocess import suite_subprocess +from wtdataset import simple_key +from wtscenario import make_scenarios +import glob + +# test_backup15.py +# Test cursor backup with a block-based incremental cursor. +class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): + bkp_home = "WT_BLOCK" + counter=0 + conn_config='cache_size=1G,log=(enabled,file_max=100K)' + logmax="100K" + max_iteration=7 + mult=0 + nops=100000 + savefirst=0 + savekey='NOTSET' + uri="table:main" + + dir='backup.dir' # Backup directory name + home_full = "WT_BLOCK_LOG_FULL" + home_incr = "WT_BLOCK_LOG_INCR" + + full_out = "./backup_block_full" + incr_out = "./backup_block_incr" + logpath = "logpath" + new_table=False + initial_backup=False + + pfx = 'test_backup' + # Set the key and value big enough that we modify a few blocks. + bigkey = 'Key' * 100 + bigval = 'Value' * 100 + + # + # Set up all the directories needed for the test. We have a full backup directory for each + # iteration and an incremental backup for each iteration. That way we can compare the full and + # incremental each time through. + # + def setup_directories(self): + for i in range(0, self.max_iteration): + remove_dir = self.home_incr + '.' + str(i) + + create_dir = self.home_incr + '.' + str(i) + '/' + self.logpath + if os.path.exists(remove_dir): + os.remove(remove_dir) + os.makedirs(create_dir) + + if i == 0: + continue + remove_dir = self.home_full + '.' + str(i) + create_dir = self.home_full + '.' + str(i) + '/' + self.logpath + if os.path.exists(remove_dir): + os.remove(remove_dir) + os.makedirs(create_dir) + + def take_full_backup(self): + if self.counter != 0: + hdir = self.home_full + '.' + str(self.counter) + else: + hdir = self.home_incr + + # + # First time through we take a full backup into the incremental directories. Otherwise only + # into the appropriate full directory. + # + buf = None + if self.initial_backup == True: + buf = 'incremental=(granularity=1M,enabled=true,this_id=ID0)' + + cursor = self.session.open_cursor('backup:', None, buf) + while True: + ret = cursor.next() + if ret != 0: + break + newfile = cursor.get_key() + + if self.counter == 0: + # Take a full bakcup into each incremental directory + for i in range(0, self.max_iteration): + copy_from = newfile + # If it is log file, prepend the path. + if ("WiredTigerLog" in newfile): + copy_to = self.home_incr + '.' + str(i) + '/' + self.logpath + else: + copy_to = self.home_incr + '.' + str(i) + shutil.copy(copy_from, copy_to) + else: + copy_from = newfile + # If it is log file, prepend the path. + if ("WiredTigerLog" in newfile): + copy_to = hdir + '/' + self.logpath + else: + copy_to = hdir + + shutil.copy(copy_from, copy_to) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + cursor.close() + + def take_incr_backup(self): + # Open the backup data source for incremental backup. + buf = 'incremental=(src_id="ID' + str(self.counter-1) + '",this_id="ID' + str(self.counter) + '")' + self.pr(buf) + bkup_c = self.session.open_cursor('backup:', None, buf) + while True: + ret = bkup_c.next() + if ret != 0: + break + newfile = bkup_c.get_key() + h = self.home_incr + '.0' + copy_from = newfile + # If it is log file, prepend the path. + if ("WiredTigerLog" in newfile): + copy_to = h + '/' + self.logpath + else: + copy_to = h + + shutil.copy(copy_from, copy_to) + first = True + config = 'incremental=(file=' + newfile + ')' + dup_cnt = 0 + incr_c = self.session.open_cursor(None, bkup_c, config) + + # For each file listed, open a duplicate backup cursor and copy the blocks. + while True: + ret = incr_c.next() + if ret != 0: + break + incrlist = incr_c.get_keys() + offset = incrlist[0] + size = incrlist[1] + curtype = incrlist[2] + # 1 is WT_BACKUP_FILE + # 2 is WT_BACKUP_RANGE + self.assertTrue(curtype == 1 or curtype == 2) + if curtype == 1: + if first == True: + h = self.home_incr + '.' + str(self.counter) + first = False + + copy_from = newfile + if ("WiredTigerLog" in newfile): + copy_to = h + '/' + self.logpath + else: + copy_to = h + shutil.copy(copy_from, copy_to) + else: + self.pr('Range copy file ' + newfile + ' offset ' + str(offset) + ' len ' + str(size)) + read_from = newfile + if self.counter > 0: + old_to = self.home_incr + '.' + str(self.counter - 1) + '/' + newfile + else: + old_to = newfile + write_to = self.home_incr + '.' + str(self.counter) + '/' + newfile + write_to = self.home_incr + '.' + str(self.counter) + '/' + newfile + rfp = open(read_from, "r+b") + self.pr('RANGE CHECK file ' + old_to + ' offset ' + str(offset) + ' len ' + str(size)) + rfp2 = open(old_to, "r+b") + rfp.seek(offset, 0) + rfp2.seek(offset, 0) + buf = rfp.read(size) + buf2 = rfp2.read(size) + self.assertNotEqual(buf, buf2) + wfp = open(write_to, "w+b") + wfp.seek(offset, 0) + wfp.write(buf) + rfp.close() + rfp2.close() + wfp.close() + dup_cnt += 1 + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + incr_c.close() + + # For each file, we want to copy the file into each of the later incremental directories + for i in range(self.counter, self.max_iteration): + h = self.home_incr + '.' + str(i) + copy_from = newfile + if ("WiredTigerLog" in newfile): + copy_to = h + '/' + self.logpath + else: + copy_to = h + shutil.copy(copy_from, copy_to) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + bkup_c.close() + + def compare_backups(self, t_uri): + # + # Run wt dump on full backup directory + # + full_backup_out = self.full_out + '.' + str(self.counter) + home_dir = self.home_full + '.' + str(self.counter) + if self.counter == 0: + home_dir = self.home + + self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=full_backup_out) + # + # Run wt dump on incremental backup directory + # + incr_backup_out = self.incr_out + '.' + str(self.counter) + home_dir = self.home_incr + '.' + str(self.counter) + self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=incr_backup_out) + + self.assertEqual(True, + compare_files(self, full_backup_out, incr_backup_out)) + + # + # Add data to the given uri. + # + def add_data(self, uri): + c = self.session.open_cursor(uri, None, None) + # The first time we want to add in a lot of data. Then after that we want to + # rapidly change a single key to create a hotspot in one block. + if self.savefirst < 2: + nops = self.nops + else: + nops = self.nops // 10 + for i in range(0, nops): + num = i + (self.mult * nops) + if self.savefirst >= 2: + key = self.savekey + else: + key = str(num) + self.bigkey + str(num) + val = str(num) + self.bigval + str(num) + c[key] = val + if self.savefirst == 0: + self.savekey = key + self.savefirst += 1 + c.close() + + # Increase the multiplier so that later calls insert unique items. + self.mult += 1 + # Increase the counter so that later backups have unique ids. + if self.initial_backup == False: + self.counter += 1 + + # + # This function will add records to the table (table:main), take incremental/full backups and + # validate the backups. + # + def add_data_validate_backups(self): + self.pr('Adding initial data') + self.initial_backup = True + self.add_data(self.uri) + self.take_full_backup() + self.initial_backup = False + self.session.checkpoint() + + # Each call now to take a full backup will make a copy into a full directory. Then + # each incremental will take an incremental backup and we can compare them. + self.add_data(self.uri) + self.session.checkpoint() + self.take_full_backup() + self.take_incr_backup() + self.compare_backups(self.uri) + + self.add_data(self.uri) + self.session.checkpoint() + self.take_incr_backup() + self.take_full_backup() + self.compare_backups(self.uri) + + self.add_data(self.uri) + self.session.checkpoint() + self.take_full_backup() + self.take_incr_backup() + self.compare_backups(self.uri) + + self.add_data(self.uri) + self.take_incr_backup() + self.take_full_backup() + self.compare_backups(self.uri) + + def test_backup15(self): + os.mkdir(self.bkp_home) + self.home = self.bkp_home + self.session.create(self.uri, "key_format=S,value_format=S") + + self.setup_directories() + + self.pr('*** Add data, checkpoint, take backups and validate ***') + self.add_data_validate_backups() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug018.py b/src/third_party/wiredtiger/test/suite/test_bug018.py index 3fc56de64e4..5e74f252a2b 100755 --- a/src/third_party/wiredtiger/test/suite/test_bug018.py +++ b/src/third_party/wiredtiger/test/suite/test_bug018.py @@ -41,6 +41,7 @@ class test_bug018(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'log=(enabled)' basename = 'bug018.' baseuri = 'file:' + basename + flist = [] uri1 = baseuri + '01.wt' uri2 = baseuri + '02.wt' @@ -50,12 +51,34 @@ class test_bug018(wttest.WiredTigerTestCase, suite_subprocess): self.skipTest('Linux-specific test skipped on ' + os.name) super(test_bug018, self).setUp() + def close_files(self): + for f in self.flist: + f.close() + + def open_files(self): + numfiles = 6 + dir = self.conn.get_home() + for i in range(1, numfiles): + fname = dir + '/file.' + str(i) + self.flist.append(open(fname, 'w')) + def create_table(self, uri): self.session.create(uri, 'key_format=S,value_format=S') return self.session.open_cursor(uri) def subprocess_bug018(self): '''Test closing multiple tables''' + # The first thing we do is open several files. We will close them later. The reason is + # that sometimes, without that, this test would fail to report an error as expected. We + # hypothesize, but could not prove (nor reproduce under strace), that after closing the + # file descriptor that an internal thread would open a file, perhaps a pre-allocated log + # file, and then would open the file descriptor we just closed. So on close, instead of + # getting an error, we would actually write to the wrong file. + # + # So we'll open some files now, and then close them before closing the one of interest to + # the test so that any stray internal file opens will use the file descriptor of one of + # the earlier files we just closed. + self.open_files() c1 = self.create_table(self.uri1) c2 = self.create_table(self.uri2) @@ -64,6 +87,7 @@ class test_bug018(wttest.WiredTigerTestCase, suite_subprocess): c2['key'] = 'value' self.session.commit_transaction() + self.close_files() # Simulate a write failure by closing the file descriptor for the second # table out from underneath WiredTiger. We do this right before # closing the connection so that the write error happens during close diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint05.py b/src/third_party/wiredtiger/test/suite/test_checkpoint05.py index 041f1e53718..58af3003a60 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint05.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint05.py @@ -76,10 +76,7 @@ class test_checkpoint05(wttest.WiredTigerTestCase): # is generous. But if WT isn't deleting checkpoints there would # be about 30x more checkpoints here. final_count = self.count_checkpoints() - - # N.B. This test is temporarily disabled because deleting checkpoints during backup - # can corrupt the backup. - # self.assertTrue (final_count < initial_count * 3) + self.assertTrue (final_count < initial_count * 3) self.session.close() diff --git a/src/third_party/wiredtiger/test/suite/test_config08.py b/src/third_party/wiredtiger/test/suite/test_config08.py new file mode 100644 index 00000000000..9fb6bc5a163 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_config08.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_config08.py +# Test the configuration that enables/disables dirty table flushing. +# + +import wiredtiger, wttest +from wtscenario import make_scenarios + +class test_config08(wttest.WiredTigerTestCase): + + logging = [ + ('log_off', dict(logging='false')), + ('log_on', dict(logging='true')), + ] + flush = [ + ('flush_off', dict(flush='false')), + ('flush_on', dict(flush='true')), + ] + + scenarios = make_scenarios(logging, flush) + + # This test varies the logging and file flush settings and therefore needs to set up its own + # connection config. Override the standard method. + def conn_config(self): + return 'create,log=(enabled={}),file_close_sync={}'.\ + format(self.logging,self.flush) + + def test_config08(self): + # Create a table with logging setting matching the connection level config. + table_params = 'key_format=i,value_format=S,log=(enabled={})'.format(self.logging) + self.uri = 'table:config_test' + + # Create a table with some data. + self.session.create(self.uri, table_params) + c = self.session.open_cursor(self.uri, None) + c[0] = 'ABC' * 4096 + c[1] = 'DEF' * 4096 + c[2] = 'GHI' * 4096 + c.close() + + # API calls that require exclusive file handles should return EBUSY if file_close_sync is + # set to false and logging is disabled. + if self.logging == 'false' and self.flush == 'false': + # WT won't allow this operation as exclsuive file handle is not possible + # with modified table. + self.assertTrue(self.raisesBusy(lambda: self.session.verify(self.uri, None)), + "was expecting API call to fail with EBUSY") + + # Taking a checkopoint should make WT happy. + self.session.checkpoint() + self.session.verify(self.uri, None) + else: + # All other combinations of configs should not return EBUSY. + self.session.verify(self.uri, None) + + # This will catch a bug if we return EBUSY from final shutdown. + self.conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 7e4a9cce9d4..f88f26b5da7 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -38,7 +38,7 @@ except ImportError: import unittest from contextlib import contextmanager -import glob, os, re, shutil, sys, time, traceback +import errno, glob, os, re, shutil, sys, time, traceback import wiredtiger, wtscenario def shortenWithEllipsis(s, maxlen): @@ -568,12 +568,13 @@ class WiredTigerTestCase(unittest.TestCase): def raisesBusy(self, expr): """ Execute the expression, returning true if a 'Resource busy' exception - is raised, returning false if no exception is raised. Some systems - report 'Device or resource busy', allow either. + is raised, returning false if no exception is raised. The actual exception + message can be different across platforms and therefore we rely on os + libraries to give use the correct exception message. Any other exception raises a test suite failure. """ return self.assertRaisesException(wiredtiger.WiredTigerError, \ - expr, exceptionString='/[Rr]esource busy/', optional=True) + expr, os.strerror(errno.EBUSY), optional=True) def assertTimestampsEqual(self, ts1, ts2): """ |