diff options
author | Luke Chen <luke.chen@mongodb.com> | 2020-02-07 02:24:42 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2020-02-07 02:24:42 +0000 |
commit | b720ff3c5a6e14eac458f539bb6b83ef46e4435a (patch) | |
tree | df90e17fec0ff96acddf1ef46296e7e0241cdc84 /src/third_party | |
parent | f60e5094bb0e582b1eef502b00580d3072ef73d4 (diff) | |
download | mongo-b720ff3c5a6e14eac458f539bb6b83ef46e4435a.tar.gz |
Import wiredtiger: 94c6c16014292f335a5094669e850125d7432b14 from branch mongodb-4.4
ref: 168c7f1b38..94c6c16014
for: 4.3.4
WT-4999 Migrate Jenkins “wiredtiger-test-format-stress-zseries” job to Evergreen
WT-5206 Return the correct checkpoint-modified list of blocks
WT-5489 page-read can race with threads locking in-memory page structures
WT-5534 Incremental backup needs to accept older metadata
WT-5537 Use correct WT_ITEM fields per memory sanitizer
Diffstat (limited to 'src/third_party')
25 files changed, 739 insertions, 387 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index cf942be2f84..45cca0ef829 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -377,6 +377,8 @@ file_config = format_meta + file_runtime_config + [ file_meta = file_config + [ Config('checkpoint', '', r''' the file checkpoint entries'''), + Config('checkpoint_backup_info', '', r''' + the incremental backup durable information'''), Config('checkpoint_lsn', '', r''' LSN of the last checkpoint'''), Config('id', '', r''' diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 6cda20d9b3d..185ed5aac8d 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -510,6 +510,7 @@ bitpos bitstring bitwise blk +blkmod bm bnd bool diff --git a/src/third_party/wiredtiger/examples/c/ex_backup_block.c b/src/third_party/wiredtiger/examples/c/ex_backup_block.c index fbae6e6da5d..422bf728540 100644 --- a/src/third_party/wiredtiger/examples/c/ex_backup_block.c +++ b/src/third_party/wiredtiger/examples/c/ex_backup_block.c @@ -136,11 +136,11 @@ setup_directories(void) } static void -add_work(WT_SESSION *session, int iter) +add_work(WT_SESSION *session, int iter, int iterj) { WT_CURSOR *cursor, *cursor2; int i; - char k[32], v[32]; + char k[64], v[64]; error_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); /* @@ -154,8 +154,8 @@ add_work(WT_SESSION *session, int iter) * Perform some operations with individual auto-commit transactions. */ for (i = 0; i < MAX_KEYS; i++) { - (void)snprintf(k, sizeof(k), "key.%d.%d", iter, i); - (void)snprintf(v, sizeof(v), "value.%d.%d", iter, i); + (void)snprintf(k, sizeof(k), "key.%d.%d.%d", iter, iterj, i); + (void)snprintf(v, sizeof(v), "value.%d.%d.%d", iter, iterj, i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); error_check(cursor->insert(cursor)); @@ -260,7 +260,8 @@ take_full_backup(WT_SESSION *session, int i) } else hdir = home_incr; if (i == 0) { - (void)snprintf(buf, sizeof(buf), "incremental=(enabled=true,this_id=ID%d)", i); + (void)snprintf( + buf, sizeof(buf), "incremental=(granularity=1M,enabled=true,this_id=ID%d)", i); error_check(session->open_cursor(session, "backup:", NULL, buf, &cursor)); } else error_check(session->open_cursor(session, "backup:", NULL, NULL, &cursor)); @@ -279,13 +280,17 @@ take_full_backup(WT_SESSION *session, int i) for (j = 0; j < MAX_ITERATIONS; j++) { (void)snprintf(h, sizeof(h), "%s.%d", home_incr, j); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); +#if 0 printf("FULL: Copy: %s\n", buf); +#endif error_check(system(buf)); } else { (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, hdir, filename); +#if 0 printf("FULL %d: Copy: %s\n", i, buf); +#endif error_check(system(buf)); } } @@ -300,13 +305,16 @@ take_incr_backup(WT_SESSION *session, int i) FILELIST *flist; WT_CURSOR *backup_cur, *incr_cur; uint64_t offset, size, type; - size_t alloc, count; + size_t alloc, count, rdsize, tmp_sz; int j, ret, rfd, wfd; - char buf[1024], h[256]; + char buf[1024], h[256], *tmp; const char *filename; + bool first; /*! [incremental backup using block transfer]*/ + tmp = NULL; + tmp_sz = 0; /* Open the backup data source for incremental backup. */ (void)snprintf(buf, sizeof(buf), "incremental=(src_id=ID%d,this_id=ID%d)", i - 1, i); error_check(session->open_cursor(session, "backup:", NULL, buf, &backup_cur)); @@ -321,51 +329,61 @@ take_incr_backup(WT_SESSION *session, int i) error_check(process_file(&flist, &count, &alloc, filename)); (void)snprintf(h, sizeof(h), "%s.0", home_incr); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); - printf("Copying backup: %s\n", buf); - error_check(system(buf)); #if 0 - (void)snprintf(buf, sizeof(buf), "%s/%s", home, filename); - printf("Open source %s for reading\n", buf); - error_check(rfd = open(buf, O_RDONLY, 0)); - (void)snprintf(h, sizeof(h), "%s.%d", home_incr, i); - (void)snprintf(buf, sizeof(buf), "%s/%s", h, filename); - printf("Open dest %s for writing\n", buf); - error_check(wfd = open(buf, O_WRONLY, 0)); + printf("Copying backup: %s\n", buf); #endif + error_check(system(buf)); + first = true; (void)snprintf(buf, sizeof(buf), "incremental=(file=%s)", filename); error_check(session->open_cursor(session, NULL, backup_cur, buf, &incr_cur)); +#if 0 printf("Taking incremental %d: File %s\n", i, filename); +#endif while ((ret = incr_cur->next(incr_cur)) == 0) { error_check(incr_cur->get_key(incr_cur, &offset, &size, &type)); - printf("Incremental %s: KEY: Off %" PRIu64 " Size: %" PRIu64 " Type: %" PRIu64 "\n", - filename, offset, size, type); scan_end_check(type == WT_BACKUP_FILE || type == WT_BACKUP_RANGE); +#if 0 + printf("Incremental %s: KEY: Off %" PRIu64 " Size: %" PRIu64 " %s\n", filename, offset, + size, type == WT_BACKUP_FILE ? "WT_BACKUP_FILE" : "WT_BACKUP_RANGE"); +#endif if (type == WT_BACKUP_RANGE) { /* * We should never get a range key after a whole file so the read file descriptor - * should be valid. If the read descriptor is valid, so it the write one. + * should be valid. If the read descriptor is valid, so is the write one. */ - scan_end_check(rfd != -1); - printf("Incremental %s: Range Offset: %" PRIu64 " Size: %" PRIu64 "\n", filename, - offset, size); + if (tmp_sz < size) { + tmp = realloc(tmp, size); + testutil_assert(tmp != NULL); + tmp_sz = size; + } + if (first) { + (void)snprintf(buf, sizeof(buf), "%s/%s", home, filename); + error_sys_check(rfd = open(buf, O_RDONLY, 0)); + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, i); + (void)snprintf(buf, sizeof(buf), "%s/%s", h, filename); + error_sys_check(wfd = open(buf, O_WRONLY, 0)); + first = false; + } + error_sys_check(lseek(rfd, (wt_off_t)offset, SEEK_SET)); - error_sys_check(read(rfd, buf, (size_t)size)); + error_sys_check(rdsize = (size_t)read(rfd, tmp, (size_t)size)); error_sys_check(lseek(wfd, (wt_off_t)offset, SEEK_SET)); - error_sys_check(write(wfd, buf, (size_t)size)); + /* Use the read size since we may have read less than the granularity. */ + error_sys_check(write(wfd, tmp, rdsize)); } else { -/* Whole file, so close both files and just copy the whole thing. */ -#if 0 - error_check(close(rfd)); - error_check(close(wfd)); -#endif + /* Whole file, so close both files and just copy the whole thing. */ + testutil_assert(first == true); rfd = wfd = -1; (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); +#if 0 printf("Incremental: Whole file copy: %s\n", buf); +#endif error_check(system(buf)); } } scan_end_check(ret == WT_NOTFOUND); + /* Done processing this file. Close incremental cursor. */ error_check(incr_cur->close(incr_cur)); /* Close file descriptors if they're open. */ @@ -386,18 +404,21 @@ take_incr_backup(WT_SESSION *session, int i) } scan_end_check(ret == WT_NOTFOUND); + /* Done processing all files. Close backup cursor. */ error_check(backup_cur->close(backup_cur)); error_check(finalize_files(flist, count)); + free(tmp); /*! [incremental backup using block transfer]*/ } int main(int argc, char *argv[]) { + struct stat sb; WT_CONNECTION *wt_conn; WT_CURSOR *backup_cur; WT_SESSION *session; - int i; + int i, j, ret; char cmd_buf[256]; (void)argc; /* Unused variable */ @@ -412,7 +433,7 @@ main(int argc, char *argv[]) error_check(session->create(session, uri, "key_format=S,value_format=S")); error_check(session->create(session, uri2, "key_format=S,value_format=S")); printf("Adding initial data\n"); - add_work(session, 0); + add_work(session, 0, 0); printf("Taking initial backup\n"); take_full_backup(session, 0); @@ -421,8 +442,12 @@ main(int argc, char *argv[]) for (i = 1; i < MAX_ITERATIONS; i++) { printf("Iteration %d: adding data\n", i); - add_work(session, i); - error_check(session->checkpoint(session, NULL)); + /* For each iteration we may add work and checkpoint multiple times. */ + for (j = 0; j < i; j++) { + add_work(session, i, j); + error_check(session->checkpoint(session, NULL)); + } + /* * The full backup here is only needed for testing and comparison purposes. A normal * incremental backup procedure would not include this. @@ -440,6 +465,20 @@ main(int argc, char *argv[]) error_check(compare_backups(i)); } + printf("Close and reopen the connection\n"); + /* + * Close and reopen the connection to illustrate the durability of id information. + */ + error_check(wt_conn->close(wt_conn, NULL)); + error_check(wiredtiger_open(home, NULL, CONN_CONFIG, &wt_conn)); + error_check(wt_conn->open_session(wt_conn, NULL, NULL, &session)); + /* + * We should have an entry for i-1 and i-2. Use the older one. + */ + (void)snprintf(cmd_buf, sizeof(cmd_buf), "incremental=(src_id=ID%d,this_id=ID%d)", i - 2, i); + error_check(session->open_cursor(session, "backup:", NULL, cmd_buf, &backup_cur)); + error_check(backup_cur->close(backup_cur)); + /* * After we're done, release resources. Test the force stop setting. */ @@ -455,6 +494,28 @@ main(int argc, char *argv[]) printf("Final comparison: dumping and comparing data\n"); error_check(compare_backups(0)); + for (i = 0; i < (int)filelist_count; ++i) { + if (last_flist[i].name == NULL) + break; + free((void *)last_flist[i].name); + } + free(last_flist); + + /* + * Reopen the connection to verify that the forced stop should remove incremental information. + */ + error_check(wiredtiger_open(home, NULL, CONN_CONFIG, &wt_conn)); + error_check(wt_conn->open_session(wt_conn, NULL, NULL, &session)); + /* + * We should not have any information. + */ + (void)snprintf(cmd_buf, sizeof(cmd_buf), "incremental=(src_id=ID%d,this_id=ID%d)", i - 2, i); + testutil_assert(session->open_cursor(session, "backup:", NULL, cmd_buf, &backup_cur) == ENOENT); + error_check(wt_conn->close(wt_conn, NULL)); + + (void)snprintf(cmd_buf, sizeof(cmd_buf), "%s/WiredTiger.backup.block", home); + ret = stat(cmd_buf, &sb); + testutil_assert(ret == -1 && errno == ENOENT); return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 6f3e41094c5..195d9c6f867 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "168c7f1b3859516115721868f312544abb0caf5b" + "commit": "94c6c16014292f335a5094669e850125d7432b14" } diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 158fc919820..e1cf8982daf 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -9,8 +9,7 @@ #include "wt_internal.h" static int __ckpt_process(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *); -static int __ckpt_update( - WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *, WT_CKPT *, WT_BLOCK_CKPT *, bool); +static int __ckpt_update(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *, WT_CKPT *, WT_BLOCK_CKPT *); /* * __wt_block_ckpt_init -- @@ -331,6 +330,7 @@ __ckpt_verify(WT_SESSION_IMPL *session, WT_CKPT *ckptbase) case WT_CKPT_DELETE | WT_CKPT_FAKE: case WT_CKPT_FAKE: break; + case WT_CKPT_ADD | WT_CKPT_BLOCK_MODS: case WT_CKPT_ADD: if (ckpt[1].name == NULL) break; @@ -570,7 +570,7 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) /* Update checkpoints marked for update. */ WT_CKPT_FOREACH (ckptbase, ckpt) if (F_ISSET(ckpt, WT_CKPT_UPDATE)) - WT_ERR(__ckpt_update(session, block, ckptbase, ckpt, ckpt->bpriv, false)); + WT_ERR(__ckpt_update(session, block, ckptbase, ckpt, ckpt->bpriv)); live_update: /* Truncate the file if that's possible. */ @@ -607,7 +607,7 @@ live_update: */ ci->ckpt_size = WT_MIN(ckpt_size, (uint64_t)block->size); - WT_ERR(__ckpt_update(session, block, ckptbase, ckpt, ci, true)); + WT_ERR(__ckpt_update(session, block, ckptbase, ckpt, ci)); } /* @@ -654,16 +654,80 @@ err: } /* + * __ckpt_add_blkmod_entry -- + * Add an offset/length entry to the bitstring based on granularity. + */ +static int +__ckpt_add_blkmod_entry( + WT_SESSION_IMPL *session, WT_BLOCK_MODS *blk_mod, wt_off_t offset, wt_off_t len) +{ + uint64_t end, start; + uint32_t end_rdup; + + WT_ASSERT(session, blk_mod->granularity != 0); + start = (uint64_t)offset / blk_mod->granularity; + end = (uint64_t)(offset + len) / blk_mod->granularity; + WT_ASSERT(session, end < UINT32_MAX); + end_rdup = WT_MAX(__wt_rduppo2((uint32_t)end, 8), WT_BLOCK_MODS_LIST_MIN); + if ((end_rdup << 3) > blk_mod->nbits) { + /* If we don't have enough, extend the buffer. */ + if (blk_mod->nbits == 0) { + WT_RET(__wt_buf_initsize(session, &blk_mod->bitstring, end_rdup)); + memset(blk_mod->bitstring.mem, 0, end_rdup); + } else + WT_RET(__wt_buf_extend(session, &blk_mod->bitstring, end_rdup)); + blk_mod->nbits = end_rdup << 3; + } + + /* Set all the bits needed to record this offset/length pair. */ + __bit_nset(blk_mod->bitstring.mem, start, end); + return (0); +} + +/* + * __ckpt_add_blk_mods -- + * Add the blocks to all valid incremental backup source identifiers. + */ +static int +__ckpt_add_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci) +{ + WT_BLOCK_MODS *blk_mod; + WT_EXT *ext; + u_int i; + + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk_mod = &ckpt->backup_blocks[i]; + /* If there is no information at this entry, we're done. */ + if (!F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) + continue; + + WT_EXT_FOREACH (ext, ci->alloc.off) + WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ext->off, ext->size)); + + if (ci->alloc.offset != WT_BLOCK_INVALID_OFFSET) + WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->alloc.offset, ci->alloc.size)); + if (ci->discard.offset != WT_BLOCK_INVALID_OFFSET) + WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->discard.offset, ci->discard.size)); + if (ci->avail.offset != WT_BLOCK_INVALID_OFFSET) + WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->avail.offset, ci->avail.size)); + } + return (0); +} + +/* * __ckpt_update -- * Update a checkpoint. */ static int -__ckpt_update(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, WT_CKPT *ckpt, - WT_BLOCK_CKPT *ci, bool is_live) +__ckpt_update( + WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci) { WT_DECL_ITEM(a); WT_DECL_RET; uint8_t *endp; + bool is_live; + + is_live = F_ISSET(ckpt, WT_CKPT_ADD); #ifdef HAVE_DIAGNOSTIC /* Check the extent list combinations for overlaps. */ @@ -723,6 +787,13 @@ __ckpt_update(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, WT_C } /* + * If this is the live system, we need to record the list of blocks written for this checkpoint + * (including the blocks we allocated to write the extent lists). + */ + if (F_ISSET(ckpt, WT_CKPT_BLOCK_MODS)) + WT_RET(__ckpt_add_blk_mods(session, ckpt, ci)); + + /* * Set the file size for the live system. * * !!! diff --git a/src/third_party/wiredtiger/src/btree/bt_import.c b/src/third_party/wiredtiger/src/btree/bt_import.c index 02f023567f5..9d63a3cc959 100644 --- a/src/third_party/wiredtiger/src/btree/bt_import.c +++ b/src/third_party/wiredtiger/src/btree/bt_import.c @@ -91,12 +91,14 @@ __wt_import(WT_SESSION_IMPL *session, const char *uri) * Build and flatten the metadata and the checkpoint list, then insert it into the metadata for * this file. * - * Strip out the checkpoint-LSN, an imported file isn't associated with any log files. Assign a + * Strip out any incremental backup information, an imported file has not been part of a backup. + * Strip out the checkpoint LSN, an imported file isn't associated with any log files. Assign a * unique file ID. */ filecfg[1] = a->data; filecfg[2] = checkpoint_list; - filecfg[3] = "checkpoint_lsn="; + filecfg[3] = "checkpoint_backup_info="; + filecfg[4] = "checkpoint_lsn="; WT_WITH_SCHEMA_LOCK(session, ret = __wt_snprintf(fileid, sizeof(fileid), "id=%" PRIu32, ++S2C(session)->next_file_id)); WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 629c5316e53..bba4a9b914b 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -415,6 +415,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, + {"checkpoint_backup_info", "string", NULL, NULL, NULL, 0}, {"checkpoint_lsn", "string", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, @@ -941,10 +942,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," "assert=(commit_timestamp=none,durable_timestamp=none," "read_timestamp=none),block_allocation=best,block_compressor=," - "cache_resident=false,checkpoint=,checkpoint_lsn=," - "checksum=uncompressed,collator=,columns=,dictionary=0," - "encryption=(keyid=,name=),format=btree,huffman_key=," - "huffman_value=,id=,ignore_in_memory_cache_size=false," + "cache_resident=false,checkpoint=,checkpoint_backup_info=," + "checkpoint_lsn=,checksum=uncompressed,collator=,columns=," + "dictionary=0,encryption=(keyid=,name=),format=btree,huffman_key=" + ",huffman_value=,id=,ignore_in_memory_cache_size=false," "internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," @@ -953,7 +954,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "prefix_compression=false,prefix_compression_min=4," "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," "value_format=u,version=(major=0,minor=0)", - confchk_file_meta, 41}, + confchk_file_meta, 42}, {"index.meta", "app_metadata=,collator=,columns=,extractor=,immutable=false," "index_key_columns=,key_format=u,source=,type=file,value_format=u", diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index ace1505e6dd..fcf46b48f95 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2671,6 +2671,12 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c WT_ERR(__wt_metadata_cursor(session, NULL)); + /* + * Load any incremental backup information. This reads the metadata so must be done after the + * turtle file is initialized. + */ + WT_ERR(__wt_backup_open(session)); + /* Start the worker threads and run recovery. */ WT_ERR(__wt_connection_workers(session, cfg)); diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index c947954e75c..1d79064186c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -38,7 +38,7 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session) WT_DATA_HANDLE *dhandle; WT_DECL_RET; char *metaconf, *tmp; - const char *base, *cfg[3]; + const char *base, *cfg[4]; dhandle = session->dhandle; base = NULL; @@ -68,30 +68,32 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session) switch (dhandle->type) { case WT_DHANDLE_TYPE_BTREE: /* - * We are stripping out the checkpoint and checkpoint_lsn information from the config - * string. We save the rest of the metadata string, that is essentially static and - * unchanging and then concatenate the new checkpoint and LSN information on each - * checkpoint. The reason is performance and avoiding a lot of calls to the config parsing - * functions during a checkpoint for information that changes in a very well known way. + * We are stripping out all checkpoint related information from the config string. We save + * the rest of the metadata string, that is essentially static and unchanging and then + * concatenate the new checkpoint related information on each checkpoint. The reason is + * performance and avoiding a lot of calls to the config parsing functions during a + * checkpoint for information that changes in a very well known way. + * + * First collapse and overwrite checkpoint information because we do not know the name of or + * how many checkpoints may be in this metadata. Similarly, for backup information, we want + * an empty category to strip out since we don't know any backup ids. Set them empty and + * call collapse to overwrite anything existing. */ cfg[0] = metaconf; cfg[1] = "checkpoint=()"; - cfg[2] = NULL; + cfg[2] = "checkpoint_backup_info=()"; + cfg[3] = NULL; WT_ERR(__wt_strdup(session, WT_CONFIG_BASE(session, file_meta), &dhandle->cfg[0])); WT_ASSERT(session, dhandle->meta_base == NULL); - /* - * First collapse and overwrite any checkpoint information because we do not know the name - * or how many checkpoints may be in this metadata. So first we have to set the string to - * the empty checkpoint string and call collapse to overwrite anything existing. - */ WT_ERR(__wt_config_collapse(session, cfg, &tmp)); /* - * Now strip out the checkpoint and checkpoint LSN items from the configuration string and - * that is now our base metadata string. + * Now strip out the checkpoint related items from the configuration string and that is now + * our base metadata string. */ cfg[0] = tmp; cfg[1] = NULL; - WT_ERR(__wt_config_merge(session, cfg, "checkpoint=,checkpoint_lsn=", &base)); + WT_ERR(__wt_config_merge( + session, cfg, "checkpoint=,checkpoint_backup_info=,checkpoint_lsn=", &base)); __wt_free(session, tmp); break; case WT_DHANDLE_TYPE_TABLE: diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 37136926060..e5c82d49a48 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -94,7 +94,6 @@ void __wt_connection_destroy(WT_CONNECTION_IMPL *conn) { WT_SESSION_IMPL *session; - u_int i; /* Check there's something to destroy. */ if (conn == NULL) @@ -132,13 +131,6 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_cond_destroy(session, &conn->lsm_manager.work_cond); /* Free allocated memory. */ - /* - * XXX we need to persist this information when we are working on making incremental backups - * persistent across restarts. - */ - for (i = 0; i < WT_BLKINCR_MAX; ++i) - __wt_free(session, conn->incr_backups[i].id_str); - __wt_free(session, conn->cfg); __wt_free(session, conn->debug_ckpt); __wt_free(session, conn->error_prefix); diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index f7e338ac9bb..df6e6b79300 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -141,6 +141,8 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) /* Close operation tracking */ WT_TRET(__wt_conn_optrack_teardown(session, false)); + __wt_backup_destroy(session); + /* Close any file handles left open. */ WT_TRET(__wt_close_connection_close(session)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index be781118895..3fdbafce445 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -19,6 +19,101 @@ static int __backup_stop(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); WT_ERR(F_ISSET(((WT_CURSOR_BACKUP *)(cursor)), WT_CURBACKUP_FORCE_STOP) ? EINVAL : 0); /* + * __wt_backup_destroy -- + * Destroy any backup information. + */ +void +__wt_backup_destroy(WT_SESSION_IMPL *session) +{ + WT_BLKINCR *blkincr; + WT_CONNECTION_IMPL *conn; + uint64_t i; + + conn = S2C(session); + /* Free any incremental backup information. */ + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blkincr = &conn->incr_backups[i]; + __wt_free(session, blkincr->id_str); + F_CLR(blkincr, WT_BLKINCR_VALID); + } + conn->incr_granularity = 0; + F_CLR(conn, WT_CONN_INCR_BACKUP); +} + +/* + * __wt_backup_open -- + * Restore any incremental backup information. We use the metadata's block information as the + * authority on whether incremental backup was in use on last shutdown. + */ +int +__wt_backup_open(WT_SESSION_IMPL *session) +{ + WT_BLKINCR *blkincr; + WT_CONFIG blkconf; + WT_CONFIG_ITEM b, k, v; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + uint64_t i; + char *config; + + conn = S2C(session); + config = NULL; + + WT_RET(__wt_metadata_search(session, WT_METAFILE_URI, &config)); + WT_ERR(__wt_config_getones(session, config, "checkpoint_backup_info", &v)); + __wt_config_subinit(session, &blkconf, &v); + /* + * Walk each item in the metadata and set up our last known global incremental information. + */ + F_CLR(conn, WT_CONN_INCR_BACKUP); + i = 0; + while (__wt_config_next(&blkconf, &k, &v) == 0) { + WT_ASSERT(session, i < WT_BLKINCR_MAX); + /* + * If we get here, we have at least one valid incremental backup. We want to set up its + * general configuration in the global table. + */ + blkincr = &conn->incr_backups[i++]; + F_SET(conn, WT_CONN_INCR_BACKUP); + WT_ERR(__wt_strndup(session, k.str, k.len, &blkincr->id_str)); + WT_ERR(__wt_config_subgets(session, &v, "granularity", &b)); + /* + * NOTE: For now the granularity is in the connection because it cannot change. We may be + * able to relax that. + */ + conn->incr_granularity = blkincr->granularity = (uint64_t)b.val; + F_SET(blkincr, WT_BLKINCR_VALID); + } + +err: + if (ret != 0 && ret != WT_NOTFOUND) + __wt_backup_destroy(session); + __wt_free(session, config); + return (ret == WT_NOTFOUND ? 0 : ret); +} + +/* + * __wt_backup_file_remove -- + * Remove the incremental and meta-data backup files. + */ +int +__wt_backup_file_remove(WT_SESSION_IMPL *session) +{ + WT_DECL_RET; + + /* + * Note that order matters for removing the incremental files. We must remove the backup file + * before removing the source file so that we always know we were a source directory while + * there's any chance of an incremental backup file existing. + */ + WT_TRET(__wt_remove_if_exists(session, WT_BACKUP_TMP, true)); + WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_BACKUP, true)); + WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_SRC, true)); + WT_TRET(__wt_remove_if_exists(session, WT_METADATA_BACKUP, true)); + return (ret); +} + +/* * __curbackup_next -- * WT_CURSOR->next method for the backup cursor type. */ @@ -71,35 +166,6 @@ err: } /* - * __backup_incr_release -- - * Free all resources relating to incremental backup. - */ -static int -__backup_incr_release(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool force) -{ - WT_BLKINCR *blk; - WT_CONNECTION_IMPL *conn; - u_int i; - - WT_UNUSED(cb); - WT_UNUSED(force); - conn = S2C(session); - /* - * Clear flags. Remove file. Release any memory information. - */ - F_CLR(conn, WT_CONN_INCR_BACKUP); - for (i = 0; i < WT_BLKINCR_MAX; ++i) { - blk = &conn->incr_backups[i]; - F_CLR(blk, WT_BLKINCR_VALID); - } - /* __wt_block_backup_remove... */ - conn->ckpt_incr_granularity = 0; - WT_RET(__wt_remove_if_exists(session, WT_BLKINCR_BACKUP, true)); - - return (0); -} - -/* * __backup_free -- * Free list resources for a backup cursor. */ @@ -115,10 +181,6 @@ __backup_free(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) } if (cb->incr_file != NULL) __wt_free(session, cb->incr_file); - if (cb->incr_src != NULL) - __wt_free(session, cb->incr_src); - if (cb->incr_this != NULL) - __wt_free(session, cb->incr_this); __wt_curbackup_free_incr(session, cb); } @@ -140,7 +202,7 @@ err: if (F_ISSET(cb, WT_CURBACKUP_FORCE_STOP)) { __wt_verbose( session, WT_VERB_BACKUP, "%s", "Releasing resources from forced stop incremental"); - __backup_incr_release(session, cb, true); + __wt_backup_destroy(session); } /* @@ -232,26 +294,6 @@ err: } /* - * __backup_get_ckpt -- - * Get the most recent checkpoint information and store it in the structure. - * - * XXX - Currently set return to static void for the compiler, when this function has real content - * it should be static int. - */ -static void -__backup_get_ckpt(WT_SESSION_IMPL *session, WT_BLKINCR *incr) -{ - WT_UNUSED(session); - WT_UNUSED(incr); - /* - * Look up the most recent checkpoint and store information about it in incr. - * - * XXX When this function has content, return a real value. return (0); - */ - return; -} - -/* * __backup_add_id -- * Add the identifier for block based incremental backup. */ @@ -262,11 +304,13 @@ __backup_add_id(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval) WT_CONNECTION_IMPL *conn; WT_DECL_RET; u_int i; + const char *ckpt; conn = S2C(session); blk = NULL; for (i = 0; i < WT_BLKINCR_MAX; ++i) { blk = &conn->incr_backups[i]; + __wt_verbose(session, WT_VERB_BACKUP, "blk[%u] flags 0x%" PRIx64, i, blk->flags); /* If it isn't use, we can use it. */ if (!F_ISSET(blk, WT_BLKINCR_INUSE)) break; @@ -281,16 +325,28 @@ __backup_add_id(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval) if (blk->id_str != NULL) __wt_verbose( session, WT_VERB_BACKUP, "Freeing and reusing backup slot with old id %s", blk->id_str); - /* Free any string that was there. */ + /* Free anything that was there. */ __wt_free(session, blk->id_str); WT_ERR(__wt_strndup(session, cval->str, cval->len, &blk->id_str)); - __wt_verbose(session, WT_VERB_BACKUP, "Using backup slot %u for id %s", i, blk->id_str); /* - * XXX This function can error in the future. - * - * WT_ERR(__backup_get_ckpt(session, blk)); + * Get the most recent checkpoint name. For now just use the one that is part of the metadata. + * We only care whether or not a checkpoint exists, so immediately free it. */ - __backup_get_ckpt(session, blk); + ret = __wt_meta_checkpoint_last_name(session, WT_METAFILE_URI, &ckpt); + __wt_free(session, ckpt); + if (ret != 0 && ret != WT_NOTFOUND) + WT_ERR(ret); + if (ret == WT_NOTFOUND) { + /* + * If we don't find any checkpoint, backup files need to be full copy. + */ + __wt_verbose(session, WT_VERB_BACKUP, "ID %s: Did not find any metadata checkpoint for %s.", + blk->id_str, WT_METAFILE_URI); + F_SET(blk, WT_BLKINCR_FULL); + } else { + __wt_verbose(session, WT_VERB_BACKUP, "Using backup slot %u for id %s", i, blk->id_str); + F_CLR(blk, WT_BLKINCR_FULL); + } F_SET(blk, WT_BLKINCR_VALID); return (0); @@ -400,12 +456,11 @@ __backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[ if (cval.val) { if (!F_ISSET(conn, WT_CONN_INCR_BACKUP)) { WT_RET(__wt_config_gets(session, cfg, "incremental.granularity", &cval)); - /* XXX may not need cb->incr_granularity */ - if (conn->ckpt_incr_granularity != 0) + if (conn->incr_granularity != 0) WT_RET_MSG(session, EINVAL, "Cannot change the incremental backup granularity"); - conn->ckpt_incr_granularity = cb->incr_granularity = (uint64_t)cval.val; + conn->incr_granularity = (uint64_t)cval.val; } - /* XXX Granularity can only be set once at the beginning */ + /* Granularity can only be set once at the beginning */ F_SET(conn, WT_CONN_INCR_BACKUP); incremental_config = true; } @@ -432,10 +487,8 @@ __backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[ if (is_dup) WT_RET_MSG(session, EINVAL, "Incremental source identifier can only be specified on a primary backup cursor"); - WT_RET(__backup_find_id(session, &cval, &cb->incr)); - /* XXX might not need this incr_src field */ - WT_RET(__wt_strndup(session, cval.str, cval.len, &cb->incr_src)); - F_SET(cb->incr, WT_BLKINCR_INUSE); + WT_RET(__backup_find_id(session, &cval, &cb->incr_src)); + F_SET(cb->incr_src, WT_BLKINCR_INUSE); incremental_config = true; } /* @@ -455,8 +508,6 @@ __backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[ WT_ERR_MSG(session, EINVAL, "Incremental identifier already exists"); WT_ERR(__backup_add_id(session, &cval)); - /* XXX might not need this incr_this field */ - WT_ERR(__wt_strndup(session, cval.str, cval.len, &cb->incr_this)); incremental_config = true; } @@ -530,8 +581,8 @@ __backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[ F_SET(cb, WT_CURBACKUP_INCR); } err: - if (ret != 0 && cb->incr != NULL) - F_CLR(cb->incr, WT_BLKINCR_INUSE); + if (ret != 0 && cb->incr_src != NULL) + F_CLR(cb->incr_src, WT_BLKINCR_INUSE); __wt_scr_free(session, &tmp); return (ret); } @@ -716,8 +767,8 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) /* If it's not a dup backup cursor, make sure one isn't open. */ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_BACKUP_DUP)); WT_WITH_HOTBACKUP_WRITE_LOCK(session, conn->hot_backup_list = NULL); - if (cb->incr != NULL) - F_CLR(cb->incr, WT_BLKINCR_INUSE); + if (cb->incr_src != NULL) + F_CLR(cb->incr_src, WT_BLKINCR_INUSE); __backup_free(session, cb); /* Remove any backup specific file. */ @@ -742,27 +793,6 @@ __backup_all(WT_SESSION_IMPL *session) } /* - * __wt_backup_file_remove -- - * Remove the incremental and meta-data backup files. - */ -int -__wt_backup_file_remove(WT_SESSION_IMPL *session) -{ - WT_DECL_RET; - - /* - * Note that order matters for removing the incremental files. We must remove the backup file - * before removing the source file so that we always know we were a source directory while - * there's any chance of an incremental backup file existing. - */ - WT_TRET(__wt_remove_if_exists(session, WT_BACKUP_TMP, true)); - WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_BACKUP, true)); - WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_SRC, true)); - WT_TRET(__wt_remove_if_exists(session, WT_METADATA_BACKUP, true)); - return (ret); -} - -/* * __backup_list_uri_append -- * Append a new file name to the list, allocate space as necessary. Called via the schema_worker * function. diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c index c8c59d3f9db..5403e2308ff 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c @@ -9,53 +9,74 @@ #include "wt_internal.h" /* - * __alloc_merge -- - * Merge two allocation lists. + * __wt_backup_load_incr -- + * Load the incremental. */ -static void -__alloc_merge( - uint64_t *a, uint64_t a_cnt, uint64_t *b, uint64_t b_cnt, uint64_t *res, uint64_t *res_cnt) +int +__wt_backup_load_incr( + WT_SESSION_IMPL *session, WT_CONFIG_ITEM *blkcfg, WT_ITEM *bitstring, uint64_t nbits) { - uint64_t total; - - for (total = 0; a_cnt > 0 || b_cnt > 0; ++total, res += 2) { - if (a_cnt > 0 && b_cnt > 0) { - if (a[0] <= b[0]) { - res[0] = a[0]; - if (a[0] + a[1] < b[0]) - res[1] = a[1]; - else { - res[1] = (b[0] + b[1]) - a[0]; - b += 2; - --b_cnt; - } - a += 2; - --a_cnt; - } else if (b[0] <= a[0]) { - res[0] = b[0]; - if (b[0] + b[1] < a[0]) - res[1] = b[1]; - else { - res[1] = (a[0] + a[1]) - b[0]; - a += 2; - --a_cnt; - } - b += 2; - --b_cnt; - } - } else if (a_cnt > 0) { - res[0] = a[0]; - res[1] = a[1]; - a += 2; - --a_cnt; - } else if (b_cnt > 0) { - res[0] = b[0]; - res[1] = b[1]; - b += 2; - --b_cnt; + if (blkcfg->len != 0) + WT_RET(__wt_nhex_to_raw(session, blkcfg->str, blkcfg->len, bitstring)); + if (bitstring->size != (nbits >> 3)) + WT_RET_MSG(session, WT_ERROR, "corrupted modified block list"); + return (0); +} + +/* + * __curbackup_incr_blkmod -- + * Get the block modifications for a tree from its metadata and fill in the backup cursor's + * information with it. + */ +static int +__curbackup_incr_blkmod(WT_SESSION_IMPL *session, WT_BTREE *btree, WT_CURSOR_BACKUP *cb) +{ + WT_CONFIG blkconf; + WT_CONFIG_ITEM b, k, v; + WT_DECL_RET; + char *config; + + WT_ASSERT(session, btree != NULL); + WT_ASSERT(session, btree->dhandle != NULL); + WT_ASSERT(session, cb->incr_src != NULL); + + WT_RET(__wt_metadata_search(session, btree->dhandle->name, &config)); + WT_ERR(__wt_config_getones(session, config, "checkpoint_backup_info", &v)); + __wt_config_subinit(session, &blkconf, &v); + while ((ret = __wt_config_next(&blkconf, &k, &v)) == 0) { + /* + * First see if we have information for this source identifier. + */ + if (WT_STRING_MATCH(cb->incr_src->id_str, k.str, k.len) == 0) + continue; + + /* + * We found a match. If we have a name, then there should be granularity and nbits. The + * granularity should be set to something. But nbits may be 0 if there are no blocks + * currently modified. + */ + WT_ERR(__wt_config_subgets(session, &v, "granularity", &b)); + cb->granularity = (uint64_t)b.val; + WT_ERR(__wt_config_subgets(session, &v, "nbits", &b)); + cb->nbits = (uint64_t)b.val; + WT_ERR(__wt_config_subgets(session, &v, "offset", &b)); + cb->offset = (uint64_t)b.val; + + /* + * We found a match. Load the block information into the cursor. + */ + ret = __wt_config_subgets(session, &v, "blocks", &b); + if (ret != WT_NOTFOUND) { + WT_ERR(__wt_backup_load_incr(session, &b, &cb->bitstring, cb->nbits)); + cb->bit_offset = 0; + cb->incr_init = true; } } - *res_cnt = total; + WT_ERR_NOTFOUND_OK(ret); + +err: + __wt_free(session, config); + return (ret == WT_NOTFOUND ? 0 : ret); } /* @@ -66,18 +87,11 @@ static int __curbackup_incr_next(WT_CURSOR *cursor) { WT_BTREE *btree; - WT_CKPT *ckpt, *ckptbase; WT_CURSOR_BACKUP *cb; WT_DECL_RET; WT_SESSION_IMPL *session; wt_off_t size; - uint64_t *a, *b, *current, *next; - uint64_t entries, total; uint32_t raw; - bool start, stop; - - ckptbase = NULL; - a = b = NULL; cb = (WT_CURSOR_BACKUP *)cursor; btree = cb->incr_cursor == NULL ? NULL : ((WT_CURSOR_BTREE *)cb->incr_cursor)->btree; @@ -86,104 +100,46 @@ __curbackup_incr_next(WT_CURSOR *cursor) F_CLR(cursor, WT_CURSTD_RAW); if (cb->incr_init) { - /* We have this object's incremental information, Check if we're done. */ - if (cb->incr_list_offset >= cb->incr_list_count - WT_BACKUP_INCR_COMPONENTS) - return (WT_NOTFOUND); + /* Look for the next chunk that had modifications. */ + while (cb->bit_offset < cb->nbits) + if (__bit_test(cb->bitstring.mem, cb->bit_offset)) + break; + else + ++cb->bit_offset; - /* - * If we returned all of the data, step to the next block, otherwise return the next chunk - * of the current block. - */ - if (cb->incr_list[cb->incr_list_offset + 1] <= cb->incr_granularity) - cb->incr_list_offset += WT_BACKUP_INCR_COMPONENTS; - else { - cb->incr_list[cb->incr_list_offset] += cb->incr_granularity; - cb->incr_list[cb->incr_list_offset + 1] -= cb->incr_granularity; - cb->incr_list[cb->incr_list_offset + 2] = WT_BACKUP_RANGE; - } - } else if (btree == NULL) { + /* We either have this object's incremental information or we're done. */ + if (cb->bit_offset >= cb->nbits) + WT_ERR(WT_NOTFOUND); + __wt_cursor_set_key(cursor, cb->offset + cb->granularity * cb->bit_offset++, + cb->granularity, WT_BACKUP_RANGE); + } else if (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL)) { /* We don't have this object's incremental information, and it's a full file copy. */ WT_ERR(__wt_fs_size(session, cb->incr_file, &size)); - cb->incr_list_count = WT_BACKUP_INCR_COMPONENTS; + cb->nbits = 0; + cb->offset = 0; + cb->bit_offset = 0; cb->incr_init = true; - cb->incr_list_offset = 0; __wt_cursor_set_key(cursor, 0, size, WT_BACKUP_FILE); } else { /* * We don't have this object's incremental information, and it's not a full file copy. Get a - * list of the checkpoints available for the file and flag the starting/stopping ones. It - * shouldn't be possible to specify checkpoints that no longer exist, but check anyway. + * list of the block modifications for the file. The block modifications are from the + * incremental identifier starting point. Walk the list looking for one with a source of our + * id. */ - ret = __wt_meta_ckptlist_get(session, cb->incr_file, false, &ckptbase); - WT_ERR(ret == WT_NOTFOUND ? ENOENT : ret); - + WT_ERR(__curbackup_incr_blkmod(session, btree, cb)); /* - * Count up the maximum number of block entries we might have to merge, and allocate a pair - * of temporary arrays in which to do the merge. + * If there is no block modification information for this file, there is no information to + * return to the user. */ - entries = 0; - WT_CKPT_FOREACH (ckptbase, ckpt) - entries += ckpt->alloc_list_entries; - WT_ERR(__wt_calloc_def(session, entries * WT_BACKUP_INCR_COMPONENTS, &a)); - WT_ERR(__wt_calloc_def(session, entries * WT_BACKUP_INCR_COMPONENTS, &b)); - - /* Merge the block lists into a final list of blocks to copy. */ - start = stop = false; - total = 0; - current = NULL; - next = a; - WT_CKPT_FOREACH (ckptbase, ckpt) { - if (strcmp(ckpt->name, cb->incr_checkpoint_start) == 0) { - start = true; - WT_ERR_ASSERT(session, ckpt->alloc_list_entries == 0, __wt_panic(session), - "incremental backup start checkpoint has allocation list blocks"); - continue; - } - if (start == true) { - if (strcmp(ckpt->name, cb->incr_checkpoint_stop) == 0) - stop = true; - - __alloc_merge( - current, total, ckpt->alloc_list, ckpt->alloc_list_entries, next, &total); - current = next; - next = next == a ? b : a; - } - - if (stop == true) - break; - } - - if (!start) - WT_ERR_MSG(session, ENOENT, "incremental backup start checkpoint %s not found", - cb->incr_checkpoint_start); - if (!stop) - WT_ERR_MSG(session, ENOENT, "incremental backup stop checkpoint %s not found", - cb->incr_checkpoint_stop); - - /* There may be nothing that needs copying. */ - if (total == 0) + if (cb->bitstring.mem == NULL) WT_ERR(WT_NOTFOUND); - - if (next == a) { - cb->incr_list = b; - b = NULL; - } else { - cb->incr_list = a; - a = NULL; - } - cb->incr_list_count = total; - cb->incr_list_offset = 0; - WT_ERR(__wt_scr_alloc(session, 0, &cb->incr_block)); - cb->incr_init = true; - - F_SET(cursor, WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT); + __wt_cursor_set_key(cursor, cb->offset + cb->granularity * cb->bit_offset++, + cb->granularity, WT_BACKUP_RANGE); } err: - __wt_free(session, a); - __wt_free(session, b); - __wt_meta_ckptlist_free(session, &ckptbase); F_SET(cursor, raw); API_END_RET(session, ret); } @@ -198,10 +154,7 @@ __wt_curbackup_free_incr(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) __wt_free(session, cb->incr_file); if (cb->incr_cursor != NULL) __wt_cursor_close(cb->incr_cursor); - __wt_free(session, cb->incr_checkpoint_start); - __wt_free(session, cb->incr_checkpoint_stop); - __wt_free(session, cb->incr_list); - __wt_scr_free(session, &cb->incr_block); + __wt_buf_free(session, &cb->bitstring); } /* @@ -213,21 +166,48 @@ __wt_curbackup_open_incr(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *o WT_CURSOR *cursor, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_BACKUP *cb, *other_cb; + WT_DECL_ITEM(open_uri); + WT_DECL_RET; cb = (WT_CURSOR_BACKUP *)cursor; other_cb = (WT_CURSOR_BACKUP *)other; - WT_UNUSED(session); cursor->key_format = WT_UNCHECKED_STRING(qqq); cursor->value_format = ""; + WT_ASSERT(session, other_cb->incr_src != NULL); + /* * Inherit from the backup cursor but reset specific functions for incremental. */ cursor->next = __curbackup_incr_next; cursor->get_key = __wt_cursor_get_key; cursor->get_value = __wt_cursor_get_value_notsup; - cb->incr_granularity = other_cb->incr_granularity; + cb->incr_src = other_cb->incr_src; + + /* All WiredTiger owned files are full file copies. */ + if (F_ISSET(other_cb->incr_src, WT_BLKINCR_FULL) || + WT_PREFIX_MATCH(cb->incr_file, "WiredTiger")) { + __wt_verbose(session, WT_VERB_BACKUP, "Forcing full file copies for id %s", + other_cb->incr_src->id_str); + F_SET(cb, WT_CURBACKUP_FORCE_FULL); + } + /* + * Set up the incremental backup information, if we are not forcing a full file copy. We need an + * open cursor on the file. Open the backup checkpoint, confirming it exists. + */ + if (!F_ISSET(cb, WT_CURBACKUP_FORCE_FULL)) { + WT_ERR(__wt_scr_alloc(session, 0, &open_uri)); + WT_ERR(__wt_buf_fmt(session, open_uri, "file:%s", cb->incr_file)); + __wt_free(session, cb->incr_file); + WT_ERR(__wt_strdup(session, open_uri->data, &cb->incr_file)); + + WT_ERR(__wt_curfile_open(session, cb->incr_file, NULL, cfg, &cb->incr_cursor)); + WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); + WT_ERR(__wt_strdup(session, cb->incr_cursor->internal_uri, &cb->incr_cursor->internal_uri)); + } else + WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); - /* XXX Return full file info for all files for now. */ - return (__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); +err: + __wt_scr_free(session, &open_uri); + return (ret); } diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index dd1dc11bb6e..50f6fcf9759 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1111,7 +1111,7 @@ __wt_ref_info_lock( */ for (;; __wt_yield()) { previous_state = ref->state; - if (previous_state != WT_REF_LOCKED && + if (previous_state != WT_REF_LOCKED && previous_state != WT_REF_READING && WT_REF_CAS_STATE(session, ref, previous_state, WT_REF_LOCKED)) break; } diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 9498eb5d6c6..2a275449284 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -299,7 +299,7 @@ struct __wt_connection_impl { uint64_t ckpt_write_pages; /* Checkpoint and incremental backup data */ - uint64_t ckpt_incr_granularity; + uint64_t incr_granularity; WT_BLKINCR incr_backups[WT_BLKINCR_MAX]; /* Connection's maximum and base write generations. */ diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 45a4ac01a9f..3ea011b8fc9 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -32,21 +32,6 @@ 0 /* uint32_t flags */ \ } -/* - * Block based incremental backup structure. These live in the connection. - */ -#define WT_BLKINCR_MAX 2 -struct __wt_blkincr { - const char *id_str; /* User's name for this backup. */ - const char *ckpt_name; /* Requires WT-5115. All checkpoints must be this name */ - void *data; -/* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_BLKINCR_INUSE 0x1u /* This entry is active */ -#define WT_BLKINCR_VALID 0x2u /* This entry is valid */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ - uint64_t flags; -}; - struct __wt_cursor_backup { WT_CURSOR iface; @@ -61,31 +46,25 @@ struct __wt_cursor_backup { size_t list_next; /* File offset-based incremental backup. */ - WT_BLKINCR *incr; /* Incremental backup in use */ - char *incr_file; /* File name */ - char *incr_src; /* Source identifier */ - char *incr_this; /* New base identifier */ - uint64_t incr_granularity; /* Maximum transfer size */ + WT_BLKINCR *incr_src; /* Incremental backup source */ + char *incr_file; /* File name */ WT_CURSOR *incr_cursor; /* File cursor */ - /* Start/stop checkpoints */ - char *incr_checkpoint_start; - char *incr_checkpoint_stop; - -#define WT_BACKUP_INCR_COMPONENTS 3 - bool incr_init; /* Cursor traversal initialized */ - uint64_t *incr_list; /* List of file offset/size/type triples */ - uint64_t incr_list_count; /* Count of file offset/size/type triples */ - uint64_t incr_list_offset; /* Current offset */ - uint64_t incr_size; /* Maximum transfer size */ - WT_ITEM *incr_block; /* Current block of data */ + + bool incr_init; /* Cursor traversal initialized */ + WT_ITEM bitstring; /* List of modified blocks */ + uint64_t nbits; /* Number of bits in bitstring */ + uint64_t offset; /* Zero bit offset in bitstring */ + uint64_t bit_offset; /* Current offset */ + uint64_t granularity; /* Length, transfer size */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CURBACKUP_DUP 0x1u /* Duplicated backup cursor */ -#define WT_CURBACKUP_FORCE_STOP 0x2u /* Force stop incremental backup */ -#define WT_CURBACKUP_INCR 0x4u /* Incremental backup cursor */ -#define WT_CURBACKUP_LOCKER 0x8u /* Hot-backup started */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_CURBACKUP_DUP 0x01u /* Duplicated backup cursor */ +#define WT_CURBACKUP_FORCE_FULL 0x02u /* Force full file copy for this cursor */ +#define WT_CURBACKUP_FORCE_STOP 0x04u /* Force stop incremental backup */ +#define WT_CURBACKUP_INCR 0x08u /* Incremental backup cursor */ +#define WT_CURBACKUP_LOCKER 0x10u /* Hot-backup started */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index c206781116c..2cf1408525e 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -84,6 +84,10 @@ extern int __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_backup_file_remove(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_backup_load_incr(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *blkcfg, + WT_ITEM *bitstring, uint64_t nbits) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_backup_open(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, @@ -1564,6 +1568,7 @@ extern void *__wt_ext_scr_alloc(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_async_stats_update(WT_SESSION_IMPL *session); +extern void __wt_backup_destroy(WT_SESSION_IMPL *session); extern void __wt_block_ckpt_destroy(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci); extern void __wt_block_configure_first_fit(WT_BLOCK *block, bool on); extern void __wt_block_ext_free(WT_SESSION_IMPL *session, WT_EXT *ext); diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index 9845dbd7f7d..b29d9665069 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -17,11 +17,10 @@ /* * Backup related WiredTiger files. */ -#define WT_BACKUP_TMP "WiredTiger.backup.tmp" /* Backup tmp file */ -#define WT_BLKINCR_BACKUP "WiredTiger.backup.block" /* Block incremental durable file */ -#define WT_METADATA_BACKUP "WiredTiger.backup" /* Hot backup file */ -#define WT_LOGINCR_BACKUP "WiredTiger.ibackup" /* Log incremental backup */ -#define WT_LOGINCR_SRC "WiredTiger.isrc" /* Log incremental source */ +#define WT_BACKUP_TMP "WiredTiger.backup.tmp" /* Backup tmp file */ +#define WT_METADATA_BACKUP "WiredTiger.backup" /* Hot backup file */ +#define WT_LOGINCR_BACKUP "WiredTiger.ibackup" /* Log incremental backup */ +#define WT_LOGINCR_SRC "WiredTiger.isrc" /* Log incremental source */ #define WT_METADATA_TURTLE "WiredTiger.turtle" /* Metadata metadata */ #define WT_METADATA_TURTLE_SET "WiredTiger.turtle.set" /* Turtle temp file */ @@ -58,6 +57,42 @@ } while (0) /* + * Block based incremental backup structure. These live in the connection. + */ +#define WT_BLKINCR_MAX 2 +struct __wt_blkincr { + const char *id_str; /* User's name for this backup. */ + uint64_t granularity; /* Granularity of this backup. */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_BLKINCR_FULL 0x1u /* There is no checkpoint, always do full file */ +#define WT_BLKINCR_INUSE 0x2u /* This entry is active */ +#define WT_BLKINCR_VALID 0x4u /* This entry is valid */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint64_t flags; +}; + +/* + * Block modifications from an incremental identifier going forward. + */ +/* + * At the default granularity, this is enough for blocks in a 2G file. + */ +#define WT_BLOCK_MODS_LIST_MIN 16 /* Initial bytes for bitmap. */ +struct __wt_block_mods { + const char *id_str; + + WT_ITEM bitstring; + uint64_t nbits; /* Number of bits in bitstring */ + + uint64_t offset; /* Zero bit offset for bitstring */ + uint64_t granularity; +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_BLOCK_MODS_VALID 0x1u /* Entry is valid */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t flags; +}; + +/* * WT_CKPT -- * Encapsulation of checkpoint information, shared by the metadata, the * btree engine, and the block manager. @@ -88,6 +123,8 @@ struct __wt_ckpt { char *block_metadata; /* Block-stored metadata */ char *block_checkpoint; /* Block-stored checkpoint */ + WT_BLOCK_MODS backup_blocks[WT_BLKINCR_MAX]; + /* Validity window */ wt_timestamp_t newest_durable_ts; wt_timestamp_t oldest_start_ts; @@ -95,9 +132,6 @@ struct __wt_ckpt { wt_timestamp_t newest_stop_ts; uint64_t newest_stop_txn; - uint64_t *alloc_list; /* Checkpoint allocation list */ - uint64_t alloc_list_entries; - WT_ITEM addr; /* Checkpoint cookie string */ WT_ITEM raw; /* Checkpoint cookie raw */ diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 1cd8753a0ac..7ae570d3e59 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -87,6 +87,8 @@ struct __wt_block_desc; typedef struct __wt_block_desc WT_BLOCK_DESC; struct __wt_block_header; typedef struct __wt_block_header WT_BLOCK_HEADER; +struct __wt_block_mods; +typedef struct __wt_block_mods WT_BLOCK_MODS; struct __wt_bloom; typedef struct __wt_bloom WT_BLOOM; struct __wt_bloom_hash; @@ -390,7 +392,7 @@ typedef uint64_t wt_timestamp_t; #include "error.h" #include "log.h" #include "lsm.h" -#include "meta.h" +#include "meta.h" /* required by block.h */ #include "optrack.h" #include "os.h" #include "reconcile.h" diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 93a01fa6abb..34c8b643c08 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -11,11 +11,79 @@ static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *); static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **); static int __ckpt_load(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *); +static int __ckpt_load_blk_mods(WT_SESSION_IMPL *, const char *, WT_CKPT *); static int __ckpt_named(WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *); static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *, bool); static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *); /* + * __ckpt_load_blk_mods -- + * Load the block information from the config string. + */ +static int +__ckpt_load_blk_mods(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt) +{ + WT_BLKINCR *blkincr; + WT_BLOCK_MODS *blk_mod; + WT_CONFIG blkconf; + WT_CONFIG_ITEM b, k, v; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + uint64_t i; + + conn = S2C(session); + if (config == NULL) + return (0); + /* + * We could be reading in a configuration from an earlier release. If the string doesn't exist + * then we're done. + */ + if ((ret = __wt_config_getones(session, config, "checkpoint_backup_info", &v)) != 0) + return (ret == WT_NOTFOUND ? 0 : ret); + __wt_config_subinit(session, &blkconf, &v); + /* + * Load block lists. Ignore any that have an id string that is not known. + * + * Remove those not known (TODO). + */ + blkincr = NULL; + while ((ret = __wt_config_next(&blkconf, &k, &v)) == 0) { + /* + * See if this is a valid backup string. + */ + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blkincr = &conn->incr_backups[i]; + if (blkincr->id_str != NULL && WT_STRING_MATCH(blkincr->id_str, k.str, k.len)) + break; + } + if (i == WT_BLKINCR_MAX) + /* + * This is the place to note that we want to remove an unknown id. + */ + continue; + + /* + * We have a valid entry. Load the block information. + */ + blk_mod = &ckpt->backup_blocks[i]; + WT_RET(__wt_strdup(session, blkincr->id_str, &blk_mod->id_str)); + WT_RET(__wt_config_subgets(session, &v, "granularity", &b)); + blk_mod->granularity = (uint64_t)b.val; + WT_RET(__wt_config_subgets(session, &v, "nbits", &b)); + blk_mod->nbits = (uint64_t)b.val; + WT_RET(__wt_config_subgets(session, &v, "offset", &b)); + blk_mod->offset = (uint64_t)b.val; + ret = __wt_config_subgets(session, &v, "blocks", &b); + WT_RET_NOTFOUND_OK(ret); + if (ret != WT_NOTFOUND) { + WT_RET(__wt_backup_load_incr(session, &b, &blk_mod->bitstring, blk_mod->nbits)); + F_SET(blk_mod, WT_BLOCK_MODS_VALID); + } + } + return (ret == WT_NOTFOUND ? 0 : ret); +} + +/* * __wt_meta_checkpoint -- * Return a file's checkpoint information. */ @@ -118,7 +186,7 @@ __ckpt_set(WT_SESSION_IMPL *session, const char *fname, const char *v, bool use_ * use the slower path through configuration parsing functions. */ config = newcfg = NULL; - str = v == NULL ? "checkpoint=(),checkpoint_lsn=" : v; + str = v == NULL ? "checkpoint=(),checkpoint_backup_info=(),checkpoint_lsn=" : v; if (use_base && session->dhandle != NULL) { WT_ERR(__wt_scr_alloc(session, 0, &tmp)); WT_ASSERT(session, strcmp(session->dhandle->name, fname) == 0); @@ -315,6 +383,75 @@ __ckpt_compare_order(const void *a, const void *b) } /* + * __ckpt_valid_blk_mods -- + * Make sure that this set of block mods reflects the current valid backup identifiers. If so, + * there is nothing to do. If not, free up old information and set it up for the current + * information. + */ +static int +__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt) +{ + WT_BLKINCR *blk; + WT_BLOCK_MODS *blk_mod; + uint64_t i; + bool free, setup; + + WT_ASSERT(session, F_ISSET(ckpt, WT_CKPT_ADD)); + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk = &S2C(session)->incr_backups[i]; + blk_mod = &ckpt->backup_blocks[i]; + + /* + * Check the state of our block list array compared to the global one. There are + * several possibilities: + * - There is no global information for this index, nothing to do but free our resources. + * - We don't have any backup information locally. Set up our entry. + * - Our entry's id string matches the current global information. We just want to add our + * information to the existing list. + * - Our entry's id string does not match the current one. It is outdated. Free old + * resources + * and then set up our entry. + */ + + /* Check if the global entry is valid at our index. */ + if (!F_ISSET(blk, WT_BLKINCR_VALID)) { + free = true; + setup = false; + } else if (F_ISSET(blk_mod, WT_BLOCK_MODS_VALID) && + WT_STRING_MATCH(blk_mod->id_str, blk->id_str, strlen(blk->id_str))) { + /* We match, keep our entry and don't set up. */ + setup = false; + free = false; + } else { + /* We don't match, free any old information. */ + free = true; + setup = true; + } + + /* Free any old information if we need to do so. */ + if (free && F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) { + __wt_free(session, blk_mod->id_str); + __wt_buf_free(session, &blk_mod->bitstring); + blk_mod->nbits = 0; + blk_mod->granularity = 0; + blk_mod->offset = 0; + F_CLR(blk_mod, WT_BLOCK_MODS_VALID); + } + + /* Set up the block list to point to the current information. */ + if (setup) { + WT_RET(__wt_strdup(session, blk->id_str, &blk_mod->id_str)); + WT_CLEAR(blk_mod->bitstring); + blk_mod->granularity = S2C(session)->incr_granularity; + blk_mod->nbits = 0; + blk_mod->offset = 0; + F_SET(blk_mod, WT_BLOCK_MODS_VALID); + } + } + return (0); +} + +/* * __wt_meta_ckptlist_get -- * Load all available checkpoint information for a file. */ @@ -378,10 +515,22 @@ __wt_meta_ckptlist_get( maxorder = ckpt->order; ckpt->order = maxorder + 1; __wt_seconds(session, &ckpt->sec); + /* + * Load most recent checkpoint backup blocks to this checkpoint. + */ + WT_ERR(__ckpt_load_blk_mods(session, config, ckpt)); WT_ERR(__wt_meta_block_metadata(session, config, ckpt)); + /* + * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of + * the checkpoint's modified blocks from the block manager. + */ F_SET(ckpt, WT_CKPT_ADD); + if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) { + F_SET(ckpt, WT_CKPT_BLOCK_MODS); + WT_ERR(__ckpt_valid_blk_mods(session, ckpt)); + } } /* Return the array to our caller. */ @@ -578,6 +727,50 @@ __wt_meta_ckptlist_to_meta(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM } /* + * __ckpt_blkmod_to_meta -- + * Add in any modification block string needed, including an empty one. + */ +static int +__ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) +{ + WT_BLOCK_MODS *blk; + WT_ITEM bitstring; + u_int i; + bool valid; + + WT_CLEAR(bitstring); + valid = false; + for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk) + if (F_ISSET(blk, WT_BLOCK_MODS_VALID)) + valid = true; + + /* + * If the existing block modifications are not valid, there is nothing to do. + */ + if (!valid) { + WT_RET(__wt_buf_catfmt(session, buf, ",checkpoint_backup_info=")); + return (0); + } + + /* + * We have at least one valid modified block list. + */ + WT_RET(__wt_buf_catfmt(session, buf, ",checkpoint_backup_info=(")); + for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk) { + if (!F_ISSET(blk, WT_BLOCK_MODS_VALID)) + continue; + WT_RET(__wt_raw_to_hex(session, blk->bitstring.data, blk->bitstring.size, &bitstring)); + WT_RET(__wt_buf_catfmt(session, buf, "%s%s=(id=%" PRIu32 ",granularity=%" PRIu64 + ",nbits=%" PRIu64 ",offset=%" PRIu64 ",blocks=%.*s)", + i == 0 ? "" : ",", blk->id_str, i, blk->granularity, blk->nbits, blk->offset, + (int)bitstring.size, (char *)bitstring.data)); + __wt_buf_free(session, &bitstring); + } + WT_RET(__wt_buf_catfmt(session, buf, ")")); + return (0); +} + +/* * __wt_meta_ckptlist_set -- * Set a file's checkpoint value from the WT_CKPT list. */ @@ -593,6 +786,10 @@ __wt_meta_ckptlist_set( WT_RET(__wt_scr_alloc(session, 1024, &buf)); WT_ERR(__wt_meta_ckptlist_to_meta(session, ckptbase, buf)); + /* Add backup block modifications for any added checkpoint. */ + WT_CKPT_FOREACH (ckptbase, ckpt) + if (F_ISSET(ckpt, WT_CKPT_ADD)) + WT_ERR(__ckpt_blkmod_to_meta(session, buf, ckpt)); has_lsn = ckptlsn != NULL; if (ckptlsn != NULL) @@ -634,6 +831,9 @@ __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep) void __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt) { + WT_BLOCK_MODS *blk_mod; + uint64_t i; + if (ckpt == NULL) return; @@ -643,6 +843,12 @@ __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt) __wt_buf_free(session, &ckpt->addr); __wt_buf_free(session, &ckpt->raw); __wt_free(session, ckpt->bpriv); + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk_mod = &ckpt->backup_blocks[i]; + __wt_buf_free(session, &blk_mod->bitstring); + __wt_free(session, blk_mod->id_str); + F_CLR(blk_mod, WT_BLOCK_MODS_VALID); + } WT_CLEAR(*ckpt); /* Clear to prepare for re-use. */ } diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 044094133ce..a7b2e740caf 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -185,7 +185,7 @@ __wt_turtle_init(WT_SESSION_IMPL *session) { WT_DECL_RET; char *metaconf, *unused_value; - bool exist_backup, exist_bincr, exist_incr, exist_isrc, exist_turtle; + bool exist_backup, exist_incr, exist_isrc, exist_turtle; bool load, loadTurtle; load = loadTurtle = false; @@ -212,17 +212,6 @@ __wt_turtle_init(WT_SESSION_IMPL *session) WT_RET(__wt_fs_exist(session, WT_LOGINCR_SRC, &exist_isrc)); WT_RET(__wt_fs_exist(session, WT_METADATA_BACKUP, &exist_backup)); WT_RET(__wt_fs_exist(session, WT_METADATA_TURTLE, &exist_turtle)); - /* - * Block incremental is different. If it exists, then we have block incremental information we - * need to keep. Mark the connection as having block-based incremental backup turned on. XXX - - * Need to call something to read it in and set this up. Maybe here, maybe not. - */ - WT_RET(__wt_fs_exist(session, WT_BLKINCR_BACKUP, &exist_bincr)); - if (exist_bincr) { - F_SET(S2C(session), WT_CONN_INCR_BACKUP); - /* Load content into some structure. Not sure this is the right place. It may be too early. - */ - } if (exist_turtle) { /* diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index e960ec03d48..200f84cc8c1 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1226,6 +1226,7 @@ __checkpoint_lock_dirty_tree_int(WT_SESSION_IMPL *session, bool is_checkpoint, b "cannot be deleted during a hot backup", ckpt->name); } + /* * Mark old checkpoints that are being deleted and figure out which trees we can skip in this * checkpoint. diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c index 4f1d1bcb1f1..44672a66854 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.c +++ b/src/third_party/wiredtiger/src/utilities/util_load.c @@ -449,7 +449,7 @@ config_update(WT_SESSION *session, char **list) if ((ret = __wt_config_merge((WT_SESSION_IMPL *)session, cfg, "filename=,id=," - "checkpoint=,checkpoint_lsn=,version=,source=,", + "checkpoint=,checkpoint_backup_info=,checkpoint_lsn=,version=,source=,", &p)) != 0) break; diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index cdc98884023..e7eaaa15de5 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -1996,22 +1996,6 @@ tasks: # At the time of writing this script, one call to underlying scripts takes about ~15 mins to finish in worst case. # We are giving an extra ~20% room for vairance in execution time. times: 80 - - # This is special task until lz4 issues are resolved for zSeries distros - - name: recovery-stress-test-without-lz4 - #set a 25 hours timeout - exec_timeout_secs: 90000 - commands: - - func: "get project" - - func: "compile wiredtiger" - vars: - posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=snappy,zlib - - func: "recovery stress test script" - vars: - # Repeat this script enough times to make this task run for 24 hours - # At the time of writing this script, one call to underlying scripts takes about 8 mins to finish in worst case. - # We are giving an extra ~20% room for vairance in execution time. - times: 120 - name: split-stress-test commands: @@ -2360,5 +2344,7 @@ buildvariants: tasks: - name: compile - name: unit-test - - name: recovery-stress-test-without-lz4 + - name: recovery-stress-test - name: split-stress-test + - name: format-stress-ppc-zseries-test + - name: format-stress-smoke-test diff --git a/src/third_party/wiredtiger/test/suite/test_backup12.py b/src/third_party/wiredtiger/test/suite/test_backup12.py index 6726164d038..1148ed84a45 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup12.py +++ b/src/third_party/wiredtiger/test/suite/test_backup12.py @@ -83,6 +83,9 @@ class test_backup12(wttest.WiredTigerTestCase, suite_subprocess): # That log file is not part of the list returned. This is a full backup # primary cursor with incremental configured. os.mkdir(self.dir) + # + # Note, this first backup is actually done before a checkpoint is taken. + # config = 'incremental=(enabled,this_id="ID1")' bkup_c = self.session.open_cursor('backup:', None, config) @@ -168,12 +171,9 @@ class test_backup12(wttest.WiredTigerTestCase, suite_subprocess): offset = incrlist[0] size = incrlist[1] curtype = incrlist[2] - self.assertEqual(offset, 0) - # For now assert WT_BACKUP_FILE (which is 1). - self.assertEqual(curtype, 1) + self.assertTrue(curtype == 1 or curtype == 2) dup_cnt += 1 dupc.close() - self.assertEqual(dup_cnt, 1) self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) shutil.copy(newfile, self.dir) self.assertEqual(ret, wiredtiger.WT_NOTFOUND) |