From 114ee790aaed1357f1d99dee1d610c26ae12858e Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Thu, 19 Dec 2019 03:46:45 +0000 Subject: Import wiredtiger: 77f9aebf23bb7fcb3911345741f20d67128f8da6 from branch mongodb-4.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ref: 81a6cd2196..77f9aebf23 for: 4.3.3 WT-4969 Remove lsm_merge configuration option WT-5006 Migrate Jenkins “wiredtiger-test-split-stress” job to Evergreen WT-5037 Move Application Developer Resources documentation in github to WT documentation WT-5077 Handle ENOENT without failure when copying the directory WT-5139 WiredTiger incremental backup API WT-5221 Bypass test_wt2853_perf in Evergreen make-check-msan-test WT-5257 Coverity analysis bug: 113971 Dereference after null check WT-5263 Prepared updates written to the lookaside file are not always read as needed WT-5309 Update format.sh script to add prefix command argument WT-5314 Avoid loading extensions that have blank name for Python tests WT-5319 Avoid clearing the saved last-key when no instantiated key --- src/third_party/wiredtiger/examples/c/Makefile.am | 1 + .../wiredtiger/examples/c/ex_backup_block.c | 460 +++++++++++++++++++++ 2 files changed, 461 insertions(+) create mode 100644 src/third_party/wiredtiger/examples/c/ex_backup_block.c (limited to 'src/third_party/wiredtiger/examples') diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index e441d45af46..5e9b91843d5 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -8,6 +8,7 @@ noinst_PROGRAMS = \ ex_all \ ex_async \ ex_backup \ + ex_backup_block \ ex_call_center \ ex_config_parse \ ex_cursor \ diff --git a/src/third_party/wiredtiger/examples/c/ex_backup_block.c b/src/third_party/wiredtiger/examples/c/ex_backup_block.c new file mode 100644 index 00000000000..fbae6e6da5d --- /dev/null +++ b/src/third_party/wiredtiger/examples/c/ex_backup_block.c @@ -0,0 +1,460 @@ +/*- + * Public Domain 2014-2019 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_backup_block.c + * demonstrates how to use block-based incremental backup. + */ +#include + +static const char *const home = "WT_BLOCK"; +static const char *const home_full = "WT_BLOCK_LOG_FULL"; +static const char *const home_incr = "WT_BLOCK_LOG_INCR"; + +static const char *const full_out = "./backup_block_full"; +static const char *const incr_out = "./backup_block_incr"; + +static const char *const uri = "table:main"; +static const char *const uri2 = "table:extra"; + +typedef struct __filelist { + const char *name; + bool exist; +} FILELIST; + +static FILELIST *last_flist = NULL; +static size_t filelist_count = 0; + +#define FLIST_INIT 16 + +#define CONN_CONFIG "create,cache_size=100MB,log=(enabled=true,file_max=100K)" +#define MAX_ITERATIONS 5 +#define MAX_KEYS 10000 + +static int +compare_backups(int i) +{ + int ret; + char buf[1024], msg[32]; + + /* + * We run 'wt dump' on both the full backup directory and the + * incremental backup directory for this iteration. Since running + * 'wt' runs recovery and makes both directories "live", we need + * a new directory for each iteration. + * + * If i == 0, we're comparing against the main, original directory + * with the final incremental directory. + */ + if (i == 0) + (void)snprintf(buf, sizeof(buf), "../../wt -R -h %s dump main > %s.%d", home, full_out, i); + else + (void)snprintf( + buf, sizeof(buf), "../../wt -R -h %s.%d dump main > %s.%d", home_full, i, full_out, i); + error_check(system(buf)); + /* + * Now run dump on the incremental directory. + */ + (void)snprintf( + buf, sizeof(buf), "../../wt -R -h %s.%d dump main > %s.%d", home_incr, i, incr_out, i); + error_check(system(buf)); + + /* + * Compare the files. + */ + (void)snprintf(buf, sizeof(buf), "cmp %s.%d %s.%d", full_out, i, incr_out, i); + ret = system(buf); + if (i == 0) + (void)snprintf(msg, sizeof(msg), "%s", "MAIN"); + else + (void)snprintf(msg, sizeof(msg), "%d", i); + printf("Iteration %s: Tables %s.%d and %s.%d %s\n", msg, full_out, i, incr_out, i, + ret == 0 ? "identical" : "differ"); + if (ret != 0) + exit(1); + + /* + * If they compare successfully, clean up. + */ + if (i != 0) { + (void)snprintf(buf, sizeof(buf), "rm -rf %s.%d %s.%d %s.%d %s.%d", home_full, i, home_incr, + i, full_out, i, incr_out, i); + error_check(system(buf)); + } + return (ret); +} + +/* + * Set up all the directories needed for the test. We have a full backup directory for each + * iteration and an incremental backup for each iteration. That way we can compare the full and + * incremental each time through. + */ +static void +setup_directories(void) +{ + int i; + char buf[1024]; + + for (i = 0; i < MAX_ITERATIONS; i++) { + /* + * For incremental backups we need 0-N. The 0 incremental directory will compare with the + * original at the end. + */ + (void)snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", home_incr, i, home_incr, i); + error_check(system(buf)); + if (i == 0) + continue; + /* + * For full backups we need 1-N. + */ + (void)snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", home_full, i, home_full, i); + error_check(system(buf)); + } +} + +static void +add_work(WT_SESSION *session, int iter) +{ + WT_CURSOR *cursor, *cursor2; + int i; + char k[32], v[32]; + + error_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + /* + * Only on even iterations add content to the extra table. This illustrates and shows that + * sometimes only some tables will be updated. + */ + cursor2 = NULL; + if (iter % 2 == 0) + error_check(session->open_cursor(session, uri2, NULL, NULL, &cursor2)); + /* + * Perform some operations with individual auto-commit transactions. + */ + for (i = 0; i < MAX_KEYS; i++) { + (void)snprintf(k, sizeof(k), "key.%d.%d", iter, i); + (void)snprintf(v, sizeof(v), "value.%d.%d", iter, i); + cursor->set_key(cursor, k); + cursor->set_value(cursor, v); + error_check(cursor->insert(cursor)); + if (cursor2 != NULL) { + cursor2->set_key(cursor2, k); + cursor2->set_value(cursor2, v); + error_check(cursor2->insert(cursor2)); + } + } + error_check(cursor->close(cursor)); + if (cursor2 != NULL) + error_check(cursor2->close(cursor2)); +} + +static int +finalize_files(FILELIST *flistp, size_t count) +{ + size_t i; + char buf[512]; + + /* + * Process files that were removed. Any file that is not marked in the previous list as existing + * in this iteration should be removed. Free all previous filenames as we go along. Then free + * the overall list. + */ + for (i = 0; i < filelist_count; ++i) { + if (last_flist[i].name == NULL) + break; + if (!last_flist[i].exist) { + (void)snprintf(buf, sizeof(buf), "rm WT_BLOCK_LOG_*/%s", last_flist[i].name); + error_check(system(buf)); + } + free((void *)last_flist[i].name); + } + free(last_flist); + + /* Set up the current list as the new previous list. */ + last_flist = flistp; + filelist_count = count; + return (0); +} + +/* + * Process a file name. Build up a list of current file names. But also process the file names from + * the previous iteration. Mark any name we see as existing so that the finalize function can remove + * any that don't exist. We walk the list each time. This is slow. + */ +static int +process_file(FILELIST **flistp, size_t *countp, size_t *allocp, const char *filename) +{ + FILELIST *flist; + size_t alloc, i, new, orig; + + /* Build up the current list, growing as needed. */ + i = *countp; + alloc = *allocp; + flist = *flistp; + if (i == alloc) { + orig = alloc * sizeof(FILELIST); + new = orig * 2; + flist = realloc(flist, new); + testutil_assert(flist != NULL); + memset(flist + alloc, 0, new - orig); + *allocp = alloc * 2; + *flistp = flist; + } + + flist[i].name = strdup(filename); + flist[i].exist = false; + ++(*countp); + + /* Check against the previous list. */ + for (i = 0; i < filelist_count; ++i) { + /* If name is NULL, we've reached the end of the list. */ + if (last_flist[i].name == NULL) + break; + if (strcmp(filename, last_flist[i].name) == 0) { + last_flist[i].exist = true; + break; + } + } + return (0); +} + +static void +take_full_backup(WT_SESSION *session, int i) +{ + FILELIST *flist; + WT_CURSOR *cursor; + size_t alloc, count; + int j, ret; + char buf[1024], h[256]; + const char *filename, *hdir; + + /* + * First time through we take a full backup into the incremental directories. Otherwise only + * into the appropriate full directory. + */ + if (i != 0) { + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); + hdir = h; + } else + hdir = home_incr; + if (i == 0) { + (void)snprintf(buf, sizeof(buf), "incremental=(enabled=true,this_id=ID%d)", i); + error_check(session->open_cursor(session, "backup:", NULL, buf, &cursor)); + } else + error_check(session->open_cursor(session, "backup:", NULL, NULL, &cursor)); + + count = 0; + alloc = FLIST_INIT; + flist = calloc(alloc, sizeof(FILELIST)); + testutil_assert(flist != NULL); + while ((ret = cursor->next(cursor)) == 0) { + error_check(cursor->get_key(cursor, &filename)); + error_check(process_file(&flist, &count, &alloc, filename)); + if (i == 0) + /* + * Take a full backup into each incremental directory. + */ + for (j = 0; j < MAX_ITERATIONS; j++) { + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + printf("FULL: Copy: %s\n", buf); + error_check(system(buf)); + } + else { + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, hdir, filename); + printf("FULL %d: Copy: %s\n", i, buf); + error_check(system(buf)); + } + } + scan_end_check(ret == WT_NOTFOUND); + error_check(cursor->close(cursor)); + error_check(finalize_files(flist, count)); +} + +static void +take_incr_backup(WT_SESSION *session, int i) +{ + FILELIST *flist; + WT_CURSOR *backup_cur, *incr_cur; + uint64_t offset, size, type; + size_t alloc, count; + int j, ret, rfd, wfd; + char buf[1024], h[256]; + const char *filename; + + /*! [incremental backup using block transfer]*/ + + /* Open the backup data source for incremental backup. */ + (void)snprintf(buf, sizeof(buf), "incremental=(src_id=ID%d,this_id=ID%d)", i - 1, i); + error_check(session->open_cursor(session, "backup:", NULL, buf, &backup_cur)); + rfd = wfd = -1; + count = 0; + alloc = FLIST_INIT; + flist = calloc(alloc, sizeof(FILELIST)); + testutil_assert(flist != NULL); + /* For each file listed, open a duplicate backup cursor and copy the blocks. */ + while ((ret = backup_cur->next(backup_cur)) == 0) { + error_check(backup_cur->get_key(backup_cur, &filename)); + error_check(process_file(&flist, &count, &alloc, filename)); + (void)snprintf(h, sizeof(h), "%s.0", home_incr); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + printf("Copying backup: %s\n", buf); + error_check(system(buf)); +#if 0 + (void)snprintf(buf, sizeof(buf), "%s/%s", home, filename); + printf("Open source %s for reading\n", buf); + error_check(rfd = open(buf, O_RDONLY, 0)); + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, i); + (void)snprintf(buf, sizeof(buf), "%s/%s", h, filename); + printf("Open dest %s for writing\n", buf); + error_check(wfd = open(buf, O_WRONLY, 0)); +#endif + + (void)snprintf(buf, sizeof(buf), "incremental=(file=%s)", filename); + error_check(session->open_cursor(session, NULL, backup_cur, buf, &incr_cur)); + printf("Taking incremental %d: File %s\n", i, filename); + while ((ret = incr_cur->next(incr_cur)) == 0) { + error_check(incr_cur->get_key(incr_cur, &offset, &size, &type)); + printf("Incremental %s: KEY: Off %" PRIu64 " Size: %" PRIu64 " Type: %" PRIu64 "\n", + filename, offset, size, type); + scan_end_check(type == WT_BACKUP_FILE || type == WT_BACKUP_RANGE); + if (type == WT_BACKUP_RANGE) { + /* + * We should never get a range key after a whole file so the read file descriptor + * should be valid. If the read descriptor is valid, so it the write one. + */ + scan_end_check(rfd != -1); + printf("Incremental %s: Range Offset: %" PRIu64 " Size: %" PRIu64 "\n", filename, + offset, size); + error_sys_check(lseek(rfd, (wt_off_t)offset, SEEK_SET)); + error_sys_check(read(rfd, buf, (size_t)size)); + error_sys_check(lseek(wfd, (wt_off_t)offset, SEEK_SET)); + error_sys_check(write(wfd, buf, (size_t)size)); + } else { +/* Whole file, so close both files and just copy the whole thing. */ +#if 0 + error_check(close(rfd)); + error_check(close(wfd)); +#endif + rfd = wfd = -1; + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + printf("Incremental: Whole file copy: %s\n", buf); + error_check(system(buf)); + } + } + scan_end_check(ret == WT_NOTFOUND); + error_check(incr_cur->close(incr_cur)); + + /* Close file descriptors if they're open. */ + if (rfd != -1) { + error_check(close(rfd)); + error_check(close(wfd)); + } + /* + * For each file, we want to copy the file into each of the later incremental directories so + * that they start out at the same for the next incremental round. We then check each + * incremental directory along the way. + */ + for (j = i; j < MAX_ITERATIONS; j++) { + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + error_check(system(buf)); + } + } + scan_end_check(ret == WT_NOTFOUND); + + error_check(backup_cur->close(backup_cur)); + error_check(finalize_files(flist, count)); + /*! [incremental backup using block transfer]*/ +} + +int +main(int argc, char *argv[]) +{ + WT_CONNECTION *wt_conn; + WT_CURSOR *backup_cur; + WT_SESSION *session; + int i; + char cmd_buf[256]; + + (void)argc; /* Unused variable */ + (void)testutil_set_progname(argv); + + (void)snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s && mkdir %s", home, home); + error_check(system(cmd_buf)); + error_check(wiredtiger_open(home, NULL, CONN_CONFIG, &wt_conn)); + + setup_directories(); + error_check(wt_conn->open_session(wt_conn, NULL, NULL, &session)); + error_check(session->create(session, uri, "key_format=S,value_format=S")); + error_check(session->create(session, uri2, "key_format=S,value_format=S")); + printf("Adding initial data\n"); + add_work(session, 0); + + printf("Taking initial backup\n"); + take_full_backup(session, 0); + + error_check(session->checkpoint(session, NULL)); + + for (i = 1; i < MAX_ITERATIONS; i++) { + printf("Iteration %d: adding data\n", i); + add_work(session, i); + error_check(session->checkpoint(session, NULL)); + /* + * The full backup here is only needed for testing and comparison purposes. A normal + * incremental backup procedure would not include this. + */ + printf("Iteration %d: taking full backup\n", i); + take_full_backup(session, i); + /* + * Taking the incremental backup also calls truncate to archive the log files, if the copies + * were successful. See that function for details on that call. + */ + printf("Iteration %d: taking incremental backup\n", i); + take_incr_backup(session, i); + + printf("Iteration %d: dumping and comparing data\n", i); + error_check(compare_backups(i)); + } + + /* + * After we're done, release resources. Test the force stop setting. + */ + (void)snprintf(cmd_buf, sizeof(cmd_buf), "incremental=(force_stop=true)"); + error_check(session->open_cursor(session, "backup:", NULL, cmd_buf, &backup_cur)); + error_check(backup_cur->close(backup_cur)); + + /* + * Close the connection. We're done and want to run the final comparison between the incremental + * and original. + */ + error_check(wt_conn->close(wt_conn, NULL)); + + printf("Final comparison: dumping and comparing data\n"); + error_check(compare_backups(0)); + + return (EXIT_SUCCESS); +} -- cgit v1.2.1