diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2019-12-19 03:46:45 +0000 |
|---|---|---|
| committer | evergreen <evergreen@mongodb.com> | 2019-12-19 03:46:45 +0000 |
| commit | 114ee790aaed1357f1d99dee1d610c26ae12858e (patch) | |
| tree | 545a23afda994a5920a62bcf9ee4dfd7d52367a7 /src/third_party | |
| parent | 6adf8ad187352b0b253d4fbb282c8af9d1e89f18 (diff) | |
| download | mongo-114ee790aaed1357f1d99dee1d610c26ae12858e.tar.gz | |
Import wiredtiger: 77f9aebf23bb7fcb3911345741f20d67128f8da6 from branch mongodb-4.4
ref: 81a6cd2196..77f9aebf23
for: 4.3.3
WT-4969 Remove lsm_merge configuration option
WT-5006 Migrate Jenkins “wiredtiger-test-split-stress” job to Evergreen
WT-5037 Move Application Developer Resources documentation in github to WT documentation
WT-5077 Handle ENOENT without failure when copying the directory
WT-5139 WiredTiger incremental backup API
WT-5221 Bypass test_wt2853_perf in Evergreen make-check-msan-test
WT-5257 Coverity analysis bug: 113971 Dereference after null check
WT-5263 Prepared updates written to the lookaside file are not always read as needed
WT-5309 Update format.sh script to add prefix command argument
WT-5314 Avoid loading extensions that have blank name for Python tests
WT-5319 Avoid clearing the saved last-key when no instantiated key
Diffstat (limited to 'src/third_party')
41 files changed, 2701 insertions, 298 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 6d9d4f1db3d..6f967a92118 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -602,9 +602,6 @@ connection_runtime_config = [ merge LSM chunks where possible''', type='boolean') ]), - Config('lsm_merge', 'true', r''' - merge LSM chunks where possible (deprecated)''', - type='boolean', undoc=True), Config('operation_timeout_ms', '0', r''' when non-zero, a requested limit on the number of elapsed real time milliseconds application threads will take to complete database operations. Time is measured from the @@ -680,6 +677,7 @@ connection_runtime_config = [ list, such as <code>"verbose=[evictserver,read]"</code>''', type='list', choices=[ 'api', + 'backup', 'block', 'checkpoint', 'checkpoint_progress', @@ -1208,6 +1206,38 @@ methods = { characters are hexadecimal encoded. These formats are compatible with the @ref util_dump and @ref util_load commands''', choices=['hex', 'json', 'print']), + Config('incremental', '', r''' + configure the cursor for block incremental backup usage. These formats + are only compatible with the backup data source; see @ref backup''', + type='category', subconfig=[ + Config('enabled', 'false', r''' + whether to configure this backup as the starting point for a subsequent + incremental backup''', + type='boolean'), + Config('file', '', r''' + the file name when opening a duplicate incremental backup cursor. + That duplicate cursor will return the block modifications relevant + to the given file name'''), + Config('force_stop', 'false', r''' + causes all block incremental backup information to be released. This is + on an open_cursor call and the resources will be released when this + cursor is closed. No other operations should be done on this open cursor''', + type='boolean'), + Config('granularity', '16MB', r''' + this setting manages the granularity of how WiredTiger maintains modification + maps internally. The larger the granularity, the smaller amount of information + WiredTiger need to maintain''', + min='1MB', max='2GB'), + Config('src_id', '', r''' + a string that identifies a previous checkpoint backup source as the source + of this incremental backup. This identifier must have already been created + by use of the 'this_id' configuration in an earlier backup. A source id is + required to begin an incremental backup'''), + Config('this_id', '', r''' + a string that identifies the current system state as a future backup source + for an incremental backup via 'src_id'. This identifier is required when opening + an incremental backup cursor and an error will be returned if one is not provided'''), + ]), Config('next_random', 'false', r''' configure the cursor to return a pseudo-random record from the object when the WT_CURSOR::next method is called; valid only for diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 9e7eb0b23ac..6cf9369fecd 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -64,7 +64,6 @@ src/config/config_check.c src/config/config_collapse.c src/config/config_def.c src/config/config_ext.c -src/config/config_upgrade.c src/conn/api_calc_modify.c src/conn/api_strerror.c src/conn/api_version.c @@ -81,6 +80,7 @@ src/conn/conn_reconfig.c src/conn/conn_stat.c src/conn/conn_sweep.c src/cursor/cur_backup.c +src/cursor/cur_backup_incr.c src/cursor/cur_bulk.c src/cursor/cur_config.c src/cursor/cur_ds.c diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 85a240550ea..3a5f70691a7 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -15,12 +15,14 @@ WT_ALIGN_CHECK WT_ATOMIC_CAS WT_ATOMIC_CAS_FUNC WT_ATOMIC_FUNC +WT_BACKUP_INVALID WT_BLOCK_DESC_SIZE WT_BLOCK_EXTLIST_VERSION_ORIG WT_BLOCK_HEADER_SIZE WT_CACHE_LINE_ALIGNMENT WT_CACHE_LINE_PAD_BEGIN WT_CACHE_LINE_PAD_END +WT_CKPT_BLOCK_MODS WT_CLOCKDIFF_NS WT_CONN_CHECK_PANIC WT_DEADLOCK diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 66480a55eec..6cda20d9b3d 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1128,6 +1128,7 @@ pvA pwrite py qdown +qqq qrrSS qsort quartile diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index e441d45af46..5e9b91843d5 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -8,6 +8,7 @@ noinst_PROGRAMS = \ ex_all \ ex_async \ ex_backup \ + ex_backup_block \ ex_call_center \ ex_config_parse \ ex_cursor \ diff --git a/src/third_party/wiredtiger/examples/c/ex_backup_block.c b/src/third_party/wiredtiger/examples/c/ex_backup_block.c new file mode 100644 index 00000000000..fbae6e6da5d --- /dev/null +++ b/src/third_party/wiredtiger/examples/c/ex_backup_block.c @@ -0,0 +1,460 @@ +/*- + * Public Domain 2014-2019 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_backup_block.c + * demonstrates how to use block-based incremental backup. + */ +#include <test_util.h> + +static const char *const home = "WT_BLOCK"; +static const char *const home_full = "WT_BLOCK_LOG_FULL"; +static const char *const home_incr = "WT_BLOCK_LOG_INCR"; + +static const char *const full_out = "./backup_block_full"; +static const char *const incr_out = "./backup_block_incr"; + +static const char *const uri = "table:main"; +static const char *const uri2 = "table:extra"; + +typedef struct __filelist { + const char *name; + bool exist; +} FILELIST; + +static FILELIST *last_flist = NULL; +static size_t filelist_count = 0; + +#define FLIST_INIT 16 + +#define CONN_CONFIG "create,cache_size=100MB,log=(enabled=true,file_max=100K)" +#define MAX_ITERATIONS 5 +#define MAX_KEYS 10000 + +static int +compare_backups(int i) +{ + int ret; + char buf[1024], msg[32]; + + /* + * We run 'wt dump' on both the full backup directory and the + * incremental backup directory for this iteration. Since running + * 'wt' runs recovery and makes both directories "live", we need + * a new directory for each iteration. + * + * If i == 0, we're comparing against the main, original directory + * with the final incremental directory. + */ + if (i == 0) + (void)snprintf(buf, sizeof(buf), "../../wt -R -h %s dump main > %s.%d", home, full_out, i); + else + (void)snprintf( + buf, sizeof(buf), "../../wt -R -h %s.%d dump main > %s.%d", home_full, i, full_out, i); + error_check(system(buf)); + /* + * Now run dump on the incremental directory. + */ + (void)snprintf( + buf, sizeof(buf), "../../wt -R -h %s.%d dump main > %s.%d", home_incr, i, incr_out, i); + error_check(system(buf)); + + /* + * Compare the files. + */ + (void)snprintf(buf, sizeof(buf), "cmp %s.%d %s.%d", full_out, i, incr_out, i); + ret = system(buf); + if (i == 0) + (void)snprintf(msg, sizeof(msg), "%s", "MAIN"); + else + (void)snprintf(msg, sizeof(msg), "%d", i); + printf("Iteration %s: Tables %s.%d and %s.%d %s\n", msg, full_out, i, incr_out, i, + ret == 0 ? "identical" : "differ"); + if (ret != 0) + exit(1); + + /* + * If they compare successfully, clean up. + */ + if (i != 0) { + (void)snprintf(buf, sizeof(buf), "rm -rf %s.%d %s.%d %s.%d %s.%d", home_full, i, home_incr, + i, full_out, i, incr_out, i); + error_check(system(buf)); + } + return (ret); +} + +/* + * Set up all the directories needed for the test. We have a full backup directory for each + * iteration and an incremental backup for each iteration. That way we can compare the full and + * incremental each time through. + */ +static void +setup_directories(void) +{ + int i; + char buf[1024]; + + for (i = 0; i < MAX_ITERATIONS; i++) { + /* + * For incremental backups we need 0-N. The 0 incremental directory will compare with the + * original at the end. + */ + (void)snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", home_incr, i, home_incr, i); + error_check(system(buf)); + if (i == 0) + continue; + /* + * For full backups we need 1-N. + */ + (void)snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", home_full, i, home_full, i); + error_check(system(buf)); + } +} + +static void +add_work(WT_SESSION *session, int iter) +{ + WT_CURSOR *cursor, *cursor2; + int i; + char k[32], v[32]; + + error_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + /* + * Only on even iterations add content to the extra table. This illustrates and shows that + * sometimes only some tables will be updated. + */ + cursor2 = NULL; + if (iter % 2 == 0) + error_check(session->open_cursor(session, uri2, NULL, NULL, &cursor2)); + /* + * Perform some operations with individual auto-commit transactions. + */ + for (i = 0; i < MAX_KEYS; i++) { + (void)snprintf(k, sizeof(k), "key.%d.%d", iter, i); + (void)snprintf(v, sizeof(v), "value.%d.%d", iter, i); + cursor->set_key(cursor, k); + cursor->set_value(cursor, v); + error_check(cursor->insert(cursor)); + if (cursor2 != NULL) { + cursor2->set_key(cursor2, k); + cursor2->set_value(cursor2, v); + error_check(cursor2->insert(cursor2)); + } + } + error_check(cursor->close(cursor)); + if (cursor2 != NULL) + error_check(cursor2->close(cursor2)); +} + +static int +finalize_files(FILELIST *flistp, size_t count) +{ + size_t i; + char buf[512]; + + /* + * Process files that were removed. Any file that is not marked in the previous list as existing + * in this iteration should be removed. Free all previous filenames as we go along. Then free + * the overall list. + */ + for (i = 0; i < filelist_count; ++i) { + if (last_flist[i].name == NULL) + break; + if (!last_flist[i].exist) { + (void)snprintf(buf, sizeof(buf), "rm WT_BLOCK_LOG_*/%s", last_flist[i].name); + error_check(system(buf)); + } + free((void *)last_flist[i].name); + } + free(last_flist); + + /* Set up the current list as the new previous list. */ + last_flist = flistp; + filelist_count = count; + return (0); +} + +/* + * Process a file name. Build up a list of current file names. But also process the file names from + * the previous iteration. Mark any name we see as existing so that the finalize function can remove + * any that don't exist. We walk the list each time. This is slow. + */ +static int +process_file(FILELIST **flistp, size_t *countp, size_t *allocp, const char *filename) +{ + FILELIST *flist; + size_t alloc, i, new, orig; + + /* Build up the current list, growing as needed. */ + i = *countp; + alloc = *allocp; + flist = *flistp; + if (i == alloc) { + orig = alloc * sizeof(FILELIST); + new = orig * 2; + flist = realloc(flist, new); + testutil_assert(flist != NULL); + memset(flist + alloc, 0, new - orig); + *allocp = alloc * 2; + *flistp = flist; + } + + flist[i].name = strdup(filename); + flist[i].exist = false; + ++(*countp); + + /* Check against the previous list. */ + for (i = 0; i < filelist_count; ++i) { + /* If name is NULL, we've reached the end of the list. */ + if (last_flist[i].name == NULL) + break; + if (strcmp(filename, last_flist[i].name) == 0) { + last_flist[i].exist = true; + break; + } + } + return (0); +} + +static void +take_full_backup(WT_SESSION *session, int i) +{ + FILELIST *flist; + WT_CURSOR *cursor; + size_t alloc, count; + int j, ret; + char buf[1024], h[256]; + const char *filename, *hdir; + + /* + * First time through we take a full backup into the incremental directories. Otherwise only + * into the appropriate full directory. + */ + if (i != 0) { + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); + hdir = h; + } else + hdir = home_incr; + if (i == 0) { + (void)snprintf(buf, sizeof(buf), "incremental=(enabled=true,this_id=ID%d)", i); + error_check(session->open_cursor(session, "backup:", NULL, buf, &cursor)); + } else + error_check(session->open_cursor(session, "backup:", NULL, NULL, &cursor)); + + count = 0; + alloc = FLIST_INIT; + flist = calloc(alloc, sizeof(FILELIST)); + testutil_assert(flist != NULL); + while ((ret = cursor->next(cursor)) == 0) { + error_check(cursor->get_key(cursor, &filename)); + error_check(process_file(&flist, &count, &alloc, filename)); + if (i == 0) + /* + * Take a full backup into each incremental directory. + */ + for (j = 0; j < MAX_ITERATIONS; j++) { + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + printf("FULL: Copy: %s\n", buf); + error_check(system(buf)); + } + else { + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, hdir, filename); + printf("FULL %d: Copy: %s\n", i, buf); + error_check(system(buf)); + } + } + scan_end_check(ret == WT_NOTFOUND); + error_check(cursor->close(cursor)); + error_check(finalize_files(flist, count)); +} + +static void +take_incr_backup(WT_SESSION *session, int i) +{ + FILELIST *flist; + WT_CURSOR *backup_cur, *incr_cur; + uint64_t offset, size, type; + size_t alloc, count; + int j, ret, rfd, wfd; + char buf[1024], h[256]; + const char *filename; + + /*! [incremental backup using block transfer]*/ + + /* Open the backup data source for incremental backup. */ + (void)snprintf(buf, sizeof(buf), "incremental=(src_id=ID%d,this_id=ID%d)", i - 1, i); + error_check(session->open_cursor(session, "backup:", NULL, buf, &backup_cur)); + rfd = wfd = -1; + count = 0; + alloc = FLIST_INIT; + flist = calloc(alloc, sizeof(FILELIST)); + testutil_assert(flist != NULL); + /* For each file listed, open a duplicate backup cursor and copy the blocks. */ + while ((ret = backup_cur->next(backup_cur)) == 0) { + error_check(backup_cur->get_key(backup_cur, &filename)); + error_check(process_file(&flist, &count, &alloc, filename)); + (void)snprintf(h, sizeof(h), "%s.0", home_incr); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + printf("Copying backup: %s\n", buf); + error_check(system(buf)); +#if 0 + (void)snprintf(buf, sizeof(buf), "%s/%s", home, filename); + printf("Open source %s for reading\n", buf); + error_check(rfd = open(buf, O_RDONLY, 0)); + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, i); + (void)snprintf(buf, sizeof(buf), "%s/%s", h, filename); + printf("Open dest %s for writing\n", buf); + error_check(wfd = open(buf, O_WRONLY, 0)); +#endif + + (void)snprintf(buf, sizeof(buf), "incremental=(file=%s)", filename); + error_check(session->open_cursor(session, NULL, backup_cur, buf, &incr_cur)); + printf("Taking incremental %d: File %s\n", i, filename); + while ((ret = incr_cur->next(incr_cur)) == 0) { + error_check(incr_cur->get_key(incr_cur, &offset, &size, &type)); + printf("Incremental %s: KEY: Off %" PRIu64 " Size: %" PRIu64 " Type: %" PRIu64 "\n", + filename, offset, size, type); + scan_end_check(type == WT_BACKUP_FILE || type == WT_BACKUP_RANGE); + if (type == WT_BACKUP_RANGE) { + /* + * We should never get a range key after a whole file so the read file descriptor + * should be valid. If the read descriptor is valid, so it the write one. + */ + scan_end_check(rfd != -1); + printf("Incremental %s: Range Offset: %" PRIu64 " Size: %" PRIu64 "\n", filename, + offset, size); + error_sys_check(lseek(rfd, (wt_off_t)offset, SEEK_SET)); + error_sys_check(read(rfd, buf, (size_t)size)); + error_sys_check(lseek(wfd, (wt_off_t)offset, SEEK_SET)); + error_sys_check(write(wfd, buf, (size_t)size)); + } else { +/* Whole file, so close both files and just copy the whole thing. */ +#if 0 + error_check(close(rfd)); + error_check(close(wfd)); +#endif + rfd = wfd = -1; + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + printf("Incremental: Whole file copy: %s\n", buf); + error_check(system(buf)); + } + } + scan_end_check(ret == WT_NOTFOUND); + error_check(incr_cur->close(incr_cur)); + + /* Close file descriptors if they're open. */ + if (rfd != -1) { + error_check(close(rfd)); + error_check(close(wfd)); + } + /* + * For each file, we want to copy the file into each of the later incremental directories so + * that they start out at the same for the next incremental round. We then check each + * incremental directory along the way. + */ + for (j = i; j < MAX_ITERATIONS; j++) { + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + error_check(system(buf)); + } + } + scan_end_check(ret == WT_NOTFOUND); + + error_check(backup_cur->close(backup_cur)); + error_check(finalize_files(flist, count)); + /*! [incremental backup using block transfer]*/ +} + +int +main(int argc, char *argv[]) +{ + WT_CONNECTION *wt_conn; + WT_CURSOR *backup_cur; + WT_SESSION *session; + int i; + char cmd_buf[256]; + + (void)argc; /* Unused variable */ + (void)testutil_set_progname(argv); + + (void)snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s && mkdir %s", home, home); + error_check(system(cmd_buf)); + error_check(wiredtiger_open(home, NULL, CONN_CONFIG, &wt_conn)); + + setup_directories(); + error_check(wt_conn->open_session(wt_conn, NULL, NULL, &session)); + error_check(session->create(session, uri, "key_format=S,value_format=S")); + error_check(session->create(session, uri2, "key_format=S,value_format=S")); + printf("Adding initial data\n"); + add_work(session, 0); + + printf("Taking initial backup\n"); + take_full_backup(session, 0); + + error_check(session->checkpoint(session, NULL)); + + for (i = 1; i < MAX_ITERATIONS; i++) { + printf("Iteration %d: adding data\n", i); + add_work(session, i); + error_check(session->checkpoint(session, NULL)); + /* + * The full backup here is only needed for testing and comparison purposes. A normal + * incremental backup procedure would not include this. + */ + printf("Iteration %d: taking full backup\n", i); + take_full_backup(session, i); + /* + * Taking the incremental backup also calls truncate to archive the log files, if the copies + * were successful. See that function for details on that call. + */ + printf("Iteration %d: taking incremental backup\n", i); + take_incr_backup(session, i); + + printf("Iteration %d: dumping and comparing data\n", i); + error_check(compare_backups(i)); + } + + /* + * After we're done, release resources. Test the force stop setting. + */ + (void)snprintf(cmd_buf, sizeof(cmd_buf), "incremental=(force_stop=true)"); + error_check(session->open_cursor(session, "backup:", NULL, cmd_buf, &backup_cur)); + error_check(backup_cur->close(backup_cur)); + + /* + * Close the connection. We're done and want to run the final comparison between the incremental + * and original. + */ + error_check(wt_conn->close(wt_conn, NULL)); + + printf("Final comparison: dumping and comparing data\n"); + error_check(compare_backups(0)); + + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 46aa3e9bfa1..ce4e98616c1 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "81a6cd219659835dd32ca563c24dadee54edef7d", + "commit": "77f9aebf23bb7fcb3911345741f20d67128f8da6", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.4" diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 958c267a7ce..337dbb2e593 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -117,7 +117,6 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_log_subconfigs, 4}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -134,7 +133,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { "\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -294,11 +293,19 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_log_flush[] = { {"sync", "string", NULL, "choices=[\"background\",\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor_incremental_subconfigs[] = { + {"enabled", "boolean", NULL, NULL, NULL, 0}, {"file", "string", NULL, NULL, NULL, 0}, + {"force_stop", "boolean", NULL, NULL, NULL, 0}, + {"granularity", "int", NULL, "min=1MB,max=2GB", NULL, 0}, + {"src_id", "string", NULL, NULL, NULL, 0}, {"this_id", "string", NULL, NULL, NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor[] = { {"append", "boolean", NULL, NULL, NULL, 0}, {"bulk", "string", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_wait", "boolean", NULL, NULL, NULL, 0}, {"dump", "string", NULL, "choices=[\"hex\",\"json\",\"print\"]", NULL, 0}, + {"incremental", "category", NULL, NULL, confchk_WT_SESSION_open_cursor_incremental_subconfigs, 6}, {"next_random", "boolean", NULL, NULL, NULL, 0}, {"next_random_sample_size", "string", NULL, NULL, NULL, 0}, {"overwrite", "boolean", NULL, NULL, NULL, 0}, {"raw", "boolean", NULL, NULL, NULL, 0}, @@ -551,8 +558,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"mmap", "boolean", NULL, NULL, NULL, 0}, - {"multiprocess", "boolean", NULL, NULL, NULL, 0}, + {"mmap", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -576,7 +582,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"use_environment", "boolean", NULL, NULL, NULL, 0}, {"use_environment_priv", "boolean", NULL, NULL, NULL, 0}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -619,8 +625,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"mmap", "boolean", NULL, NULL, NULL, 0}, - {"multiprocess", "boolean", NULL, NULL, NULL, 0}, + {"mmap", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -644,7 +649,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"use_environment", "boolean", NULL, NULL, NULL, 0}, {"use_environment_priv", "boolean", NULL, NULL, NULL, 0}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -687,8 +692,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"mmap", "boolean", NULL, NULL, NULL, 0}, - {"multiprocess", "boolean", NULL, NULL, NULL, 0}, + {"mmap", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -710,7 +714,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -753,8 +757,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"mmap", "boolean", NULL, NULL, NULL, 0}, - {"multiprocess", "boolean", NULL, NULL, NULL, 0}, + {"mmap", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -776,7 +779,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -820,13 +823,13 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),io_capacity=(total=0),log=(archive=true," "os_cache_dirty_pct=0,prealloc=true,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," + "lsm_manager=(merge=true,worker_thread_max=4)," "operation_timeout_ms=0,operation_tracking=(enabled=false," "path=\".\"),shared_cache=(chunk=10MB,name=,quota=0,reserve=0," "size=500MB),statistics=none,statistics_log=(json=false," "on_close=false,sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,verbose=", - confchk_WT_CONNECTION_reconfigure, 27}, + confchk_WT_CONNECTION_reconfigure, 26}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", "commit_timestamp=,durable_timestamp=,force=false," @@ -887,10 +890,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.log_printf", "", NULL, 0}, {"WT_SESSION.open_cursor", "append=false,bulk=false,checkpoint=,checkpoint_wait=true,dump=," - "next_random=false,next_random_sample_size=0,overwrite=true," - "raw=false,read_once=false,readonly=false,skip_sort_check=false," - "statistics=,target=", - confchk_WT_SESSION_open_cursor, 14}, + "incremental=(enabled=false,file=,force_stop=false," + "granularity=16MB,src_id=,this_id=),next_random=false," + "next_random_sample_size=0,overwrite=true,raw=false," + "read_once=false,readonly=false,skip_sort_check=false,statistics=" + ",target=", + confchk_WT_SESSION_open_cursor, 15}, {"WT_SESSION.prepare_transaction", "prepare_timestamp=", confchk_WT_SESSION_prepare_transaction, 1}, {"WT_SESSION.query_timestamp", "get=read", confchk_WT_SESSION_query_timestamp, 1}, @@ -995,8 +1000,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "io_capacity=(total=0),log=(archive=true,compressor=," "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_timeout_ms=0," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "multiprocess=false,operation_timeout_ms=0," "operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," @@ -1005,7 +1010,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,write_through=", - confchk_wiredtiger_open, 51}, + confchk_wiredtiger_open, 50}, {"wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1025,8 +1030,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "io_capacity=(total=0),log=(archive=true,compressor=," "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_timeout_ms=0," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "multiprocess=false,operation_timeout_ms=0," "operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," @@ -1035,7 +1040,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 52}, + confchk_wiredtiger_open_all, 51}, {"wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1053,8 +1058,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "io_capacity=(total=0),log=(archive=true,compressor=," "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_timeout_ms=0," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "multiprocess=false,operation_timeout_ms=0," "operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," @@ -1062,7 +1067,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),verbose=,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 46}, + confchk_wiredtiger_open_basecfg, 45}, {"wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1080,8 +1085,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "io_capacity=(total=0),log=(archive=true,compressor=," "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_timeout_ms=0," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "multiprocess=false,operation_timeout_ms=0," "operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," @@ -1089,7 +1094,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),verbose=,write_through=", - confchk_wiredtiger_open_usercfg, 45}, + confchk_wiredtiger_open_usercfg, 44}, {NULL, NULL, NULL, 0}}; int diff --git a/src/third_party/wiredtiger/src/config/config_upgrade.c b/src/third_party/wiredtiger/src/config/config_upgrade.c deleted file mode 100644 index be67fa0c3f3..00000000000 --- a/src/third_party/wiredtiger/src/config/config_upgrade.c +++ /dev/null @@ -1,31 +0,0 @@ -/*- - * Copyright (c) 2014-2019 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -/* - * __wt_config_upgrade -- - * Upgrade a configuration string by appended the replacement version. - */ -int -__wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) -{ - WT_CONFIG_ITEM v; - const char *config; - - config = buf->data; - - /* - * wiredtiger_open: - * lsm_merge=boolean -> lsm_manager=(merge=boolean) - */ - if (__wt_config_getones(session, config, "lsm_merge", &v) != WT_NOTFOUND) - WT_RET(__wt_buf_catfmt(session, buf, ",lsm_manager=(merge=%s)", v.val ? "true" : "false")); - - return (0); -} diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 7e8a898193f..2b5f1fb7ae3 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1428,9 +1428,6 @@ __conn_config_file( /* Check any version. */ WT_ERR(__conn_config_check_version(session, cbuf->data)); - /* Upgrade the configuration string. */ - WT_ERR(__wt_config_upgrade(session, cbuf)); - /* Check the configuration information. */ WT_ERR(__wt_config_check(session, is_user ? WT_CONFIG_REF(session, wiredtiger_open_usercfg) : WT_CONFIG_REF(session, wiredtiger_open_basecfg), @@ -1519,7 +1516,6 @@ __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf) /* Upgrade the configuration string. */ WT_ERR(__wt_buf_setstr(session, cbuf, env_config)); - WT_ERR(__wt_config_upgrade(session, cbuf)); /* Check the configuration information. */ WT_ERR(__wt_config_check(session, WT_CONFIG_REF(session, wiredtiger_open), env_config, 0)); @@ -1825,14 +1821,15 @@ typedef struct { int __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { - static const WT_NAME_FLAG verbtypes[] = {{"api", WT_VERB_API}, {"block", WT_VERB_BLOCK}, - {"checkpoint", WT_VERB_CHECKPOINT}, {"checkpoint_progress", WT_VERB_CHECKPOINT_PROGRESS}, - {"compact", WT_VERB_COMPACT}, {"compact_progress", WT_VERB_COMPACT_PROGRESS}, - {"error_returns", WT_VERB_ERROR_RETURNS}, {"evict", WT_VERB_EVICT}, - {"evict_stuck", WT_VERB_EVICT_STUCK}, {"evictserver", WT_VERB_EVICTSERVER}, - {"fileops", WT_VERB_FILEOPS}, {"handleops", WT_VERB_HANDLEOPS}, {"log", WT_VERB_LOG}, - {"lookaside", WT_VERB_LOOKASIDE}, {"lookaside_activity", WT_VERB_LOOKASIDE_ACTIVITY}, - {"lsm", WT_VERB_LSM}, {"lsm_manager", WT_VERB_LSM_MANAGER}, {"metadata", WT_VERB_METADATA}, + static const WT_NAME_FLAG verbtypes[] = {{"api", WT_VERB_API}, {"backup", WT_VERB_BACKUP}, + {"block", WT_VERB_BLOCK}, {"checkpoint", WT_VERB_CHECKPOINT}, + {"checkpoint_progress", WT_VERB_CHECKPOINT_PROGRESS}, {"compact", WT_VERB_COMPACT}, + {"compact_progress", WT_VERB_COMPACT_PROGRESS}, {"error_returns", WT_VERB_ERROR_RETURNS}, + {"evict", WT_VERB_EVICT}, {"evict_stuck", WT_VERB_EVICT_STUCK}, + {"evictserver", WT_VERB_EVICTSERVER}, {"fileops", WT_VERB_FILEOPS}, + {"handleops", WT_VERB_HANDLEOPS}, {"log", WT_VERB_LOG}, {"lookaside", WT_VERB_LOOKASIDE}, + {"lookaside_activity", WT_VERB_LOOKASIDE_ACTIVITY}, {"lsm", WT_VERB_LSM}, + {"lsm_manager", WT_VERB_LSM_MANAGER}, {"metadata", WT_VERB_METADATA}, {"mutex", WT_VERB_MUTEX}, {"overflow", WT_VERB_OVERFLOW}, {"read", WT_VERB_READ}, {"rebalance", WT_VERB_REBALANCE}, {"reconcile", WT_VERB_RECONCILE}, {"recovery", WT_VERB_RECOVERY}, {"recovery_progress", WT_VERB_RECOVERY_PROGRESS}, diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index e5c82d49a48..37136926060 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -94,6 +94,7 @@ void __wt_connection_destroy(WT_CONNECTION_IMPL *conn) { WT_SESSION_IMPL *session; + u_int i; /* Check there's something to destroy. */ if (conn == NULL) @@ -131,6 +132,13 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_cond_destroy(session, &conn->lsm_manager.work_cond); /* Free allocated memory. */ + /* + * XXX we need to persist this information when we are working on making incremental backups + * persistent across restarts. + */ + for (i = 0; i < WT_BLKINCR_MAX; ++i) + __wt_free(session, conn->incr_backups[i].id_str); + __wt_free(session, conn->cfg); __wt_free(session, conn->debug_ckpt); __wt_free(session, conn->error_prefix); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 4869bcb3b71..cb7e11a64cc 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -11,9 +11,9 @@ static int __backup_all(WT_SESSION_IMPL *); static int __backup_list_append(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *); static int __backup_list_uri_append(WT_SESSION_IMPL *, const char *, bool *); -static int __backup_start(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, bool, const char *[]); +static int __backup_start( + WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, WT_CURSOR_BACKUP *, const char *[]); static int __backup_stop(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); -static int __backup_uri(WT_SESSION_IMPL *, const char *[], bool, bool *, bool *); /* * __curbackup_next -- @@ -28,6 +28,7 @@ __curbackup_next(WT_CURSOR *cursor) cb = (WT_CURSOR_BACKUP *)cursor; CURSOR_API_CALL(cursor, session, next, NULL); + WT_CURSOR_BACKUP_CHECK_STOP(cb); if (cb->list == NULL || cb->list[cb->next] == NULL) { F_CLR(cursor, WT_CURSTD_KEY_SET); @@ -57,6 +58,7 @@ __curbackup_reset(WT_CURSOR *cursor) cb = (WT_CURSOR_BACKUP *)cursor; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, reset, NULL); + WT_CURSOR_BACKUP_CHECK_STOP(cb); cb->next = 0; F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); @@ -66,6 +68,35 @@ err: } /* + * __backup_incr_release -- + * Free all resources relating to incremental backup. + */ +static int +__backup_incr_release(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool force) +{ + WT_BLKINCR *blk; + WT_CONNECTION_IMPL *conn; + u_int i; + + WT_UNUSED(cb); + WT_UNUSED(force); + conn = S2C(session); + /* + * Clear flags. Remove file. Release any memory information. + */ + F_CLR(conn, WT_CONN_INCR_BACKUP); + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk = &conn->incr_backups[i]; + F_CLR(blk, WT_BLKINCR_VALID); + } + /* __wt_block_backup_remove... */ + conn->ckpt_incr_granularity = 0; + WT_RET(__wt_remove_if_exists(session, WT_BLKINCR_BACKUP, true)); + + return (0); +} + +/* * __backup_free -- * Free list resources for a backup cursor. */ @@ -79,6 +110,13 @@ __backup_free(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) __wt_free(session, cb->list[i]); __wt_free(session, cb->list); } + if (cb->incr_file != NULL) + __wt_free(session, cb->incr_file); + if (cb->incr_src != NULL) + __wt_free(session, cb->incr_src); + if (cb->incr_this != NULL) + __wt_free(session, cb->incr_this); + __wt_curbackup_free_incr(session, cb); } /* @@ -96,6 +134,12 @@ __curbackup_close(WT_CURSOR *cursor) CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); err: + if (F_ISSET(cb, WT_CURBACKUP_FORCE_STOP)) { + __wt_verbose( + session, WT_VERB_BACKUP, "%s", "Releasing resources from forced stop incremental"); + __backup_incr_release(session, cb, true); + } + /* * When starting a hot backup, we serialize hot backup cursors and set the connection's * hot-backup flag. Once that's done, we set the cursor's backup-locker flag, implying the @@ -147,7 +191,7 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, __wt_cursor_reopen_notsup, /* reopen */ __curbackup_close); /* close */ WT_CURSOR *cursor; - WT_CURSOR_BACKUP *cb; + WT_CURSOR_BACKUP *cb, *othercb; WT_DECL_RET; WT_STATIC_ASSERT(offsetof(WT_CURSOR_BACKUP, iface) == 0); @@ -157,19 +201,23 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, *cursor = iface; cursor->session = (WT_SESSION *)session; cursor->key_format = "S"; /* Return the file names as the key. */ - cursor->value_format = ""; /* No value. */ + cursor->value_format = ""; /* No value, for now. */ session->bkp_cursor = cb; + othercb = (WT_CURSOR_BACKUP *)other; + if (othercb != NULL) + WT_CURSOR_BACKUP_CHECK_STOP(othercb); /* * Start the backup and fill in the cursor's list. Acquire the schema lock, we need a consistent * view when creating a copy. */ WT_WITH_CHECKPOINT_LOCK( - session, WT_WITH_SCHEMA_LOCK(session, ret = __backup_start(session, cb, other != NULL, cfg))); + session, WT_WITH_SCHEMA_LOCK(session, ret = __backup_start(session, cb, othercb, cfg))); WT_ERR(ret); - - WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); + WT_ERR(cb->incr_file == NULL ? + __wt_cursor_init(cursor, uri, NULL, cfg, cursorp) : + __wt_curbackup_open_incr(session, uri, other, cursor, cfg, cursorp)); if (0) { err: @@ -181,6 +229,105 @@ err: } /* + * __backup_get_ckpt -- + * Get the most recent checkpoint information and store it in the structure. + * + * XXX - Currently set return to static void for the compiler, when this function has real content + * it should be static int. + */ +static void +__backup_get_ckpt(WT_SESSION_IMPL *session, WT_BLKINCR *incr) +{ + WT_UNUSED(session); + WT_UNUSED(incr); + /* + * Look up the most recent checkpoint and store information about it in incr. + * + * XXX When this function has content, return a real value. return (0); + */ + return; +} + +/* + * __backup_add_id -- + * Add the identifier for block based incremental backup. + */ +static int +__backup_add_id(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval) +{ + WT_BLKINCR *blk; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + u_int i; + + conn = S2C(session); + blk = NULL; + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk = &conn->incr_backups[i]; + /* If it isn't use, we can use it. */ + if (!F_ISSET(blk, WT_BLKINCR_INUSE)) + break; + } + /* + * We didn't find an entry. This should not happen. + */ + if (i == WT_BLKINCR_MAX) + WT_PANIC_RET(session, WT_NOTFOUND, "Could not find an incremental backup slot to use"); + + /* Use the slot. */ + if (blk->id_str != NULL) + __wt_verbose( + session, WT_VERB_BACKUP, "Freeing and reusing backup slot with old id %s", blk->id_str); + /* Free any string that was there. */ + __wt_free(session, blk->id_str); + WT_ERR(__wt_strndup(session, cval->str, cval->len, &blk->id_str)); + __wt_verbose(session, WT_VERB_BACKUP, "Using backup slot %u for id %s", i, blk->id_str); + /* + * XXX This function can error in the future. + * + * WT_ERR(__backup_get_ckpt(session, blk)); + */ + __backup_get_ckpt(session, blk); + F_SET(blk, WT_BLKINCR_VALID); + return (0); + +err: + if (blk != NULL) + __wt_free(session, blk->id_str); + return (ret); +} + +/* + * __backup_find_id -- + * Find the source identifier for block based incremental backup. Error if it is not a valid id. + */ +static int +__backup_find_id(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_BLKINCR **incrp) +{ + WT_BLKINCR *blk; + WT_CONNECTION_IMPL *conn; + u_int i; + + conn = S2C(session); + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk = &conn->incr_backups[i]; + /* If it isn't valid, skip it. */ + if (!F_ISSET(blk, WT_BLKINCR_VALID)) + continue; + if (WT_STRING_MATCH(blk->id_str, cval->str, cval->len)) { + if (F_ISSET(blk, WT_BLKINCR_INUSE)) + WT_RET_MSG(session, EINVAL, "Incremental backup structure already in use"); + if (incrp != NULL) + *incrp = blk; + __wt_verbose(session, WT_VERB_BACKUP, "Found backup slot %u for id %s", i, blk->id_str); + return (0); + } + } + __wt_verbose(session, WT_VERB_BACKUP, "Did not find %.*s", (int)cval->len, cval->str); + return (WT_NOTFOUND); +} + +/* * __backup_log_append -- * Append log files needed for backup. */ @@ -208,21 +355,203 @@ err: } /* + * __backup_config -- + * Backup configuration. + */ +static int +__backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[], + WT_CURSOR_BACKUP *othercb, bool *foundp, bool *log_only, bool *incr_only) +{ + WT_CONFIG targetconf; + WT_CONFIG_ITEM cval, k, v; + WT_CONNECTION_IMPL *conn; + WT_DECL_ITEM(tmp); + WT_DECL_RET; + const char *uri; + bool incremental_config, is_dup, log_config, target_list; + + *foundp = *incr_only = *log_only = false; + + conn = S2C(session); + incremental_config = log_config = false; + is_dup = othercb != NULL; + + /* + * Per-file offset incremental hot backup configurations take a starting checkpoint and optional + * maximum transfer size, and the subsequent duplicate cursors take a file object. + */ + WT_RET_NOTFOUND_OK(__wt_config_gets(session, cfg, "incremental.force_stop", &cval)); + if (cval.val) { + /* + * If we're force stopping incremental backup, set the flag. The resources involved in + * incremental backup will be released on cursor close and that is the only expected usage + * for this cursor. + */ + if (is_dup) + WT_RET_MSG(session, EINVAL, + "Incremental force stop can only be specified on a primary backup cursor"); + F_SET(cb, WT_CURBACKUP_FORCE_STOP); + return (0); + } + WT_RET_NOTFOUND_OK(__wt_config_gets(session, cfg, "incremental.enabled", &cval)); + if (cval.val) { + if (!F_ISSET(conn, WT_CONN_INCR_BACKUP)) { + WT_RET(__wt_config_gets(session, cfg, "incremental.granularity", &cval)); + /* XXX may not need cb->incr_granularity */ + if (conn->ckpt_incr_granularity != 0) + WT_RET_MSG(session, EINVAL, "Cannot change the incremental backup granularity"); + conn->ckpt_incr_granularity = cb->incr_granularity = (uint64_t)cval.val; + } + /* XXX Granularity can only be set once at the beginning */ + F_SET(conn, WT_CONN_INCR_BACKUP); + incremental_config = true; + } + + /* + * Specifying an incremental file means we're opening a duplicate backup cursor. + */ + WT_RET(__wt_config_gets(session, cfg, "incremental.file", &cval)); + if (cval.len != 0) { + if (!is_dup) + WT_RET_MSG(session, EINVAL, + "Incremental file name can only be specified on a duplicate backup cursor"); + WT_RET(__wt_strndup(session, cval.str, cval.len, &cb->incr_file)); + incremental_config = true; + } + + /* + * See if we have a source identifier. We must process the source identifier before processing + * the 'this' identifier. That will mark which source is in use so that we can use any slot that + * is not in use as a new source starting point for this identifier. + */ + WT_RET(__wt_config_gets(session, cfg, "incremental.src_id", &cval)); + if (cval.len != 0) { + if (is_dup) + WT_RET_MSG(session, EINVAL, + "Incremental source identifier can only be specified on a primary backup cursor"); + WT_RET(__backup_find_id(session, &cval, &cb->incr)); + /* XXX might not need this incr_src field */ + WT_RET(__wt_strndup(session, cval.str, cval.len, &cb->incr_src)); + F_SET(cb->incr, WT_BLKINCR_INUSE); + incremental_config = true; + } + /* + * Use WT_ERR from here out because we need to clear the in use flag on error now. + */ + + /* + * Look for a new checkpoint name to retain and mark as a starting point. + */ + WT_ERR(__wt_config_gets(session, cfg, "incremental.this_id", &cval)); + if (cval.len != 0) { + if (is_dup) + WT_ERR_MSG(session, EINVAL, + "Incremental identifier can only be specified on a primary backup cursor"); + ret = __backup_find_id(session, &cval, NULL); + if (ret != WT_NOTFOUND) + WT_ERR_MSG(session, EINVAL, "Incremental identifier already exists"); + + WT_ERR(__backup_add_id(session, &cval)); + /* XXX might not need this incr_this field */ + WT_ERR(__wt_strndup(session, cval.str, cval.len, &cb->incr_this)); + incremental_config = true; + } + + /* + * If we find a non-empty target configuration string, we have a job, otherwise it's not our + * problem. + */ + WT_ERR(__wt_config_gets(session, cfg, "target", &cval)); + __wt_config_subinit(session, &targetconf, &cval); + for (target_list = false; (ret = __wt_config_next(&targetconf, &k, &v)) == 0; + target_list = true) { + /* If it is our first time through, allocate. */ + if (!target_list) { + *foundp = true; + WT_ERR(__wt_scr_alloc(session, 512, &tmp)); + } + + WT_ERR(__wt_buf_fmt(session, tmp, "%.*s", (int)k.len, k.str)); + uri = tmp->data; + if (v.len != 0) + WT_ERR_MSG(session, EINVAL, "%s: invalid backup target: URIs may need quoting", uri); + + /* + * Handle log targets. We do not need to go through the schema worker, just call the + * function to append them. Set log_only only if it is our only URI target. + */ + if (WT_PREFIX_MATCH(uri, "log:")) { + log_config = true; + *log_only = !target_list; + WT_ERR(__backup_log_append(session, session->bkp_cursor, false)); + } else if (is_dup) + WT_ERR_MSG( + session, EINVAL, "duplicate backup cursor cannot be used for non-log target"); + else { + *log_only = false; + + /* + * If backing up individual tables, we have to include indexes, which may involve + * opening those indexes. Acquire the table lock in write mode for that case. + */ + WT_WITH_TABLE_WRITE_LOCK(session, + ret = __wt_schema_worker(session, uri, NULL, __backup_list_uri_append, cfg, 0)); + WT_ERR(ret); + } + } + WT_ERR_NOTFOUND_OK(ret); + + /* + * Compatibility checking. + * + * Log archive cannot mix with log-file based incremental backups (but if a duplicate cursor, + * archiving has been temporarily suspended). + * + * Duplicate backup cursors are only for log targets or block-based incremental backups. But log + * targets don't make sense with block-based incremental backup. + */ + if (!is_dup && log_config && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE)) + WT_ERR_MSG(session, EINVAL, + "incremental log file backup not possible when automatic log archival configured"); + if (is_dup && (!incremental_config && !log_config)) + WT_ERR_MSG(session, EINVAL, + "duplicate backup cursor must be for block-based incremental or logging backup"); + if (incremental_config && (log_config || target_list)) + WT_ERR_MSG( + session, EINVAL, "block-based incremental backup incompatible with a list of targets"); + + if (incremental_config) { + if (is_dup && !F_ISSET(othercb, WT_CURBACKUP_INCR)) + WT_ERR_MSG(session, EINVAL, + "Incremental duplicate cursor must have an incremental primary backup cursor"); + F_SET(cb, WT_CURBACKUP_INCR); + } +err: + if (ret != 0 && cb->incr != NULL) + F_CLR(cb->incr, WT_BLKINCR_INUSE); + __wt_scr_free(session, &tmp); + return (ret); +} + +/* * __backup_start -- * Start a backup. */ static int -__backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, const char *cfg[]) +__backup_start( + WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, WT_CURSOR_BACKUP *othercb, const char *cfg[]) { + WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_FSTREAM *srcfs; const char *dest; - bool exist, log_only, target_list; + bool exist, is_dup, is_incr, log_only, target_list; conn = S2C(session); srcfs = NULL; dest = NULL; + is_dup = othercb != NULL; cb->next = 0; cb->list = NULL; @@ -240,6 +569,21 @@ __backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, cons if (F_ISSET(session, WT_SESSION_BACKUP_DUP) && is_dup) WT_RET_MSG(session, EINVAL, "there is already a duplicate backup cursor open"); + /* + * We want to check for forced stopping early before we do anything else. If it is set, we just + * set a flag and we're done. Actions will be performed on cursor close. + */ + WT_RET_NOTFOUND_OK(__wt_config_gets(session, cfg, "incremental.force_stop", &cval)); + if (cval.val) { + /* + * If we're force stopping incremental backup, set the flag. The resources involved in + * incremental backup will be released on cursor close and that is the only expected usage + * for this cursor. + */ + F_SET(cb, WT_CURBACKUP_FORCE_STOP); + return (0); + } + if (!is_dup) { /* * The hot backup copy is done outside of WiredTiger, which means file blocks can't be freed @@ -260,10 +604,10 @@ __backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, cons /* * Create a temporary backup file. This must be opened before generating the list of targets - * in backup_uri. This file will later be renamed to the correct name depending on whether - * or not we're doing an incremental backup. We need a temp file so that if we fail or crash - * while filling it, the existence of a partial file doesn't confuse restarting in the - * source database. + * in backup_config. This file will later be renamed to the correct name depending on + * whether or not we're doing an incremental backup. We need a temp file so that if we fail + * or crash while filling it, the existence of a partial file doesn't confuse restarting in + * the source database. */ WT_ERR(__wt_fopen(session, WT_BACKUP_TMP, WT_FS_OPEN_CREATE, WT_STREAM_WRITE, &cb->bfs)); } @@ -273,14 +617,11 @@ __backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, cons * database objects and log files to the list. */ target_list = false; - WT_ERR(__backup_uri(session, cfg, is_dup, &target_list, &log_only)); + WT_ERR(__backup_config(session, cb, cfg, othercb, &target_list, &log_only, &is_incr)); /* - * For a duplicate cursor, all the work is done in backup_uri. The only usage accepted is - * "target=("log:")" so error if not log only. + * For a duplicate cursor, all the work is done in backup_config. */ if (is_dup) { - if (!log_only) - WT_ERR_MSG(session, EINVAL, "duplicate backup cursor must be for logs only"); F_SET(cb, WT_CURBACKUP_DUP); F_SET(session, WT_SESSION_BACKUP_DUP); goto done; @@ -317,12 +658,12 @@ __backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, cons * We also open an incremental backup source file so that we can detect a crash with an * incremental backup existing in the source directory versus an improper destination. */ - dest = WT_INCREMENTAL_BACKUP; - WT_ERR(__wt_fopen(session, WT_INCREMENTAL_SRC, WT_FS_OPEN_CREATE, WT_STREAM_WRITE, &srcfs)); - WT_ERR(__backup_list_append(session, cb, WT_INCREMENTAL_BACKUP)); + dest = WT_LOGINCR_BACKUP; + WT_ERR(__wt_fopen(session, WT_LOGINCR_SRC, WT_FS_OPEN_CREATE, WT_STREAM_WRITE, &srcfs)); + WT_ERR(__backup_list_append(session, cb, dest)); } else { dest = WT_METADATA_BACKUP; - WT_ERR(__backup_list_append(session, cb, WT_METADATA_BACKUP)); + WT_ERR(__backup_list_append(session, cb, dest)); WT_ERR(__wt_fs_exist(session, WT_BASECONFIG, &exist)); if (exist) WT_ERR(__backup_list_append(session, cb, WT_BASECONFIG)); @@ -372,6 +713,8 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) /* If it's not a dup backup cursor, make sure one isn't open. */ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_BACKUP_DUP)); WT_WITH_HOTBACKUP_WRITE_LOCK(session, conn->hot_backup_list = NULL); + if (cb->incr != NULL) + F_CLR(cb->incr, WT_BLKINCR_INUSE); __backup_free(session, cb); /* Remove any backup specific file. */ @@ -396,75 +739,6 @@ __backup_all(WT_SESSION_IMPL *session) } /* - * __backup_uri -- - * Backup a list of objects. - */ -static int -__backup_uri(WT_SESSION_IMPL *session, const char *cfg[], bool is_dup, bool *foundp, bool *log_only) -{ - WT_CONFIG targetconf; - WT_CONFIG_ITEM cval, k, v; - WT_DECL_ITEM(tmp); - WT_DECL_RET; - const char *uri; - bool target_list; - - *foundp = *log_only = false; - - /* - * If we find a non-empty target configuration string, we have a job, otherwise it's not our - * problem. - */ - WT_RET(__wt_config_gets(session, cfg, "target", &cval)); - __wt_config_subinit(session, &targetconf, &cval); - for (target_list = false; (ret = __wt_config_next(&targetconf, &k, &v)) == 0; - target_list = true) { - /* If it is our first time through, allocate. */ - if (!target_list) { - *foundp = true; - WT_ERR(__wt_scr_alloc(session, 512, &tmp)); - } - - WT_ERR(__wt_buf_fmt(session, tmp, "%.*s", (int)k.len, k.str)); - uri = tmp->data; - if (v.len != 0) - WT_ERR_MSG(session, EINVAL, "%s: invalid backup target: URIs may need quoting", uri); - - /* - * Handle log targets. We do not need to go through the schema worker, just call the - * function to append them. Set log_only only if it is our only URI target. - */ - if (WT_PREFIX_MATCH(uri, "log:")) { - /* - * Log archive cannot mix with incremental backup, don't let that happen. If we're a - * duplicate cursor archiving is already temporarily suspended. - */ - if (!is_dup && FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ARCHIVE)) - WT_ERR_MSG(session, EINVAL, - "incremental backup not possible when " - "automatic log archival configured"); - *log_only = !target_list; - WT_ERR(__backup_log_append(session, session->bkp_cursor, false)); - } else { - *log_only = false; - - /* - * If backing up individual tables, we have to include indexes, which may involve - * opening those indexes. Acquire the table lock in write mode for that case. - */ - WT_WITH_TABLE_WRITE_LOCK(session, - ret = __wt_schema_worker(session, uri, NULL, __backup_list_uri_append, cfg, 0)); - WT_ERR(ret); - } - } - WT_ERR_NOTFOUND_OK(ret); - -err: - __wt_scr_free(session, &tmp); - return (ret); -} - -/* * __wt_backup_file_remove -- * Remove the incremental and meta-data backup files. */ @@ -479,8 +753,8 @@ __wt_backup_file_remove(WT_SESSION_IMPL *session) * there's any chance of an incremental backup file existing. */ WT_TRET(__wt_remove_if_exists(session, WT_BACKUP_TMP, true)); - WT_TRET(__wt_remove_if_exists(session, WT_INCREMENTAL_BACKUP, true)); - WT_TRET(__wt_remove_if_exists(session, WT_INCREMENTAL_SRC, true)); + WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_BACKUP, true)); + WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_SRC, true)); WT_TRET(__wt_remove_if_exists(session, WT_METADATA_BACKUP, true)); return (ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c new file mode 100644 index 00000000000..c8c59d3f9db --- /dev/null +++ b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c @@ -0,0 +1,233 @@ +/*- + * Copyright (c) 2014-2019 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __alloc_merge -- + * Merge two allocation lists. + */ +static void +__alloc_merge( + uint64_t *a, uint64_t a_cnt, uint64_t *b, uint64_t b_cnt, uint64_t *res, uint64_t *res_cnt) +{ + uint64_t total; + + for (total = 0; a_cnt > 0 || b_cnt > 0; ++total, res += 2) { + if (a_cnt > 0 && b_cnt > 0) { + if (a[0] <= b[0]) { + res[0] = a[0]; + if (a[0] + a[1] < b[0]) + res[1] = a[1]; + else { + res[1] = (b[0] + b[1]) - a[0]; + b += 2; + --b_cnt; + } + a += 2; + --a_cnt; + } else if (b[0] <= a[0]) { + res[0] = b[0]; + if (b[0] + b[1] < a[0]) + res[1] = b[1]; + else { + res[1] = (a[0] + a[1]) - b[0]; + a += 2; + --a_cnt; + } + b += 2; + --b_cnt; + } + } else if (a_cnt > 0) { + res[0] = a[0]; + res[1] = a[1]; + a += 2; + --a_cnt; + } else if (b_cnt > 0) { + res[0] = b[0]; + res[1] = b[1]; + b += 2; + --b_cnt; + } + } + *res_cnt = total; +} + +/* + * __curbackup_incr_next -- + * WT_CURSOR->next method for the btree cursor type when configured with incremental_backup. + */ +static int +__curbackup_incr_next(WT_CURSOR *cursor) +{ + WT_BTREE *btree; + WT_CKPT *ckpt, *ckptbase; + WT_CURSOR_BACKUP *cb; + WT_DECL_RET; + WT_SESSION_IMPL *session; + wt_off_t size; + uint64_t *a, *b, *current, *next; + uint64_t entries, total; + uint32_t raw; + bool start, stop; + + ckptbase = NULL; + a = b = NULL; + + cb = (WT_CURSOR_BACKUP *)cursor; + btree = cb->incr_cursor == NULL ? NULL : ((WT_CURSOR_BTREE *)cb->incr_cursor)->btree; + raw = F_MASK(cursor, WT_CURSTD_RAW); + CURSOR_API_CALL(cursor, session, get_value, btree); + F_CLR(cursor, WT_CURSTD_RAW); + + if (cb->incr_init) { + /* We have this object's incremental information, Check if we're done. */ + if (cb->incr_list_offset >= cb->incr_list_count - WT_BACKUP_INCR_COMPONENTS) + return (WT_NOTFOUND); + + /* + * If we returned all of the data, step to the next block, otherwise return the next chunk + * of the current block. + */ + if (cb->incr_list[cb->incr_list_offset + 1] <= cb->incr_granularity) + cb->incr_list_offset += WT_BACKUP_INCR_COMPONENTS; + else { + cb->incr_list[cb->incr_list_offset] += cb->incr_granularity; + cb->incr_list[cb->incr_list_offset + 1] -= cb->incr_granularity; + cb->incr_list[cb->incr_list_offset + 2] = WT_BACKUP_RANGE; + } + } else if (btree == NULL) { + /* We don't have this object's incremental information, and it's a full file copy. */ + WT_ERR(__wt_fs_size(session, cb->incr_file, &size)); + + cb->incr_list_count = WT_BACKUP_INCR_COMPONENTS; + cb->incr_init = true; + cb->incr_list_offset = 0; + __wt_cursor_set_key(cursor, 0, size, WT_BACKUP_FILE); + } else { + /* + * We don't have this object's incremental information, and it's not a full file copy. Get a + * list of the checkpoints available for the file and flag the starting/stopping ones. It + * shouldn't be possible to specify checkpoints that no longer exist, but check anyway. + */ + ret = __wt_meta_ckptlist_get(session, cb->incr_file, false, &ckptbase); + WT_ERR(ret == WT_NOTFOUND ? ENOENT : ret); + + /* + * Count up the maximum number of block entries we might have to merge, and allocate a pair + * of temporary arrays in which to do the merge. + */ + entries = 0; + WT_CKPT_FOREACH (ckptbase, ckpt) + entries += ckpt->alloc_list_entries; + WT_ERR(__wt_calloc_def(session, entries * WT_BACKUP_INCR_COMPONENTS, &a)); + WT_ERR(__wt_calloc_def(session, entries * WT_BACKUP_INCR_COMPONENTS, &b)); + + /* Merge the block lists into a final list of blocks to copy. */ + start = stop = false; + total = 0; + current = NULL; + next = a; + WT_CKPT_FOREACH (ckptbase, ckpt) { + if (strcmp(ckpt->name, cb->incr_checkpoint_start) == 0) { + start = true; + WT_ERR_ASSERT(session, ckpt->alloc_list_entries == 0, __wt_panic(session), + "incremental backup start checkpoint has allocation list blocks"); + continue; + } + if (start == true) { + if (strcmp(ckpt->name, cb->incr_checkpoint_stop) == 0) + stop = true; + + __alloc_merge( + current, total, ckpt->alloc_list, ckpt->alloc_list_entries, next, &total); + current = next; + next = next == a ? b : a; + } + + if (stop == true) + break; + } + + if (!start) + WT_ERR_MSG(session, ENOENT, "incremental backup start checkpoint %s not found", + cb->incr_checkpoint_start); + if (!stop) + WT_ERR_MSG(session, ENOENT, "incremental backup stop checkpoint %s not found", + cb->incr_checkpoint_stop); + + /* There may be nothing that needs copying. */ + if (total == 0) + WT_ERR(WT_NOTFOUND); + + if (next == a) { + cb->incr_list = b; + b = NULL; + } else { + cb->incr_list = a; + a = NULL; + } + cb->incr_list_count = total; + cb->incr_list_offset = 0; + WT_ERR(__wt_scr_alloc(session, 0, &cb->incr_block)); + cb->incr_init = true; + + F_SET(cursor, WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT); + } + +err: + __wt_free(session, a); + __wt_free(session, b); + __wt_meta_ckptlist_free(session, &ckptbase); + F_SET(cursor, raw); + API_END_RET(session, ret); +} + +/* + * __wt_curbackup_free_incr -- + * Free the duplicate backup cursor for a file-based incremental backup. + */ +void +__wt_curbackup_free_incr(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) +{ + __wt_free(session, cb->incr_file); + if (cb->incr_cursor != NULL) + __wt_cursor_close(cb->incr_cursor); + __wt_free(session, cb->incr_checkpoint_start); + __wt_free(session, cb->incr_checkpoint_stop); + __wt_free(session, cb->incr_list); + __wt_scr_free(session, &cb->incr_block); +} + +/* + * __wt_curbackup_open_incr -- + * Initialize the duplicate backup cursor for a file-based incremental backup. + */ +int +__wt_curbackup_open_incr(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, + WT_CURSOR *cursor, const char *cfg[], WT_CURSOR **cursorp) +{ + WT_CURSOR_BACKUP *cb, *other_cb; + + cb = (WT_CURSOR_BACKUP *)cursor; + other_cb = (WT_CURSOR_BACKUP *)other; + WT_UNUSED(session); + cursor->key_format = WT_UNCHECKED_STRING(qqq); + cursor->value_format = ""; + + /* + * Inherit from the backup cursor but reset specific functions for incremental. + */ + cursor->next = __curbackup_incr_next; + cursor->get_key = __wt_cursor_get_key; + cursor->get_value = __wt_cursor_get_value_notsup; + cb->incr_granularity = other_cb->incr_granularity; + + /* XXX Return full file info for all files for now. */ + return (__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); +} diff --git a/src/third_party/wiredtiger/src/docs/devdoc-index.dox b/src/third_party/wiredtiger/src/docs/devdoc-index.dox index 7ada556aa1a..d88826cb52d 100644 --- a/src/third_party/wiredtiger/src/docs/devdoc-index.dox +++ b/src/third_party/wiredtiger/src/docs/devdoc-index.dox @@ -1,13 +1,12 @@ /*! @page devdoc-index Developer Documentation +@subpage devdoc-schema + Most applications begin to make use of WiredTiger by creating a table (or other data object) to store their data in. Create is one of several schema operations available in WiredTiger. -For more information on how schema operations are implemented in WiredTiger, -see: - -- @subpage devdoc-schema +@subpage devdoc-dhandle-lifecycle An internal structure called Data Handle (dhandle) is used to represent and access a table in WiredTiger. A dhandle gets created when a table is accessed @@ -15,8 +14,32 @@ for the first time. It is kept in a global list and is shared across the sessions. When a dhandle is not needed anymore and has been idle for a while, it is closed and destroyed, releasing all the resources associated with it. -For more information on the lifecycle of a dhandle, see: +@subpage devdoc-statistics + +WiredTiger can generate statistics that are useful for providing information +necessary when performance tuning your WiredTiger application. Here we focus +on analyzing and reviewing the data generated by the statistics logging +functionality. + +@subpage devdoc-optrack + +The purpose of operation tracking is to visualize WiredTiger's execution so +that correlations between performance anomalies are easily spotted. This +operation tracking tutorial provides a general overview of operation tracking +and describes ways to visualize the data in fine detail. + +@subpage devdoc-perf + +Linux `perf` is a tool that allows counting and sampling of various events in +the hardware and in the kernel. Hardware events are available via performance +monitoring units (PMU); they measure CPU cycles, cache misses, branches, etc. +Kernel events include scheduling context switches, page faults and block I/O. +Here we provide a quick cheat sheet of how to use `perf` with WiredTiger. + +@subpage devdoc-perf-flamegraphs -- @subpage devdoc-dhandle-lifecycle +Why is my CPU busy? FlameGraphs help visually summarize on-CPU call stacks and +allow for the quick identification of hot code paths. Here we explain how to +generate FlameGraphs from WiredTiger `perf` data. */ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox b/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox new file mode 100644 index 00000000000..360b3b6de47 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox @@ -0,0 +1,351 @@ +/*! @page devdoc-optrack Operation Tracking + +# Overview + +This tutorial will walk you through using operation tracking on one of `wtperf` +workloads: from preparing the workload for most effective data collection, to +gathering, visualizing and interpreting the execution logs. + +## Why use operation tracking? + +Analysis of WiredTiger JIRA tickets revealed that in diagnosing performance +bugs engineers rely mostly on information that shows which functions were +executed over time and across threads. They seek to find correlations between +performance anomalies, such as long latencies or periods of low throughput, and +events on these timelines. The purpose of operation tracking is to visualize +the execution so that these correlations could be easily spotted. + +## How does operation tracking work? + +Operation tracking allows you to put a pair of macros `WT_TRACK_OP_INIT` and +`WT_TRACK_OP_END` into any function in WiredTiger whose invocation time you +want to be measured and recorded. As your program runs, the `WT_TRACK_OP_INIT` +macro will produce the function entry timestamp and `WT_TRACK_OP_END` macro +will produce the function exit timestamp. These timestamps, along with the +function name, will be recorded in a set of log files. It is typical to put the +`WT_TRACK_OP_INIT` in the beginning of the function and the `WT_TRACK_OP_END` +macro at every exit point from the function, but you can put them anywhere +within the function body. The only condition required in order to correctly +visualize the logs is that each `WT_TRACK_OP_INIT` has a matching +`WT_TRACK_OP_END`. It is totally okay to insert tracking macros into nested +functions. + +## Will operation tracking introduce runtime overhead? + +The answer depends on your workload, and how many functions you track. For +`wtperf` suite the largest overhead of 30% was observed on the CPU-bound +workload `small-btree`. For I/O-bound workloads, such as +`evict-btree-stress-multi`, the overhead is negligible, unless you decided to +track short frequently executed functions. + +# Prerequisites + +## Prepare your workload + +Modify your benchmark so that it produces the desired behavior with as little +running time as possible. The longer the benchmark runs the more data it will +produce, and the more difficult it will be for you to go over all this data. + +In our running example with the `evict-btree-stress-multi.wtperf` workload, we +separate the database creation phase from the work phase. We first create the +database with operation tracking disabled. Then we enable operation tracking +and run the work phase. So operation tracking logs are generated during the +work phase only. We also reduce the work phase to 30 seconds, to make sure that +the volume of information is manageable. + +## Install the python modules + +To generate data visualizations, you will need the python modules bokeh, +matplotlib and pandas installed. You can install them via the following +commands (if you don't use pip modify for your python module manager): + + pip install bokeh matplotlib pandas + +## Enable operation tracking +To enable operation tracking, you need to add the following option to your +connection configuration string. + + operation_tracking=(enabled=true,path=.) + +For example, if you are running `wtperf`, you would add it to the `conn_config` +string: + + ./wtperf -c conn_config="operation_tracking=(enabled=true,path=.)" + +The "path" argument determines where the log files, produced by operation +tracking, will be stored. Now you can run your workload. + +# Working with log files + +## Becoming familiar with the operation tracking log files + +After the run is complete, the path directory will have a bunch of files of the +following format: + + optrack.00000<pid>.000000000<sid> + +and the file + + optrack-map.00000<pid> + +where `pid` is the process ID and `sid` is the session ID. + +The first kind of files are the actual log files. The second is the map file. +Log files are generated in the binary format, because this is a lot more +efficient than writing strings, and the map file is needed to decode them. + +## Processing the log files + +Now the binary log files need to be converted into the text format. To do that +use the script `wt_optrack_decode.py` in the WiredTiger tree. We will refer to +the path to your WiredTiger tree as WT. Suppose that the process ID that +generated the operation tracking files is 25660. Then you'd run the decode +script like so: + + % WT/tools/optrack/wt_optrack_decode.py -m optrack-map.0000025660 optrack.0000025660.00000000* + +As the script runs you will see lots of output on the screen reporting the +progress through the parsing process. One kind of output you might see is +something like this: + + TSC_NSEC ratio parsed: 2.8000 + +This refers to the clock cycles to nanoseconds ratio encoded into the log +files. In the example above, we had 2.8 ticks per nanosecond, suggesting that +the workload was run on a 2.8 GHz processor. This ratio is used later to +convert the timestamps, which are recorded in clock cycles for efficiency, to +nanoseconds. + +In the end, you should see lots of output files whose names look like the +optrack log files, but with the suffixes `-internal.txt` and `-external.txt`. +The "internal" files are the log files for WT internal sessions (such as +eviction threads). The "external" files are for the sessions created by the +client application. + +Now that the files have been decoded, it's time to visualize the data. This can +be done using the script find-latency-spikes.py located in the `tools/optrack` +directory of the WiredTiger tree. To process your text files from our example, +you would run this script like so: + + % WT/tools/optrack/find-latency-spikes.py optrack.0000025660.00000000*.txt + +As the script runs, you will probably see messages similar to this one: + + Processing file optrack.0000025660.0000000026-external.txt + Your data may have errors. Check the file .optrack.0000025660.0000000026-external.txt.log for details. + +This means that as the script was processing the log file, it found some +inconsistencies, for example there was a function entry timestamp, but no +corresponding function exit, or vice versa. Sometimes these errors are benign. +For example, the log file will never have an exit timestamp from a +session-closing function, because operation tracking is torn down as the +session closes. In other cases, an error may indicate that there is a mistake +in how the operation-tracking macros were inserted. For example, the programmer +may have forgotten to insert a macro where there is an early return from a +function. Examining the log file mentioned in the error message will give you +more details on the error. + +The data processing script will not fail or exit if it encounters these error +messages, it will proceed despite them, attempting to visualize the portion of +the log that did not contain the errors. The function timestamps implicated in +the errors will simply be dropped. + +At the very end of the processing, you will see messages like this: + + Normalizing data... + Generating timeline charts... 99% complete + Generating outlier histograms... 100% complete + +This is how you know the processing has completed. The script takes a while to +run, but it is parallelizable, so that is something that can be done in the +future to speed it up. + +# Looking at the data: Part 1 + +In the directory where you ran the data processing script, you will have the +file `WT-outliers.html` and the directory `BUCKET-FILES` to which that HTML +file refers. If you would like to look at the visualization on another machine, +transfer both the HTML file and the directory to that machine. Otherwise, just +open `WT-outliers.html` locally in your browser. + +To get an idea of what kind of output you will see, take a look at [this +example data](http://www.ece.ubc.ca/~sasha/TUTORIAL-DEMO/) generated by the +workload described earlier in this tutorial. Please open the URL, because we +will now walk over what it shows. + +The main page shows a number of outlier histograms. Each histogram corresponds +to a function that was tracked by the operation tracking system in WT. The +x-axis shows the execution timeline (in nanoseconds). The y-axis shows how many +abnormally long executions of the function occurred during very period of the +execution. + + + +You can click on outlier bars and look at the detailed visualization of the +period during which the abnormally long function invocations occurred. But +before we do that, let's talk about what is considered an "outlier" or an +"abnormally long execution" and how to control these thresholds. As annotations +on the histograms show, by default a function invocation is considered an +outlier if its duration is longer than two standard deviations above average. +(The average is computed for this function over the entire execution). If you +would like a more precise control of outlier thresholds, you can do that in a +configuration file that is read by the execution script. + +Before we talk about other visualization screens, let's actually generate a +visualization with some more meaningful outlier thresholds than two standard +deviations. This way, it will be easier to navigate the data, and we will be +able to observe some very interesting behavior that gives us a clue as to why +eviction is sometimes slow. (If you can't wait to find out, skip over the next +step.) + +## Getting a more precise control of the outlier thresholds + +Let's learn how to configure custom outlier thresholds. The easiest way to +begin is to grab a sample configuration file and edit it to your needs: + + % cp WT/tools/optrack/sample.config . + +Here is how that file looks: + + # Units in which timestamps are recorded + 1000000000 + # Outlier threshold for individual functions + __curfile_reset 100 ms + __curfile_search 50 ms + __curfile_update 3 ms + __evict_lru_pages 75 ms + __evict_lru_walk 20 ms + __evict_page 75 ms + __evict_walk 20 ms + __wt_cache_eviction_worker 10 ms + __wt_cond_wait_signal 5 s + __wt_cursor_set_valuev 10 ms + +In that file all lines prefixed with a # are comments. + +The first non-comment line specifies the units in which your timestamps are +recorded. The units are specified by providing a value indicating how many time +units there are in a second. By default operation scripts generate timestamps +in nanoseconds, hence you see a value of 1000000000 on the first line in our +sample file. If your time units were, say, milliseconds, you'd change that +value to 1000, and so on. + +The remaining non-comment lines in the file provide the outlier thresholds for +any functions you care to have a custom threshold. The format is: + + <function name> <value> <unit> + +For example, to specify a threshold of 15 milliseconds for the +`__wt_cache_eviction_worker` function, you'd insert the following line: + + __wt_cache_eviction_worker 15 ms + +Valid units are nanoseconds (ns), microseconds (us), milliseconds (ms) and +seconds (s). + +If you don't specify a threshold for a function the default of two standard +deviations will be used. Let's actually configure some more meaningful outlier +thresholds in our `sample.config` file and re-run the visualization script. Here +is the modified `sample.config`: + + # Units in which timestamps are recorded + 1000000000 + # Outlier threshold for individual functions + __curfile_search 30 ms + __evict_lru_pages 250 ms + __evict_lru_walk 1 ms + __evict_page 15 ms + __evict_walk 1 ms + __wt_cache_eviction_worker 15 ms + __wt_cond_wait_signal 10 s + +We rerun the data processing script with the configuration file argument, like +so: + + % WT/tools/optrack/find-latency-spikes.py -c sample.config optrack.0000025660.00000000*.txt + +The new charts can be accessed +[here](http://www.ece.ubc.ca/~sasha/TUTORIAL-DEMO-CONFIG/). + +# Looking at the data: Part 2 + +Let's examine the visualized data in more detail. Please click on [this +URL](http://www.ece.ubc.ca/~sasha/TUTORIAL-DEMO-CONFIG/): + +Although I insert the screenshots here, you will have more fun if you access it +directly. + +Now you can see the outlier charts complying with the configuration parameters +we supplied. For example, on the chart for the `__wt_cache_eviction_worker`, we +see only the intervals where that function took longer than 15 ms to complete. + + + +Let's click on one of those intervals to examine what happened there. I am +going to click on the tallest bar in the chart, which will take me to a +detailed output of function call activity by thread for interval 137 at [this +URL](http://www.ece.ubc.ca/~sasha/TUTORIAL-DEMO-CONFIG/BUCKET-FILES/bucket-137.html). + +There is a lot of graphics there, so the URL might take a few seconds to load. + +At the very top of the page you see a small navigation bar. The bar tells you +where you are in the execution (the red marker). By clicking on other intervals +within the bar you can navigate to other intervals. For example, if you wanted +to look at the execution interval located after the current one, you will just +click on the white bar following the current, red-highlighted, bar. + + + +Next you see a legend that shows all functions that were called during this +execution interval and their corresponding colours. + + + +Then you will see the most interesting information: function calls across +threads that occurred during this period. Durations and hierarchy of function +calls is preserved, meaning that longer functions will be shown with longer +bars and if function A called function B, B will be shown on top of A. + +By hovering over the colored bars corresponding to functions, you will see a +box appear telling you the name of the function, how long it took and its +original timestamp in your operation tracking log file. This way, if you are +suspicious about the correctness of your log or want to go back to it for some +reason, you know what to look for. + +In our example visualization, if we scroll down to the function sequences for +external threads, we will quickly see a few instances where +__wt_cache_eviction_worker took longer than 15 ms, for example here: + + + +As we can see, the threads are simply waiting on the condition variable inside +the eviction worker. To try and understand why, we might want to scroll up and +look at the activity of internal threads during this period. Interestingly +enough, most of the internal threads are also waiting on the condition variable +during this period. + + + +Looking at the internal thread with session id 1 shows something suspicious. + + + +During this period where all threads are waiting, this thread shows no activity +at all, whereas prior and after that "dead" period it was making regular calls +to `__evict_walk`. Perhaps that thread was unscheduled or stopped calling +__evict_walk for other reasons. Perhaps other threads were dependent on work +performed within `__evict_walk` to make forward progress. + +To better understand why `__evict_walk` was interrupted for almost 10ms we +might want to go back and insert operation tracking macros inside the functions +called by `__evict_walk`, to see if the thread was doing some other work during +this time or was simply unscheduled. + +# Summary + +Operation tracking is great for debugging performance problems where you are +interested in understanding the activity of the workload over time and across +threads. To effectively navigate large quantities of data it is best to +reproduce your performance problem in a short run. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-perf-flamegraphs.dox b/src/third_party/wiredtiger/src/docs/devdoc-perf-flamegraphs.dox new file mode 100644 index 00000000000..1e5bc1ad4fd --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-perf-flamegraphs.dox @@ -0,0 +1,74 @@ +/*! @page devdoc-perf-flamegraphs CPU Flame Graphs + +# Introduction + +[FlameGraphs](http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html), +invented by [Brendan Gregg](http://www.brendangregg.com), help visually +summarize on-CPU call stacks. + +Below is an image of a FlameGraph generated from a 10-second run of the +evict-btree-stress-multi wtperf job: + + + +To see it in action, download the non-redacted interactive version +[here](https://github.com/wiredtiger/wiredtiger/wiki/images/FlameGraph-evict-stress-multi.svg) +and open it in your web browser. + +Quoting from [Brendan Gregg's +website](http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html), here is +how to interpret the output of the FlameGraph: + +- Each box represents a function in the stack (a "stack frame"). +- The y-axis shows stack depth (number of frames on the stack). The top box + shows the function that was on-CPU. Everything beneath that is ancestry. The + function beneath a function is its parent, just like the stack traces shown + earlier. (Some flame graph implementations prefer to invert the order and use + an "icicle layout", so flames look upside down.) +- The x-axis spans the sample population. It does not show the passing of time + from left to right, as most graphs do. The left to right ordering has no + meaning (it's sorted alphabetically to maximize frame merging). +- The width of the box shows the total time it was on-CPU or part of an + ancestry that was on-CPU (based on sample count). Functions with wide boxes + may consume more CPU per execution than those with narrow boxes, or, they may + simply be called more often. The call count is not shown (or known via + sampling). + +__Note:__ If you generate a FlameGraph yourself and then open it in a browser, +the image will be interactive, allowing you to hover over the functions to see +their full names. + +# Generating FlameGraphs + +To generate FlameGraphs, you first need to run perf and generate a perf.data +file using the `perf record -g` command. [Click here](@ref devdoc-perf) for +instructions explaining how to do this with WiredTiger. Don't forget the `-g` +option: you need it in order to record the call stacks! + + # Clone the FlameGraph repository + git clone https://github.com/brendangregg/FlameGraph + cd FlameGraph + + # Place perf.data into the current directory, then run these commands. + # Run them on the same machine where you ran `perf record` so perf can + # resolve the symbols. + perf script | ./stackcollapse-perf.pl > out.perf-folded + ./flamegraph.pl out.perf-folded > perf-kernel.svg + +Then open the resultant `svg` file in your web browser. + +Here is another FlameGraph for a full run of the `many-stable-stress.wtperf` +job: + + + +Download the non-redacted interactive version +[here](https://github.com/wiredtiger/wiredtiger/wiki/images/FlameGraph-many-tables-stress.svg) +and open it in your web browser. + +If you see 'unknown' call stacks in your FlameGraph, this means that perf could +not resolve the symbols. To fix this, install the versions of the libraries or +the OS kernel with symbols included, link against those libraries and re-gather +the `perf` output. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-perf.dox b/src/third_party/wiredtiger/src/docs/devdoc-perf.dox new file mode 100644 index 00000000000..4739fba897f --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-perf.dox @@ -0,0 +1,160 @@ +/*! @page devdoc-perf Performance Monitoring + +# Overview + +Linux `perf` is a tool that allows counting and sampling of various events in +the hardware and in the kernel. Hardware events are available via performance +monitoring units (PMU); they measure CPU cycles, cache misses, branches, etc. +Kernel events include scheduling context switches, page faults, block I/O, etc. + +`perf` is good for answering these and similar questions: + +- What is the breakdown of CPU time (or cache misses or branch mispredictions) + by function? +- What was the total number of page faults generated by two different runs of + my program? +- How many context switches occurred during the execution? +- How many times did my program issued a `sched_yield` system call? +- Which functions triggered most page faults? +- How much block I/O did my program generate? + +Despite [claims that perf can also tell you where and why your program was +blocking](https://perf.wiki.kernel.org/index.php/Tutorial#Profiling_sleep_times), +I was not able to get it to do so after many attempts. For alternative ways to +perform off-CPU analysis, read [this post by Brendan +Gregg](http://www.brendangregg.com/offcpuanalysis.html) or use WiredTiger's +@ref devdoc-optrack. + +What follows is a quick cheat sheet of how to use `perf` with WiredTiger. Most +of the information in this cheat sheet comes from [this excellent tutorial by +Brendan Gregg](http://www.brendangregg.com/perf.html). If you need more +information, please refer to that source. + +# Running perf with WiredTiger + +Build WiredTiger as usual. You do not need to add any special compilation +flags. + +To run `perf` with WiredTiger, you simply insert the name of the `perf` command +before the WiredTiger executable. For example, if you want to record default +`perf` statistics for a `wtperf` job, you'd run: + + perf stat wtperf <wtperf options> + +If you want to profile CPU time of `wtperf`, you'd run: + + perf record wtperf <wtperf options> + +Any environment variables you care about will go before the `perf` command. + +If you wanted to run `perf` on an already running WiredTiger process, you +supply a `-p <PID>` option to either of these commands: + + perf record -p <PID> + +or + + perf stat -p <PID> + +# Counting events + +The command `perf stat` will report the total count of events for the entire +execution. + + perf stat wtperf <wtperf options> + +By default it will report things like clock cycles, context switches, CPU +migrations, page faults, cycles, instructions, branches and branch misses. + +Here is a sample output: + + 55275.953046 task-clock (msec) # 1.466 CPUs utilized + 7,349,023 context-switches # 0.133 M/sec + 38,219 cpu-migrations # 0.691 K/sec + 160,562 page-faults # 0.003 M/sec + 173,665,597,703 cycles # 3.142 GHz + 90,146,578,819 instructions # 0.52 insn per cycle + 22,199,890,600 branches # 401.619 M/sec + 387,770,656 branch-misses # 1.75% of all branches + + 37.710693533 seconds time elapsed + +Alternatively, you can decide what events you want to include. For example, if +you wanted to count last-level cache (LLC) misses, one way to do this is like +so: + + perf stat -e LLC-load-misses wtperf <wtperf options> + +Or if you wanted all default events and all cache events, run: + + perf stat -d wtperf <wtperf options> + +To learn about all events you can count or sample with perf, use the command: + + perf list + +To count all system calls by type, run: + + perf stat -e 'syscalls:sys_enter_*' + +# Sampling events (profiling) + +Unlike `perf stat`, which counts all events, `perf record` samples events. The +simplest way to invoke it is like so: + + perf record <command> + +This will sample on-CPU functions. + +# Examining the output + +By default, the output will be placed into a file named `perf.data`. To examine +it, run: + + perf report + +`perf report` will show, for each sampled event, the call stacks that were +recorded at the time of sampling. The call stacks will be sorted from the most +frequently occurring to the least frequently occurring. + +At the time of this writing, the default sampling frequency is 4000; that is, a +sample is taken after every 4000 events. This number may change, so to be sure +check the setting in the header in the `perf` output, like so: + + perf report --header + +If you want to change the output file, use the `-o` option: + + perf record -o <new_output_file> <command> + +And run the reporting with the `-i` option: + + perf report -i <new_output_file> + +Here is a way to look at the output with all the call stacks expanded: + + perf report -n --stdio + +# More examples + +Sample on-cpu functions with call stacks: + + perf record -g <command> + +Sample as above, but using a specific sampling frequency (99 hertz): + + perf record -F 99 -g <command> + +Sample last-level cache load misses with call stacks: + + perf record -e LLC-load-misses -g <command> + +Sample cache misses for a running process: + + perf record -e LLC-load-misses -g -p <PID> + +Sample page faults: + + perf record -e page-faults -g + +*/ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-schema.dox b/src/third_party/wiredtiger/src/docs/devdoc-schema.dox index 509fbf16d67..dd59aa2535b 100644 --- a/src/third_party/wiredtiger/src/docs/devdoc-schema.dox +++ b/src/third_party/wiredtiger/src/docs/devdoc-schema.dox @@ -1,4 +1,4 @@ -/*! @page devdoc-schema Schema operations +/*! @page devdoc-schema Schema Operations A schema defines the format of the application data in WiredTiger. WiredTiger supports various types of schemas (See @ref schema for more diff --git a/src/third_party/wiredtiger/src/docs/devdoc-statistics.dox b/src/third_party/wiredtiger/src/docs/devdoc-statistics.dox new file mode 100644 index 00000000000..d8befa61136 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-statistics.dox @@ -0,0 +1,165 @@ +/*! @page devdoc-statistics Statistics Logging + +# Introduction + +The following content describes a subset of the statistics features available +in WiredTiger. For more complete information see the [statistics] +(http://source.wiredtiger.com/2.0.1/statistics.html) page in the WiredTiger +documentation. This content focuses on analyzing and reviewing the data +generated by the statistics logging functionality. + +# Enabling statistics + +By default statistics are disabled in WiredTiger - this is because tracking +statistics has a performance overhead. WiredTiger has two modes for tracking +statistics `fast` and `all`. Fast statistics are those that don't incur a +significant performance overhead. The effect on performance of enabling +statistics varies depending on the database and access characteristics. To +enable statistics you should include either `statistics=[fast,clear]` or +`statistics=[all,clear]` in the configuration string to `wiredtiger_open`. + +Alternatively you can set the `WIREDTIGER_CONFIG` environment variable to +include the relevant configuration string. Note that the environment variable +is checked at `wiredtiger_open` time - changing the environment variable after +WiredTiger is already running will not alter the behavior of WiredTiger. + +# Accessing statistics + +WiredTiger provides two methods for accessing statistics: using a `statistics:` +uri in the WiredTiger `WT_CURSOR` API and using the built in statistics +logging functionality. The remainder of this document assumes the use of the +statistics logging functionality as the tools provided parse the output that +it generates. + +Enabling statistics logging causes WiredTiger to generate files containing +statistics information in the database directory. To enable statistics logging +include statistics_log=(wait=30) in the configuration string to `wiredtiger_open`. + +The wait option configures how often the statistics are written to the log file +in seconds. Gathering and writing statistics has overhead - so this period +should not be too small. The default location for statistics log files is in +the WiredTiger database directory. The files are named `WiredTigerStat.XX`, +where `XX` corresponds to the current hour. A new file is created for each +hour. If your application runs for more than 24 hours and the file names wrap, +the statistics logging output is appended to the already existing files. +Statistics log file names are configurable. The output in the statistics log +files is textual and human readable. + +# Visualizing statistics + +WiredTiger provides several tools that generate visual representations of +statistics based on statistics log files. + +## Data Driven Documents (D3) graph + +[D3](http://d3js.org/) is a standard for generating interactive graphs in a +HTML page. There is a python program in the WiredTiger source tree at +`tools/wtstats.py` that can be used to generate a D3 html page based on +WiredTiger statistics log(s). In order to use the Python script you need to +have Python installed and a recent version of the Python `nvd3` library. + +To install `nvd3`: + + $ git clone https://github.com/areski/python-nvd3.git + $ cd python-nvd3/ + $ sudo python setup.py install + +It is not sufficient to run `pip install python-nvd3` - the version provided +by the library is too old. + +Once you have the prerequisites installed you can generate a graph that +contains your statistics by running: + + $ cd ~/work/wiredtiger + $ python ./tools/wtstats.py <path/to/db/directory/WiredTigerStat.*> + $ ls wtstats.html + wtstats.html + +There are several optional arguments. Run with `-h` or `--help` to see them. +For example, `-o file` or `--output file` redirects where the HTML output goes. +Also `--focus` generates a chart that contains a focus slider to allow you to +zoom in on particular time periods. + +You can open the generated HTML document in your browser, and see the generated +statistics. The keys in the graph are clickable - double clicking on one of the +keys will show only that statistic. Single clicking will enable/disable a +particular statistic. + +## Gnuplot graphs + +WiredTiger provides a python application that plots relevant statistics on +individual graphs using gnuplot. The following commands will generate graphs of +this type: + + $ cd ~/work/wiredtiger + $ python tools/statlog.py <path/to/db/directory/WiredTigerStat.*> + $ ls reports + raw report.010.png report.020.png report.030.png + report.001.png report.011.png report.021.png report.031.png + report.002.png report.012.png report.022.png report.032.png + report.003.png report.013.png report.023.png report.033.png + report.004.png report.014.png report.024.png report.034.png + report.005.png report.015.png report.025.png report.035.png + report.006.png report.016.png report.026.png report.036.png + report.007.png report.017.png report.027.png report.037.png + report.008.png report.018.png report.028.png report.038.png + report.009.png report.019.png report.029.png report.039.png + +Each of the images contains a graph for a single statistic over time. + +# A complete example + +The following steps take you through the process of generating a D3 graph of +WiredTiger statistics based on the statistics generated from running an +application included in the WiredTiger tree called `wtperf`. These instructions +assume you have all the necessary tools to build WiredTiger, Python and +python-nvd3 installed. + + $ echo "First get and build a working WiredTiger install" + $ cd work + $ wget http://source.wiredtiger.com/releases/wiredtiger-2.0.1.tar.bz2 + $ tar jxf wiredtiger-2.0.1.tar.bz2 && cd wiredtiger-2.0.1 + $ ./configure && make -j 12 + $ echo "Run an application that will generate statistics" + $ mkdir WT_TEST && ./bench/wtperf/wtperf \ + -O bench/wtperf/runners/update-lsm.wtperf \ + -o "conn_config=\"cache_size=1G,statistics=[fast,clear],statistics_log=(wait=30)\"" + $ echo "Now generate a graph from the statistics" + $ python ./tools/wtstats.py WT_TEST/WiredTigerStat.* + +The above steps will have generated a file called `wtstats.html` open this file +in a web browser and review the statistics. The page you see should look +something like: + + + +If you double click on the "btree cursor insert calls per second" key, it will +show just that statistic in the window. Depending on the test you ran (I ran a +modified version of the `update-large-lsm.wtperf` configuration), the graph may +look something like: + + + +This is an interesting graph. We can do some analysis on the meaning of the +results: + +- From 0 to about 180 seconds there are about 130k inserts a second. This +probably corresponds to the time during which the cache isn't full, so we get a +lot of work done. + +- Between 240 and 510 seconds we see approximately 110k inserts per second - +this is probably the rate we will sustain once the cache is full. + +- At 510 seconds we see a huge spike to 210k inserts a second - this +corresponds to the time in the wtperf application where we are about to switch +from populating a database, to running the workload phase. Since this is an LSM +tree, the inserts here are the result of merge operations. + +- The remainder of the time we sustain approximately 80k inserts a second - this +is probably the steady state during the workload phase of the benchmark. + +From this particular statistic we can gather that it's unlikely the cursor +insert rate that is a bottle neck in this particular test - since we could +achieve an insert rate over double the steady state insert rate when required. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 0677b92028d..1f93969ca69 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -7,6 +7,7 @@ Atomicity BLOBs CFLAGS CPPFLAGS +CPUs CRC Cheng Christoph @@ -21,6 +22,7 @@ Datastore DbCursor DbEnv DbMultiple +Durations EAGAIN EB EBUSY @@ -32,6 +34,8 @@ EmpId Encryptors Facebook Facebook's +FlameGraph +FlameGraphs FlexeLint FreeBSD FreeBSD's @@ -45,11 +49,13 @@ HyperDex HyperLevelDB IEC JDK +JIRA JavaScript KMS LD LDFLAGS LIBS +LLC LLVM LOGREC LRU @@ -67,9 +73,11 @@ Mewhort MongoDB Multithreaded NOTFOUND +NSEC NUMA NoSQL OPTYPE +PMU PPC PRELOAD README @@ -84,6 +92,7 @@ Rrx SCons Seward's SiH +TSC TXT UNC URIs @@ -125,6 +134,7 @@ benchmarking bigram bindir bitstring +bokeh bool boolean booleans @@ -159,9 +169,12 @@ conn const control's copydoc +cp cpp +cpu crashless crc +curfile cursortype customerABC cv @@ -190,8 +203,8 @@ destructors dev devdoc dhandle -dhandles dhandle's +dhandles disjunction disjunctions distclean @@ -239,6 +252,8 @@ filesystem filesystems fillfactor firstname +flamegraph +flamegraphs fnv forw fput @@ -266,20 +281,22 @@ htmlinclude huffman hugepage icount +ie iflag indices init +insn intl inuse io ip -ie je jemalloc jitter jni jrx json +jxf kb keyexist keyfmt @@ -289,6 +306,7 @@ keyvalue kvs lang lastname +latencies le len li @@ -305,6 +323,7 @@ logc lookup lookups lrtf +lru lsm lsn lt @@ -315,6 +334,7 @@ malloc malloc'd marshalled marshalling +matplotlib maxintlitem maxintlpage maxleafitem @@ -327,12 +347,14 @@ memp metadata metatracking minkey +mispredictions mixin mixins mkdir mmap mpool mpoolfile +msec msg msgs multi @@ -363,6 +385,7 @@ nosync notgranted notyet nowait +ns nul num numa @@ -377,6 +400,7 @@ os outlier outliers ovfl +parallelizable pareto pcoll pdf @@ -435,14 +459,17 @@ sHq sanitization scalable scanf +sched schemas scons +screenshots secretkey selectable seqname seqno serializable sess +sid skiplist spinlock spinlocks @@ -463,8 +490,11 @@ subdatabases subdirectory subpage substring +sudo superset +svg sys +syscalls sz tRuE tablename @@ -474,6 +504,8 @@ td th thang tid +timeline +timelines timestamp timestamps todo @@ -488,6 +520,7 @@ tt txn txnid txns +txt typedef uint ul @@ -507,11 +540,13 @@ utf util valign valuefmt +valuev vec versa vm vpmsum warmup +wget whitespace wiredtiger workQ diff --git a/src/third_party/wiredtiger/src/docs/style/wiredtiger.css b/src/third_party/wiredtiger/src/docs/style/wiredtiger.css index b16b21f2029..984e525f42f 100644 --- a/src/third_party/wiredtiger/src/docs/style/wiredtiger.css +++ b/src/third_party/wiredtiger/src/docs/style/wiredtiger.css @@ -20,6 +20,12 @@ a { color: #3D578C; } +img { + display: block; + max-width: 100%; + height: auto; +} + .tablist { width: 800px; } diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 18199997a19..5099bc4dce9 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -838,10 +838,6 @@ __wt_row_leaf_key_info( *ikeyp = NULL; if (cellp != NULL) *cellp = WT_PAGE_REF_OFFSET(page, WT_CELL_DECODE_OFFSET(v)); - if (datap != NULL) { - *(void **)datap = NULL; - *sizep = 0; - } return (false); case WT_K_FLAG: /* Encoded key: no instantiated key, no cell. */ @@ -999,6 +995,9 @@ __wt_row_leaf_value_cell(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, size_t size; void *copy, *key; + size = 0; /* -Werror=maybe-uninitialized */ + key = NULL; /* -Werror=maybe-uninitialized */ + /* If we already have an unpacked key cell, use it. */ if (kpack != NULL) vcell = (WT_CELL *)((uint8_t *)kpack->cell + __wt_cell_total_len(kpack)); diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index db4b2e9b41e..ab810ee86fb 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -298,6 +298,10 @@ struct __wt_connection_impl { uint64_t ckpt_write_bytes; uint64_t ckpt_write_pages; + /* Checkpoint and incremental backup data */ + uint64_t ckpt_incr_granularity; + WT_BLKINCR incr_backups[WT_BLKINCR_MAX]; + /* Connection's maximum and base write generations. */ uint64_t max_write_gen; uint64_t base_write_gen; @@ -441,40 +445,41 @@ struct __wt_connection_impl { /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_VERB_API 0x000000001u -#define WT_VERB_BLOCK 0x000000002u -#define WT_VERB_CHECKPOINT 0x000000004u -#define WT_VERB_CHECKPOINT_PROGRESS 0x000000008u -#define WT_VERB_COMPACT 0x000000010u -#define WT_VERB_COMPACT_PROGRESS 0x000000020u -#define WT_VERB_ERROR_RETURNS 0x000000040u -#define WT_VERB_EVICT 0x000000080u -#define WT_VERB_EVICTSERVER 0x000000100u -#define WT_VERB_EVICT_STUCK 0x000000200u -#define WT_VERB_FILEOPS 0x000000400u -#define WT_VERB_HANDLEOPS 0x000000800u -#define WT_VERB_LOG 0x000001000u -#define WT_VERB_LOOKASIDE 0x000002000u -#define WT_VERB_LOOKASIDE_ACTIVITY 0x000004000u -#define WT_VERB_LSM 0x000008000u -#define WT_VERB_LSM_MANAGER 0x000010000u -#define WT_VERB_METADATA 0x000020000u -#define WT_VERB_MUTEX 0x000040000u -#define WT_VERB_OVERFLOW 0x000080000u -#define WT_VERB_READ 0x000100000u -#define WT_VERB_REBALANCE 0x000200000u -#define WT_VERB_RECONCILE 0x000400000u -#define WT_VERB_RECOVERY 0x000800000u -#define WT_VERB_RECOVERY_PROGRESS 0x001000000u -#define WT_VERB_SALVAGE 0x002000000u -#define WT_VERB_SHARED_CACHE 0x004000000u -#define WT_VERB_SPLIT 0x008000000u -#define WT_VERB_TEMPORARY 0x010000000u -#define WT_VERB_THREAD_GROUP 0x020000000u -#define WT_VERB_TIMESTAMP 0x040000000u -#define WT_VERB_TRANSACTION 0x080000000u -#define WT_VERB_VERIFY 0x100000000u -#define WT_VERB_VERSION 0x200000000u -#define WT_VERB_WRITE 0x400000000u +#define WT_VERB_BACKUP 0x000000002u +#define WT_VERB_BLOCK 0x000000004u +#define WT_VERB_CHECKPOINT 0x000000008u +#define WT_VERB_CHECKPOINT_PROGRESS 0x000000010u +#define WT_VERB_COMPACT 0x000000020u +#define WT_VERB_COMPACT_PROGRESS 0x000000040u +#define WT_VERB_ERROR_RETURNS 0x000000080u +#define WT_VERB_EVICT 0x000000100u +#define WT_VERB_EVICTSERVER 0x000000200u +#define WT_VERB_EVICT_STUCK 0x000000400u +#define WT_VERB_FILEOPS 0x000000800u +#define WT_VERB_HANDLEOPS 0x000001000u +#define WT_VERB_LOG 0x000002000u +#define WT_VERB_LOOKASIDE 0x000004000u +#define WT_VERB_LOOKASIDE_ACTIVITY 0x000008000u +#define WT_VERB_LSM 0x000010000u +#define WT_VERB_LSM_MANAGER 0x000020000u +#define WT_VERB_METADATA 0x000040000u +#define WT_VERB_MUTEX 0x000080000u +#define WT_VERB_OVERFLOW 0x000100000u +#define WT_VERB_READ 0x000200000u +#define WT_VERB_REBALANCE 0x000400000u +#define WT_VERB_RECONCILE 0x000800000u +#define WT_VERB_RECOVERY 0x001000000u +#define WT_VERB_RECOVERY_PROGRESS 0x002000000u +#define WT_VERB_SALVAGE 0x004000000u +#define WT_VERB_SHARED_CACHE 0x008000000u +#define WT_VERB_SPLIT 0x010000000u +#define WT_VERB_TEMPORARY 0x020000000u +#define WT_VERB_THREAD_GROUP 0x040000000u +#define WT_VERB_TIMESTAMP 0x080000000u +#define WT_VERB_TRANSACTION 0x100000000u +#define WT_VERB_VERIFY 0x200000000u +#define WT_VERB_VERSION 0x400000000u +#define WT_VERB_WRITE 0x800000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t verbose; @@ -506,34 +511,35 @@ struct __wt_connection_impl { WT_FILE_SYSTEM *file_system; /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CONN_CACHE_CURSORS 0x0000001u -#define WT_CONN_CACHE_POOL 0x0000002u -#define WT_CONN_CKPT_SYNC 0x0000004u -#define WT_CONN_CLOSING 0x0000008u -#define WT_CONN_CLOSING_NO_MORE_OPENS 0x0000010u -#define WT_CONN_CLOSING_TIMESTAMP 0x0000020u -#define WT_CONN_COMPATIBILITY 0x0000040u -#define WT_CONN_DATA_CORRUPTION 0x0000080u -#define WT_CONN_EVICTION_NO_LOOKASIDE 0x0000100u -#define WT_CONN_EVICTION_RUN 0x0000200u -#define WT_CONN_IN_MEMORY 0x0000400u -#define WT_CONN_LEAK_MEMORY 0x0000800u -#define WT_CONN_LOOKASIDE_OPEN 0x0001000u -#define WT_CONN_LSM_MERGE 0x0002000u -#define WT_CONN_OPTRACK 0x0004000u -#define WT_CONN_PANIC 0x0008000u -#define WT_CONN_READONLY 0x0010000u -#define WT_CONN_RECONFIGURING 0x0020000u -#define WT_CONN_RECOVERING 0x0040000u -#define WT_CONN_SALVAGE 0x0080000u -#define WT_CONN_SERVER_ASYNC 0x0100000u -#define WT_CONN_SERVER_CAPACITY 0x0200000u -#define WT_CONN_SERVER_CHECKPOINT 0x0400000u -#define WT_CONN_SERVER_LOG 0x0800000u -#define WT_CONN_SERVER_LSM 0x1000000u -#define WT_CONN_SERVER_STATISTICS 0x2000000u -#define WT_CONN_SERVER_SWEEP 0x4000000u -#define WT_CONN_WAS_BACKUP 0x8000000u +#define WT_CONN_CACHE_CURSORS 0x00000001u +#define WT_CONN_CACHE_POOL 0x00000002u +#define WT_CONN_CKPT_SYNC 0x00000004u +#define WT_CONN_CLOSING 0x00000008u +#define WT_CONN_CLOSING_NO_MORE_OPENS 0x00000010u +#define WT_CONN_CLOSING_TIMESTAMP 0x00000020u +#define WT_CONN_COMPATIBILITY 0x00000040u +#define WT_CONN_DATA_CORRUPTION 0x00000080u +#define WT_CONN_EVICTION_NO_LOOKASIDE 0x00000100u +#define WT_CONN_EVICTION_RUN 0x00000200u +#define WT_CONN_INCR_BACKUP 0x00000400u +#define WT_CONN_IN_MEMORY 0x00000800u +#define WT_CONN_LEAK_MEMORY 0x00001000u +#define WT_CONN_LOOKASIDE_OPEN 0x00002000u +#define WT_CONN_LSM_MERGE 0x00004000u +#define WT_CONN_OPTRACK 0x00008000u +#define WT_CONN_PANIC 0x00010000u +#define WT_CONN_READONLY 0x00020000u +#define WT_CONN_RECONFIGURING 0x00040000u +#define WT_CONN_RECOVERING 0x00080000u +#define WT_CONN_SALVAGE 0x00100000u +#define WT_CONN_SERVER_ASYNC 0x00200000u +#define WT_CONN_SERVER_CAPACITY 0x00400000u +#define WT_CONN_SERVER_CHECKPOINT 0x00800000u +#define WT_CONN_SERVER_LOG 0x01000000u +#define WT_CONN_SERVER_LSM 0x02000000u +#define WT_CONN_SERVER_STATISTICS 0x04000000u +#define WT_CONN_SERVER_SWEEP 0x08000000u +#define WT_CONN_WAS_BACKUP 0x10000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index b3d32ad8417..39740cc2be4 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -32,6 +32,21 @@ 0 /* uint32_t flags */ \ } +/* + * Block based incremental backup structure. These live in the connection. + */ +#define WT_BLKINCR_MAX 2 +struct __wt_blkincr { + const char *id_str; /* User's name for this backup. */ + const char *ckpt_name; /* Requires WT-5115. All checkpoints must be this name */ + void *data; +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_BLKINCR_INUSE 0x1u /* This entry is active */ +#define WT_BLKINCR_VALID 0x2u /* This entry is valid */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint64_t flags; +}; + struct __wt_cursor_backup { WT_CURSOR iface; @@ -43,12 +58,36 @@ struct __wt_cursor_backup { size_t list_allocated; size_t list_next; + /* File offset-based incremental backup. */ + WT_BLKINCR *incr; /* Incremental backup in use */ + char *incr_file; /* File name */ + char *incr_src; /* Source identifier */ + char *incr_this; /* New base identifier */ + uint64_t incr_granularity; /* Maximum transfer size */ + + WT_CURSOR *incr_cursor; /* File cursor */ + /* Start/stop checkpoints */ + char *incr_checkpoint_start; + char *incr_checkpoint_stop; + +#define WT_BACKUP_INCR_COMPONENTS 3 + bool incr_init; /* Cursor traversal initialized */ + uint64_t *incr_list; /* List of file offset/size/type triples */ + uint64_t incr_list_count; /* Count of file offset/size/type triples */ + uint64_t incr_list_offset; /* Current offset */ + uint64_t incr_size; /* Maximum transfer size */ + WT_ITEM *incr_block; /* Current block of data */ + /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CURBACKUP_DUP 0x1u /* Duplicated backup cursor */ -#define WT_CURBACKUP_LOCKER 0x2u /* Hot-backup started */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_CURBACKUP_DUP 0x1u /* Duplicated backup cursor */ +#define WT_CURBACKUP_FORCE_STOP 0x2u /* Force stop incremental backup */ +#define WT_CURBACKUP_INCR 0x4u /* Incremental backup cursor */ +#define WT_CURBACKUP_LOCKER 0x8u /* Hot-backup started */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; +#define WT_CURSOR_BACKUP_CHECK_STOP(cursor) \ + WT_RET(F_ISSET(((WT_CURSOR_BACKUP *)(cursor)), WT_CURBACKUP_FORCE_STOP) ? EINVAL : 0); #define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)(cursor))->maxid) struct __wt_cursor_btree { diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 3eb7583ecc6..5e530b8aaa1 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -408,8 +408,6 @@ extern int __wt_config_subgetraw(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_subgets(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -471,6 +469,9 @@ extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const ch extern int __wt_count_birthmarks(WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curbackup_open_incr(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, + WT_CURSOR *cursor, const char *cfg[], WT_CURSOR **cursorp) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curbulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool bitmap, bool skip_sort_check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], @@ -1594,6 +1595,7 @@ extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session); extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); +extern void __wt_curbackup_free_incr(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb); extern void __wt_cursor_close(WT_CURSOR *cursor); extern void __wt_cursor_key_order_reset(WT_CURSOR_BTREE *cbt); extern void __wt_cursor_reopen(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle); diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index 574c9400f8f..9845dbd7f7d 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -14,10 +14,14 @@ #define WT_USERCONFIG "WiredTiger.config" /* User configuration */ -#define WT_BACKUP_TMP "WiredTiger.backup.tmp" /* Backup tmp file */ -#define WT_METADATA_BACKUP "WiredTiger.backup" /* Hot backup file */ -#define WT_INCREMENTAL_BACKUP "WiredTiger.ibackup" /* Incremental backup */ -#define WT_INCREMENTAL_SRC "WiredTiger.isrc" /* Incremental source */ +/* + * Backup related WiredTiger files. + */ +#define WT_BACKUP_TMP "WiredTiger.backup.tmp" /* Backup tmp file */ +#define WT_BLKINCR_BACKUP "WiredTiger.backup.block" /* Block incremental durable file */ +#define WT_METADATA_BACKUP "WiredTiger.backup" /* Hot backup file */ +#define WT_LOGINCR_BACKUP "WiredTiger.ibackup" /* Log incremental backup */ +#define WT_LOGINCR_SRC "WiredTiger.isrc" /* Log incremental source */ #define WT_METADATA_TURTLE "WiredTiger.turtle" /* Metadata metadata */ #define WT_METADATA_TURTLE_SET "WiredTiger.turtle.set" /* Turtle temp file */ @@ -91,16 +95,20 @@ struct __wt_ckpt { wt_timestamp_t newest_stop_ts; uint64_t newest_stop_txn; + uint64_t *alloc_list; /* Checkpoint allocation list */ + uint64_t alloc_list_entries; + WT_ITEM addr; /* Checkpoint cookie string */ WT_ITEM raw; /* Checkpoint cookie raw */ void *bpriv; /* Block manager private */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CKPT_ADD 0x1u /* Checkpoint to be added */ -#define WT_CKPT_DELETE 0x2u /* Checkpoint to be deleted */ -#define WT_CKPT_FAKE 0x4u /* Checkpoint is a fake */ -#define WT_CKPT_UPDATE 0x8u /* Checkpoint requires update */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_CKPT_ADD 0x01u /* Checkpoint to be added */ +#define WT_CKPT_BLOCK_MODS 0x02u /* Return list of modified blocks */ +#define WT_CKPT_DELETE 0x04u /* Checkpoint to be deleted */ +#define WT_CKPT_FAKE 0x08u /* Checkpoint is a fake */ +#define WT_CKPT_UPDATE 0x10u /* Checkpoint requires update */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index aec3014e788..e8a872178dd 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1064,6 +1064,33 @@ struct __wt_session { * non-printing characters are hexadecimal encoded. These formats are compatible with the * @ref util_dump and @ref util_load commands., a string\, chosen from the following * options: \c "hex"\, \c "json"\, \c "print"; default empty.} + * @config{incremental = (, configure the cursor for block incremental backup usage. These + * formats are only compatible with the backup data source; see @ref backup., a set of + * related configuration options defined below.} + * @config{ enabled, + * whether to configure this backup as the starting point for a subsequent incremental + * backup., a boolean flag; default \c false.} + * @config{ file, the + * file name when opening a duplicate incremental backup cursor. That duplicate cursor will + * return the block modifications relevant to the given file name., a string; default + * empty.} + * @config{ force_stop, causes all block incremental backup + * information to be released. This is on an open_cursor call and the resources will be + * released when this cursor is closed. No other operations should be done on this open + * cursor., a boolean flag; default \c false.} + * @config{ granularity, + * this setting manages the granularity of how WiredTiger maintains modification maps + * internally. The larger the granularity\, the smaller amount of information WiredTiger + * need to maintain., an integer between 1MB and 2GB; default \c 16MB.} + * @config{ src_id, a string that identifies a previous checkpoint + * backup source as the source of this incremental backup. This identifier must have + * already been created by use of the 'this_id' configuration in an earlier backup. A + * source id is required to begin an incremental backup., a string; default empty.} + * @config{ this_id, a string that identifies the current system + * state as a future backup source for an incremental backup via 'src_id'. This identifier + * is required when opening an incremental backup cursor and an error will be returned if + * one is not provided., a string; default empty.} + * @config{ ),,} * @config{next_random, configure the cursor to return a pseudo-random record from the * object when the WT_CURSOR::next method is called; valid only for row-store cursors. See * @ref cursor_random for details., a boolean flag; default \c false.} @@ -2335,14 +2362,14 @@ struct __wt_connection { * ),,} * @config{verbose, enable messages for various events. Options are given as a list\, such * as <code>"verbose=[evictserver\,read]"</code>., a list\, with values chosen from the - * following options: \c "api"\, \c "block"\, \c "checkpoint"\, \c "checkpoint_progress"\, - * \c "compact"\, \c "compact_progress"\, \c "error_returns"\, \c "evict"\, \c - * "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c - * "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c - * "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, - * \c "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, - * \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c - * "write"; default empty.} + * following options: \c "api"\, \c "backup"\, \c "block"\, \c "checkpoint"\, \c + * "checkpoint_progress"\, \c "compact"\, \c "compact_progress"\, \c "error_returns"\, \c + * "evict"\, \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c + * "log"\, \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c + * "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, + * \c "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, + * \c "temporary"\, \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c "verify"\, \c + * "version"\, \c "write"; default empty.} * @configend * @errors */ @@ -3017,13 +3044,13 @@ struct __wt_connection { * information., a boolean flag; default \c false.} * @config{verbose, enable messages for various events. Options are given as a list\, such as * <code>"verbose=[evictserver\,read]"</code>., a list\, with values chosen from the following - * options: \c "api"\, \c "block"\, \c "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c - * "compact_progress"\, \c "error_returns"\, \c "evict"\, \c "evict_stuck"\, \c "evictserver"\, \c - * "fileops"\, \c "handleops"\, \c "log"\, \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c - * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c - * "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c - * "split"\, \c "temporary"\, \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c "verify"\, - * \c "version"\, \c "write"; default empty.} + * options: \c "api"\, \c "backup"\, \c "block"\, \c "checkpoint"\, \c "checkpoint_progress"\, \c + * "compact"\, \c "compact_progress"\, \c "error_returns"\, \c "evict"\, \c "evict_stuck"\, \c + * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c "lookaside"\, \c + * "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, + * \c "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c + * "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c + * "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; default empty.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to files. Ignored on * non-Windows systems. Options are given as a list\, such as <code>"write_through=[data]"</code>. * Configuring \c write_through requires care\, see @ref tuning_system_buffer_cache_direct_io for @@ -4787,6 +4814,18 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * @addtogroup wt * @{ */ +/*! + * @name Incremental backup types + * @anchor backup_types + * @{ + */ +/*! invalid type */ +#define WT_BACKUP_INVALID 0 +/*! whole file */ +#define WT_BACKUP_FILE 1 +/*! file range */ +#define WT_BACKUP_RANGE 2 +/*! @} */ /*! * @name Log record and operation types diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 2b281443f21..1cd8753a0ac 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -77,6 +77,8 @@ struct __wt_async_op_impl; typedef struct __wt_async_op_impl WT_ASYNC_OP_IMPL; struct __wt_async_worker_state; typedef struct __wt_async_worker_state WT_ASYNC_WORKER_STATE; +struct __wt_blkincr; +typedef struct __wt_blkincr WT_BLKINCR; struct __wt_block; typedef struct __wt_block WT_BLOCK; struct __wt_block_ckpt; diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index db790537c79..044094133ce 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -185,7 +185,7 @@ __wt_turtle_init(WT_SESSION_IMPL *session) { WT_DECL_RET; char *metaconf, *unused_value; - bool exist_backup, exist_incr, exist_isrc, exist_turtle; + bool exist_backup, exist_bincr, exist_incr, exist_isrc, exist_turtle; bool load, loadTurtle; load = loadTurtle = false; @@ -208,10 +208,22 @@ __wt_turtle_init(WT_SESSION_IMPL *session) * turtle file and an incremental backup file, that is an error. Otherwise, if there's already a * turtle file, we're done. */ - WT_RET(__wt_fs_exist(session, WT_INCREMENTAL_BACKUP, &exist_incr)); - WT_RET(__wt_fs_exist(session, WT_INCREMENTAL_SRC, &exist_isrc)); + WT_RET(__wt_fs_exist(session, WT_LOGINCR_BACKUP, &exist_incr)); + WT_RET(__wt_fs_exist(session, WT_LOGINCR_SRC, &exist_isrc)); WT_RET(__wt_fs_exist(session, WT_METADATA_BACKUP, &exist_backup)); WT_RET(__wt_fs_exist(session, WT_METADATA_TURTLE, &exist_turtle)); + /* + * Block incremental is different. If it exists, then we have block incremental information we + * need to keep. Mark the connection as having block-based incremental backup turned on. XXX - + * Need to call something to read it in and set this up. Maybe here, maybe not. + */ + WT_RET(__wt_fs_exist(session, WT_BLKINCR_BACKUP, &exist_bincr)); + if (exist_bincr) { + F_SET(S2C(session), WT_CONN_INCR_BACKUP); + /* Load content into some structure. Not sure this is the right place. It may be too early. + */ + } + if (exist_turtle) { /* * Failure to read means a bad turtle file. Remove it and create a new turtle file. diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index 2150bf63559..451d4f2faae 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -189,6 +189,13 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v list_prepared = true; if (upd->start_ts > max_ts) max_ts = upd->start_ts; + + /* + * Track the oldest update not on the page, used to decide whether reads can use the + * page image, hence using the start rather than the durable timestamp. + */ + if (upd->start_ts < r->min_skipped_ts) + r->min_skipped_ts = upd->start_ts; continue; } } @@ -231,10 +238,8 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v skipped_birthmark = true; /* - * Track the oldest update not on the page. - * - * This is used to decide whether reads can use the page image, hence using the start - * rather than the durable timestamp. + * Track the oldest update not on the page, used to decide whether reads can use the + * page image, hence using the start rather than the durable timestamp. */ if (upd_select->upd == NULL && upd->start_ts < r->min_skipped_ts) r->min_skipped_ts = upd->start_ts; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 3b3d13770d2..e52b86da055 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -1568,6 +1568,7 @@ __rec_split_write_supd( key->data = WT_INSERT_KEY(supd->ins); key->size = WT_INSERT_KEY_SIZE(supd->ins); } + WT_ASSERT(session, next != NULL); WT_ERR(__wt_compare(session, btree->collator, key, &next->key, &cmp)); if (cmp >= 0) break; diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/util.c b/src/third_party/wiredtiger/test/csuite/random_directio/util.c index 40de5d49f36..a9ad564c160 100644 --- a/src/third_party/wiredtiger/test/csuite/random_directio/util.c +++ b/src/third_party/wiredtiger/test/csuite/random_directio/util.c @@ -95,6 +95,14 @@ copy_directory(const char *fromdir, const char *todir, bool directio) testutil_check(__wt_snprintf(fromfile, sizeof(fromfile), "%s/%s", fromdir, dp->d_name)); testutil_check(__wt_snprintf(tofile, sizeof(tofile), "%s/%s", todir, dp->d_name)); rfd = open(fromfile, O_RDONLY | openflags, 0); + /* + * The child process may have been stopped during a drop and WiredTiger drop will do an + * unlink call followed by syncing the directory. It is possible for the signal to have been + * delivered in between those calls so the file may no longer exist but reading the + * directory will still return its entry. Handle that case and skip the file if it happens. + */ + if (rfd < 0 && errno == ENOENT) + continue; testutil_assertfmt(rfd >= 0, "Open of source %s failed with %d\n", fromfile, errno); wfd = open(tofile, O_WRONLY | O_CREAT, 0666); testutil_assertfmt(wfd >= 0, "Open of dest %s failed with %d\n", tofile, errno); diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index e28772c915b..971d6b7bcea 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -283,7 +283,7 @@ tasks: posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - func: "make check all" vars: - test_env_vars: MSAN_OPTIONS=abort_on_error=1:disable_coredump=0 MSAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + test_env_vars: MSAN_OPTIONS=abort_on_error=1:disable_coredump=0 MSAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer TESTUTIL_SLOW_MACHINE=1 - name: make-check-asan-test depends_on: @@ -1784,6 +1784,20 @@ tasks: ./time_shift_test.sh /usr/local/lib/faketime/libfaketimeMT.so.1 0-1 2>&1 + - name: split-stress-test + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + configure_env_vars: CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" + - command: shell.exec + params: + working_dir: "wiredtiger/bench/workgen/runner" + script: | + set -o errexit + set -o verbose + for i in {1..10}; do ${python_binary|python} split_stress.py; done + buildvariants: - name: ubuntu1804 display_name: Ubuntu 18.04 @@ -1813,6 +1827,7 @@ buildvariants: - name: wtperf-test - name: ftruncate-test - name: long-test + - name: split-stress-test - name: ubuntu1804-python3 display_name: Ubuntu 18.04 (Python3) @@ -1965,3 +1980,4 @@ buildvariants: tasks: - name: compile - name: unit-test + - name: split-stress-test diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index 9bdc55a4d69..3a140bcfc05 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -18,9 +18,10 @@ trap 'onintr' 2 usage() { echo "usage: $0 [-aFSv] [-c config] " - echo " [-h home] [-j parallel-jobs] [-n total-jobs] [-t minutes] [format-configuration]" + echo " [-b format-binary] [-h home] [-j parallel-jobs] [-n total-jobs] [-t minutes] [format-configuration]" echo echo " -a abort/recovery testing (defaults to off)" + echo " -b binary format binary (defaults to "./t")" echo " -c config format configuration file (defaults to CONFIG.stress)" echo " -F quit on first failure (defaults to off)" echo " -h home run directory (defaults to .)" @@ -74,12 +75,16 @@ parallel_jobs=8 smoke_test=0 total_jobs=0 verbose=0 +format_binary="./t" while :; do case "$1" in -a) abort_test=1 shift ;; + -b) + format_binary="$2" + shift ; shift ;; -c) config="$2" shift ; shift ;; @@ -155,16 +160,17 @@ cd $(dirname $0) || exit 1 # local. [[ $config_found -eq 0 ]] && [[ -f "$config" ]] && config="$PWD/$config" -# Find the format binary. Builds are normally in the WiredTiger source tree, in which case it's -# in the same directory as format.sh, else it's in the build_posix tree. If the build is in the -# build_posix tree, move there, we have to run in the directory where the format binary lives -# because the format binary "knows" the wt utility is two directory levels above it. -format_binary="./t" -[[ -x $format_binary ]] || { +# Find the last part of format_binary, which is format binary file. Builds are normally in the +# WiredTiger source tree, in which case it's in the same directory as format.sh, else it's in +# the build_posix tree. If the build is in the build_posix tree, move there, we have to run in +# the directory where the format binary lives because the format binary "knows" the wt utility +# is two directory levels above it. + +[[ -x ${format_binary##* } ]] || { build_posix_directory="../../build_posix/test/format" [[ ! -d $build_posix_directory ]] || cd $build_posix_directory || exit 1 - [[ -x $format_binary ]] || { - echo "$name: format program \"$format_binary\" not found" + [[ -x ${format_binary##* } ]] || { + echo "$name: format program \"${format_binary##* }\" not found" exit 1 } } diff --git a/src/third_party/wiredtiger/test/format/snap.c b/src/third_party/wiredtiger/test/format/snap.c index 15df14b71dc..b304faa7376 100644 --- a/src/third_party/wiredtiger/test/format/snap.c +++ b/src/third_party/wiredtiger/test/format/snap.c @@ -397,6 +397,12 @@ snap_repeat_txn(WT_CURSOR *cursor, TINFO *tinfo) if (current->opid != tinfo->opid) break; + /* + * The transaction is not yet resolved, so the rules are as if the transaction has + * committed. Note we are NOT checking if reads are repeatable based on the chosen + * timestamp. This is because we expect snapshot isolation to work even in the presence of + * other threads of control committing in our past, until the transaction resolves. + */ if (snap_repeat_ok_commit(tinfo, current)) WT_RET(snap_verify(cursor, tinfo, current)); } diff --git a/src/third_party/wiredtiger/test/suite/test_backup06.py b/src/third_party/wiredtiger/test/suite/test_backup06.py index d4efba4c6f0..d9db39d241d 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup06.py +++ b/src/third_party/wiredtiger/test/suite/test_backup06.py @@ -34,6 +34,12 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet +try: + # Windows does not getrlimit/setrlimit so we must catch the resource + # module load. + import resource +except: + None # test_backup06.py # Test that opening a backup cursor does not open file handles. @@ -80,11 +86,21 @@ class test_backup06(wttest.WiredTigerTestCase, suite_subprocess): # Test that the open handle count does not change. def test_cursor_open_handles(self): + if os.name == "nt": + self.skipTest('Unix specific test skipped on Windows') + + limits = resource.getrlimit(resource.RLIMIT_NOFILE) + if limits[0] < 1024: + new = (1024, limits[1]) + resource.setrlimit(resource.RLIMIT_NOFILE, new) self.populate_many() # Close and reopen the connection so the populate dhandles are # not in the list. self.reopen_conn() + new = (limits[0], limits[1]) + resource.setrlimit(resource.RLIMIT_NOFILE, new) + # Confirm that opening a backup cursor does not open # file handles. stat_cursor = self.session.open_cursor('statistics:', None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_backup10.py b/src/third_party/wiredtiger/test/suite/test_backup10.py index 3682db7cf01..b8806c439dc 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup10.py +++ b/src/third_party/wiredtiger/test/suite/test_backup10.py @@ -38,7 +38,6 @@ from wtscenario import make_scenarios class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): dir='backup.dir' # Backup directory name logmax="100K" - newuri="table:newtable" uri="table:test" nops=100 @@ -95,7 +94,6 @@ class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): if ret != 0: break newfile = bkup_c.get_key() - self.assertNotEqual(newfile, self.newuri) sz = os.path.getsize(newfile) self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) shutil.copy(newfile, self.dir) @@ -130,6 +128,7 @@ class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): diff = dup_set.difference(orig_set) self.assertEqual(len(diff), 1) self.assertTrue(log3 in dup_set) + self.assertFalse(log3 in orig_set) # Test a few error cases now. # - We cannot make multiple duplcate backup cursors. @@ -148,7 +147,7 @@ class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): dupc.close() # Test we must use the log target. - msg = "/must be for logs/" + msg = "/must be for /" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.assertEquals(self.session.open_cursor(None, bkup_c, None), 0), msg) diff --git a/src/third_party/wiredtiger/test/suite/test_backup11.py b/src/third_party/wiredtiger/test/suite/test_backup11.py new file mode 100644 index 00000000000..f1622e9d2bc --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup11.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +import os, shutil +from helper import compare_files +from suite_subprocess import suite_subprocess +from wtdataset import simple_key +from wtscenario import make_scenarios + +# test_backup11.py +# Test cursor backup with a duplicate backup cursor. +class test_backup11(wttest.WiredTigerTestCase, suite_subprocess): + dir='backup.dir' # Backup directory name + logmax="100K" + uri="table:test" + nops=100 + + pfx = 'test_backup' + + # ('archiving', dict(archive='true')), + # ('not-archiving', dict(archive='false')), + scenarios = make_scenarios([ + ('archiving', dict(archive='true')), + ]) + + # Create a large cache, otherwise this test runs quite slowly. + def conn_config(self): + return 'cache_size=1G,log=(archive=%s,' % self.archive + \ + 'enabled,file_max=%s)' % self.logmax + + def add_data(self): + log2 = "WiredTigerLog.0000000002" + log3 = "WiredTigerLog.0000000003" + + self.session.create(self.uri, "key_format=S,value_format=S") + + # Insert small amounts of data at a time stopping after we + # cross into log file 2. + loop = 0 + c = self.session.open_cursor(self.uri) + while not os.path.exists(log2): + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + self.session.checkpoint() + c.close() + return loop + + def test_backup11(self): + + loop = self.add_data() + + # Open up the backup cursor. This causes a new log file to be created. + # That log file is not part of the list returned. This is a full backup + # primary cursor with incremental configured. + os.mkdir(self.dir) + config = 'incremental=(enabled,this_id="ID1")' + bkup_c = self.session.open_cursor('backup:', None, config) + + # Add some data that will appear in log file 3. + c = self.session.open_cursor(self.uri) + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + c.close() + self.session.log_flush('sync=on') + self.session.checkpoint() + + # Now copy the files returned by the backup cursor. + orig_logs = [] + while True: + ret = bkup_c.next() + if ret != 0: + break + newfile = bkup_c.get_key() + sz = os.path.getsize(newfile) + self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + if "WiredTigerLog" in newfile: + orig_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + + # Now open a duplicate backup cursor. + # We *can* use a log target duplicate on an incremental primary backup so that + # a backup process can get all the log files that occur while that primary cursor + # is open. + config = 'target=("log:")' + dupc = self.session.open_cursor(None, bkup_c, config) + dup_logs = [] + while True: + ret = dupc.next() + if ret != 0: + break + newfile = dupc.get_key() + self.assertTrue("WiredTigerLog" in newfile) + sz = os.path.getsize(newfile) + if (newfile not in orig_logs): + self.pr('DUP: Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + # Record all log files returned for later verification. + dup_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + dupc.close() + bkup_c.close() + + # Add more data + c = self.session.open_cursor(self.uri) + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + c.close() + self.session.log_flush('sync=on') + self.session.checkpoint() + + # Test a few error cases now. + # - Incremental filename must be on duplicate, not primary. + # - An incremental duplicate must have an incremental primary. + # - We cannot make multiple incremental duplcate backup cursors. + # - We cannot duplicate the duplicate backup cursor. + # - We cannot mix block incremental with a log target on the same duplicate. + # - Incremental ids must be on primary, not duplicate. + # - Force stop must be on primary, not duplicate. + + # - Incremental filename must be on duplicate, not primary. + # Test this first because we currently do not have a primary open. + config = 'incremental=(file=test.wt)' + msg = "/file name can only be specified on a duplicate/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor('backup:', + None, config), 0), msg) + + # Open a non-incremental full backup cursor. + # - An incremental duplicate must have an incremental primary. + bkup_c = self.session.open_cursor('backup:', None, None) + msg = "/must have an incremental primary/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + bkup_c.close() + + config = 'incremental=(src_id="ID1",this_id="ID2")' + bkup_c = self.session.open_cursor('backup:', None, config) + + self.pr("Opened backup for error testing") + + # Now test all the error cases with an incremental primary open. + # - We cannot make multiple incremental duplcate backup cursors. + # - We cannot duplicate the duplicate backup cursor. + config = 'incremental=(file=test.wt)' + dupc = self.session.open_cursor(None, bkup_c, config) + msg = "/already a duplicate backup cursor open/" + self.pr("Test multiple dups") + self.pr("=========") + # Test multiple duplicate backup cursors. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + # Test duplicate of duplicate backup cursor. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + dupc, config), 0), msg) + dupc.close() + + # - A duplicate cursor must specify incremental or log target. + self.pr("Test dup and file target") + self.pr("=========") + msg = "/cannot be used for/" + config = 'target=("file:test.wt")' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + + # - We cannot mix block incremental with a log target on the same duplicate. + self.pr("Test mixed targets") + self.pr("=========") + config = 'incremental=(file=test.wt),target=("log:")' + msg = "/incremental backup incompatible/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + + # - Incremental ids must be on primary, not duplicate. + self.pr("Test ids on dups") + self.pr("=========") + config = 'incremental=(src_id="ID1")' + msg = "/specified on a primary/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + config = 'incremental=(this_id="ID1")' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + + # - Force stop must be on primary, not duplicate. + #self.pr("Test force stop") + #self.pr("=========") + #config = 'incremental=(force_stop=true)' + #print "config is " + config + #self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + # lambda:self.assertEquals(self.session.open_cursor(None, + # bkup_c, config), 0), msg) + + bkup_c.close() + + # After the full backup, open and recover the backup database. + #backup_conn = self.wiredtiger_open(self.dir) + #backup_conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_backup12.py b/src/third_party/wiredtiger/test/suite/test_backup12.py new file mode 100644 index 00000000000..6726164d038 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup12.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +import os, shutil +from helper import compare_files +from suite_subprocess import suite_subprocess +from wtdataset import simple_key +from wtscenario import make_scenarios + +# test_backup12.py +# Test cursor backup with a block-based incremental cursor. +class test_backup12(wttest.WiredTigerTestCase, suite_subprocess): + dir='backup.dir' # Backup directory name + logmax="100K" + uri="table:test" + nops=100 + + pfx = 'test_backup' + + # ('archiving', dict(archive='true')), + # ('not-archiving', dict(archive='false')), + scenarios = make_scenarios([ + ('archiving', dict(archive='true')), + ]) + + # Create a large cache, otherwise this test runs quite slowly. + def conn_config(self): + return 'cache_size=1G,log=(archive=%s,' % self.archive + \ + 'enabled,file_max=%s)' % self.logmax + + def add_data(self): + log2 = "WiredTigerLog.0000000002" + log3 = "WiredTigerLog.0000000003" + + self.session.create(self.uri, "key_format=S,value_format=S") + + # Insert small amounts of data at a time stopping after we + # cross into log file 2. + loop = 0 + c = self.session.open_cursor(self.uri) + while not os.path.exists(log2): + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + self.session.checkpoint() + c.close() + return loop + + def test_backup12(self): + + loop = self.add_data() + + # Open up the backup cursor. This causes a new log file to be created. + # That log file is not part of the list returned. This is a full backup + # primary cursor with incremental configured. + os.mkdir(self.dir) + config = 'incremental=(enabled,this_id="ID1")' + bkup_c = self.session.open_cursor('backup:', None, config) + + # Add some data that will appear in log file 3. + c = self.session.open_cursor(self.uri) + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + c.close() + self.session.log_flush('sync=on') + self.session.checkpoint() + + # Now copy the files returned by the backup cursor. + orig_logs = [] + while True: + ret = bkup_c.next() + if ret != 0: + break + newfile = bkup_c.get_key() + sz = os.path.getsize(newfile) + self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + if "WiredTigerLog" in newfile: + orig_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + + # Now open a duplicate backup cursor. + # We *can* use a log target duplicate on an incremental primary backup so that + # a backup process can get all the log files that occur while that primary cursor + # is open. + config = 'target=("log:")' + dupc = self.session.open_cursor(None, bkup_c, config) + dup_logs = [] + while True: + ret = dupc.next() + if ret != 0: + break + newfile = dupc.get_key() + self.assertTrue("WiredTigerLog" in newfile) + sz = os.path.getsize(newfile) + if (newfile not in orig_logs): + self.pr('DUP: Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + # Record all log files returned for later verification. + dup_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + dupc.close() + bkup_c.close() + + # Add more data. + c = self.session.open_cursor(self.uri) + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + c.close() + self.session.log_flush('sync=on') + self.session.checkpoint() + + # Now do an incremental backup. + config = 'incremental=(src_id="ID1",this_id="ID2")' + bkup_c = self.session.open_cursor('backup:', None, config) + self.pr('Open backup cursor ID1') + while True: + ret = bkup_c.next() + if ret != 0: + break + newfile = bkup_c.get_key() + config = 'incremental=(file=' + newfile + ')' + self.pr('Open incremental cursor with ' + config) + dup_cnt = 0 + dupc = self.session.open_cursor(None, bkup_c, config) + while True: + ret = dupc.next() + if ret != 0: + break + incrlist = dupc.get_keys() + offset = incrlist[0] + size = incrlist[1] + curtype = incrlist[2] + self.assertEqual(offset, 0) + # For now assert WT_BACKUP_FILE (which is 1). + self.assertEqual(curtype, 1) + dup_cnt += 1 + dupc.close() + self.assertEqual(dup_cnt, 1) + self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + bkup_c.close() + + # After the full backup, open and recover the backup database. + backup_conn = self.wiredtiger_open(self.dir) + backup_conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 4f97ac2aff7..668b53a7334 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -160,7 +160,7 @@ class TestSuiteConnection(object): class ExtensionList(list): skipIfMissing = False def extension(self, dirname, name, extarg=None): - if name != None and name != 'none': + if name and name != 'none': ext = '' if extarg == None else '=' + extarg self.append(dirname + '/' + name + ext) diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index 3442e8edcec..734026683c9 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -148,6 +148,18 @@ typedef struct { } while (0) /* + * error_sys_check -- + * Complain and quit if a function call fails. A special name because it appears in the + * documentation. Allow any non-negative values. + */ +#define error_sys_check(call) \ + do { \ + int __r; \ + if ((__r = (int)(call)) < 0 && __r != ENOTSUP) \ + testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call); \ + } while (0) + +/* * error_check -- * Complain and quit if a function call fails. A special name because it appears in the * documentation. Ignore ENOTSUP to allow library calls which might not be included in any |