diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2020-01-02 06:49:42 +0000 |
|---|---|---|
| committer | evergreen <evergreen@mongodb.com> | 2020-01-02 06:49:42 +0000 |
| commit | 4019b32248c7ceb767d15be1e8b78874b4f7ccb7 (patch) | |
| tree | b1852259c2de5863f4616234ab06fee39d705e72 | |
| parent | b2ef306184845e22c6ab585b1e59d1c828193ca8 (diff) | |
| download | mongo-4019b32248c7ceb767d15be1e8b78874b4f7ccb7.tar.gz | |
Import wiredtiger: 714078029d530f6438fd170f963a9ee27c7b517c from branch mongodb-4.2
ref: a95c73b589..714078029d
for: 4.2.3
WT-4919 Add debug mode which allocates exact amount of memory in __wt_realloc_def
WT-4921 Add debug mode option that slows checkpoint creation
WT-4969 Remove lsm_merge configuration option
WT-4997 Migrate Jenkins “wiredtiger-test-format-stress” job to Evergreen
WT-4998 Migrate Jenkins “wiredtiger-test-format-stress-ppc” job to Evergreen
WT-5000 Migrate Jenkins “wiredtiger-test-format-stress-sanitizer” job to Evergreen
WT-5004 Migrate Jenkins “wiredtiger-test-checkpoint-stress” job to Evergreen
WT-5005 Migrate Jenkins “wiredtiger-test-recovery-stress” job to Evergreen
WT-5006 Migrate Jenkins “wiredtiger-test-split-stress” job to Evergreen
WT-5037 Move Application Developer Resources documentation in github to WT documentation
WT-5077 Handle ENOENT without failure when copying the directory
WT-5090 Log a message when rolling a thread back because it holds the oldest timestamp
WT-5112 Handling goto labels with multiple words in s_goto.py
WT-5139 WiredTiger incremental backup API
WT-5221 Bypass test_wt2853_perf in Evergreen make-check-msan-test
WT-5257 Coverity analysis bug: 113971 Dereference after null check
WT-5263 Prepared updates written to the lookaside file are not always read as needed
WT-5309 Update format.sh script to add prefix command argument
WT-5312 Add thread doing random cursor operations to format's cache size calculation
WT-5314 Avoid loading extensions that have blank name for Python tests
WT-5319 Avoid clearing the saved last-key when no instantiated key
WT-5324 WiredTiger API error handling fixes
WT-5327 Backup cursor API can fail to restore WT_SESSION name information
WT-5330 Read-only transactions are being rolled back in MongoDB
WT-5360 Add a template for building a glossary in developer docs
WT-5365 Remove format's long-running transaction support
83 files changed, 3618 insertions, 851 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index f4d0d31dab0..ab5f3ab49cc 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -493,7 +493,6 @@ t = env.Program("t_format", "test/format/bulk.c", "test/format/compact.c", "test/format/config.c", - "test/format/lrt.c", "test/format/ops.c", "test/format/rebalance.c", "test/format/random.c", diff --git a/src/third_party/wiredtiger/bench/wtperf/split_heavy.wtperf b/src/third_party/wiredtiger/bench/wtperf/split_heavy.wtperf new file mode 100644 index 00000000000..4f11340c095 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/split_heavy.wtperf @@ -0,0 +1,13 @@ +conn_config="cache_size=100MB,log=(enabled=false),statistics=[fast],statistics_log=(wait=5,json=false)" +table_config="leaf_page_max=8k,internal_page_max=8k,leaf_item_max=1433,internal_item_max=3100,type=file,memory_page_max=1MB,split_deepen_min_child=100" +table_count=3 +icount=50000 +populate_threads=1 +random_range=100000000 +checkpoint_threads=0 +report_interval=5 +run_time=10 +threads=((count=20,inserts=1)) +value_sz=200 +key_sz=64 +reopen_connection=false
\ No newline at end of file diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 697d59c8dcd..fb1df282cdb 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -401,7 +401,7 @@ worker_async(void *arg) break; goto op_err; default: - op_err: +op_err: lprintf(wtperf, ret, 0, "%s failed for: %s, range: %" PRIu64, op_name(op), key_buf, wtperf_value_range(wtperf)); goto err; /* can't happen */ @@ -831,7 +831,7 @@ worker(void *arg) if (ret == WT_NOTFOUND) break; - op_err: +op_err: if (ret == WT_ROLLBACK && (ops_per_txn != 0 || opts->log_like_table)) { /* * If we are running with explicit transactions configured and we hit a WT_ROLLBACK, diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 6d9d4f1db3d..d6280b7f0ec 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -501,10 +501,19 @@ connection_runtime_config = [ is not limited to not skewing newest, not favoring leaf pages, and modifying the eviction score mechanism.''', type='boolean'), + Config('realloc_exact', 'false', r''' + if true, reallocation of memory will only provide the exact + amount requested. This will help with spotting memory allocation + issues more easily.''', + type='boolean'), Config('rollback_error', '0', r''' return a WT_ROLLBACK error from a transaction operation about every Nth operation to simulate a collision''', min='0', max='10M'), + Config('slow_checkpoint', 'false', r''' + if true, slow down checkpoint creation by slowing down internal + page processing.''', + type='boolean'), Config('table_logging', 'false', r''' if true, write transaction related information to the log for all operations, even operations for tables with logging turned off. @@ -602,9 +611,6 @@ connection_runtime_config = [ merge LSM chunks where possible''', type='boolean') ]), - Config('lsm_merge', 'true', r''' - merge LSM chunks where possible (deprecated)''', - type='boolean', undoc=True), Config('operation_timeout_ms', '0', r''' when non-zero, a requested limit on the number of elapsed real time milliseconds application threads will take to complete database operations. Time is measured from the @@ -680,6 +686,7 @@ connection_runtime_config = [ list, such as <code>"verbose=[evictserver,read]"</code>''', type='list', choices=[ 'api', + 'backup', 'block', 'checkpoint', 'checkpoint_progress', @@ -1208,6 +1215,38 @@ methods = { characters are hexadecimal encoded. These formats are compatible with the @ref util_dump and @ref util_load commands''', choices=['hex', 'json', 'print']), + Config('incremental', '', r''' + configure the cursor for block incremental backup usage. These formats + are only compatible with the backup data source; see @ref backup''', + type='category', subconfig=[ + Config('enabled', 'false', r''' + whether to configure this backup as the starting point for a subsequent + incremental backup''', + type='boolean'), + Config('file', '', r''' + the file name when opening a duplicate incremental backup cursor. + That duplicate cursor will return the block modifications relevant + to the given file name'''), + Config('force_stop', 'false', r''' + causes all block incremental backup information to be released. This is + on an open_cursor call and the resources will be released when this + cursor is closed. No other operations should be done on this open cursor''', + type='boolean'), + Config('granularity', '16MB', r''' + this setting manages the granularity of how WiredTiger maintains modification + maps internally. The larger the granularity, the smaller amount of information + WiredTiger need to maintain''', + min='1MB', max='2GB'), + Config('src_id', '', r''' + a string that identifies a previous checkpoint backup source as the source + of this incremental backup. This identifier must have already been created + by use of the 'this_id' configuration in an earlier backup. A source id is + required to begin an incremental backup'''), + Config('this_id', '', r''' + a string that identifies the current system state as a future backup source + for an incremental backup via 'src_id'. This identifier is required when opening + an incremental backup cursor and an error will be returned if one is not provided'''), + ]), Config('next_random', 'false', r''' configure the cursor to return a pseudo-random record from the object when the WT_CURSOR::next method is called; valid only for diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 9e7eb0b23ac..6cf9369fecd 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -64,7 +64,6 @@ src/config/config_check.c src/config/config_collapse.c src/config/config_def.c src/config/config_ext.c -src/config/config_upgrade.c src/conn/api_calc_modify.c src/conn/api_strerror.c src/conn/api_version.c @@ -81,6 +80,7 @@ src/conn/conn_reconfig.c src/conn/conn_stat.c src/conn/conn_sweep.c src/cursor/cur_backup.c +src/cursor/cur_backup_incr.c src/cursor/cur_bulk.c src/cursor/cur_config.c src/cursor/cur_ds.c diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 85a240550ea..01fa0a7846b 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -1,12 +1,16 @@ # List of WiredTiger #defines that are "unused", but it's OK. API_CALL API_CALL_NOCONF +API_END API_SESSION_INIT +API_SESSION_POP +API_SESSION_PUSH FLD_MASK JOINABLE_CURSOR_CALL_CHECK LF_MASK LLONG_MAX LLONG_MIN +SESSION_API_PREPARE_CHECK TXN_API_CALL TXN_API_CALL_NOCONF TXN_API_END @@ -15,12 +19,14 @@ WT_ALIGN_CHECK WT_ATOMIC_CAS WT_ATOMIC_CAS_FUNC WT_ATOMIC_FUNC +WT_BACKUP_INVALID WT_BLOCK_DESC_SIZE WT_BLOCK_EXTLIST_VERSION_ORIG WT_BLOCK_HEADER_SIZE WT_CACHE_LINE_ALIGNMENT WT_CACHE_LINE_PAD_BEGIN WT_CACHE_LINE_PAD_END +WT_CKPT_BLOCK_MODS WT_CLOCKDIFF_NS WT_CONN_CHECK_PANIC WT_DEADLOCK diff --git a/src/third_party/wiredtiger/dist/s_funcs.list b/src/third_party/wiredtiger/dist/s_funcs.list index 0218937fffc..54928fc03b4 100644 --- a/src/third_party/wiredtiger/dist/s_funcs.list +++ b/src/third_party/wiredtiger/dist/s_funcs.list @@ -34,6 +34,7 @@ __wt_stat_join_aggregate __wt_stat_join_clear_all __wt_stream_set_no_buffer __wt_try_readlock +__wt_txn_err_chk wiredtiger_calc_modify wiredtiger_config_parser_open wiredtiger_config_validate diff --git a/src/third_party/wiredtiger/dist/s_goto.py b/src/third_party/wiredtiger/dist/s_goto.py index 032084168cc..a7f35488603 100644 --- a/src/third_party/wiredtiger/dist/s_goto.py +++ b/src/third_party/wiredtiger/dist/s_goto.py @@ -6,7 +6,7 @@ import re, sys # 1. Zero or more whitespace characters. # 2. One or more lowercase ASCII characters. # 3. Colon character. -p = re.compile('^\s*[a-z]+:$') +p = re.compile('^\s*[a-z_]+:$') for line in sys.stdin: m = p.search(line) if m is not None: diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 66480a55eec..6cda20d9b3d 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1128,6 +1128,7 @@ pvA pwrite py qdown +qqq qrrSS qsort quartile diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index e441d45af46..5e9b91843d5 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -8,6 +8,7 @@ noinst_PROGRAMS = \ ex_all \ ex_async \ ex_backup \ + ex_backup_block \ ex_call_center \ ex_config_parse \ ex_cursor \ diff --git a/src/third_party/wiredtiger/examples/c/ex_backup_block.c b/src/third_party/wiredtiger/examples/c/ex_backup_block.c new file mode 100644 index 00000000000..fbae6e6da5d --- /dev/null +++ b/src/third_party/wiredtiger/examples/c/ex_backup_block.c @@ -0,0 +1,460 @@ +/*- + * Public Domain 2014-2019 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_backup_block.c + * demonstrates how to use block-based incremental backup. + */ +#include <test_util.h> + +static const char *const home = "WT_BLOCK"; +static const char *const home_full = "WT_BLOCK_LOG_FULL"; +static const char *const home_incr = "WT_BLOCK_LOG_INCR"; + +static const char *const full_out = "./backup_block_full"; +static const char *const incr_out = "./backup_block_incr"; + +static const char *const uri = "table:main"; +static const char *const uri2 = "table:extra"; + +typedef struct __filelist { + const char *name; + bool exist; +} FILELIST; + +static FILELIST *last_flist = NULL; +static size_t filelist_count = 0; + +#define FLIST_INIT 16 + +#define CONN_CONFIG "create,cache_size=100MB,log=(enabled=true,file_max=100K)" +#define MAX_ITERATIONS 5 +#define MAX_KEYS 10000 + +static int +compare_backups(int i) +{ + int ret; + char buf[1024], msg[32]; + + /* + * We run 'wt dump' on both the full backup directory and the + * incremental backup directory for this iteration. Since running + * 'wt' runs recovery and makes both directories "live", we need + * a new directory for each iteration. + * + * If i == 0, we're comparing against the main, original directory + * with the final incremental directory. + */ + if (i == 0) + (void)snprintf(buf, sizeof(buf), "../../wt -R -h %s dump main > %s.%d", home, full_out, i); + else + (void)snprintf( + buf, sizeof(buf), "../../wt -R -h %s.%d dump main > %s.%d", home_full, i, full_out, i); + error_check(system(buf)); + /* + * Now run dump on the incremental directory. + */ + (void)snprintf( + buf, sizeof(buf), "../../wt -R -h %s.%d dump main > %s.%d", home_incr, i, incr_out, i); + error_check(system(buf)); + + /* + * Compare the files. + */ + (void)snprintf(buf, sizeof(buf), "cmp %s.%d %s.%d", full_out, i, incr_out, i); + ret = system(buf); + if (i == 0) + (void)snprintf(msg, sizeof(msg), "%s", "MAIN"); + else + (void)snprintf(msg, sizeof(msg), "%d", i); + printf("Iteration %s: Tables %s.%d and %s.%d %s\n", msg, full_out, i, incr_out, i, + ret == 0 ? "identical" : "differ"); + if (ret != 0) + exit(1); + + /* + * If they compare successfully, clean up. + */ + if (i != 0) { + (void)snprintf(buf, sizeof(buf), "rm -rf %s.%d %s.%d %s.%d %s.%d", home_full, i, home_incr, + i, full_out, i, incr_out, i); + error_check(system(buf)); + } + return (ret); +} + +/* + * Set up all the directories needed for the test. We have a full backup directory for each + * iteration and an incremental backup for each iteration. That way we can compare the full and + * incremental each time through. + */ +static void +setup_directories(void) +{ + int i; + char buf[1024]; + + for (i = 0; i < MAX_ITERATIONS; i++) { + /* + * For incremental backups we need 0-N. The 0 incremental directory will compare with the + * original at the end. + */ + (void)snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", home_incr, i, home_incr, i); + error_check(system(buf)); + if (i == 0) + continue; + /* + * For full backups we need 1-N. + */ + (void)snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", home_full, i, home_full, i); + error_check(system(buf)); + } +} + +static void +add_work(WT_SESSION *session, int iter) +{ + WT_CURSOR *cursor, *cursor2; + int i; + char k[32], v[32]; + + error_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + /* + * Only on even iterations add content to the extra table. This illustrates and shows that + * sometimes only some tables will be updated. + */ + cursor2 = NULL; + if (iter % 2 == 0) + error_check(session->open_cursor(session, uri2, NULL, NULL, &cursor2)); + /* + * Perform some operations with individual auto-commit transactions. + */ + for (i = 0; i < MAX_KEYS; i++) { + (void)snprintf(k, sizeof(k), "key.%d.%d", iter, i); + (void)snprintf(v, sizeof(v), "value.%d.%d", iter, i); + cursor->set_key(cursor, k); + cursor->set_value(cursor, v); + error_check(cursor->insert(cursor)); + if (cursor2 != NULL) { + cursor2->set_key(cursor2, k); + cursor2->set_value(cursor2, v); + error_check(cursor2->insert(cursor2)); + } + } + error_check(cursor->close(cursor)); + if (cursor2 != NULL) + error_check(cursor2->close(cursor2)); +} + +static int +finalize_files(FILELIST *flistp, size_t count) +{ + size_t i; + char buf[512]; + + /* + * Process files that were removed. Any file that is not marked in the previous list as existing + * in this iteration should be removed. Free all previous filenames as we go along. Then free + * the overall list. + */ + for (i = 0; i < filelist_count; ++i) { + if (last_flist[i].name == NULL) + break; + if (!last_flist[i].exist) { + (void)snprintf(buf, sizeof(buf), "rm WT_BLOCK_LOG_*/%s", last_flist[i].name); + error_check(system(buf)); + } + free((void *)last_flist[i].name); + } + free(last_flist); + + /* Set up the current list as the new previous list. */ + last_flist = flistp; + filelist_count = count; + return (0); +} + +/* + * Process a file name. Build up a list of current file names. But also process the file names from + * the previous iteration. Mark any name we see as existing so that the finalize function can remove + * any that don't exist. We walk the list each time. This is slow. + */ +static int +process_file(FILELIST **flistp, size_t *countp, size_t *allocp, const char *filename) +{ + FILELIST *flist; + size_t alloc, i, new, orig; + + /* Build up the current list, growing as needed. */ + i = *countp; + alloc = *allocp; + flist = *flistp; + if (i == alloc) { + orig = alloc * sizeof(FILELIST); + new = orig * 2; + flist = realloc(flist, new); + testutil_assert(flist != NULL); + memset(flist + alloc, 0, new - orig); + *allocp = alloc * 2; + *flistp = flist; + } + + flist[i].name = strdup(filename); + flist[i].exist = false; + ++(*countp); + + /* Check against the previous list. */ + for (i = 0; i < filelist_count; ++i) { + /* If name is NULL, we've reached the end of the list. */ + if (last_flist[i].name == NULL) + break; + if (strcmp(filename, last_flist[i].name) == 0) { + last_flist[i].exist = true; + break; + } + } + return (0); +} + +static void +take_full_backup(WT_SESSION *session, int i) +{ + FILELIST *flist; + WT_CURSOR *cursor; + size_t alloc, count; + int j, ret; + char buf[1024], h[256]; + const char *filename, *hdir; + + /* + * First time through we take a full backup into the incremental directories. Otherwise only + * into the appropriate full directory. + */ + if (i != 0) { + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); + hdir = h; + } else + hdir = home_incr; + if (i == 0) { + (void)snprintf(buf, sizeof(buf), "incremental=(enabled=true,this_id=ID%d)", i); + error_check(session->open_cursor(session, "backup:", NULL, buf, &cursor)); + } else + error_check(session->open_cursor(session, "backup:", NULL, NULL, &cursor)); + + count = 0; + alloc = FLIST_INIT; + flist = calloc(alloc, sizeof(FILELIST)); + testutil_assert(flist != NULL); + while ((ret = cursor->next(cursor)) == 0) { + error_check(cursor->get_key(cursor, &filename)); + error_check(process_file(&flist, &count, &alloc, filename)); + if (i == 0) + /* + * Take a full backup into each incremental directory. + */ + for (j = 0; j < MAX_ITERATIONS; j++) { + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + printf("FULL: Copy: %s\n", buf); + error_check(system(buf)); + } + else { + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, hdir, filename); + printf("FULL %d: Copy: %s\n", i, buf); + error_check(system(buf)); + } + } + scan_end_check(ret == WT_NOTFOUND); + error_check(cursor->close(cursor)); + error_check(finalize_files(flist, count)); +} + +static void +take_incr_backup(WT_SESSION *session, int i) +{ + FILELIST *flist; + WT_CURSOR *backup_cur, *incr_cur; + uint64_t offset, size, type; + size_t alloc, count; + int j, ret, rfd, wfd; + char buf[1024], h[256]; + const char *filename; + + /*! [incremental backup using block transfer]*/ + + /* Open the backup data source for incremental backup. */ + (void)snprintf(buf, sizeof(buf), "incremental=(src_id=ID%d,this_id=ID%d)", i - 1, i); + error_check(session->open_cursor(session, "backup:", NULL, buf, &backup_cur)); + rfd = wfd = -1; + count = 0; + alloc = FLIST_INIT; + flist = calloc(alloc, sizeof(FILELIST)); + testutil_assert(flist != NULL); + /* For each file listed, open a duplicate backup cursor and copy the blocks. */ + while ((ret = backup_cur->next(backup_cur)) == 0) { + error_check(backup_cur->get_key(backup_cur, &filename)); + error_check(process_file(&flist, &count, &alloc, filename)); + (void)snprintf(h, sizeof(h), "%s.0", home_incr); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + printf("Copying backup: %s\n", buf); + error_check(system(buf)); +#if 0 + (void)snprintf(buf, sizeof(buf), "%s/%s", home, filename); + printf("Open source %s for reading\n", buf); + error_check(rfd = open(buf, O_RDONLY, 0)); + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, i); + (void)snprintf(buf, sizeof(buf), "%s/%s", h, filename); + printf("Open dest %s for writing\n", buf); + error_check(wfd = open(buf, O_WRONLY, 0)); +#endif + + (void)snprintf(buf, sizeof(buf), "incremental=(file=%s)", filename); + error_check(session->open_cursor(session, NULL, backup_cur, buf, &incr_cur)); + printf("Taking incremental %d: File %s\n", i, filename); + while ((ret = incr_cur->next(incr_cur)) == 0) { + error_check(incr_cur->get_key(incr_cur, &offset, &size, &type)); + printf("Incremental %s: KEY: Off %" PRIu64 " Size: %" PRIu64 " Type: %" PRIu64 "\n", + filename, offset, size, type); + scan_end_check(type == WT_BACKUP_FILE || type == WT_BACKUP_RANGE); + if (type == WT_BACKUP_RANGE) { + /* + * We should never get a range key after a whole file so the read file descriptor + * should be valid. If the read descriptor is valid, so it the write one. + */ + scan_end_check(rfd != -1); + printf("Incremental %s: Range Offset: %" PRIu64 " Size: %" PRIu64 "\n", filename, + offset, size); + error_sys_check(lseek(rfd, (wt_off_t)offset, SEEK_SET)); + error_sys_check(read(rfd, buf, (size_t)size)); + error_sys_check(lseek(wfd, (wt_off_t)offset, SEEK_SET)); + error_sys_check(write(wfd, buf, (size_t)size)); + } else { +/* Whole file, so close both files and just copy the whole thing. */ +#if 0 + error_check(close(rfd)); + error_check(close(wfd)); +#endif + rfd = wfd = -1; + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + printf("Incremental: Whole file copy: %s\n", buf); + error_check(system(buf)); + } + } + scan_end_check(ret == WT_NOTFOUND); + error_check(incr_cur->close(incr_cur)); + + /* Close file descriptors if they're open. */ + if (rfd != -1) { + error_check(close(rfd)); + error_check(close(wfd)); + } + /* + * For each file, we want to copy the file into each of the later incremental directories so + * that they start out at the same for the next incremental round. We then check each + * incremental directory along the way. + */ + for (j = i; j < MAX_ITERATIONS; j++) { + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); + error_check(system(buf)); + } + } + scan_end_check(ret == WT_NOTFOUND); + + error_check(backup_cur->close(backup_cur)); + error_check(finalize_files(flist, count)); + /*! [incremental backup using block transfer]*/ +} + +int +main(int argc, char *argv[]) +{ + WT_CONNECTION *wt_conn; + WT_CURSOR *backup_cur; + WT_SESSION *session; + int i; + char cmd_buf[256]; + + (void)argc; /* Unused variable */ + (void)testutil_set_progname(argv); + + (void)snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s && mkdir %s", home, home); + error_check(system(cmd_buf)); + error_check(wiredtiger_open(home, NULL, CONN_CONFIG, &wt_conn)); + + setup_directories(); + error_check(wt_conn->open_session(wt_conn, NULL, NULL, &session)); + error_check(session->create(session, uri, "key_format=S,value_format=S")); + error_check(session->create(session, uri2, "key_format=S,value_format=S")); + printf("Adding initial data\n"); + add_work(session, 0); + + printf("Taking initial backup\n"); + take_full_backup(session, 0); + + error_check(session->checkpoint(session, NULL)); + + for (i = 1; i < MAX_ITERATIONS; i++) { + printf("Iteration %d: adding data\n", i); + add_work(session, i); + error_check(session->checkpoint(session, NULL)); + /* + * The full backup here is only needed for testing and comparison purposes. A normal + * incremental backup procedure would not include this. + */ + printf("Iteration %d: taking full backup\n", i); + take_full_backup(session, i); + /* + * Taking the incremental backup also calls truncate to archive the log files, if the copies + * were successful. See that function for details on that call. + */ + printf("Iteration %d: taking incremental backup\n", i); + take_incr_backup(session, i); + + printf("Iteration %d: dumping and comparing data\n", i); + error_check(compare_backups(i)); + } + + /* + * After we're done, release resources. Test the force stop setting. + */ + (void)snprintf(cmd_buf, sizeof(cmd_buf), "incremental=(force_stop=true)"); + error_check(session->open_cursor(session, "backup:", NULL, cmd_buf, &backup_cur)); + error_check(backup_cur->close(backup_cur)); + + /* + * Close the connection. We're done and want to run the final comparison between the incremental + * and original. + */ + error_check(wt_conn->close(wt_conn, NULL)); + + printf("Final comparison: dumping and comparing data\n"); + error_check(compare_backups(0)); + + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index fc2b4a1a4a8..b7d02227069 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "a95c73b58971e3dbea0287e08053f4b6894bd8c0", + "commit": "714078029d530f6438fd170f963a9ee27c7b517c", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/async/async_op.c b/src/third_party/wiredtiger/src/async/async_op.c index 5ba9af81055..88c6fde49c3 100644 --- a/src/third_party/wiredtiger/src/async/async_op.c +++ b/src/third_party/wiredtiger/src/async/async_op.c @@ -36,7 +36,7 @@ __async_set_key(WT_ASYNC_OP *asyncop, ...) c = &asyncop->c; va_start(ap, asyncop); - __wt_cursor_set_keyv(c, c->flags, ap); + WT_IGNORE_RET(__wt_cursor_set_keyv(c, c->flags, ap)); if (!WT_DATA_IN_ITEM(&c->key) && !WT_CURSOR_RECNO(c)) c->saved_err = __wt_buf_set(O2S((WT_ASYNC_OP_IMPL *)asyncop), &c->key, c->key.data, c->key.size); @@ -71,7 +71,7 @@ __async_set_value(WT_ASYNC_OP *asyncop, ...) c = &asyncop->c; va_start(ap, asyncop); - __wt_cursor_set_valuev(c, ap); + WT_IGNORE_RET(__wt_cursor_set_valuev(c, ap)); /* Copy the data, if it is pointing at data elsewhere. */ if (!WT_DATA_IN_ITEM(&c->value)) c->saved_err = diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index d4abddbd9dd..9502c70c218 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -57,7 +57,7 @@ __cursor_fix_append_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) cbt->v = 0; cbt->iface.value.data = &cbt->v; } else { - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) { cbt->v = 0; @@ -111,7 +111,7 @@ new_page: if (cbt->ins != NULL && cbt->recno != WT_INSERT_RECNO(cbt->ins)) cbt->ins = NULL; if (cbt->ins != NULL) - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); @@ -145,12 +145,12 @@ __cursor_var_append_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) for (;;) { cbt->ins = WT_SKIP_NEXT(cbt->ins); - new_page: +new_page: if (cbt->ins == NULL) return (WT_NOTFOUND); __cursor_set_recno(cbt, WT_INSERT_RECNO(cbt->ins)); - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) continue; @@ -209,8 +209,8 @@ __cursor_var_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) return (WT_NOTFOUND); __cursor_set_recno(cbt, cbt->recno + 1); - new_page: - restart_read: +new_page: +restart_read: /* Find the matching WT_COL slot. */ if ((cip = __col_var_search(cbt->ref, cbt->recno, &rle_start)) == NULL) return (WT_NOTFOUND); @@ -337,9 +337,9 @@ __cursor_row_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) if (cbt->ins != NULL) cbt->ins = WT_SKIP_NEXT(cbt->ins); - new_insert: +new_insert: cbt->iter_retry = WT_CBT_RETRY_INSERT; - restart_read_insert: +restart_read_insert: if ((ins = cbt->ins) != NULL) { WT_RET(__wt_txn_read(session, ins->upd, &upd)); if (upd == NULL) @@ -373,7 +373,7 @@ __cursor_row_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) cbt->iter_retry = WT_CBT_RETRY_PAGE; cbt->slot = cbt->row_iteration_slot / 2 - 1; - restart_read_page: +restart_read_page: rip = &page->pg_row[cbt->slot]; WT_RET(__wt_txn_read(session, WT_ROW_UPDATE(page, rip), &upd)); if (upd != NULL && upd->type == WT_UPDATE_TOMBSTONE) { diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 4f514db6495..049e9f48bc5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -198,7 +198,7 @@ __cursor_fix_append_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) cbt->iface.value.data = &cbt->v; } else { upd = NULL; - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) { cbt->v = 0; @@ -252,7 +252,7 @@ new_page: cbt->ins = NULL; upd = NULL; if (cbt->ins != NULL) - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); @@ -286,12 +286,12 @@ __cursor_var_append_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) for (;;) { WT_RET(__cursor_skip_prev(cbt)); - new_page: +new_page: if (cbt->ins == NULL) return (WT_NOTFOUND); __cursor_set_recno(cbt, WT_INSERT_RECNO(cbt->ins)); - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) continue; @@ -348,11 +348,11 @@ __cursor_var_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) for (;;) { __cursor_set_recno(cbt, cbt->recno - 1); - new_page: +new_page: if (cbt->recno < cbt->ref->ref_recno) return (WT_NOTFOUND); - restart_read: +restart_read: /* Find the matching WT_COL slot. */ if ((cip = __col_var_search(cbt->ref, cbt->recno, &rle_start)) == NULL) return (WT_NOTFOUND); @@ -488,9 +488,9 @@ __cursor_row_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) if (cbt->ins != NULL) WT_RET(__cursor_skip_prev(cbt)); - new_insert: +new_insert: cbt->iter_retry = WT_CBT_RETRY_INSERT; - restart_read_insert: +restart_read_insert: if ((ins = cbt->ins) != NULL) { WT_RET(__wt_txn_read(session, ins->upd, &upd)); if (upd == NULL) @@ -526,7 +526,7 @@ __cursor_row_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) cbt->iter_retry = WT_CBT_RETRY_PAGE; cbt->slot = cbt->row_iteration_slot / 2 - 1; - restart_read_page: +restart_read_page: rip = &page->pg_row[cbt->slot]; WT_RET(__wt_txn_read(session, WT_ROW_UPDATE(page, rip), &upd)); if (upd != NULL && upd->type == WT_UPDATE_TOMBSTONE) { diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 687c44bc3fa..0f6b569b755 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -1142,7 +1142,7 @@ err: * subsequent iteration can succeed, we cannot return success.) */ if (0) { - search_notfound: +search_notfound: ret = WT_NOTFOUND; if (!iterating && !positioned && F_ISSET(cursor, WT_CURSTD_OVERWRITE)) ret = 0; diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 81adc9de7a1..ddf47a43c2e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -754,7 +754,7 @@ read: continue; } - skip_evict: +skip_evict: /* * If we read the page and are configured to not trash the cache, and no other thread * has already used the page, set the read generation so the page is evicted soon. diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index ea54d449576..d918b5d856c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -917,7 +917,7 @@ __slvg_col_range_overlap(WT_SESSION_IMPL *session, uint32_t a_slot, uint32_t b_s * Case #5: a_trk is a superset of b_trk and a_trk is more desirable -- discard b_trk. */ if (a_trk->trk_gen > b_trk->trk_gen) { - delete_b: +delete_b: /* * After page and overflow reconciliation, one (and only one) * page can reference an overflow record. But, if we split a @@ -1512,7 +1512,7 @@ __slvg_row_range_overlap(WT_SESSION_IMPL *session, uint32_t a_slot, uint32_t b_s * Case #5: a_trk is a superset of b_trk and a_trk is more desirable -- discard b_trk. */ if (a_trk->trk_gen > b_trk->trk_gen) { - delete_b: +delete_b: /* * After page and overflow reconciliation, one (and only one) * page can reference an overflow record. But, if we split a diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 037fb5cdcb6..f7297729967 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -271,6 +271,9 @@ __wt_sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) if (WT_PAGE_IS_INTERNAL(page)) { internal_bytes += page->memory_footprint; ++internal_pages; + /* Slow down checkpoints. */ + if (F_ISSET(conn, WT_CONN_DEBUG_SLOW_CKPT)) + __wt_sleep(0, 10000); } else { leaf_bytes += page->memory_footprint; ++leaf_pages; diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 7685547b351..175a4010db0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -423,7 +423,7 @@ __verify_tree(WT_SESSION_IMPL *session, WT_REF *ref, WT_CELL_UNPACK *addr_unpack goto recno_chk; case WT_PAGE_COL_VAR: recno = ref->ref_recno; - recno_chk: +recno_chk: if (recno != vs->record_total + 1) WT_RET_MSG(session, WT_ERROR, "page at %s has a starting record of %" PRIu64 " when the expected starting record is %" PRIu64, @@ -473,7 +473,7 @@ __verify_tree(WT_SESSION_IMPL *session, WT_REF *ref, WT_CELL_UNPACK *addr_unpack case WT_PAGE_COL_INT: case WT_PAGE_ROW_INT: if (addr_unpack->raw != WT_CELL_ADDR_INT) - celltype_err: +celltype_err: WT_RET_MSG(session, WT_ERROR, "page at %s, of type %s, is referenced in " "its parent by a cell of type %s", diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index 2d6654ebd43..29a89231cf3 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -557,7 +557,7 @@ __verify_dsk_row( current->size = prefix + unpack->size; } - key_compare: +key_compare: /* * Compare the current key against the last key. * @@ -770,7 +770,7 @@ __verify_dsk_col_var( goto match_err; } else if (cell_type == WT_CELL_VALUE && last.data != NULL && last.size == unpack->size && memcmp(last.data, unpack->data, last.size) == 0) - match_err: +match_err: WT_RET_VRFY(session, "data entries %" PRIu32 " and %" PRIu32 " on page at %s are identical and should " "have been run-length encoded", diff --git a/src/third_party/wiredtiger/src/btree/row_key.c b/src/third_party/wiredtiger/src/btree/row_key.c index d0524dfe5a3..caaca56eb88 100644 --- a/src/third_party/wiredtiger/src/btree/row_key.c +++ b/src/third_party/wiredtiger/src/btree/row_key.c @@ -168,7 +168,7 @@ __wt_row_leaf_key_work( #endif for (slot_offset = 0;;) { if (0) { - switch_and_jump: +switch_and_jump: /* Switching to a forward roll. */ WT_ASSERT(session, direction == BACKWARD); direction = FORWARD; diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index 26328dee4d6..aa427d93e44 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -533,7 +533,7 @@ leaf_only: * read-mostly workload. Check that case and get out fast. */ if (0) { - leaf_match: +leaf_match: cbt->compare = 0; cbt->slot = WT_ROW_SLOT(page, rip); return (0); diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 958c267a7ce..a24841fe67e 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -55,8 +55,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_compatibility_sub static const WT_CONFIG_CHECK confchk_wiredtiger_open_debug_mode_subconfigs[] = { {"checkpoint_retention", "int", NULL, "min=0,max=1024", NULL, 0}, - {"eviction", "boolean", NULL, NULL, NULL, 0}, + {"eviction", "boolean", NULL, NULL, NULL, 0}, {"realloc_exact", "boolean", NULL, NULL, NULL, 0}, {"rollback_error", "int", NULL, "min=0,max=10M", NULL, 0}, + {"slow_checkpoint", "boolean", NULL, NULL, NULL, 0}, {"table_logging", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_eviction_subconfigs[] = { @@ -105,7 +106,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"compatibility", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"error_prefix", "string", NULL, NULL, NULL, 0}, {"eviction", "category", NULL, NULL, confchk_wiredtiger_open_eviction_subconfigs, 2}, {"eviction_checkpoint_target", "int", NULL, "min=0,max=10TB", NULL, 0}, @@ -117,7 +118,6 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_log_subconfigs, 4}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -134,7 +134,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { "\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -294,11 +294,19 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_log_flush[] = { {"sync", "string", NULL, "choices=[\"background\",\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor_incremental_subconfigs[] = { + {"enabled", "boolean", NULL, NULL, NULL, 0}, {"file", "string", NULL, NULL, NULL, 0}, + {"force_stop", "boolean", NULL, NULL, NULL, 0}, + {"granularity", "int", NULL, "min=1MB,max=2GB", NULL, 0}, + {"src_id", "string", NULL, NULL, NULL, 0}, {"this_id", "string", NULL, NULL, NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor[] = { {"append", "boolean", NULL, NULL, NULL, 0}, {"bulk", "string", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_wait", "boolean", NULL, NULL, NULL, 0}, {"dump", "string", NULL, "choices=[\"hex\",\"json\",\"print\"]", NULL, 0}, + {"incremental", "category", NULL, NULL, confchk_WT_SESSION_open_cursor_incremental_subconfigs, 6}, {"next_random", "boolean", NULL, NULL, NULL, 0}, {"next_random_sample_size", "string", NULL, NULL, NULL, 0}, {"overwrite", "boolean", NULL, NULL, NULL, 0}, {"raw", "boolean", NULL, NULL, NULL, 0}, @@ -534,7 +542,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -551,8 +559,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"mmap", "boolean", NULL, NULL, NULL, 0}, - {"multiprocess", "boolean", NULL, NULL, NULL, 0}, + {"mmap", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -576,7 +583,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"use_environment", "boolean", NULL, NULL, NULL, 0}, {"use_environment_priv", "boolean", NULL, NULL, NULL, 0}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -602,7 +609,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -619,8 +626,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"mmap", "boolean", NULL, NULL, NULL, 0}, - {"multiprocess", "boolean", NULL, NULL, NULL, 0}, + {"mmap", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -644,7 +650,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"use_environment", "boolean", NULL, NULL, NULL, 0}, {"use_environment_priv", "boolean", NULL, NULL, NULL, 0}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -670,7 +676,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -687,8 +693,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"mmap", "boolean", NULL, NULL, NULL, 0}, - {"multiprocess", "boolean", NULL, NULL, NULL, 0}, + {"mmap", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -710,7 +715,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -736,7 +741,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -753,8 +758,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, - {"lsm_merge", "boolean", NULL, NULL, NULL, 0}, {"mmap", "boolean", NULL, NULL, NULL, 0}, - {"multiprocess", "boolean", NULL, NULL, NULL, 0}, + {"mmap", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {"operation_tracking", "category", NULL, NULL, confchk_wiredtiger_open_operation_tracking_subconfigs, 2}, @@ -776,7 +780,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, {"verbose", "list", NULL, - "choices=[\"api\",\"block\",\"checkpoint\"," + "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," "\"checkpoint_progress\",\"compact\",\"compact_progress\"," "\"error_returns\",\"evict\",\"evict_stuck\",\"evictserver\"," "\"fileops\",\"handleops\",\"log\",\"lookaside\"," @@ -813,20 +817,21 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_overflow=(file_max=0),cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),compatibility=(release=)," "debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",file_manager=(close_handle_minimum=250,close_idle_time=30," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),error_prefix=,eviction=(threads_max=8," + "threads_min=1),eviction_checkpoint_target=1," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95," + "file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),io_capacity=(total=0),log=(archive=true," "os_cache_dirty_pct=0,prealloc=true,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," + "lsm_manager=(merge=true,worker_thread_max=4)," "operation_timeout_ms=0,operation_tracking=(enabled=false," "path=\".\"),shared_cache=(chunk=10MB,name=,quota=0,reserve=0," "size=500MB),statistics=none,statistics_log=(json=false," "on_close=false,sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,verbose=", - confchk_WT_CONNECTION_reconfigure, 27}, + confchk_WT_CONNECTION_reconfigure, 26}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", "commit_timestamp=,durable_timestamp=,force=false," @@ -887,10 +892,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.log_printf", "", NULL, 0}, {"WT_SESSION.open_cursor", "append=false,bulk=false,checkpoint=,checkpoint_wait=true,dump=," - "next_random=false,next_random_sample_size=0,overwrite=true," - "raw=false,read_once=false,readonly=false,skip_sort_check=false," - "statistics=,target=", - confchk_WT_SESSION_open_cursor, 14}, + "incremental=(enabled=false,file=,force_stop=false," + "granularity=16MB,src_id=,this_id=),next_random=false," + "next_random_sample_size=0,overwrite=true,raw=false," + "read_once=false,readonly=false,skip_sort_check=false,statistics=" + ",target=", + confchk_WT_SESSION_open_cursor, 15}, {"WT_SESSION.prepare_transaction", "prepare_timestamp=", confchk_WT_SESSION_prepare_transaction, 1}, {"WT_SESSION.query_timestamp", "get=read", confchk_WT_SESSION_query_timestamp, 1}, @@ -984,10 +991,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," "debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -995,8 +1002,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "io_capacity=(total=0),log=(archive=true,compressor=," "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_timeout_ms=0," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "multiprocess=false,operation_timeout_ms=0," "operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," @@ -1005,7 +1012,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,write_through=", - confchk_wiredtiger_open, 51}, + confchk_wiredtiger_open, 50}, {"wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1014,10 +1021,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," "debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1025,8 +1032,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "io_capacity=(total=0),log=(archive=true,compressor=," "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_timeout_ms=0," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "multiprocess=false,operation_timeout_ms=0," "operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," @@ -1035,7 +1042,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 52}, + confchk_wiredtiger_open_all, 51}, {"wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1043,18 +1050,18 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," "io_capacity=(total=0),log=(archive=true,compressor=," "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_timeout_ms=0," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "multiprocess=false,operation_timeout_ms=0," "operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," @@ -1062,7 +1069,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),verbose=,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 46}, + confchk_wiredtiger_open_basecfg, 45}, {"wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1070,18 +1077,18 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," "io_capacity=(total=0),log=(archive=true,compressor=," "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_timeout_ms=0," + "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," + "multiprocess=false,operation_timeout_ms=0," "operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," @@ -1089,7 +1096,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),verbose=,write_through=", - confchk_wiredtiger_open_usercfg, 45}, + confchk_wiredtiger_open_usercfg, 44}, {NULL, NULL, NULL, 0}}; int diff --git a/src/third_party/wiredtiger/src/config/config_upgrade.c b/src/third_party/wiredtiger/src/config/config_upgrade.c deleted file mode 100644 index be67fa0c3f3..00000000000 --- a/src/third_party/wiredtiger/src/config/config_upgrade.c +++ /dev/null @@ -1,31 +0,0 @@ -/*- - * Copyright (c) 2014-2019 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -/* - * __wt_config_upgrade -- - * Upgrade a configuration string by appended the replacement version. - */ -int -__wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) -{ - WT_CONFIG_ITEM v; - const char *config; - - config = buf->data; - - /* - * wiredtiger_open: - * lsm_merge=boolean -> lsm_manager=(merge=boolean) - */ - if (__wt_config_getones(session, config, "lsm_merge", &v) != WT_NOTFOUND) - WT_RET(__wt_buf_catfmt(session, buf, ",lsm_manager=(merge=%s)", v.val ? "true" : "false")); - - return (0); -} diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 7e8a898193f..7013ca58238 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1428,9 +1428,6 @@ __conn_config_file( /* Check any version. */ WT_ERR(__conn_config_check_version(session, cbuf->data)); - /* Upgrade the configuration string. */ - WT_ERR(__wt_config_upgrade(session, cbuf)); - /* Check the configuration information. */ WT_ERR(__wt_config_check(session, is_user ? WT_CONFIG_REF(session, wiredtiger_open_usercfg) : WT_CONFIG_REF(session, wiredtiger_open_basecfg), @@ -1519,7 +1516,6 @@ __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf) /* Upgrade the configuration string. */ WT_ERR(__wt_buf_setstr(session, cbuf, env_config)); - WT_ERR(__wt_config_upgrade(session, cbuf)); /* Check the configuration information. */ WT_ERR(__wt_config_check(session, WT_CONFIG_REF(session, wiredtiger_open), env_config, 0)); @@ -1800,9 +1796,21 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) else F_CLR(cache, WT_CACHE_EVICT_DEBUG_MODE); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.realloc_exact", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_DEBUG_REALLOC_EXACT); + else + F_CLR(conn, WT_CONN_DEBUG_REALLOC_EXACT); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.rollback_error", &cval)); txn_global->debug_rollback = (uint64_t)cval.val; + WT_RET(__wt_config_gets(session, cfg, "debug_mode.slow_checkpoint", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_DEBUG_SLOW_CKPT); + else + F_CLR(conn, WT_CONN_DEBUG_SLOW_CKPT); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.table_logging", &cval)); if (cval.val) FLD_SET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE); @@ -1825,14 +1833,15 @@ typedef struct { int __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { - static const WT_NAME_FLAG verbtypes[] = {{"api", WT_VERB_API}, {"block", WT_VERB_BLOCK}, - {"checkpoint", WT_VERB_CHECKPOINT}, {"checkpoint_progress", WT_VERB_CHECKPOINT_PROGRESS}, - {"compact", WT_VERB_COMPACT}, {"compact_progress", WT_VERB_COMPACT_PROGRESS}, - {"error_returns", WT_VERB_ERROR_RETURNS}, {"evict", WT_VERB_EVICT}, - {"evict_stuck", WT_VERB_EVICT_STUCK}, {"evictserver", WT_VERB_EVICTSERVER}, - {"fileops", WT_VERB_FILEOPS}, {"handleops", WT_VERB_HANDLEOPS}, {"log", WT_VERB_LOG}, - {"lookaside", WT_VERB_LOOKASIDE}, {"lookaside_activity", WT_VERB_LOOKASIDE_ACTIVITY}, - {"lsm", WT_VERB_LSM}, {"lsm_manager", WT_VERB_LSM_MANAGER}, {"metadata", WT_VERB_METADATA}, + static const WT_NAME_FLAG verbtypes[] = {{"api", WT_VERB_API}, {"backup", WT_VERB_BACKUP}, + {"block", WT_VERB_BLOCK}, {"checkpoint", WT_VERB_CHECKPOINT}, + {"checkpoint_progress", WT_VERB_CHECKPOINT_PROGRESS}, {"compact", WT_VERB_COMPACT}, + {"compact_progress", WT_VERB_COMPACT_PROGRESS}, {"error_returns", WT_VERB_ERROR_RETURNS}, + {"evict", WT_VERB_EVICT}, {"evict_stuck", WT_VERB_EVICT_STUCK}, + {"evictserver", WT_VERB_EVICTSERVER}, {"fileops", WT_VERB_FILEOPS}, + {"handleops", WT_VERB_HANDLEOPS}, {"log", WT_VERB_LOG}, {"lookaside", WT_VERB_LOOKASIDE}, + {"lookaside_activity", WT_VERB_LOOKASIDE_ACTIVITY}, {"lsm", WT_VERB_LSM}, + {"lsm_manager", WT_VERB_LSM_MANAGER}, {"metadata", WT_VERB_METADATA}, {"mutex", WT_VERB_MUTEX}, {"overflow", WT_VERB_OVERFLOW}, {"read", WT_VERB_READ}, {"rebalance", WT_VERB_REBALANCE}, {"reconcile", WT_VERB_RECONCILE}, {"recovery", WT_VERB_RECOVERY}, {"recovery_progress", WT_VERB_RECOVERY_PROGRESS}, diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index e5c82d49a48..37136926060 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -94,6 +94,7 @@ void __wt_connection_destroy(WT_CONNECTION_IMPL *conn) { WT_SESSION_IMPL *session; + u_int i; /* Check there's something to destroy. */ if (conn == NULL) @@ -131,6 +132,13 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_cond_destroy(session, &conn->lsm_manager.work_cond); /* Free allocated memory. */ + /* + * XXX we need to persist this information when we are working on making incremental backups + * persistent across restarts. + */ + for (i = 0; i < WT_BLKINCR_MAX; ++i) + __wt_free(session, conn->incr_backups[i].id_str); + __wt_free(session, conn->cfg); __wt_free(session, conn->debug_ckpt); __wt_free(session, conn->error_prefix); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 4869bcb3b71..be781118895 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -11,9 +11,12 @@ static int __backup_all(WT_SESSION_IMPL *); static int __backup_list_append(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *); static int __backup_list_uri_append(WT_SESSION_IMPL *, const char *, bool *); -static int __backup_start(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, bool, const char *[]); +static int __backup_start( + WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, WT_CURSOR_BACKUP *, const char *[]); static int __backup_stop(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); -static int __backup_uri(WT_SESSION_IMPL *, const char *[], bool, bool *, bool *); + +#define WT_CURSOR_BACKUP_CHECK_STOP(cursor) \ + WT_ERR(F_ISSET(((WT_CURSOR_BACKUP *)(cursor)), WT_CURBACKUP_FORCE_STOP) ? EINVAL : 0); /* * __curbackup_next -- @@ -28,6 +31,7 @@ __curbackup_next(WT_CURSOR *cursor) cb = (WT_CURSOR_BACKUP *)cursor; CURSOR_API_CALL(cursor, session, next, NULL); + WT_CURSOR_BACKUP_CHECK_STOP(cb); if (cb->list == NULL || cb->list[cb->next] == NULL) { F_CLR(cursor, WT_CURSTD_KEY_SET); @@ -57,6 +61,7 @@ __curbackup_reset(WT_CURSOR *cursor) cb = (WT_CURSOR_BACKUP *)cursor; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, reset, NULL); + WT_CURSOR_BACKUP_CHECK_STOP(cb); cb->next = 0; F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); @@ -66,6 +71,35 @@ err: } /* + * __backup_incr_release -- + * Free all resources relating to incremental backup. + */ +static int +__backup_incr_release(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool force) +{ + WT_BLKINCR *blk; + WT_CONNECTION_IMPL *conn; + u_int i; + + WT_UNUSED(cb); + WT_UNUSED(force); + conn = S2C(session); + /* + * Clear flags. Remove file. Release any memory information. + */ + F_CLR(conn, WT_CONN_INCR_BACKUP); + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk = &conn->incr_backups[i]; + F_CLR(blk, WT_BLKINCR_VALID); + } + /* __wt_block_backup_remove... */ + conn->ckpt_incr_granularity = 0; + WT_RET(__wt_remove_if_exists(session, WT_BLKINCR_BACKUP, true)); + + return (0); +} + +/* * __backup_free -- * Free list resources for a backup cursor. */ @@ -79,6 +113,13 @@ __backup_free(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) __wt_free(session, cb->list[i]); __wt_free(session, cb->list); } + if (cb->incr_file != NULL) + __wt_free(session, cb->incr_file); + if (cb->incr_src != NULL) + __wt_free(session, cb->incr_src); + if (cb->incr_this != NULL) + __wt_free(session, cb->incr_this); + __wt_curbackup_free_incr(session, cb); } /* @@ -96,6 +137,12 @@ __curbackup_close(WT_CURSOR *cursor) CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); err: + if (F_ISSET(cb, WT_CURBACKUP_FORCE_STOP)) { + __wt_verbose( + session, WT_VERB_BACKUP, "%s", "Releasing resources from forced stop incremental"); + __backup_incr_release(session, cb, true); + } + /* * When starting a hot backup, we serialize hot backup cursors and set the connection's * hot-backup flag. Once that's done, we set the cursor's backup-locker flag, implying the @@ -147,7 +194,7 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, __wt_cursor_reopen_notsup, /* reopen */ __curbackup_close); /* close */ WT_CURSOR *cursor; - WT_CURSOR_BACKUP *cb; + WT_CURSOR_BACKUP *cb, *othercb; WT_DECL_RET; WT_STATIC_ASSERT(offsetof(WT_CURSOR_BACKUP, iface) == 0); @@ -157,19 +204,23 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, *cursor = iface; cursor->session = (WT_SESSION *)session; cursor->key_format = "S"; /* Return the file names as the key. */ - cursor->value_format = ""; /* No value. */ + cursor->value_format = ""; /* No value, for now. */ session->bkp_cursor = cb; + othercb = (WT_CURSOR_BACKUP *)other; + if (othercb != NULL) + WT_CURSOR_BACKUP_CHECK_STOP(othercb); /* * Start the backup and fill in the cursor's list. Acquire the schema lock, we need a consistent * view when creating a copy. */ WT_WITH_CHECKPOINT_LOCK( - session, WT_WITH_SCHEMA_LOCK(session, ret = __backup_start(session, cb, other != NULL, cfg))); + session, WT_WITH_SCHEMA_LOCK(session, ret = __backup_start(session, cb, othercb, cfg))); WT_ERR(ret); - - WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); + WT_ERR(cb->incr_file == NULL ? + __wt_cursor_init(cursor, uri, NULL, cfg, cursorp) : + __wt_curbackup_open_incr(session, uri, other, cursor, cfg, cursorp)); if (0) { err: @@ -181,6 +232,105 @@ err: } /* + * __backup_get_ckpt -- + * Get the most recent checkpoint information and store it in the structure. + * + * XXX - Currently set return to static void for the compiler, when this function has real content + * it should be static int. + */ +static void +__backup_get_ckpt(WT_SESSION_IMPL *session, WT_BLKINCR *incr) +{ + WT_UNUSED(session); + WT_UNUSED(incr); + /* + * Look up the most recent checkpoint and store information about it in incr. + * + * XXX When this function has content, return a real value. return (0); + */ + return; +} + +/* + * __backup_add_id -- + * Add the identifier for block based incremental backup. + */ +static int +__backup_add_id(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval) +{ + WT_BLKINCR *blk; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + u_int i; + + conn = S2C(session); + blk = NULL; + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk = &conn->incr_backups[i]; + /* If it isn't use, we can use it. */ + if (!F_ISSET(blk, WT_BLKINCR_INUSE)) + break; + } + /* + * We didn't find an entry. This should not happen. + */ + if (i == WT_BLKINCR_MAX) + WT_PANIC_RET(session, WT_NOTFOUND, "Could not find an incremental backup slot to use"); + + /* Use the slot. */ + if (blk->id_str != NULL) + __wt_verbose( + session, WT_VERB_BACKUP, "Freeing and reusing backup slot with old id %s", blk->id_str); + /* Free any string that was there. */ + __wt_free(session, blk->id_str); + WT_ERR(__wt_strndup(session, cval->str, cval->len, &blk->id_str)); + __wt_verbose(session, WT_VERB_BACKUP, "Using backup slot %u for id %s", i, blk->id_str); + /* + * XXX This function can error in the future. + * + * WT_ERR(__backup_get_ckpt(session, blk)); + */ + __backup_get_ckpt(session, blk); + F_SET(blk, WT_BLKINCR_VALID); + return (0); + +err: + if (blk != NULL) + __wt_free(session, blk->id_str); + return (ret); +} + +/* + * __backup_find_id -- + * Find the source identifier for block based incremental backup. Error if it is not a valid id. + */ +static int +__backup_find_id(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_BLKINCR **incrp) +{ + WT_BLKINCR *blk; + WT_CONNECTION_IMPL *conn; + u_int i; + + conn = S2C(session); + for (i = 0; i < WT_BLKINCR_MAX; ++i) { + blk = &conn->incr_backups[i]; + /* If it isn't valid, skip it. */ + if (!F_ISSET(blk, WT_BLKINCR_VALID)) + continue; + if (WT_STRING_MATCH(blk->id_str, cval->str, cval->len)) { + if (F_ISSET(blk, WT_BLKINCR_INUSE)) + WT_RET_MSG(session, EINVAL, "Incremental backup structure already in use"); + if (incrp != NULL) + *incrp = blk; + __wt_verbose(session, WT_VERB_BACKUP, "Found backup slot %u for id %s", i, blk->id_str); + return (0); + } + } + __wt_verbose(session, WT_VERB_BACKUP, "Did not find %.*s", (int)cval->len, cval->str); + return (WT_NOTFOUND); +} + +/* * __backup_log_append -- * Append log files needed for backup. */ @@ -208,21 +358,203 @@ err: } /* + * __backup_config -- + * Backup configuration. + */ +static int +__backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[], + WT_CURSOR_BACKUP *othercb, bool *foundp, bool *log_only, bool *incr_only) +{ + WT_CONFIG targetconf; + WT_CONFIG_ITEM cval, k, v; + WT_CONNECTION_IMPL *conn; + WT_DECL_ITEM(tmp); + WT_DECL_RET; + const char *uri; + bool incremental_config, is_dup, log_config, target_list; + + *foundp = *incr_only = *log_only = false; + + conn = S2C(session); + incremental_config = log_config = false; + is_dup = othercb != NULL; + + /* + * Per-file offset incremental hot backup configurations take a starting checkpoint and optional + * maximum transfer size, and the subsequent duplicate cursors take a file object. + */ + WT_RET_NOTFOUND_OK(__wt_config_gets(session, cfg, "incremental.force_stop", &cval)); + if (cval.val) { + /* + * If we're force stopping incremental backup, set the flag. The resources involved in + * incremental backup will be released on cursor close and that is the only expected usage + * for this cursor. + */ + if (is_dup) + WT_RET_MSG(session, EINVAL, + "Incremental force stop can only be specified on a primary backup cursor"); + F_SET(cb, WT_CURBACKUP_FORCE_STOP); + return (0); + } + WT_RET_NOTFOUND_OK(__wt_config_gets(session, cfg, "incremental.enabled", &cval)); + if (cval.val) { + if (!F_ISSET(conn, WT_CONN_INCR_BACKUP)) { + WT_RET(__wt_config_gets(session, cfg, "incremental.granularity", &cval)); + /* XXX may not need cb->incr_granularity */ + if (conn->ckpt_incr_granularity != 0) + WT_RET_MSG(session, EINVAL, "Cannot change the incremental backup granularity"); + conn->ckpt_incr_granularity = cb->incr_granularity = (uint64_t)cval.val; + } + /* XXX Granularity can only be set once at the beginning */ + F_SET(conn, WT_CONN_INCR_BACKUP); + incremental_config = true; + } + + /* + * Specifying an incremental file means we're opening a duplicate backup cursor. + */ + WT_RET(__wt_config_gets(session, cfg, "incremental.file", &cval)); + if (cval.len != 0) { + if (!is_dup) + WT_RET_MSG(session, EINVAL, + "Incremental file name can only be specified on a duplicate backup cursor"); + WT_RET(__wt_strndup(session, cval.str, cval.len, &cb->incr_file)); + incremental_config = true; + } + + /* + * See if we have a source identifier. We must process the source identifier before processing + * the 'this' identifier. That will mark which source is in use so that we can use any slot that + * is not in use as a new source starting point for this identifier. + */ + WT_RET(__wt_config_gets(session, cfg, "incremental.src_id", &cval)); + if (cval.len != 0) { + if (is_dup) + WT_RET_MSG(session, EINVAL, + "Incremental source identifier can only be specified on a primary backup cursor"); + WT_RET(__backup_find_id(session, &cval, &cb->incr)); + /* XXX might not need this incr_src field */ + WT_RET(__wt_strndup(session, cval.str, cval.len, &cb->incr_src)); + F_SET(cb->incr, WT_BLKINCR_INUSE); + incremental_config = true; + } + /* + * Use WT_ERR from here out because we need to clear the in use flag on error now. + */ + + /* + * Look for a new checkpoint name to retain and mark as a starting point. + */ + WT_ERR(__wt_config_gets(session, cfg, "incremental.this_id", &cval)); + if (cval.len != 0) { + if (is_dup) + WT_ERR_MSG(session, EINVAL, + "Incremental identifier can only be specified on a primary backup cursor"); + ret = __backup_find_id(session, &cval, NULL); + if (ret != WT_NOTFOUND) + WT_ERR_MSG(session, EINVAL, "Incremental identifier already exists"); + + WT_ERR(__backup_add_id(session, &cval)); + /* XXX might not need this incr_this field */ + WT_ERR(__wt_strndup(session, cval.str, cval.len, &cb->incr_this)); + incremental_config = true; + } + + /* + * If we find a non-empty target configuration string, we have a job, otherwise it's not our + * problem. + */ + WT_ERR(__wt_config_gets(session, cfg, "target", &cval)); + __wt_config_subinit(session, &targetconf, &cval); + for (target_list = false; (ret = __wt_config_next(&targetconf, &k, &v)) == 0; + target_list = true) { + /* If it is our first time through, allocate. */ + if (!target_list) { + *foundp = true; + WT_ERR(__wt_scr_alloc(session, 512, &tmp)); + } + + WT_ERR(__wt_buf_fmt(session, tmp, "%.*s", (int)k.len, k.str)); + uri = tmp->data; + if (v.len != 0) + WT_ERR_MSG(session, EINVAL, "%s: invalid backup target: URIs may need quoting", uri); + + /* + * Handle log targets. We do not need to go through the schema worker, just call the + * function to append them. Set log_only only if it is our only URI target. + */ + if (WT_PREFIX_MATCH(uri, "log:")) { + log_config = true; + *log_only = !target_list; + WT_ERR(__backup_log_append(session, session->bkp_cursor, false)); + } else if (is_dup) + WT_ERR_MSG( + session, EINVAL, "duplicate backup cursor cannot be used for non-log target"); + else { + *log_only = false; + + /* + * If backing up individual tables, we have to include indexes, which may involve + * opening those indexes. Acquire the table lock in write mode for that case. + */ + WT_WITH_TABLE_WRITE_LOCK(session, + ret = __wt_schema_worker(session, uri, NULL, __backup_list_uri_append, cfg, 0)); + WT_ERR(ret); + } + } + WT_ERR_NOTFOUND_OK(ret); + + /* + * Compatibility checking. + * + * Log archive cannot mix with log-file based incremental backups (but if a duplicate cursor, + * archiving has been temporarily suspended). + * + * Duplicate backup cursors are only for log targets or block-based incremental backups. But log + * targets don't make sense with block-based incremental backup. + */ + if (!is_dup && log_config && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE)) + WT_ERR_MSG(session, EINVAL, + "incremental log file backup not possible when automatic log archival configured"); + if (is_dup && (!incremental_config && !log_config)) + WT_ERR_MSG(session, EINVAL, + "duplicate backup cursor must be for block-based incremental or logging backup"); + if (incremental_config && (log_config || target_list)) + WT_ERR_MSG( + session, EINVAL, "block-based incremental backup incompatible with a list of targets"); + + if (incremental_config) { + if (is_dup && !F_ISSET(othercb, WT_CURBACKUP_INCR)) + WT_ERR_MSG(session, EINVAL, + "Incremental duplicate cursor must have an incremental primary backup cursor"); + F_SET(cb, WT_CURBACKUP_INCR); + } +err: + if (ret != 0 && cb->incr != NULL) + F_CLR(cb->incr, WT_BLKINCR_INUSE); + __wt_scr_free(session, &tmp); + return (ret); +} + +/* * __backup_start -- * Start a backup. */ static int -__backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, const char *cfg[]) +__backup_start( + WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, WT_CURSOR_BACKUP *othercb, const char *cfg[]) { + WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_FSTREAM *srcfs; const char *dest; - bool exist, log_only, target_list; + bool exist, is_dup, is_incr, log_only, target_list; conn = S2C(session); srcfs = NULL; dest = NULL; + is_dup = othercb != NULL; cb->next = 0; cb->list = NULL; @@ -240,6 +572,21 @@ __backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, cons if (F_ISSET(session, WT_SESSION_BACKUP_DUP) && is_dup) WT_RET_MSG(session, EINVAL, "there is already a duplicate backup cursor open"); + /* + * We want to check for forced stopping early before we do anything else. If it is set, we just + * set a flag and we're done. Actions will be performed on cursor close. + */ + WT_RET_NOTFOUND_OK(__wt_config_gets(session, cfg, "incremental.force_stop", &cval)); + if (cval.val) { + /* + * If we're force stopping incremental backup, set the flag. The resources involved in + * incremental backup will be released on cursor close and that is the only expected usage + * for this cursor. + */ + F_SET(cb, WT_CURBACKUP_FORCE_STOP); + return (0); + } + if (!is_dup) { /* * The hot backup copy is done outside of WiredTiger, which means file blocks can't be freed @@ -260,10 +607,10 @@ __backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, cons /* * Create a temporary backup file. This must be opened before generating the list of targets - * in backup_uri. This file will later be renamed to the correct name depending on whether - * or not we're doing an incremental backup. We need a temp file so that if we fail or crash - * while filling it, the existence of a partial file doesn't confuse restarting in the - * source database. + * in backup_config. This file will later be renamed to the correct name depending on + * whether or not we're doing an incremental backup. We need a temp file so that if we fail + * or crash while filling it, the existence of a partial file doesn't confuse restarting in + * the source database. */ WT_ERR(__wt_fopen(session, WT_BACKUP_TMP, WT_FS_OPEN_CREATE, WT_STREAM_WRITE, &cb->bfs)); } @@ -273,14 +620,11 @@ __backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, cons * database objects and log files to the list. */ target_list = false; - WT_ERR(__backup_uri(session, cfg, is_dup, &target_list, &log_only)); + WT_ERR(__backup_config(session, cb, cfg, othercb, &target_list, &log_only, &is_incr)); /* - * For a duplicate cursor, all the work is done in backup_uri. The only usage accepted is - * "target=("log:")" so error if not log only. + * For a duplicate cursor, all the work is done in backup_config. */ if (is_dup) { - if (!log_only) - WT_ERR_MSG(session, EINVAL, "duplicate backup cursor must be for logs only"); F_SET(cb, WT_CURBACKUP_DUP); F_SET(session, WT_SESSION_BACKUP_DUP); goto done; @@ -317,12 +661,12 @@ __backup_start(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool is_dup, cons * We also open an incremental backup source file so that we can detect a crash with an * incremental backup existing in the source directory versus an improper destination. */ - dest = WT_INCREMENTAL_BACKUP; - WT_ERR(__wt_fopen(session, WT_INCREMENTAL_SRC, WT_FS_OPEN_CREATE, WT_STREAM_WRITE, &srcfs)); - WT_ERR(__backup_list_append(session, cb, WT_INCREMENTAL_BACKUP)); + dest = WT_LOGINCR_BACKUP; + WT_ERR(__wt_fopen(session, WT_LOGINCR_SRC, WT_FS_OPEN_CREATE, WT_STREAM_WRITE, &srcfs)); + WT_ERR(__backup_list_append(session, cb, dest)); } else { dest = WT_METADATA_BACKUP; - WT_ERR(__backup_list_append(session, cb, WT_METADATA_BACKUP)); + WT_ERR(__backup_list_append(session, cb, dest)); WT_ERR(__wt_fs_exist(session, WT_BASECONFIG, &exist)); if (exist) WT_ERR(__backup_list_append(session, cb, WT_BASECONFIG)); @@ -372,6 +716,8 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) /* If it's not a dup backup cursor, make sure one isn't open. */ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_BACKUP_DUP)); WT_WITH_HOTBACKUP_WRITE_LOCK(session, conn->hot_backup_list = NULL); + if (cb->incr != NULL) + F_CLR(cb->incr, WT_BLKINCR_INUSE); __backup_free(session, cb); /* Remove any backup specific file. */ @@ -396,75 +742,6 @@ __backup_all(WT_SESSION_IMPL *session) } /* - * __backup_uri -- - * Backup a list of objects. - */ -static int -__backup_uri(WT_SESSION_IMPL *session, const char *cfg[], bool is_dup, bool *foundp, bool *log_only) -{ - WT_CONFIG targetconf; - WT_CONFIG_ITEM cval, k, v; - WT_DECL_ITEM(tmp); - WT_DECL_RET; - const char *uri; - bool target_list; - - *foundp = *log_only = false; - - /* - * If we find a non-empty target configuration string, we have a job, otherwise it's not our - * problem. - */ - WT_RET(__wt_config_gets(session, cfg, "target", &cval)); - __wt_config_subinit(session, &targetconf, &cval); - for (target_list = false; (ret = __wt_config_next(&targetconf, &k, &v)) == 0; - target_list = true) { - /* If it is our first time through, allocate. */ - if (!target_list) { - *foundp = true; - WT_ERR(__wt_scr_alloc(session, 512, &tmp)); - } - - WT_ERR(__wt_buf_fmt(session, tmp, "%.*s", (int)k.len, k.str)); - uri = tmp->data; - if (v.len != 0) - WT_ERR_MSG(session, EINVAL, "%s: invalid backup target: URIs may need quoting", uri); - - /* - * Handle log targets. We do not need to go through the schema worker, just call the - * function to append them. Set log_only only if it is our only URI target. - */ - if (WT_PREFIX_MATCH(uri, "log:")) { - /* - * Log archive cannot mix with incremental backup, don't let that happen. If we're a - * duplicate cursor archiving is already temporarily suspended. - */ - if (!is_dup && FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ARCHIVE)) - WT_ERR_MSG(session, EINVAL, - "incremental backup not possible when " - "automatic log archival configured"); - *log_only = !target_list; - WT_ERR(__backup_log_append(session, session->bkp_cursor, false)); - } else { - *log_only = false; - - /* - * If backing up individual tables, we have to include indexes, which may involve - * opening those indexes. Acquire the table lock in write mode for that case. - */ - WT_WITH_TABLE_WRITE_LOCK(session, - ret = __wt_schema_worker(session, uri, NULL, __backup_list_uri_append, cfg, 0)); - WT_ERR(ret); - } - } - WT_ERR_NOTFOUND_OK(ret); - -err: - __wt_scr_free(session, &tmp); - return (ret); -} - -/* * __wt_backup_file_remove -- * Remove the incremental and meta-data backup files. */ @@ -479,8 +756,8 @@ __wt_backup_file_remove(WT_SESSION_IMPL *session) * there's any chance of an incremental backup file existing. */ WT_TRET(__wt_remove_if_exists(session, WT_BACKUP_TMP, true)); - WT_TRET(__wt_remove_if_exists(session, WT_INCREMENTAL_BACKUP, true)); - WT_TRET(__wt_remove_if_exists(session, WT_INCREMENTAL_SRC, true)); + WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_BACKUP, true)); + WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_SRC, true)); WT_TRET(__wt_remove_if_exists(session, WT_METADATA_BACKUP, true)); return (ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c new file mode 100644 index 00000000000..c8c59d3f9db --- /dev/null +++ b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c @@ -0,0 +1,233 @@ +/*- + * Copyright (c) 2014-2019 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __alloc_merge -- + * Merge two allocation lists. + */ +static void +__alloc_merge( + uint64_t *a, uint64_t a_cnt, uint64_t *b, uint64_t b_cnt, uint64_t *res, uint64_t *res_cnt) +{ + uint64_t total; + + for (total = 0; a_cnt > 0 || b_cnt > 0; ++total, res += 2) { + if (a_cnt > 0 && b_cnt > 0) { + if (a[0] <= b[0]) { + res[0] = a[0]; + if (a[0] + a[1] < b[0]) + res[1] = a[1]; + else { + res[1] = (b[0] + b[1]) - a[0]; + b += 2; + --b_cnt; + } + a += 2; + --a_cnt; + } else if (b[0] <= a[0]) { + res[0] = b[0]; + if (b[0] + b[1] < a[0]) + res[1] = b[1]; + else { + res[1] = (a[0] + a[1]) - b[0]; + a += 2; + --a_cnt; + } + b += 2; + --b_cnt; + } + } else if (a_cnt > 0) { + res[0] = a[0]; + res[1] = a[1]; + a += 2; + --a_cnt; + } else if (b_cnt > 0) { + res[0] = b[0]; + res[1] = b[1]; + b += 2; + --b_cnt; + } + } + *res_cnt = total; +} + +/* + * __curbackup_incr_next -- + * WT_CURSOR->next method for the btree cursor type when configured with incremental_backup. + */ +static int +__curbackup_incr_next(WT_CURSOR *cursor) +{ + WT_BTREE *btree; + WT_CKPT *ckpt, *ckptbase; + WT_CURSOR_BACKUP *cb; + WT_DECL_RET; + WT_SESSION_IMPL *session; + wt_off_t size; + uint64_t *a, *b, *current, *next; + uint64_t entries, total; + uint32_t raw; + bool start, stop; + + ckptbase = NULL; + a = b = NULL; + + cb = (WT_CURSOR_BACKUP *)cursor; + btree = cb->incr_cursor == NULL ? NULL : ((WT_CURSOR_BTREE *)cb->incr_cursor)->btree; + raw = F_MASK(cursor, WT_CURSTD_RAW); + CURSOR_API_CALL(cursor, session, get_value, btree); + F_CLR(cursor, WT_CURSTD_RAW); + + if (cb->incr_init) { + /* We have this object's incremental information, Check if we're done. */ + if (cb->incr_list_offset >= cb->incr_list_count - WT_BACKUP_INCR_COMPONENTS) + return (WT_NOTFOUND); + + /* + * If we returned all of the data, step to the next block, otherwise return the next chunk + * of the current block. + */ + if (cb->incr_list[cb->incr_list_offset + 1] <= cb->incr_granularity) + cb->incr_list_offset += WT_BACKUP_INCR_COMPONENTS; + else { + cb->incr_list[cb->incr_list_offset] += cb->incr_granularity; + cb->incr_list[cb->incr_list_offset + 1] -= cb->incr_granularity; + cb->incr_list[cb->incr_list_offset + 2] = WT_BACKUP_RANGE; + } + } else if (btree == NULL) { + /* We don't have this object's incremental information, and it's a full file copy. */ + WT_ERR(__wt_fs_size(session, cb->incr_file, &size)); + + cb->incr_list_count = WT_BACKUP_INCR_COMPONENTS; + cb->incr_init = true; + cb->incr_list_offset = 0; + __wt_cursor_set_key(cursor, 0, size, WT_BACKUP_FILE); + } else { + /* + * We don't have this object's incremental information, and it's not a full file copy. Get a + * list of the checkpoints available for the file and flag the starting/stopping ones. It + * shouldn't be possible to specify checkpoints that no longer exist, but check anyway. + */ + ret = __wt_meta_ckptlist_get(session, cb->incr_file, false, &ckptbase); + WT_ERR(ret == WT_NOTFOUND ? ENOENT : ret); + + /* + * Count up the maximum number of block entries we might have to merge, and allocate a pair + * of temporary arrays in which to do the merge. + */ + entries = 0; + WT_CKPT_FOREACH (ckptbase, ckpt) + entries += ckpt->alloc_list_entries; + WT_ERR(__wt_calloc_def(session, entries * WT_BACKUP_INCR_COMPONENTS, &a)); + WT_ERR(__wt_calloc_def(session, entries * WT_BACKUP_INCR_COMPONENTS, &b)); + + /* Merge the block lists into a final list of blocks to copy. */ + start = stop = false; + total = 0; + current = NULL; + next = a; + WT_CKPT_FOREACH (ckptbase, ckpt) { + if (strcmp(ckpt->name, cb->incr_checkpoint_start) == 0) { + start = true; + WT_ERR_ASSERT(session, ckpt->alloc_list_entries == 0, __wt_panic(session), + "incremental backup start checkpoint has allocation list blocks"); + continue; + } + if (start == true) { + if (strcmp(ckpt->name, cb->incr_checkpoint_stop) == 0) + stop = true; + + __alloc_merge( + current, total, ckpt->alloc_list, ckpt->alloc_list_entries, next, &total); + current = next; + next = next == a ? b : a; + } + + if (stop == true) + break; + } + + if (!start) + WT_ERR_MSG(session, ENOENT, "incremental backup start checkpoint %s not found", + cb->incr_checkpoint_start); + if (!stop) + WT_ERR_MSG(session, ENOENT, "incremental backup stop checkpoint %s not found", + cb->incr_checkpoint_stop); + + /* There may be nothing that needs copying. */ + if (total == 0) + WT_ERR(WT_NOTFOUND); + + if (next == a) { + cb->incr_list = b; + b = NULL; + } else { + cb->incr_list = a; + a = NULL; + } + cb->incr_list_count = total; + cb->incr_list_offset = 0; + WT_ERR(__wt_scr_alloc(session, 0, &cb->incr_block)); + cb->incr_init = true; + + F_SET(cursor, WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT); + } + +err: + __wt_free(session, a); + __wt_free(session, b); + __wt_meta_ckptlist_free(session, &ckptbase); + F_SET(cursor, raw); + API_END_RET(session, ret); +} + +/* + * __wt_curbackup_free_incr -- + * Free the duplicate backup cursor for a file-based incremental backup. + */ +void +__wt_curbackup_free_incr(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) +{ + __wt_free(session, cb->incr_file); + if (cb->incr_cursor != NULL) + __wt_cursor_close(cb->incr_cursor); + __wt_free(session, cb->incr_checkpoint_start); + __wt_free(session, cb->incr_checkpoint_stop); + __wt_free(session, cb->incr_list); + __wt_scr_free(session, &cb->incr_block); +} + +/* + * __wt_curbackup_open_incr -- + * Initialize the duplicate backup cursor for a file-based incremental backup. + */ +int +__wt_curbackup_open_incr(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, + WT_CURSOR *cursor, const char *cfg[], WT_CURSOR **cursorp) +{ + WT_CURSOR_BACKUP *cb, *other_cb; + + cb = (WT_CURSOR_BACKUP *)cursor; + other_cb = (WT_CURSOR_BACKUP *)other; + WT_UNUSED(session); + cursor->key_format = WT_UNCHECKED_STRING(qqq); + cursor->value_format = ""; + + /* + * Inherit from the backup cursor but reset specific functions for incremental. + */ + cursor->next = __curbackup_incr_next; + cursor->get_key = __wt_cursor_get_key; + cursor->get_value = __wt_cursor_get_value_notsup; + cb->incr_granularity = other_cb->incr_granularity; + + /* XXX Return full file info for all files for now. */ + return (__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); +} diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index 73690788dfb..9006690402b 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -136,11 +136,11 @@ format: } /* - * __curdump_set_key -- + * __curdump_set_keyv -- * WT_CURSOR->set_key for dump cursors. */ -static void -__curdump_set_key(WT_CURSOR *cursor, ...) +static int +__curdump_set_keyv(WT_CURSOR *cursor, va_list ap) { WT_CURSOR *child; WT_CURSOR_DUMP *cdump; @@ -150,18 +150,15 @@ __curdump_set_key(WT_CURSOR *cursor, ...) const uint8_t *up; const char *p; bool json; - va_list ap; cdump = (WT_CURSOR_DUMP *)cursor; child = cdump->child; CURSOR_API_CALL(cursor, session, set_key, NULL); - va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSTD_RAW)) p = va_arg(ap, WT_ITEM *)->data; else p = va_arg(ap, const char *); - va_end(ap); json = F_ISSET(cursor, WT_CURSTD_DUMP_JSON); if (json) @@ -188,7 +185,21 @@ err: cursor->saved_err = ret; F_CLR(cursor, WT_CURSTD_KEY_SET); } - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __curdump_set_key -- + * WT_CURSOR->set_key for dump cursors. + */ +static void +__curdump_set_key(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curdump_set_keyv(cursor, ap)); + va_end(ap); } /* @@ -238,29 +249,26 @@ err: } /* - * __curdump_set_value -- + * __curdump_set_valuev -- * WT_CURSOR->set_value for dump cursors. */ -static void -__curdump_set_value(WT_CURSOR *cursor, ...) +static int +__curdump_set_valuev(WT_CURSOR *cursor, va_list ap) { WT_CURSOR *child; WT_CURSOR_DUMP *cdump; WT_DECL_RET; WT_SESSION_IMPL *session; const char *p; - va_list ap; cdump = (WT_CURSOR_DUMP *)cursor; child = cdump->child; CURSOR_API_CALL(cursor, session, set_value, NULL); - va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSTD_RAW)) p = va_arg(ap, WT_ITEM *)->data; else p = va_arg(ap, const char *); - va_end(ap); if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) WT_ERR(__wt_json_to_item(session, p, cursor->value_format, @@ -275,7 +283,21 @@ err: cursor->saved_err = ret; F_CLR(cursor, WT_CURSTD_VALUE_SET); } - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __curdump_set_value -- + * WT_CURSOR->set_value for dump cursors. + */ +static void +__curdump_set_value(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curdump_set_valuev(cursor, ap)); + va_end(ap); } /* Pass through a call to the underlying cursor. */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index e675392939c..8de5878653e 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -29,22 +29,38 @@ err: } /* - * __curindex_set_value -- + * __curindex_set_valuev -- * WT_CURSOR->set_value implementation for index cursors. */ -static void -__curindex_set_value(WT_CURSOR *cursor, ...) +static int +__curindex_set_valuev(WT_CURSOR *cursor, va_list ap) { WT_DECL_RET; WT_SESSION_IMPL *session; + WT_UNUSED(ap); + JOINABLE_CURSOR_API_CALL(cursor, session, set_value, NULL); WT_ERR_MSG(session, ENOTSUP, "WT_CURSOR.set_value not supported for index cursors"); err: cursor->saved_err = ret; F_CLR(cursor, WT_CURSTD_VALUE_SET); - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __curindex_set_value -- + * WT_CURSOR->set_value implementation for index cursors. + */ +static void +__curindex_set_value(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curindex_set_valuev(cursor, ap)); + va_end(ap); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index 4db139a8633..fedf1b07640 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -120,35 +120,46 @@ err: } /* - * __curstat_set_key -- + * __curstat_set_keyv -- * WT_CURSOR->set_key for statistics cursors. */ -static void -__curstat_set_key(WT_CURSOR *cursor, ...) +static int +__curstat_set_keyv(WT_CURSOR *cursor, va_list ap) { WT_CURSOR_STAT *cst; WT_DECL_RET; WT_ITEM *item; WT_SESSION_IMPL *session; - va_list ap; cst = (WT_CURSOR_STAT *)cursor; CURSOR_API_CALL(cursor, session, set_key, NULL); F_CLR(cursor, WT_CURSTD_KEY_SET); - va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSTD_RAW)) { item = va_arg(ap, WT_ITEM *); ret = __wt_struct_unpack(session, item->data, item->size, cursor->key_format, &cst->key); } else cst->key = va_arg(ap, int); - va_end(ap); if ((cursor->saved_err = ret) == 0) F_SET(cursor, WT_CURSTD_KEY_EXT); err: - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __curstat_set_key -- + * WT_CURSOR->set_key for statistics cursors. + */ +static void +__curstat_set_key(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curstat_set_keyv(cursor, ap)); + va_end(ap); } /* @@ -716,7 +727,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, c WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); if (0) { - config_err: +config_err: WT_ERR_MSG(session, EINVAL, "cursor's statistics configuration doesn't match the " "database statistics configuration"); diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 6140b453f86..7ff9273bf6d 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -253,7 +253,7 @@ __wt_cursor_set_key(WT_CURSOR *cursor, ...) va_list ap; va_start(ap, cursor); - __wt_cursor_set_keyv(cursor, cursor->flags, ap); + WT_IGNORE_RET(__wt_cursor_set_keyv(cursor, cursor->flags, ap)); va_end(ap); } @@ -377,7 +377,7 @@ err: * __wt_cursor_set_keyv -- * WT_CURSOR->set_key default implementation. */ -void +int __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) { WT_DECL_RET; @@ -453,7 +453,7 @@ err: } else __wt_free(session, tmp.mem); } - API_END(session, ret); + API_END_RET(session, ret); } /* @@ -516,7 +516,7 @@ __wt_cursor_set_value(WT_CURSOR *cursor, ...) va_list ap; va_start(ap, cursor); - __wt_cursor_set_valuev(cursor, ap); + WT_IGNORE_RET(__wt_cursor_set_valuev(cursor, ap)); va_end(ap); } @@ -524,7 +524,7 @@ __wt_cursor_set_value(WT_CURSOR *cursor, ...) * __wt_cursor_set_valuev -- * WT_CURSOR->set_value worker implementation. */ -void +int __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap) { WT_DECL_RET; @@ -588,7 +588,7 @@ err: __wt_free(session, tmp.mem); } - API_END(session, ret); + API_END_RET(session, ret); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 94acee0592e..a5052532542 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -16,8 +16,11 @@ static int __curtable_update(WT_CURSOR *cursor); WT_CURSOR **__cp; \ u_int __i; \ for (__i = 0, __cp = (ctable)->cg_cursors; __i < WT_COLGROUPS((ctable)->table); \ - __i++, __cp++) \ + __i++, __cp++) { \ WT_TRET((*__cp)->f(*__cp)); \ + if (ret != 0 && ret != WT_NOTFOUND) \ + goto err; \ + } \ } while (0) /* Cursor type for custom extractor callback. */ @@ -224,7 +227,7 @@ __wt_curtable_set_key(WT_CURSOR *cursor, ...) primary = *cp++; va_start(ap, cursor); - __wt_cursor_set_keyv(primary, cursor->flags, ap); + WT_IGNORE_RET(__wt_cursor_set_keyv(primary, cursor->flags, ap)); va_end(ap); if (!F_ISSET(primary, WT_CURSTD_KEY_SET)) @@ -240,11 +243,11 @@ __wt_curtable_set_key(WT_CURSOR *cursor, ...) } /* - * __wt_curtable_set_value -- + * __curtable_set_valuev -- * WT_CURSOR->set_value implementation for tables. */ -void -__wt_curtable_set_value(WT_CURSOR *cursor, ...) +static int +__curtable_set_valuev(WT_CURSOR *cursor, va_list ap) { WT_CURSOR **cp; WT_CURSOR_TABLE *ctable; @@ -252,12 +255,10 @@ __wt_curtable_set_value(WT_CURSOR *cursor, ...) WT_ITEM *item, *tmp; WT_SESSION_IMPL *session; u_int i; - va_list ap; ctable = (WT_CURSOR_TABLE *)cursor; JOINABLE_CURSOR_API_CALL(cursor, session, set_value, NULL); - va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSOR_RAW_OK | WT_CURSTD_DUMP_JSON)) { item = va_arg(ap, WT_ITEM *); cursor->value.data = item->data; @@ -292,7 +293,6 @@ __wt_curtable_set_value(WT_CURSOR *cursor, ...) } } } - va_end(ap); for (i = 0, cp = ctable->cg_cursors; i < WT_COLGROUPS(ctable->table); i++, cp++) if (ret == 0) @@ -303,7 +303,21 @@ __wt_curtable_set_value(WT_CURSOR *cursor, ...) } err: - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __wt_curtable_set_value -- + * WT_CURSOR->set_value implementation for tables. + */ +void +__wt_curtable_set_value(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curtable_set_valuev(cursor, ap)); + va_end(ap); } /* diff --git a/src/third_party/wiredtiger/src/docs/devdoc-glossary.dox b/src/third_party/wiredtiger/src/docs/devdoc-glossary.dox new file mode 100644 index 00000000000..4cb762cbaa7 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-glossary.dox @@ -0,0 +1,30 @@ +/*! @page devdoc-glossary Developer Documentation Glossary of Terms + +WiredTiger has a lot of domain specific nomenclature - this page attempts +to decode it. This is intended for those navigating the WiredTiger source +tree - it describes terms internal to the storage engine. + +@section devdoc-glossary-general General Terms Used in WiredTiger + +<table> +<caption id="general_terms">General Term Table</caption> +<tr><th>Term <th>Definition +<tr><td>Hello<td>The typical next word is World +</table> + +@section devdoc-glossary-checkpoint Terms Related to Checkpoints + +<table> +<caption id="checkpoint_terms">Checkpoint Term Table</caption> +<tr><th>Term <th>Definition +<tr><td>Hello<td>The typical next word is World +</table> +@section devdoc-glossary-transactions Terms Related to Transactions + +<table> +<caption id="transaction_terms">Transaction Term Table</caption> +<tr><th>Term <th>Definition +<tr><td>Hello<td>The typical next word is World +</table> + +*/ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-index.dox b/src/third_party/wiredtiger/src/docs/devdoc-index.dox index 7ada556aa1a..f7707653207 100644 --- a/src/third_party/wiredtiger/src/docs/devdoc-index.dox +++ b/src/third_party/wiredtiger/src/docs/devdoc-index.dox @@ -1,13 +1,16 @@ /*! @page devdoc-index Developer Documentation +@subpage devdoc-glossary + +WiredTiger assigns specific meanings to certain words. Here we decode them. + +@subpage devdoc-schema + Most applications begin to make use of WiredTiger by creating a table (or other data object) to store their data in. Create is one of several schema operations available in WiredTiger. -For more information on how schema operations are implemented in WiredTiger, -see: - -- @subpage devdoc-schema +@subpage devdoc-dhandle-lifecycle An internal structure called Data Handle (dhandle) is used to represent and access a table in WiredTiger. A dhandle gets created when a table is accessed @@ -15,8 +18,32 @@ for the first time. It is kept in a global list and is shared across the sessions. When a dhandle is not needed anymore and has been idle for a while, it is closed and destroyed, releasing all the resources associated with it. -For more information on the lifecycle of a dhandle, see: +@subpage devdoc-statistics + +WiredTiger can generate statistics that are useful for providing information +necessary when performance tuning your WiredTiger application. Here we focus +on analyzing and reviewing the data generated by the statistics logging +functionality. + +@subpage devdoc-optrack + +The purpose of operation tracking is to visualize WiredTiger's execution so +that correlations between performance anomalies are easily spotted. This +operation tracking tutorial provides a general overview of operation tracking +and describes ways to visualize the data in fine detail. + +@subpage devdoc-perf + +Linux `perf` is a tool that allows counting and sampling of various events in +the hardware and in the kernel. Hardware events are available via performance +monitoring units (PMU); they measure CPU cycles, cache misses, branches, etc. +Kernel events include scheduling context switches, page faults and block I/O. +Here we provide a quick cheat sheet of how to use `perf` with WiredTiger. + +@subpage devdoc-perf-flamegraphs -- @subpage devdoc-dhandle-lifecycle +Why is my CPU busy? FlameGraphs help visually summarize on-CPU call stacks and +allow for the quick identification of hot code paths. Here we explain how to +generate FlameGraphs from WiredTiger `perf` data. */ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox b/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox new file mode 100644 index 00000000000..360b3b6de47 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-optrack.dox @@ -0,0 +1,351 @@ +/*! @page devdoc-optrack Operation Tracking + +# Overview + +This tutorial will walk you through using operation tracking on one of `wtperf` +workloads: from preparing the workload for most effective data collection, to +gathering, visualizing and interpreting the execution logs. + +## Why use operation tracking? + +Analysis of WiredTiger JIRA tickets revealed that in diagnosing performance +bugs engineers rely mostly on information that shows which functions were +executed over time and across threads. They seek to find correlations between +performance anomalies, such as long latencies or periods of low throughput, and +events on these timelines. The purpose of operation tracking is to visualize +the execution so that these correlations could be easily spotted. + +## How does operation tracking work? + +Operation tracking allows you to put a pair of macros `WT_TRACK_OP_INIT` and +`WT_TRACK_OP_END` into any function in WiredTiger whose invocation time you +want to be measured and recorded. As your program runs, the `WT_TRACK_OP_INIT` +macro will produce the function entry timestamp and `WT_TRACK_OP_END` macro +will produce the function exit timestamp. These timestamps, along with the +function name, will be recorded in a set of log files. It is typical to put the +`WT_TRACK_OP_INIT` in the beginning of the function and the `WT_TRACK_OP_END` +macro at every exit point from the function, but you can put them anywhere +within the function body. The only condition required in order to correctly +visualize the logs is that each `WT_TRACK_OP_INIT` has a matching +`WT_TRACK_OP_END`. It is totally okay to insert tracking macros into nested +functions. + +## Will operation tracking introduce runtime overhead? + +The answer depends on your workload, and how many functions you track. For +`wtperf` suite the largest overhead of 30% was observed on the CPU-bound +workload `small-btree`. For I/O-bound workloads, such as +`evict-btree-stress-multi`, the overhead is negligible, unless you decided to +track short frequently executed functions. + +# Prerequisites + +## Prepare your workload + +Modify your benchmark so that it produces the desired behavior with as little +running time as possible. The longer the benchmark runs the more data it will +produce, and the more difficult it will be for you to go over all this data. + +In our running example with the `evict-btree-stress-multi.wtperf` workload, we +separate the database creation phase from the work phase. We first create the +database with operation tracking disabled. Then we enable operation tracking +and run the work phase. So operation tracking logs are generated during the +work phase only. We also reduce the work phase to 30 seconds, to make sure that +the volume of information is manageable. + +## Install the python modules + +To generate data visualizations, you will need the python modules bokeh, +matplotlib and pandas installed. You can install them via the following +commands (if you don't use pip modify for your python module manager): + + pip install bokeh matplotlib pandas + +## Enable operation tracking +To enable operation tracking, you need to add the following option to your +connection configuration string. + + operation_tracking=(enabled=true,path=.) + +For example, if you are running `wtperf`, you would add it to the `conn_config` +string: + + ./wtperf -c conn_config="operation_tracking=(enabled=true,path=.)" + +The "path" argument determines where the log files, produced by operation +tracking, will be stored. Now you can run your workload. + +# Working with log files + +## Becoming familiar with the operation tracking log files + +After the run is complete, the path directory will have a bunch of files of the +following format: + + optrack.00000<pid>.000000000<sid> + +and the file + + optrack-map.00000<pid> + +where `pid` is the process ID and `sid` is the session ID. + +The first kind of files are the actual log files. The second is the map file. +Log files are generated in the binary format, because this is a lot more +efficient than writing strings, and the map file is needed to decode them. + +## Processing the log files + +Now the binary log files need to be converted into the text format. To do that +use the script `wt_optrack_decode.py` in the WiredTiger tree. We will refer to +the path to your WiredTiger tree as WT. Suppose that the process ID that +generated the operation tracking files is 25660. Then you'd run the decode +script like so: + + % WT/tools/optrack/wt_optrack_decode.py -m optrack-map.0000025660 optrack.0000025660.00000000* + +As the script runs you will see lots of output on the screen reporting the +progress through the parsing process. One kind of output you might see is +something like this: + + TSC_NSEC ratio parsed: 2.8000 + +This refers to the clock cycles to nanoseconds ratio encoded into the log +files. In the example above, we had 2.8 ticks per nanosecond, suggesting that +the workload was run on a 2.8 GHz processor. This ratio is used later to +convert the timestamps, which are recorded in clock cycles for efficiency, to +nanoseconds. + +In the end, you should see lots of output files whose names look like the +optrack log files, but with the suffixes `-internal.txt` and `-external.txt`. +The "internal" files are the log files for WT internal sessions (such as +eviction threads). The "external" files are for the sessions created by the +client application. + +Now that the files have been decoded, it's time to visualize the data. This can +be done using the script find-latency-spikes.py located in the `tools/optrack` +directory of the WiredTiger tree. To process your text files from our example, +you would run this script like so: + + % WT/tools/optrack/find-latency-spikes.py optrack.0000025660.00000000*.txt + +As the script runs, you will probably see messages similar to this one: + + Processing file optrack.0000025660.0000000026-external.txt + Your data may have errors. Check the file .optrack.0000025660.0000000026-external.txt.log for details. + +This means that as the script was processing the log file, it found some +inconsistencies, for example there was a function entry timestamp, but no +corresponding function exit, or vice versa. Sometimes these errors are benign. +For example, the log file will never have an exit timestamp from a +session-closing function, because operation tracking is torn down as the +session closes. In other cases, an error may indicate that there is a mistake +in how the operation-tracking macros were inserted. For example, the programmer +may have forgotten to insert a macro where there is an early return from a +function. Examining the log file mentioned in the error message will give you +more details on the error. + +The data processing script will not fail or exit if it encounters these error +messages, it will proceed despite them, attempting to visualize the portion of +the log that did not contain the errors. The function timestamps implicated in +the errors will simply be dropped. + +At the very end of the processing, you will see messages like this: + + Normalizing data... + Generating timeline charts... 99% complete + Generating outlier histograms... 100% complete + +This is how you know the processing has completed. The script takes a while to +run, but it is parallelizable, so that is something that can be done in the +future to speed it up. + +# Looking at the data: Part 1 + +In the directory where you ran the data processing script, you will have the +file `WT-outliers.html` and the directory `BUCKET-FILES` to which that HTML +file refers. If you would like to look at the visualization on another machine, +transfer both the HTML file and the directory to that machine. Otherwise, just +open `WT-outliers.html` locally in your browser. + +To get an idea of what kind of output you will see, take a look at [this +example data](http://www.ece.ubc.ca/~sasha/TUTORIAL-DEMO/) generated by the +workload described earlier in this tutorial. Please open the URL, because we +will now walk over what it shows. + +The main page shows a number of outlier histograms. Each histogram corresponds +to a function that was tracked by the operation tracking system in WT. The +x-axis shows the execution timeline (in nanoseconds). The y-axis shows how many +abnormally long executions of the function occurred during very period of the +execution. + + + +You can click on outlier bars and look at the detailed visualization of the +period during which the abnormally long function invocations occurred. But +before we do that, let's talk about what is considered an "outlier" or an +"abnormally long execution" and how to control these thresholds. As annotations +on the histograms show, by default a function invocation is considered an +outlier if its duration is longer than two standard deviations above average. +(The average is computed for this function over the entire execution). If you +would like a more precise control of outlier thresholds, you can do that in a +configuration file that is read by the execution script. + +Before we talk about other visualization screens, let's actually generate a +visualization with some more meaningful outlier thresholds than two standard +deviations. This way, it will be easier to navigate the data, and we will be +able to observe some very interesting behavior that gives us a clue as to why +eviction is sometimes slow. (If you can't wait to find out, skip over the next +step.) + +## Getting a more precise control of the outlier thresholds + +Let's learn how to configure custom outlier thresholds. The easiest way to +begin is to grab a sample configuration file and edit it to your needs: + + % cp WT/tools/optrack/sample.config . + +Here is how that file looks: + + # Units in which timestamps are recorded + 1000000000 + # Outlier threshold for individual functions + __curfile_reset 100 ms + __curfile_search 50 ms + __curfile_update 3 ms + __evict_lru_pages 75 ms + __evict_lru_walk 20 ms + __evict_page 75 ms + __evict_walk 20 ms + __wt_cache_eviction_worker 10 ms + __wt_cond_wait_signal 5 s + __wt_cursor_set_valuev 10 ms + +In that file all lines prefixed with a # are comments. + +The first non-comment line specifies the units in which your timestamps are +recorded. The units are specified by providing a value indicating how many time +units there are in a second. By default operation scripts generate timestamps +in nanoseconds, hence you see a value of 1000000000 on the first line in our +sample file. If your time units were, say, milliseconds, you'd change that +value to 1000, and so on. + +The remaining non-comment lines in the file provide the outlier thresholds for +any functions you care to have a custom threshold. The format is: + + <function name> <value> <unit> + +For example, to specify a threshold of 15 milliseconds for the +`__wt_cache_eviction_worker` function, you'd insert the following line: + + __wt_cache_eviction_worker 15 ms + +Valid units are nanoseconds (ns), microseconds (us), milliseconds (ms) and +seconds (s). + +If you don't specify a threshold for a function the default of two standard +deviations will be used. Let's actually configure some more meaningful outlier +thresholds in our `sample.config` file and re-run the visualization script. Here +is the modified `sample.config`: + + # Units in which timestamps are recorded + 1000000000 + # Outlier threshold for individual functions + __curfile_search 30 ms + __evict_lru_pages 250 ms + __evict_lru_walk 1 ms + __evict_page 15 ms + __evict_walk 1 ms + __wt_cache_eviction_worker 15 ms + __wt_cond_wait_signal 10 s + +We rerun the data processing script with the configuration file argument, like +so: + + % WT/tools/optrack/find-latency-spikes.py -c sample.config optrack.0000025660.00000000*.txt + +The new charts can be accessed +[here](http://www.ece.ubc.ca/~sasha/TUTORIAL-DEMO-CONFIG/). + +# Looking at the data: Part 2 + +Let's examine the visualized data in more detail. Please click on [this +URL](http://www.ece.ubc.ca/~sasha/TUTORIAL-DEMO-CONFIG/): + +Although I insert the screenshots here, you will have more fun if you access it +directly. + +Now you can see the outlier charts complying with the configuration parameters +we supplied. For example, on the chart for the `__wt_cache_eviction_worker`, we +see only the intervals where that function took longer than 15 ms to complete. + + + +Let's click on one of those intervals to examine what happened there. I am +going to click on the tallest bar in the chart, which will take me to a +detailed output of function call activity by thread for interval 137 at [this +URL](http://www.ece.ubc.ca/~sasha/TUTORIAL-DEMO-CONFIG/BUCKET-FILES/bucket-137.html). + +There is a lot of graphics there, so the URL might take a few seconds to load. + +At the very top of the page you see a small navigation bar. The bar tells you +where you are in the execution (the red marker). By clicking on other intervals +within the bar you can navigate to other intervals. For example, if you wanted +to look at the execution interval located after the current one, you will just +click on the white bar following the current, red-highlighted, bar. + + + +Next you see a legend that shows all functions that were called during this +execution interval and their corresponding colours. + + + +Then you will see the most interesting information: function calls across +threads that occurred during this period. Durations and hierarchy of function +calls is preserved, meaning that longer functions will be shown with longer +bars and if function A called function B, B will be shown on top of A. + +By hovering over the colored bars corresponding to functions, you will see a +box appear telling you the name of the function, how long it took and its +original timestamp in your operation tracking log file. This way, if you are +suspicious about the correctness of your log or want to go back to it for some +reason, you know what to look for. + +In our example visualization, if we scroll down to the function sequences for +external threads, we will quickly see a few instances where +__wt_cache_eviction_worker took longer than 15 ms, for example here: + + + +As we can see, the threads are simply waiting on the condition variable inside +the eviction worker. To try and understand why, we might want to scroll up and +look at the activity of internal threads during this period. Interestingly +enough, most of the internal threads are also waiting on the condition variable +during this period. + + + +Looking at the internal thread with session id 1 shows something suspicious. + + + +During this period where all threads are waiting, this thread shows no activity +at all, whereas prior and after that "dead" period it was making regular calls +to `__evict_walk`. Perhaps that thread was unscheduled or stopped calling +__evict_walk for other reasons. Perhaps other threads were dependent on work +performed within `__evict_walk` to make forward progress. + +To better understand why `__evict_walk` was interrupted for almost 10ms we +might want to go back and insert operation tracking macros inside the functions +called by `__evict_walk`, to see if the thread was doing some other work during +this time or was simply unscheduled. + +# Summary + +Operation tracking is great for debugging performance problems where you are +interested in understanding the activity of the workload over time and across +threads. To effectively navigate large quantities of data it is best to +reproduce your performance problem in a short run. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-perf-flamegraphs.dox b/src/third_party/wiredtiger/src/docs/devdoc-perf-flamegraphs.dox new file mode 100644 index 00000000000..1e5bc1ad4fd --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-perf-flamegraphs.dox @@ -0,0 +1,74 @@ +/*! @page devdoc-perf-flamegraphs CPU Flame Graphs + +# Introduction + +[FlameGraphs](http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html), +invented by [Brendan Gregg](http://www.brendangregg.com), help visually +summarize on-CPU call stacks. + +Below is an image of a FlameGraph generated from a 10-second run of the +evict-btree-stress-multi wtperf job: + + + +To see it in action, download the non-redacted interactive version +[here](https://github.com/wiredtiger/wiredtiger/wiki/images/FlameGraph-evict-stress-multi.svg) +and open it in your web browser. + +Quoting from [Brendan Gregg's +website](http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html), here is +how to interpret the output of the FlameGraph: + +- Each box represents a function in the stack (a "stack frame"). +- The y-axis shows stack depth (number of frames on the stack). The top box + shows the function that was on-CPU. Everything beneath that is ancestry. The + function beneath a function is its parent, just like the stack traces shown + earlier. (Some flame graph implementations prefer to invert the order and use + an "icicle layout", so flames look upside down.) +- The x-axis spans the sample population. It does not show the passing of time + from left to right, as most graphs do. The left to right ordering has no + meaning (it's sorted alphabetically to maximize frame merging). +- The width of the box shows the total time it was on-CPU or part of an + ancestry that was on-CPU (based on sample count). Functions with wide boxes + may consume more CPU per execution than those with narrow boxes, or, they may + simply be called more often. The call count is not shown (or known via + sampling). + +__Note:__ If you generate a FlameGraph yourself and then open it in a browser, +the image will be interactive, allowing you to hover over the functions to see +their full names. + +# Generating FlameGraphs + +To generate FlameGraphs, you first need to run perf and generate a perf.data +file using the `perf record -g` command. [Click here](@ref devdoc-perf) for +instructions explaining how to do this with WiredTiger. Don't forget the `-g` +option: you need it in order to record the call stacks! + + # Clone the FlameGraph repository + git clone https://github.com/brendangregg/FlameGraph + cd FlameGraph + + # Place perf.data into the current directory, then run these commands. + # Run them on the same machine where you ran `perf record` so perf can + # resolve the symbols. + perf script | ./stackcollapse-perf.pl > out.perf-folded + ./flamegraph.pl out.perf-folded > perf-kernel.svg + +Then open the resultant `svg` file in your web browser. + +Here is another FlameGraph for a full run of the `many-stable-stress.wtperf` +job: + + + +Download the non-redacted interactive version +[here](https://github.com/wiredtiger/wiredtiger/wiki/images/FlameGraph-many-tables-stress.svg) +and open it in your web browser. + +If you see 'unknown' call stacks in your FlameGraph, this means that perf could +not resolve the symbols. To fix this, install the versions of the libraries or +the OS kernel with symbols included, link against those libraries and re-gather +the `perf` output. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-perf.dox b/src/third_party/wiredtiger/src/docs/devdoc-perf.dox new file mode 100644 index 00000000000..4739fba897f --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-perf.dox @@ -0,0 +1,160 @@ +/*! @page devdoc-perf Performance Monitoring + +# Overview + +Linux `perf` is a tool that allows counting and sampling of various events in +the hardware and in the kernel. Hardware events are available via performance +monitoring units (PMU); they measure CPU cycles, cache misses, branches, etc. +Kernel events include scheduling context switches, page faults, block I/O, etc. + +`perf` is good for answering these and similar questions: + +- What is the breakdown of CPU time (or cache misses or branch mispredictions) + by function? +- What was the total number of page faults generated by two different runs of + my program? +- How many context switches occurred during the execution? +- How many times did my program issued a `sched_yield` system call? +- Which functions triggered most page faults? +- How much block I/O did my program generate? + +Despite [claims that perf can also tell you where and why your program was +blocking](https://perf.wiki.kernel.org/index.php/Tutorial#Profiling_sleep_times), +I was not able to get it to do so after many attempts. For alternative ways to +perform off-CPU analysis, read [this post by Brendan +Gregg](http://www.brendangregg.com/offcpuanalysis.html) or use WiredTiger's +@ref devdoc-optrack. + +What follows is a quick cheat sheet of how to use `perf` with WiredTiger. Most +of the information in this cheat sheet comes from [this excellent tutorial by +Brendan Gregg](http://www.brendangregg.com/perf.html). If you need more +information, please refer to that source. + +# Running perf with WiredTiger + +Build WiredTiger as usual. You do not need to add any special compilation +flags. + +To run `perf` with WiredTiger, you simply insert the name of the `perf` command +before the WiredTiger executable. For example, if you want to record default +`perf` statistics for a `wtperf` job, you'd run: + + perf stat wtperf <wtperf options> + +If you want to profile CPU time of `wtperf`, you'd run: + + perf record wtperf <wtperf options> + +Any environment variables you care about will go before the `perf` command. + +If you wanted to run `perf` on an already running WiredTiger process, you +supply a `-p <PID>` option to either of these commands: + + perf record -p <PID> + +or + + perf stat -p <PID> + +# Counting events + +The command `perf stat` will report the total count of events for the entire +execution. + + perf stat wtperf <wtperf options> + +By default it will report things like clock cycles, context switches, CPU +migrations, page faults, cycles, instructions, branches and branch misses. + +Here is a sample output: + + 55275.953046 task-clock (msec) # 1.466 CPUs utilized + 7,349,023 context-switches # 0.133 M/sec + 38,219 cpu-migrations # 0.691 K/sec + 160,562 page-faults # 0.003 M/sec + 173,665,597,703 cycles # 3.142 GHz + 90,146,578,819 instructions # 0.52 insn per cycle + 22,199,890,600 branches # 401.619 M/sec + 387,770,656 branch-misses # 1.75% of all branches + + 37.710693533 seconds time elapsed + +Alternatively, you can decide what events you want to include. For example, if +you wanted to count last-level cache (LLC) misses, one way to do this is like +so: + + perf stat -e LLC-load-misses wtperf <wtperf options> + +Or if you wanted all default events and all cache events, run: + + perf stat -d wtperf <wtperf options> + +To learn about all events you can count or sample with perf, use the command: + + perf list + +To count all system calls by type, run: + + perf stat -e 'syscalls:sys_enter_*' + +# Sampling events (profiling) + +Unlike `perf stat`, which counts all events, `perf record` samples events. The +simplest way to invoke it is like so: + + perf record <command> + +This will sample on-CPU functions. + +# Examining the output + +By default, the output will be placed into a file named `perf.data`. To examine +it, run: + + perf report + +`perf report` will show, for each sampled event, the call stacks that were +recorded at the time of sampling. The call stacks will be sorted from the most +frequently occurring to the least frequently occurring. + +At the time of this writing, the default sampling frequency is 4000; that is, a +sample is taken after every 4000 events. This number may change, so to be sure +check the setting in the header in the `perf` output, like so: + + perf report --header + +If you want to change the output file, use the `-o` option: + + perf record -o <new_output_file> <command> + +And run the reporting with the `-i` option: + + perf report -i <new_output_file> + +Here is a way to look at the output with all the call stacks expanded: + + perf report -n --stdio + +# More examples + +Sample on-cpu functions with call stacks: + + perf record -g <command> + +Sample as above, but using a specific sampling frequency (99 hertz): + + perf record -F 99 -g <command> + +Sample last-level cache load misses with call stacks: + + perf record -e LLC-load-misses -g <command> + +Sample cache misses for a running process: + + perf record -e LLC-load-misses -g -p <PID> + +Sample page faults: + + perf record -e page-faults -g + +*/ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-schema.dox b/src/third_party/wiredtiger/src/docs/devdoc-schema.dox index 509fbf16d67..dd59aa2535b 100644 --- a/src/third_party/wiredtiger/src/docs/devdoc-schema.dox +++ b/src/third_party/wiredtiger/src/docs/devdoc-schema.dox @@ -1,4 +1,4 @@ -/*! @page devdoc-schema Schema operations +/*! @page devdoc-schema Schema Operations A schema defines the format of the application data in WiredTiger. WiredTiger supports various types of schemas (See @ref schema for more diff --git a/src/third_party/wiredtiger/src/docs/devdoc-statistics.dox b/src/third_party/wiredtiger/src/docs/devdoc-statistics.dox new file mode 100644 index 00000000000..d8befa61136 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-statistics.dox @@ -0,0 +1,165 @@ +/*! @page devdoc-statistics Statistics Logging + +# Introduction + +The following content describes a subset of the statistics features available +in WiredTiger. For more complete information see the [statistics] +(http://source.wiredtiger.com/2.0.1/statistics.html) page in the WiredTiger +documentation. This content focuses on analyzing and reviewing the data +generated by the statistics logging functionality. + +# Enabling statistics + +By default statistics are disabled in WiredTiger - this is because tracking +statistics has a performance overhead. WiredTiger has two modes for tracking +statistics `fast` and `all`. Fast statistics are those that don't incur a +significant performance overhead. The effect on performance of enabling +statistics varies depending on the database and access characteristics. To +enable statistics you should include either `statistics=[fast,clear]` or +`statistics=[all,clear]` in the configuration string to `wiredtiger_open`. + +Alternatively you can set the `WIREDTIGER_CONFIG` environment variable to +include the relevant configuration string. Note that the environment variable +is checked at `wiredtiger_open` time - changing the environment variable after +WiredTiger is already running will not alter the behavior of WiredTiger. + +# Accessing statistics + +WiredTiger provides two methods for accessing statistics: using a `statistics:` +uri in the WiredTiger `WT_CURSOR` API and using the built in statistics +logging functionality. The remainder of this document assumes the use of the +statistics logging functionality as the tools provided parse the output that +it generates. + +Enabling statistics logging causes WiredTiger to generate files containing +statistics information in the database directory. To enable statistics logging +include statistics_log=(wait=30) in the configuration string to `wiredtiger_open`. + +The wait option configures how often the statistics are written to the log file +in seconds. Gathering and writing statistics has overhead - so this period +should not be too small. The default location for statistics log files is in +the WiredTiger database directory. The files are named `WiredTigerStat.XX`, +where `XX` corresponds to the current hour. A new file is created for each +hour. If your application runs for more than 24 hours and the file names wrap, +the statistics logging output is appended to the already existing files. +Statistics log file names are configurable. The output in the statistics log +files is textual and human readable. + +# Visualizing statistics + +WiredTiger provides several tools that generate visual representations of +statistics based on statistics log files. + +## Data Driven Documents (D3) graph + +[D3](http://d3js.org/) is a standard for generating interactive graphs in a +HTML page. There is a python program in the WiredTiger source tree at +`tools/wtstats.py` that can be used to generate a D3 html page based on +WiredTiger statistics log(s). In order to use the Python script you need to +have Python installed and a recent version of the Python `nvd3` library. + +To install `nvd3`: + + $ git clone https://github.com/areski/python-nvd3.git + $ cd python-nvd3/ + $ sudo python setup.py install + +It is not sufficient to run `pip install python-nvd3` - the version provided +by the library is too old. + +Once you have the prerequisites installed you can generate a graph that +contains your statistics by running: + + $ cd ~/work/wiredtiger + $ python ./tools/wtstats.py <path/to/db/directory/WiredTigerStat.*> + $ ls wtstats.html + wtstats.html + +There are several optional arguments. Run with `-h` or `--help` to see them. +For example, `-o file` or `--output file` redirects where the HTML output goes. +Also `--focus` generates a chart that contains a focus slider to allow you to +zoom in on particular time periods. + +You can open the generated HTML document in your browser, and see the generated +statistics. The keys in the graph are clickable - double clicking on one of the +keys will show only that statistic. Single clicking will enable/disable a +particular statistic. + +## Gnuplot graphs + +WiredTiger provides a python application that plots relevant statistics on +individual graphs using gnuplot. The following commands will generate graphs of +this type: + + $ cd ~/work/wiredtiger + $ python tools/statlog.py <path/to/db/directory/WiredTigerStat.*> + $ ls reports + raw report.010.png report.020.png report.030.png + report.001.png report.011.png report.021.png report.031.png + report.002.png report.012.png report.022.png report.032.png + report.003.png report.013.png report.023.png report.033.png + report.004.png report.014.png report.024.png report.034.png + report.005.png report.015.png report.025.png report.035.png + report.006.png report.016.png report.026.png report.036.png + report.007.png report.017.png report.027.png report.037.png + report.008.png report.018.png report.028.png report.038.png + report.009.png report.019.png report.029.png report.039.png + +Each of the images contains a graph for a single statistic over time. + +# A complete example + +The following steps take you through the process of generating a D3 graph of +WiredTiger statistics based on the statistics generated from running an +application included in the WiredTiger tree called `wtperf`. These instructions +assume you have all the necessary tools to build WiredTiger, Python and +python-nvd3 installed. + + $ echo "First get and build a working WiredTiger install" + $ cd work + $ wget http://source.wiredtiger.com/releases/wiredtiger-2.0.1.tar.bz2 + $ tar jxf wiredtiger-2.0.1.tar.bz2 && cd wiredtiger-2.0.1 + $ ./configure && make -j 12 + $ echo "Run an application that will generate statistics" + $ mkdir WT_TEST && ./bench/wtperf/wtperf \ + -O bench/wtperf/runners/update-lsm.wtperf \ + -o "conn_config=\"cache_size=1G,statistics=[fast,clear],statistics_log=(wait=30)\"" + $ echo "Now generate a graph from the statistics" + $ python ./tools/wtstats.py WT_TEST/WiredTigerStat.* + +The above steps will have generated a file called `wtstats.html` open this file +in a web browser and review the statistics. The page you see should look +something like: + + + +If you double click on the "btree cursor insert calls per second" key, it will +show just that statistic in the window. Depending on the test you ran (I ran a +modified version of the `update-large-lsm.wtperf` configuration), the graph may +look something like: + + + +This is an interesting graph. We can do some analysis on the meaning of the +results: + +- From 0 to about 180 seconds there are about 130k inserts a second. This +probably corresponds to the time during which the cache isn't full, so we get a +lot of work done. + +- Between 240 and 510 seconds we see approximately 110k inserts per second - +this is probably the rate we will sustain once the cache is full. + +- At 510 seconds we see a huge spike to 210k inserts a second - this +corresponds to the time in the wtperf application where we are about to switch +from populating a database, to running the workload phase. Since this is an LSM +tree, the inserts here are the result of merge operations. + +- The remainder of the time we sustain approximately 80k inserts a second - this +is probably the steady state during the workload phase of the benchmark. + +From this particular statistic we can gather that it's unlikely the cursor +insert rate that is a bottle neck in this particular test - since we could +achieve an insert rate over double the steady state insert rate when required. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 0677b92028d..1f93969ca69 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -7,6 +7,7 @@ Atomicity BLOBs CFLAGS CPPFLAGS +CPUs CRC Cheng Christoph @@ -21,6 +22,7 @@ Datastore DbCursor DbEnv DbMultiple +Durations EAGAIN EB EBUSY @@ -32,6 +34,8 @@ EmpId Encryptors Facebook Facebook's +FlameGraph +FlameGraphs FlexeLint FreeBSD FreeBSD's @@ -45,11 +49,13 @@ HyperDex HyperLevelDB IEC JDK +JIRA JavaScript KMS LD LDFLAGS LIBS +LLC LLVM LOGREC LRU @@ -67,9 +73,11 @@ Mewhort MongoDB Multithreaded NOTFOUND +NSEC NUMA NoSQL OPTYPE +PMU PPC PRELOAD README @@ -84,6 +92,7 @@ Rrx SCons Seward's SiH +TSC TXT UNC URIs @@ -125,6 +134,7 @@ benchmarking bigram bindir bitstring +bokeh bool boolean booleans @@ -159,9 +169,12 @@ conn const control's copydoc +cp cpp +cpu crashless crc +curfile cursortype customerABC cv @@ -190,8 +203,8 @@ destructors dev devdoc dhandle -dhandles dhandle's +dhandles disjunction disjunctions distclean @@ -239,6 +252,8 @@ filesystem filesystems fillfactor firstname +flamegraph +flamegraphs fnv forw fput @@ -266,20 +281,22 @@ htmlinclude huffman hugepage icount +ie iflag indices init +insn intl inuse io ip -ie je jemalloc jitter jni jrx json +jxf kb keyexist keyfmt @@ -289,6 +306,7 @@ keyvalue kvs lang lastname +latencies le len li @@ -305,6 +323,7 @@ logc lookup lookups lrtf +lru lsm lsn lt @@ -315,6 +334,7 @@ malloc malloc'd marshalled marshalling +matplotlib maxintlitem maxintlpage maxleafitem @@ -327,12 +347,14 @@ memp metadata metatracking minkey +mispredictions mixin mixins mkdir mmap mpool mpoolfile +msec msg msgs multi @@ -363,6 +385,7 @@ nosync notgranted notyet nowait +ns nul num numa @@ -377,6 +400,7 @@ os outlier outliers ovfl +parallelizable pareto pcoll pdf @@ -435,14 +459,17 @@ sHq sanitization scalable scanf +sched schemas scons +screenshots secretkey selectable seqname seqno serializable sess +sid skiplist spinlock spinlocks @@ -463,8 +490,11 @@ subdatabases subdirectory subpage substring +sudo superset +svg sys +syscalls sz tRuE tablename @@ -474,6 +504,8 @@ td th thang tid +timeline +timelines timestamp timestamps todo @@ -488,6 +520,7 @@ tt txn txnid txns +txt typedef uint ul @@ -507,11 +540,13 @@ utf util valign valuefmt +valuev vec versa vm vpmsum warmup +wget whitespace wiredtiger workQ diff --git a/src/third_party/wiredtiger/src/docs/style/wiredtiger.css b/src/third_party/wiredtiger/src/docs/style/wiredtiger.css index b16b21f2029..984e525f42f 100644 --- a/src/third_party/wiredtiger/src/docs/style/wiredtiger.css +++ b/src/third_party/wiredtiger/src/docs/style/wiredtiger.css @@ -20,6 +20,12 @@ a { color: #3D578C; } +img { + display: block; + max-width: 100%; + height: auto; +} + .tablist { width: 800px; } diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 0faaacc710c..3393bf36501 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -2300,6 +2300,10 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d if (ret == WT_ROLLBACK) { --cache->evict_aggressive_score; WT_STAT_CONN_INCR(session, txn_fail_cache); + if (timer) + WT_IGNORE_RET(__wt_msg(session, + "Application thread returned a rollback error because " + "it was forced to evict and eviction is stuck")); } WT_ERR(ret); } diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 533f276b15c..95946dfbc65 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -32,24 +32,31 @@ #define WT_SINGLE_THREAD_CHECK_STOP(s) #endif +#define API_SESSION_PUSH(s, h, n, dh) \ + WT_DATA_HANDLE *__olddh = (s)->dhandle; \ + const char *__oldname = (s)->name; \ + (s)->dhandle = (dh); \ + (s)->name = (s)->lastop = #h "." #n +#define API_SESSION_POP(s) \ + (s)->dhandle = __olddh; \ + (s)->name = __oldname + /* Standard entry points to the API: declares/initializes local variables. */ -#define API_SESSION_INIT(s, h, n, dh) \ - WT_TRACK_OP_DECL; \ - WT_DATA_HANDLE *__olddh = (s)->dhandle; \ - const char *__oldname = (s)->name; \ - (s)->dhandle = (dh); \ - (s)->name = (s)->lastop = #h "." #n; \ - /* \ - * No code before this line, otherwise error handling won't be \ - * correct. \ - */ \ - WT_TRACK_OP_INIT(s); \ - WT_SINGLE_THREAD_CHECK_START(s); \ - __wt_op_timer_start(s); \ - WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ - /* Reset wait time if this isn't an API reentry. */ \ - if (__oldname == NULL) \ - (s)->cache_wait_us = 0; \ +#define API_SESSION_INIT(s, h, n, dh) \ + WT_TRACK_OP_DECL; \ + API_SESSION_PUSH(s, h, n, dh); \ + /* \ + * No code before this line, otherwise error handling won't be \ + * correct. \ + */ \ + WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ + WT_SINGLE_THREAD_CHECK_START(s); \ + WT_ERR(__wt_txn_err_chk(s)); \ + WT_TRACK_OP_INIT(s); \ + __wt_op_timer_start(s); \ + /* Reset wait time if this isn't an API reentry. */ \ + if (__oldname == NULL) \ + (s)->cache_wait_us = 0; \ __wt_verbose((s), WT_VERB_API, "%s", "CALL: " #h ":" #n) #define API_CALL_NOCONF(s, h, n, dh) \ @@ -75,8 +82,7 @@ * No code after this line, otherwise error handling \ * won't be correct. \ */ \ - (s)->dhandle = __olddh; \ - (s)->name = __oldname; \ + API_SESSION_POP(s); \ } \ } \ while (0) @@ -115,7 +121,7 @@ if (__autotxn) { \ if (F_ISSET(&(s)->txn, WT_TXN_AUTOCOMMIT)) \ F_CLR(&(s)->txn, WT_TXN_AUTOCOMMIT); \ - else if ((ret) == 0 && !F_ISSET(&(s)->txn, WT_TXN_ERROR)) \ + else if ((ret) == 0) \ (ret) = __wt_txn_commit((s), NULL); \ else { \ if (retry) \ @@ -167,24 +173,33 @@ #define SESSION_API_CALL_PREPARE_ALLOWED(s, n, config, cfg) \ API_CALL(s, WT_SESSION, n, NULL, config, cfg) -#define SESSION_API_CALL(s, n, config, cfg) \ - API_CALL(s, WT_SESSION, n, NULL, config, cfg); \ - WT_ERR(__wt_txn_context_prepare_check((s))) +#define SESSION_API_PREPARE_CHECK(s, h, n) \ + do { \ + int __prepare_ret; \ + API_SESSION_PUSH(s, WT_SESSION, n, NULL); \ + __prepare_ret = __wt_txn_context_prepare_check(s); \ + API_SESSION_POP(s); \ + WT_RET(__prepare_ret); \ + } while (0) + +#define SESSION_API_CALL(s, n, config, cfg) \ + SESSION_API_PREPARE_CHECK(s, WT_SESSION, n); \ + API_CALL(s, WT_SESSION, n, NULL, config, cfg) #define SESSION_API_CALL_NOCONF(s, n) API_CALL_NOCONF(s, WT_SESSION, n, NULL) #define SESSION_API_CALL_NOCONF_PREPARE_NOT_ALLOWED(s, n) \ - API_CALL_NOCONF(s, WT_SESSION, n, NULL); \ - WT_ERR(__wt_txn_context_prepare_check((s))) + SESSION_API_PREPARE_CHECK(s, WT_SESSION, n); \ + API_CALL_NOCONF(s, WT_SESSION, n, NULL) -#define SESSION_TXN_API_CALL(s, n, config, cfg) \ - TXN_API_CALL(s, WT_SESSION, n, NULL, config, cfg); \ - WT_ERR(__wt_txn_context_prepare_check((s))) +#define SESSION_TXN_API_CALL(s, n, config, cfg) \ + SESSION_API_PREPARE_CHECK(s, WT_SESSION, n); \ + TXN_API_CALL(s, WT_SESSION, n, NULL, config, cfg) #define CURSOR_API_CALL(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ + SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ API_CALL_NOCONF(s, WT_CURSOR, n, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ - WT_ERR(__wt_txn_context_prepare_check((s))); \ if (F_ISSET(cur, WT_CURSTD_CACHED)) \ WT_ERR(__wt_cursor_cached(cur)) @@ -206,11 +221,10 @@ CURSOR_API_CALL_PREPARE_ALLOWED(cur, s, n, bt); \ JOINABLE_CURSOR_CALL_CHECK(cur) -#define CURSOR_REMOVE_API_CALL(cur, s, bt) \ - (s) = (WT_SESSION_IMPL *)(cur)->session; \ - TXN_API_CALL_NOCONF( \ - s, WT_CURSOR, remove, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ - WT_ERR(__wt_txn_context_prepare_check((s))) +#define CURSOR_REMOVE_API_CALL(cur, s, bt) \ + (s) = (WT_SESSION_IMPL *)(cur)->session; \ + SESSION_API_PREPARE_CHECK(s, WT_CURSOR, remove); \ + TXN_API_CALL_NOCONF(s, WT_CURSOR, remove, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle) #define JOINABLE_CURSOR_REMOVE_API_CALL(cur, s, bt) \ CURSOR_REMOVE_API_CALL(cur, s, bt); \ @@ -218,16 +232,16 @@ #define CURSOR_UPDATE_API_CALL_BTREE(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ + SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ TXN_API_CALL_NOCONF(s, WT_CURSOR, n, ((WT_BTREE *)(bt))->dhandle); \ - WT_ERR(__wt_txn_context_prepare_check((s))); \ if (F_ISSET(S2C(s), WT_CONN_IN_MEMORY) && !F_ISSET((WT_BTREE *)(bt), WT_BTREE_IGNORE_CACHE) && \ __wt_cache_full(s)) \ WT_ERR(WT_CACHE_FULL); #define CURSOR_UPDATE_API_CALL(cur, s, n) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - TXN_API_CALL_NOCONF(s, WT_CURSOR, n, NULL); \ - WT_ERR(__wt_txn_context_prepare_check((s))) + SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ + TXN_API_CALL_NOCONF(s, WT_CURSOR, n, NULL) #define JOINABLE_CURSOR_UPDATE_API_CALL(cur, s, n) \ CURSOR_UPDATE_API_CALL(cur, s, n); \ diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 18199997a19..5099bc4dce9 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -838,10 +838,6 @@ __wt_row_leaf_key_info( *ikeyp = NULL; if (cellp != NULL) *cellp = WT_PAGE_REF_OFFSET(page, WT_CELL_DECODE_OFFSET(v)); - if (datap != NULL) { - *(void **)datap = NULL; - *sizep = 0; - } return (false); case WT_K_FLAG: /* Encoded key: no instantiated key, no cell. */ @@ -999,6 +995,9 @@ __wt_row_leaf_value_cell(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, size_t size; void *copy, *key; + size = 0; /* -Werror=maybe-uninitialized */ + key = NULL; /* -Werror=maybe-uninitialized */ + /* If we already have an unpacked key cell, use it. */ if (kpack != NULL) vcell = (WT_CELL *)((uint8_t *)kpack->cell + __wt_cell_total_len(kpack)); diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index db4b2e9b41e..f77712972b6 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -298,6 +298,10 @@ struct __wt_connection_impl { uint64_t ckpt_write_bytes; uint64_t ckpt_write_pages; + /* Checkpoint and incremental backup data */ + uint64_t ckpt_incr_granularity; + WT_BLKINCR incr_backups[WT_BLKINCR_MAX]; + /* Connection's maximum and base write generations. */ uint64_t max_write_gen; uint64_t base_write_gen; @@ -441,40 +445,41 @@ struct __wt_connection_impl { /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_VERB_API 0x000000001u -#define WT_VERB_BLOCK 0x000000002u -#define WT_VERB_CHECKPOINT 0x000000004u -#define WT_VERB_CHECKPOINT_PROGRESS 0x000000008u -#define WT_VERB_COMPACT 0x000000010u -#define WT_VERB_COMPACT_PROGRESS 0x000000020u -#define WT_VERB_ERROR_RETURNS 0x000000040u -#define WT_VERB_EVICT 0x000000080u -#define WT_VERB_EVICTSERVER 0x000000100u -#define WT_VERB_EVICT_STUCK 0x000000200u -#define WT_VERB_FILEOPS 0x000000400u -#define WT_VERB_HANDLEOPS 0x000000800u -#define WT_VERB_LOG 0x000001000u -#define WT_VERB_LOOKASIDE 0x000002000u -#define WT_VERB_LOOKASIDE_ACTIVITY 0x000004000u -#define WT_VERB_LSM 0x000008000u -#define WT_VERB_LSM_MANAGER 0x000010000u -#define WT_VERB_METADATA 0x000020000u -#define WT_VERB_MUTEX 0x000040000u -#define WT_VERB_OVERFLOW 0x000080000u -#define WT_VERB_READ 0x000100000u -#define WT_VERB_REBALANCE 0x000200000u -#define WT_VERB_RECONCILE 0x000400000u -#define WT_VERB_RECOVERY 0x000800000u -#define WT_VERB_RECOVERY_PROGRESS 0x001000000u -#define WT_VERB_SALVAGE 0x002000000u -#define WT_VERB_SHARED_CACHE 0x004000000u -#define WT_VERB_SPLIT 0x008000000u -#define WT_VERB_TEMPORARY 0x010000000u -#define WT_VERB_THREAD_GROUP 0x020000000u -#define WT_VERB_TIMESTAMP 0x040000000u -#define WT_VERB_TRANSACTION 0x080000000u -#define WT_VERB_VERIFY 0x100000000u -#define WT_VERB_VERSION 0x200000000u -#define WT_VERB_WRITE 0x400000000u +#define WT_VERB_BACKUP 0x000000002u +#define WT_VERB_BLOCK 0x000000004u +#define WT_VERB_CHECKPOINT 0x000000008u +#define WT_VERB_CHECKPOINT_PROGRESS 0x000000010u +#define WT_VERB_COMPACT 0x000000020u +#define WT_VERB_COMPACT_PROGRESS 0x000000040u +#define WT_VERB_ERROR_RETURNS 0x000000080u +#define WT_VERB_EVICT 0x000000100u +#define WT_VERB_EVICTSERVER 0x000000200u +#define WT_VERB_EVICT_STUCK 0x000000400u +#define WT_VERB_FILEOPS 0x000000800u +#define WT_VERB_HANDLEOPS 0x000001000u +#define WT_VERB_LOG 0x000002000u +#define WT_VERB_LOOKASIDE 0x000004000u +#define WT_VERB_LOOKASIDE_ACTIVITY 0x000008000u +#define WT_VERB_LSM 0x000010000u +#define WT_VERB_LSM_MANAGER 0x000020000u +#define WT_VERB_METADATA 0x000040000u +#define WT_VERB_MUTEX 0x000080000u +#define WT_VERB_OVERFLOW 0x000100000u +#define WT_VERB_READ 0x000200000u +#define WT_VERB_REBALANCE 0x000400000u +#define WT_VERB_RECONCILE 0x000800000u +#define WT_VERB_RECOVERY 0x001000000u +#define WT_VERB_RECOVERY_PROGRESS 0x002000000u +#define WT_VERB_SALVAGE 0x004000000u +#define WT_VERB_SHARED_CACHE 0x008000000u +#define WT_VERB_SPLIT 0x010000000u +#define WT_VERB_TEMPORARY 0x020000000u +#define WT_VERB_THREAD_GROUP 0x040000000u +#define WT_VERB_TIMESTAMP 0x080000000u +#define WT_VERB_TRANSACTION 0x100000000u +#define WT_VERB_VERIFY 0x200000000u +#define WT_VERB_VERSION 0x400000000u +#define WT_VERB_WRITE 0x800000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t verbose; @@ -506,34 +511,37 @@ struct __wt_connection_impl { WT_FILE_SYSTEM *file_system; /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CONN_CACHE_CURSORS 0x0000001u -#define WT_CONN_CACHE_POOL 0x0000002u -#define WT_CONN_CKPT_SYNC 0x0000004u -#define WT_CONN_CLOSING 0x0000008u -#define WT_CONN_CLOSING_NO_MORE_OPENS 0x0000010u -#define WT_CONN_CLOSING_TIMESTAMP 0x0000020u -#define WT_CONN_COMPATIBILITY 0x0000040u -#define WT_CONN_DATA_CORRUPTION 0x0000080u -#define WT_CONN_EVICTION_NO_LOOKASIDE 0x0000100u -#define WT_CONN_EVICTION_RUN 0x0000200u -#define WT_CONN_IN_MEMORY 0x0000400u -#define WT_CONN_LEAK_MEMORY 0x0000800u -#define WT_CONN_LOOKASIDE_OPEN 0x0001000u -#define WT_CONN_LSM_MERGE 0x0002000u -#define WT_CONN_OPTRACK 0x0004000u -#define WT_CONN_PANIC 0x0008000u -#define WT_CONN_READONLY 0x0010000u -#define WT_CONN_RECONFIGURING 0x0020000u -#define WT_CONN_RECOVERING 0x0040000u -#define WT_CONN_SALVAGE 0x0080000u -#define WT_CONN_SERVER_ASYNC 0x0100000u -#define WT_CONN_SERVER_CAPACITY 0x0200000u -#define WT_CONN_SERVER_CHECKPOINT 0x0400000u -#define WT_CONN_SERVER_LOG 0x0800000u -#define WT_CONN_SERVER_LSM 0x1000000u -#define WT_CONN_SERVER_STATISTICS 0x2000000u -#define WT_CONN_SERVER_SWEEP 0x4000000u -#define WT_CONN_WAS_BACKUP 0x8000000u +#define WT_CONN_CACHE_CURSORS 0x00000001u +#define WT_CONN_CACHE_POOL 0x00000002u +#define WT_CONN_CKPT_SYNC 0x00000004u +#define WT_CONN_CLOSING 0x00000008u +#define WT_CONN_CLOSING_NO_MORE_OPENS 0x00000010u +#define WT_CONN_CLOSING_TIMESTAMP 0x00000020u +#define WT_CONN_COMPATIBILITY 0x00000040u +#define WT_CONN_DATA_CORRUPTION 0x00000080u +#define WT_CONN_DEBUG_REALLOC_EXACT 0x00000100u +#define WT_CONN_DEBUG_SLOW_CKPT 0x00000200u +#define WT_CONN_EVICTION_NO_LOOKASIDE 0x00000400u +#define WT_CONN_EVICTION_RUN 0x00000800u +#define WT_CONN_INCR_BACKUP 0x00001000u +#define WT_CONN_IN_MEMORY 0x00002000u +#define WT_CONN_LEAK_MEMORY 0x00004000u +#define WT_CONN_LOOKASIDE_OPEN 0x00008000u +#define WT_CONN_LSM_MERGE 0x00010000u +#define WT_CONN_OPTRACK 0x00020000u +#define WT_CONN_PANIC 0x00040000u +#define WT_CONN_READONLY 0x00080000u +#define WT_CONN_RECONFIGURING 0x00100000u +#define WT_CONN_RECOVERING 0x00200000u +#define WT_CONN_SALVAGE 0x00400000u +#define WT_CONN_SERVER_ASYNC 0x00800000u +#define WT_CONN_SERVER_CAPACITY 0x01000000u +#define WT_CONN_SERVER_CHECKPOINT 0x02000000u +#define WT_CONN_SERVER_LOG 0x04000000u +#define WT_CONN_SERVER_LSM 0x08000000u +#define WT_CONN_SERVER_STATISTICS 0x10000000u +#define WT_CONN_SERVER_SWEEP 0x20000000u +#define WT_CONN_WAS_BACKUP 0x40000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index b3d32ad8417..45a4ac01a9f 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -32,24 +32,62 @@ 0 /* uint32_t flags */ \ } +/* + * Block based incremental backup structure. These live in the connection. + */ +#define WT_BLKINCR_MAX 2 +struct __wt_blkincr { + const char *id_str; /* User's name for this backup. */ + const char *ckpt_name; /* Requires WT-5115. All checkpoints must be this name */ + void *data; +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_BLKINCR_INUSE 0x1u /* This entry is active */ +#define WT_BLKINCR_VALID 0x2u /* This entry is valid */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint64_t flags; +}; + struct __wt_cursor_backup { WT_CURSOR iface; size_t next; /* Cursor position */ WT_FSTREAM *bfs; /* Backup file stream */ - uint32_t maxid; /* Maximum log file ID seen */ + +#define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)(cursor))->maxid) + uint32_t maxid; /* Maximum log file ID seen */ char **list; /* List of files to be copied. */ size_t list_allocated; size_t list_next; + /* File offset-based incremental backup. */ + WT_BLKINCR *incr; /* Incremental backup in use */ + char *incr_file; /* File name */ + char *incr_src; /* Source identifier */ + char *incr_this; /* New base identifier */ + uint64_t incr_granularity; /* Maximum transfer size */ + + WT_CURSOR *incr_cursor; /* File cursor */ + /* Start/stop checkpoints */ + char *incr_checkpoint_start; + char *incr_checkpoint_stop; + +#define WT_BACKUP_INCR_COMPONENTS 3 + bool incr_init; /* Cursor traversal initialized */ + uint64_t *incr_list; /* List of file offset/size/type triples */ + uint64_t incr_list_count; /* Count of file offset/size/type triples */ + uint64_t incr_list_offset; /* Current offset */ + uint64_t incr_size; /* Maximum transfer size */ + WT_ITEM *incr_block; /* Current block of data */ + /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CURBACKUP_DUP 0x1u /* Duplicated backup cursor */ -#define WT_CURBACKUP_LOCKER 0x2u /* Hot-backup started */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_CURBACKUP_DUP 0x1u /* Duplicated backup cursor */ +#define WT_CURBACKUP_FORCE_STOP 0x2u /* Force stop incremental backup */ +#define WT_CURBACKUP_INCR 0x4u /* Incremental backup cursor */ +#define WT_CURBACKUP_LOCKER 0x8u /* Hot-backup started */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; -#define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)(cursor))->maxid) struct __wt_cursor_btree { WT_CURSOR iface; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 3eb7583ecc6..f3710895938 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -408,8 +408,6 @@ extern int __wt_config_subgetraw(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_subgets(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -471,6 +469,9 @@ extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const ch extern int __wt_count_birthmarks(WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curbackup_open_incr(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, + WT_CURSOR *cursor, const char *cfg[], WT_CURSOR **cursorp) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curbulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool bitmap, bool skip_sort_check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], @@ -550,6 +551,10 @@ extern int __wt_cursor_reopen_notsup(WT_CURSOR *cursor, bool check_only) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp, bool *valid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curstat_colgroup_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], @@ -1594,18 +1599,17 @@ extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session); extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); +extern void __wt_curbackup_free_incr(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb); extern void __wt_cursor_close(WT_CURSOR *cursor); extern void __wt_cursor_key_order_reset(WT_CURSOR_BTREE *cbt); extern void __wt_cursor_reopen(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle); extern void __wt_cursor_set_key(WT_CURSOR *cursor, ...); extern void __wt_cursor_set_key_notsup(WT_CURSOR *cursor, ...); -extern void __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap); extern void __wt_cursor_set_notsup(WT_CURSOR *cursor); extern void __wt_cursor_set_raw_key(WT_CURSOR *cursor, WT_ITEM *key); extern void __wt_cursor_set_raw_value(WT_CURSOR *cursor, WT_ITEM *value); extern void __wt_cursor_set_value(WT_CURSOR *cursor, ...); extern void __wt_cursor_set_value_notsup(WT_CURSOR *cursor, ...); -extern void __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap); extern void __wt_curstat_cache_walk(WT_SESSION_IMPL *session); extern void __wt_curstat_dsrc_final(WT_CURSOR_STAT *cst); extern void __wt_curtable_set_key(WT_CURSOR *cursor, ...); @@ -1958,6 +1962,8 @@ static inline int __wt_txn_context_check(WT_SESSION_IMPL *session, bool requires WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_context_prepare_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_txn_err_chk(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_id_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_idle_cache_check(WT_SESSION_IMPL *session) diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index 574c9400f8f..9845dbd7f7d 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -14,10 +14,14 @@ #define WT_USERCONFIG "WiredTiger.config" /* User configuration */ -#define WT_BACKUP_TMP "WiredTiger.backup.tmp" /* Backup tmp file */ -#define WT_METADATA_BACKUP "WiredTiger.backup" /* Hot backup file */ -#define WT_INCREMENTAL_BACKUP "WiredTiger.ibackup" /* Incremental backup */ -#define WT_INCREMENTAL_SRC "WiredTiger.isrc" /* Incremental source */ +/* + * Backup related WiredTiger files. + */ +#define WT_BACKUP_TMP "WiredTiger.backup.tmp" /* Backup tmp file */ +#define WT_BLKINCR_BACKUP "WiredTiger.backup.block" /* Block incremental durable file */ +#define WT_METADATA_BACKUP "WiredTiger.backup" /* Hot backup file */ +#define WT_LOGINCR_BACKUP "WiredTiger.ibackup" /* Log incremental backup */ +#define WT_LOGINCR_SRC "WiredTiger.isrc" /* Log incremental source */ #define WT_METADATA_TURTLE "WiredTiger.turtle" /* Metadata metadata */ #define WT_METADATA_TURTLE_SET "WiredTiger.turtle.set" /* Turtle temp file */ @@ -91,16 +95,20 @@ struct __wt_ckpt { wt_timestamp_t newest_stop_ts; uint64_t newest_stop_txn; + uint64_t *alloc_list; /* Checkpoint allocation list */ + uint64_t alloc_list_entries; + WT_ITEM addr; /* Checkpoint cookie string */ WT_ITEM raw; /* Checkpoint cookie raw */ void *bpriv; /* Block manager private */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CKPT_ADD 0x1u /* Checkpoint to be added */ -#define WT_CKPT_DELETE 0x2u /* Checkpoint to be deleted */ -#define WT_CKPT_FAKE 0x4u /* Checkpoint is a fake */ -#define WT_CKPT_UPDATE 0x8u /* Checkpoint requires update */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_CKPT_ADD 0x01u /* Checkpoint to be added */ +#define WT_CKPT_BLOCK_MODS 0x02u /* Return list of modified blocks */ +#define WT_CKPT_DELETE 0x04u /* Checkpoint to be deleted */ +#define WT_CKPT_FAKE 0x08u /* Checkpoint is a fake */ +#define WT_CKPT_UPDATE 0x10u /* Checkpoint requires update */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index c303edc9488..7d5c99e7980 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -110,11 +110,14 @@ * Common case allocate-and-grow function. Starts by allocating the requested number of items * (at least 10), then doubles each time the list needs to grow. */ -#define __wt_realloc_def(session, sizep, number, addr) \ - (((number) * sizeof(**(addr)) <= *(sizep)) ? \ - 0 : \ - __wt_realloc( \ - session, sizep, WT_MAX(*(sizep)*2, WT_MAX(10, (number)) * sizeof(**(addr))), addr)) +#define __wt_realloc_def(session, sizep, number, addr) \ + (((number) * sizeof(**(addr)) <= *(sizep)) ? \ + 0 : \ + __wt_realloc(session, sizep, (F_ISSET(S2C(session), WT_CONN_DEBUG_REALLOC_EXACT)) ? \ + (number) * sizeof(**(addr)) : \ + WT_MAX(*(sizep)*2, WT_MAX(10, (number)) * sizeof(**(addr))), \ + addr)) + /* * Our internal free function clears the underlying address atomically so there is a smaller chance * of racing threads seeing intermediate results while a structure is being free'd. (That would be a diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index d739e78cf28..65780e73c9d 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -142,32 +142,6 @@ __wt_snprintf_len_incr(char *buf, size_t size, size_t *retsizep, const char *fmt } /* - * __wt_txn_context_prepare_check -- - * Return an error if the current transaction is in the prepare state. - */ -static inline int -__wt_txn_context_prepare_check(WT_SESSION_IMPL *session) -{ - if (F_ISSET(&session->txn, WT_TXN_PREPARE)) - WT_RET_MSG(session, EINVAL, "%s: not permitted in a prepared transaction", session->name); - return (0); -} - -/* - * __wt_txn_context_check -- - * Complain if a transaction is/isn't running. - */ -static inline int -__wt_txn_context_check(WT_SESSION_IMPL *session, bool requires_txn) -{ - if (requires_txn && !F_ISSET(&session->txn, WT_TXN_RUNNING)) - WT_RET_MSG(session, EINVAL, "%s: only permitted in a running transaction", session->name); - if (!requires_txn && F_ISSET(&session->txn, WT_TXN_RUNNING)) - WT_RET_MSG(session, EINVAL, "%s: not permitted in a running transaction", session->name); - return (0); -} - -/* * __wt_spin_backoff -- * Back off while spinning for a resource. This is used to avoid busy waiting loops that can * consume enough CPU to block real work being done. The algorithm spins a few times, then diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 757627dff97..332f3a3735c 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -35,6 +35,50 @@ __wt_ref_cas_state_int(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t old_state } /* + * __wt_txn_context_prepare_check -- + * Return an error if the current transaction is in the prepare state. + */ +static inline int +__wt_txn_context_prepare_check(WT_SESSION_IMPL *session) +{ + if (F_ISSET(&session->txn, WT_TXN_PREPARE)) + WT_RET_MSG(session, EINVAL, "not permitted in a prepared transaction"); + return (0); +} + +/* + * __wt_txn_context_check -- + * Complain if a transaction is/isn't running. + */ +static inline int +__wt_txn_context_check(WT_SESSION_IMPL *session, bool requires_txn) +{ + if (requires_txn && !F_ISSET(&session->txn, WT_TXN_RUNNING)) + WT_RET_MSG(session, EINVAL, "only permitted in a running transaction"); + if (!requires_txn && F_ISSET(&session->txn, WT_TXN_RUNNING)) + WT_RET_MSG(session, EINVAL, "not permitted in a running transaction"); + return (0); +} + +/* + * __wt_txn_err_chk -- + * Check the transaction hasn't already failed. + */ +static inline int +__wt_txn_err_chk(WT_SESSION_IMPL *session) +{ + /* Allow transaction rollback, but nothing else. */ + if (!F_ISSET(&(session->txn), WT_TXN_ERROR) || + strcmp(session->name, "rollback_transaction") != 0) + return (0); + +#ifdef HAVE_DIAGNOSTIC + WT_ASSERT(session, !F_ISSET(&(session->txn), WT_TXN_ERROR)); +#endif + WT_RET_MSG(session, EINVAL, "additional transaction operations attempted after error"); +} + +/* * __wt_txn_timestamp_flags -- * Set transaction related timestamp flags. */ diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index aec3014e788..a50e617705b 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1064,6 +1064,33 @@ struct __wt_session { * non-printing characters are hexadecimal encoded. These formats are compatible with the * @ref util_dump and @ref util_load commands., a string\, chosen from the following * options: \c "hex"\, \c "json"\, \c "print"; default empty.} + * @config{incremental = (, configure the cursor for block incremental backup usage. These + * formats are only compatible with the backup data source; see @ref backup., a set of + * related configuration options defined below.} + * @config{ enabled, + * whether to configure this backup as the starting point for a subsequent incremental + * backup., a boolean flag; default \c false.} + * @config{ file, the + * file name when opening a duplicate incremental backup cursor. That duplicate cursor will + * return the block modifications relevant to the given file name., a string; default + * empty.} + * @config{ force_stop, causes all block incremental backup + * information to be released. This is on an open_cursor call and the resources will be + * released when this cursor is closed. No other operations should be done on this open + * cursor., a boolean flag; default \c false.} + * @config{ granularity, + * this setting manages the granularity of how WiredTiger maintains modification maps + * internally. The larger the granularity\, the smaller amount of information WiredTiger + * need to maintain., an integer between 1MB and 2GB; default \c 16MB.} + * @config{ src_id, a string that identifies a previous checkpoint + * backup source as the source of this incremental backup. This identifier must have + * already been created by use of the 'this_id' configuration in an earlier backup. A + * source id is required to begin an incremental backup., a string; default empty.} + * @config{ this_id, a string that identifies the current system + * state as a future backup source for an incremental backup via 'src_id'. This identifier + * is required when opening an incremental backup cursor and an error will be returned if + * one is not provided., a string; default empty.} + * @config{ ),,} * @config{next_random, configure the cursor to return a pseudo-random record from the * object when the WT_CURSOR::next method is called; valid only for row-store cursors. See * @ref cursor_random for details., a boolean flag; default \c false.} @@ -2179,14 +2206,19 @@ struct __wt_connection { * algorithms to change skew to force lookaside eviction to happen more aggressively. This * includes but is not limited to not skewing newest\, not favoring leaf pages\, and * modifying the eviction score mechanism., a boolean flag; default \c false.} - * @config{ rollback_error, return a WT_ROLLBACK error from a - * transaction operation about every Nth operation to simulate a collision., an integer - * between 0 and 10M; default \c 0.} - * @config{ table_logging, if - * true\, write transaction related information to the log for all operations\, even - * operations for tables with logging turned off. This setting introduces a log format - * change that may break older versions of WiredTiger. These operations are informational - * and skipped in recovery., a boolean flag; default \c false.} + * @config{ realloc_exact, if true\, reallocation of memory will only + * provide the exact amount requested. This will help with spotting memory allocation + * issues more easily., a boolean flag; default \c false.} + * @config{ + * rollback_error, return a WT_ROLLBACK error from a transaction operation about every Nth + * operation to simulate a collision., an integer between 0 and 10M; default \c 0.} + * @config{ slow_checkpoint, if true\, slow down checkpoint creation + * by slowing down internal page processing., a boolean flag; default \c false.} + * @config{ table_logging, if true\, write transaction related + * information to the log for all operations\, even operations for tables with logging + * turned off. This setting introduces a log format change that may break older versions of + * WiredTiger. These operations are informational and skipped in recovery., a boolean flag; + * default \c false.} * @config{ ),,} * @config{error_prefix, prefix string for error messages., a string; default empty.} * @config{eviction = (, eviction configuration options., a set of related configuration @@ -2335,14 +2367,14 @@ struct __wt_connection { * ),,} * @config{verbose, enable messages for various events. Options are given as a list\, such * as <code>"verbose=[evictserver\,read]"</code>., a list\, with values chosen from the - * following options: \c "api"\, \c "block"\, \c "checkpoint"\, \c "checkpoint_progress"\, - * \c "compact"\, \c "compact_progress"\, \c "error_returns"\, \c "evict"\, \c - * "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c - * "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c - * "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, - * \c "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, - * \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c - * "write"; default empty.} + * following options: \c "api"\, \c "backup"\, \c "block"\, \c "checkpoint"\, \c + * "checkpoint_progress"\, \c "compact"\, \c "compact_progress"\, \c "error_returns"\, \c + * "evict"\, \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c + * "log"\, \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c + * "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, + * \c "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, + * \c "temporary"\, \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c "verify"\, \c + * "version"\, \c "write"; default empty.} * @configend * @errors */ @@ -2780,14 +2812,18 @@ struct __wt_connection { * skew to force lookaside eviction to happen more aggressively. This includes but is not limited * to not skewing newest\, not favoring leaf pages\, and modifying the eviction score mechanism., a * boolean flag; default \c false.} - * @config{ rollback_error, return a - * WT_ROLLBACK error from a transaction operation about every Nth operation to simulate a - * collision., an integer between 0 and 10M; default \c 0.} - * @config{ - * table_logging, if true\, write transaction related information to the log for all operations\, - * even operations for tables with logging turned off. This setting introduces a log format change - * that may break older versions of WiredTiger. These operations are informational and skipped in - * recovery., a boolean flag; default \c false.} + * @config{ realloc_exact, if true\, + * reallocation of memory will only provide the exact amount requested. This will help with + * spotting memory allocation issues more easily., a boolean flag; default \c false.} + * @config{ rollback_error, return a WT_ROLLBACK error from a transaction + * operation about every Nth operation to simulate a collision., an integer between 0 and 10M; + * default \c 0.} + * @config{ slow_checkpoint, if true\, slow down checkpoint + * creation by slowing down internal page processing., a boolean flag; default \c false.} + * @config{ table_logging, if true\, write transaction related information to + * the log for all operations\, even operations for tables with logging turned off. This setting + * introduces a log format change that may break older versions of WiredTiger. These operations are + * informational and skipped in recovery., a boolean flag; default \c false.} * @config{ ),,} * @config{direct_io, Use \c O_DIRECT on POSIX systems\, and \c FILE_FLAG_NO_BUFFERING on Windows to * access files. Options are given as a list\, such as <code>"direct_io=[data]"</code>. Configuring @@ -3017,13 +3053,13 @@ struct __wt_connection { * information., a boolean flag; default \c false.} * @config{verbose, enable messages for various events. Options are given as a list\, such as * <code>"verbose=[evictserver\,read]"</code>., a list\, with values chosen from the following - * options: \c "api"\, \c "block"\, \c "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c - * "compact_progress"\, \c "error_returns"\, \c "evict"\, \c "evict_stuck"\, \c "evictserver"\, \c - * "fileops"\, \c "handleops"\, \c "log"\, \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c - * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c - * "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c - * "split"\, \c "temporary"\, \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c "verify"\, - * \c "version"\, \c "write"; default empty.} + * options: \c "api"\, \c "backup"\, \c "block"\, \c "checkpoint"\, \c "checkpoint_progress"\, \c + * "compact"\, \c "compact_progress"\, \c "error_returns"\, \c "evict"\, \c "evict_stuck"\, \c + * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c "lookaside"\, \c + * "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, + * \c "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c + * "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c + * "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; default empty.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to files. Ignored on * non-Windows systems. Options are given as a list\, such as <code>"write_through=[data]"</code>. * Configuring \c write_through requires care\, see @ref tuning_system_buffer_cache_direct_io for @@ -4787,6 +4823,18 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * @addtogroup wt * @{ */ +/*! + * @name Incremental backup types + * @anchor backup_types + * @{ + */ +/*! invalid type */ +#define WT_BACKUP_INVALID 0 +/*! whole file */ +#define WT_BACKUP_FILE 1 +/*! file range */ +#define WT_BACKUP_RANGE 2 +/*! @} */ /*! * @name Log record and operation types diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 2b281443f21..1cd8753a0ac 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -77,6 +77,8 @@ struct __wt_async_op_impl; typedef struct __wt_async_op_impl WT_ASYNC_OP_IMPL; struct __wt_async_worker_state; typedef struct __wt_async_worker_state WT_ASYNC_WORKER_STATE; +struct __wt_blkincr; +typedef struct __wt_blkincr WT_BLKINCR; struct __wt_block; typedef struct __wt_block WT_BLOCK; struct __wt_block_ckpt; diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index db790537c79..044094133ce 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -185,7 +185,7 @@ __wt_turtle_init(WT_SESSION_IMPL *session) { WT_DECL_RET; char *metaconf, *unused_value; - bool exist_backup, exist_incr, exist_isrc, exist_turtle; + bool exist_backup, exist_bincr, exist_incr, exist_isrc, exist_turtle; bool load, loadTurtle; load = loadTurtle = false; @@ -208,10 +208,22 @@ __wt_turtle_init(WT_SESSION_IMPL *session) * turtle file and an incremental backup file, that is an error. Otherwise, if there's already a * turtle file, we're done. */ - WT_RET(__wt_fs_exist(session, WT_INCREMENTAL_BACKUP, &exist_incr)); - WT_RET(__wt_fs_exist(session, WT_INCREMENTAL_SRC, &exist_isrc)); + WT_RET(__wt_fs_exist(session, WT_LOGINCR_BACKUP, &exist_incr)); + WT_RET(__wt_fs_exist(session, WT_LOGINCR_SRC, &exist_isrc)); WT_RET(__wt_fs_exist(session, WT_METADATA_BACKUP, &exist_backup)); WT_RET(__wt_fs_exist(session, WT_METADATA_TURTLE, &exist_turtle)); + /* + * Block incremental is different. If it exists, then we have block incremental information we + * need to keep. Mark the connection as having block-based incremental backup turned on. XXX - + * Need to call something to read it in and set this up. Maybe here, maybe not. + */ + WT_RET(__wt_fs_exist(session, WT_BLKINCR_BACKUP, &exist_bincr)); + if (exist_bincr) { + F_SET(S2C(session), WT_CONN_INCR_BACKUP); + /* Load content into some structure. Not sure this is the right place. It may be too early. + */ + } + if (exist_turtle) { /* * Failure to read means a bad turtle file. Remove it and create a new turtle file. diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index ca6b0b4adaa..4b9510d372a 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -731,7 +731,7 @@ __wt_rec_col_var( */ WT_ERR(__wt_dsk_cell_data_ref(session, WT_PAGE_COL_VAR, vpack, orig)); - record_loop: +record_loop: /* * Generate on-page entries: loop repeat records, looking for WT_INSERT entries matching the * record number. The WT_INSERT lists are in sorted order, so only need check the next one. diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index 43360d702c5..dbef848c3f3 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -989,7 +989,7 @@ build: /* Update compression state. */ __rec_key_state_update(r, ovfl_key); - leaf_insert: +leaf_insert: /* Write any K/V pairs inserted into the page after this key. */ if ((ins = WT_SKIP_FIRST(WT_ROW_INSERT(page, rip))) != NULL) WT_ERR(__rec_row_leaf_insert(session, r, ins)); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index 2150bf63559..451d4f2faae 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -189,6 +189,13 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v list_prepared = true; if (upd->start_ts > max_ts) max_ts = upd->start_ts; + + /* + * Track the oldest update not on the page, used to decide whether reads can use the + * page image, hence using the start rather than the durable timestamp. + */ + if (upd->start_ts < r->min_skipped_ts) + r->min_skipped_ts = upd->start_ts; continue; } } @@ -231,10 +238,8 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v skipped_birthmark = true; /* - * Track the oldest update not on the page. - * - * This is used to decide whether reads can use the page image, hence using the start - * rather than the durable timestamp. + * Track the oldest update not on the page, used to decide whether reads can use the + * page image, hence using the start rather than the durable timestamp. */ if (upd_select->upd == NULL && upd->start_ts < r->min_skipped_ts) r->min_skipped_ts = upd->start_ts; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 3b3d13770d2..e52b86da055 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -1568,6 +1568,7 @@ __rec_split_write_supd( key->data = WT_INSERT_KEY(supd->ins); key->size = WT_INSERT_KEY_SIZE(supd->ins); } + WT_ASSERT(session, next != NULL); WT_ERR(__wt_compare(session, btree->collator, key, &next->key, &cmp)); if (cmp >= 0) break; diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 098acb87c60..d956c9692d9 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -1643,10 +1643,13 @@ __session_commit_transaction(WT_SESSION *wt_session, const char *config) WT_ERR(__wt_txn_context_check(session, true)); - if (F_ISSET(txn, WT_TXN_ERROR) && txn->mod_count != 0) - WT_ERR_MSG(session, EINVAL, "failed transaction requires rollback%s%s", + /* Permit the commit if the transaction failed, but was read-only. */ + if (F_ISSET(txn, WT_TXN_ERROR) && txn->mod_count != 0) { + __wt_err(session, EINVAL, "failed transaction requires rollback%s%s", txn->rollback_reason == NULL ? "" : ": ", txn->rollback_reason == NULL ? "" : txn->rollback_reason); + ret = EINVAL; + } if (ret == 0) ret = __wt_txn_commit(session, cfg); @@ -1656,7 +1659,6 @@ __session_commit_transaction(WT_SESSION *wt_session, const char *config) } err: - WT_ASSERT(session, WT_SESSION_TXN_STATE(session)->id == WT_TXN_NONE); API_END_RET(session, ret); } @@ -1669,7 +1671,6 @@ __session_prepare_transaction(WT_SESSION *wt_session, const char *config) { WT_DECL_RET; WT_SESSION_IMPL *session; - WT_TXN *txn; session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, prepare_transaction, config, cfg); @@ -1678,19 +1679,9 @@ __session_prepare_transaction(WT_SESSION *wt_session, const char *config) WT_ERR(__wt_txn_context_check(session, true)); - /* - * A failed transaction cannot be prepared, as it cannot guarantee a subsequent commit. - */ - txn = &session->txn; - if (F_ISSET(txn, WT_TXN_ERROR) && txn->mod_count != 0) - WT_ERR_MSG(session, EINVAL, "failed transaction requires rollback%s%s", - txn->rollback_reason == NULL ? "" : ": ", - txn->rollback_reason == NULL ? "" : txn->rollback_reason); - WT_ERR(__wt_txn_prepare(session, cfg)); err: - WT_ASSERT(session, WT_SESSION_TXN_STATE(session)->id == WT_TXN_NONE); API_END_RET(session, ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index e8e97b2df57..0fae8db558d 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -892,6 +892,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) prepare = F_ISSET(txn, WT_TXN_PREPARE); readonly = txn->mod_count == 0; + /* Permit the commit if the transaction failed, but was read-only. */ WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || txn->mod_count == 0); @@ -1131,7 +1132,8 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) txn_prepared_updates_count = 0; WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); - WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || txn->mod_count == 0); + WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR)); + /* * A transaction should not have updated any of the logged tables, if debug mode logging is not * turned on. @@ -1605,15 +1607,16 @@ __wt_txn_is_blocking(WT_SESSION_IMPL *session) return (false); /* - * Check the oldest transaction ID of either the current transaction ID or the snapshot. Using - * the snapshot potentially means rolling back a read-only transaction, which MongoDB can't - * (yet) handle. For this reason, don't use the snapshot unless there's also a transaction ID - * or we're configured to time out thread operations (a way to confirm our caller is prepared - * for rollback). + * MongoDB can't (yet) handle rolling back read only transactions. For this reason, don't check + * unless there's at least one update or we're configured to time out thread operations (a way + * to confirm our caller is prepared for rollback). */ + if (txn->mod_count == 0 && !__wt_op_timer_fired(session)) + return (false); + + /* Check the oldest transaction ID of either the current transaction ID or the snapshot. */ txn_oldest = txn->id; if (F_ISSET(txn, WT_TXN_HAS_SNAPSHOT) && txn->snap_min != WT_TXN_NONE && - (txn_oldest != WT_TXN_NONE || __wt_op_timer_fired(session)) && (txn_oldest == WT_TXN_NONE || WT_TXNID_LT(txn->snap_min, txn_oldest))) txn_oldest = txn->snap_min; return (txn_oldest == conn->txn_global.oldest_id ? diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/util.c b/src/third_party/wiredtiger/test/csuite/random_directio/util.c index 40de5d49f36..a9ad564c160 100644 --- a/src/third_party/wiredtiger/test/csuite/random_directio/util.c +++ b/src/third_party/wiredtiger/test/csuite/random_directio/util.c @@ -95,6 +95,14 @@ copy_directory(const char *fromdir, const char *todir, bool directio) testutil_check(__wt_snprintf(fromfile, sizeof(fromfile), "%s/%s", fromdir, dp->d_name)); testutil_check(__wt_snprintf(tofile, sizeof(tofile), "%s/%s", todir, dp->d_name)); rfd = open(fromfile, O_RDONLY | openflags, 0); + /* + * The child process may have been stopped during a drop and WiredTiger drop will do an + * unlink call followed by syncing the directory. It is possible for the signal to have been + * delivered in between those calls so the file may no longer exist but reading the + * directory will still return its entry. Handle that case and skip the file if it happens. + */ + if (rfd < 0 && errno == ENOENT) + continue; testutil_assertfmt(rfd >= 0, "Open of source %s failed with %d\n", fromfile, errno); wfd = open(tofile, O_WRONLY | O_CREAT, 0666); testutil_assertfmt(wfd >= 0, "Open of dest %s failed with %d\n", tofile, errno); diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index ffa46247106..ea5eeecf75c 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -486,7 +486,7 @@ thread_ts_run(void *arg) printf("SET STABLE: %" PRIx64 " %" PRIu64 "\n", oldest_ts, oldest_ts); } } else - ts_wait: +ts_wait: __wt_sleep(0, 1000); } /* NOTREACHED */ diff --git a/src/third_party/wiredtiger/test/csuite/scope/main.c b/src/third_party/wiredtiger/test/csuite/scope/main.c index 57947fcf166..5a0c50f5a48 100644 --- a/src/third_party/wiredtiger/test/csuite/scope/main.c +++ b/src/third_party/wiredtiger/test/csuite/scope/main.c @@ -50,42 +50,78 @@ handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const ch static WT_EVENT_HANDLER event_handler = {handle_error, NULL, NULL, NULL}; +#define SET_KEY \ + do { \ + if (recno) \ + cursor->set_key(cursor, (uint64_t)1); \ + else { \ + strcpy(keybuf, KEY); \ + cursor->set_key(cursor, keybuf); \ + } \ + } while (0) +#define SET_VALUE \ + do { \ + strcpy(valuebuf, VALUE); \ + if (vstring) \ + cursor->set_value(cursor, valuebuf); \ + else { \ + vu.size = strlen(vu.data = valuebuf); \ + cursor->set_value(cursor, &vu); \ + } \ + } while (0) + static void cursor_scope_ops(WT_SESSION *session, const char *uri) { struct { const char *op; - enum { INSERT, MODIFY, SEARCH, SEARCH_NEAR, REMOVE, REMOVE_POS, RESERVE, UPDATE } func; - const char *config; + enum { + INSERT_GET_KEY, + INSERT_GET_VALUE, + MODIFY, + SEARCH, + SEARCH_NEAR, + REMOVE_GET_KEY, + REMOVE_GET_VALUE, + REMOVE_POS, + RESERVE, + UPDATE + } func; } * op, ops[] = {/* * The ops order is specific: insert has to happen first so * other operations are possible, and remove has to be last. */ { - "insert", INSERT, NULL, + "insert", INSERT_GET_KEY, + }, + { + "insert", INSERT_GET_VALUE, + }, + { + "search", SEARCH, }, { - "search", SEARCH, NULL, + "search", SEARCH_NEAR, }, { - "search", SEARCH_NEAR, NULL, + "reserve", RESERVE, }, { - "reserve", RESERVE, NULL, + "insert", MODIFY, }, { - "insert", MODIFY, NULL, + "update", UPDATE, }, { - "update", UPDATE, NULL, + "remove", REMOVE_GET_KEY, }, { - "remove", REMOVE, NULL, + "remove", REMOVE_GET_VALUE, }, { - "remove", REMOVE_POS, NULL, + "remove", REMOVE_POS, }, - {NULL, INSERT, NULL}}; + {NULL, INSERT_GET_KEY}}; WT_CURSOR *cursor; #define MODIFY_ENTRIES 2 WT_MODIFY entries[MODIFY_ENTRIES]; @@ -94,48 +130,43 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) const char *key, *vs; char keybuf[100], valuebuf[100]; int exact; - bool recno, vstring; - - /* - * Modify and reserve require a transaction, modify requires snapshot isolation. - */ - testutil_check(session->begin_transaction(session, "isolation=snapshot")); + bool recno, rollback, vstring; cursor = NULL; for (op = ops; op->op != NULL; op++) { - key = vs = NULL; - memset(&vu, 0, sizeof(vu)); + /* Open a cursor, track key/value formats. */ + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + recno = strcmp(cursor->key_format, "r") == 0; + vstring = strcmp(cursor->value_format, "S") == 0; + + /* Remove any leftover key/value pair, start fresh. */ + SET_KEY; + testutil_check(cursor->remove(cursor)); + + /* If not an insert operation, make sure there's a key/value pair to operate on. */ + if (op->func != INSERT_GET_KEY && op->func != INSERT_GET_VALUE) { + SET_KEY; + SET_VALUE; + testutil_check(cursor->insert(cursor)); + } + /* Discard that cursor, we'll open one inside the transaction. */ + testutil_check(cursor->close(cursor)); - /* Open a cursor. */ - if (cursor != NULL) - testutil_check(cursor->close(cursor)); - testutil_check(session->open_cursor(session, uri, NULL, op->config, &cursor)); + /* Modify and reserve require a transaction, modify requires snapshot isolation. */ + testutil_check(session->begin_transaction(session, "isolation=snapshot")); + rollback = false; - /* Operations change based on the key/value formats. */ + /* Open a cursor, track key/value formats. */ + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); recno = strcmp(cursor->key_format, "r") == 0; vstring = strcmp(cursor->value_format, "S") == 0; - /* Modify is only possible with "item" values. */ - if (vstring && op->func == MODIFY) - continue; - /* * Set up application buffers so we can detect overwrites or failure to copy application * information into library memory. */ - if (recno) - cursor->set_key(cursor, (uint64_t)1); - else { - strcpy(keybuf, KEY); - cursor->set_key(cursor, keybuf); - } - strcpy(valuebuf, VALUE); - if (vstring) - cursor->set_value(cursor, valuebuf); - else { - vu.size = strlen(vu.data = valuebuf); - cursor->set_value(cursor, &vu); - } + SET_KEY; + SET_VALUE; /* * The application must keep key and value memory valid until the next operation that @@ -144,7 +175,8 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) * Modifying either the key or value buffers is not permitted. */ switch (op->func) { - case INSERT: + case INSERT_GET_KEY: + case INSERT_GET_VALUE: testutil_check(cursor->insert(cursor)); break; case MODIFY: @@ -173,7 +205,8 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) */ testutil_check(cursor->search(cursor)); /* FALLTHROUGH */ - case REMOVE: + case REMOVE_GET_KEY: + case REMOVE_GET_VALUE: testutil_check(cursor->remove(cursor)); break; case RESERVE: @@ -194,20 +227,36 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) /* * Check that get_key/get_value behave as expected after the operation. */ + key = vs = NULL; + keyr = 37; + memset(&vu, 0, sizeof(vu)); switch (op->func) { - case INSERT: - case REMOVE: + case INSERT_GET_KEY: + case REMOVE_GET_KEY: /* * Insert and remove configured with a search key do not position the cursor and have no * key or value. * - * There should be two error messages, ignore them. + * There should be two error messages, ignore them, and errors require rollback. */ - ignore_errors = 2; + ignore_errors = 1; + rollback = true; if (recno) testutil_assert(cursor->get_key(cursor, &keyr) != 0); else testutil_assert(cursor->get_key(cursor, &key) != 0); + testutil_assert(ignore_errors == 0); + break; + case INSERT_GET_VALUE: + case REMOVE_GET_VALUE: + /* + * Insert and remove configured with a search key do not position the cursor and have no + * key or value. + * + * There should be two error messages, ignore them, and errors require rollback. + */ + ignore_errors = 1; + rollback = true; if (vstring) testutil_assert(cursor->get_value(cursor, &vs) != 0); else @@ -218,7 +267,7 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) /* * Remove configured with a cursor position has a key, but no value. * - * There should be one error message, ignore it. + * There should be one error message, ignore it, and errors require rollback. */ if (recno) { testutil_assert(cursor->get_key(cursor, &keyr) == 0); @@ -229,6 +278,7 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) testutil_assert(strcmp(key, KEY) == 0); } ignore_errors = 1; + rollback = true; if (vstring) testutil_assert(cursor->get_value(cursor, &vs) != 0); else @@ -267,25 +317,11 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) break; } - /* - * We have more than one remove operation, add the key back in. - */ - if (op->func == REMOVE || op->func == REMOVE_POS) { - if (recno) - cursor->set_key(cursor, (uint64_t)1); - else { - strcpy(keybuf, KEY); - cursor->set_key(cursor, keybuf); - } - strcpy(valuebuf, VALUE); - if (vstring) - cursor->set_value(cursor, valuebuf); - else { - vu.size = strlen(vu.data = valuebuf); - cursor->set_value(cursor, &vu); - } - testutil_check(cursor->insert(cursor)); - } + if (rollback) + testutil_check(session->rollback_transaction(session, NULL)); + else + testutil_check(session->commit_transaction(session, NULL)); + testutil_check(cursor->close(cursor)); } } diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index e28772c915b..d728e9f4e96 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -97,6 +97,36 @@ functions: set -o verbose echo '#undef HAVE_FTRUNCATE' >> wiredtiger_config.h - *make_wiredtiger + "compile wiredtiger address sanitizer": + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + shell: bash + script: | + set -o errexit + set -o verbose + sh reconf + if [ "$OS" != "Windows_NT" ]; then + CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fno-omit-frame-pointer -fsanitize=address" \ + ../configure ${configure_python_setting|} \ + --enable-diagnostic --with-builtins=lz4,snappy,zlib + fi + - *make_wiredtiger + "compile wiredtiger with builtins": + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + shell: bash + script: | + set -o errexit + set -o verbose + sh reconf + if [ "$OS" != "Windows_NT" ]; then + CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" \ + ../configure ${configure_python_setting|} \ + --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib + fi + - *make_wiredtiger "make check directory": command: shell.exec params: @@ -124,7 +154,7 @@ functions: set -o verbose ${test_env_vars|} ${python_binary|python} ../test/suite/run.py ${unit_test_args|-v 2} ${smp_command|} 2>&1 - "test format": + "format test": command: shell.exec params: working_dir: "wiredtiger/build_posix/test/format" @@ -132,32 +162,89 @@ functions: set -o errexit set -o verbose for i in $(seq ${times|1}); do - ./t -1 -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} + ./t -1 -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} 2>&1 + done + "format test script": + command: shell.exec + params: + working_dir: "wiredtiger/test/format" + script: | + set -o errexit + set -o verbose + for i in $(seq ${times|1}); do + ${test_env_vars|} ${format_test_setting|} ./format.sh ${smp_command|} ${format_test_script_args|} 2>&1 done "many dbs test": command: shell.exec - parms: + params: working_dir: "wiredtiger/build_posix/test/manydbs" script: | set -o errexit set -o verbose - ${test_env_vars|} ./t ${many_db_args|} + ${test_env_vars|} ./t ${many_db_args|} 2>&1 "thread test": command: shell.exec - parms: + params: working_dir: "wiredtiger/build_posix/test/thread" script: | set -o errexit set -o verbose - ${test_env_vars|} ./t ${thread_test_args|} + ${test_env_vars|} ./t ${thread_test_args|} 2>&1 "random abort test": command: shell.exec - parms: + params: working_dir: "wiredtiger/build_posix/test/csuite" script: | set -o errexit set -o verbose - ${test_env_vars|} ./test_random_abort ${random_abort_args|} + ${test_env_vars|} ./test_random_abort ${random_abort_args|} 2>&1 + "timestamp abort test": + command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/csuite" + script: | + set -o errexit + set -o verbose + ${test_env_vars|} ./test_timestamp_abort ${timestamp_abort_args|} 2>&1 + "truncated log test": + command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/csuite" + script: | + set -o errexit + set -o verbose + ${test_env_vars|} ./test_truncated_log ${truncated_log_args|} 2>&1 + "recovery stress test script": + command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/csuite" + script: | + set -o errexit + set -o verbose + + for i in $(seq ${times|1}); do + # Run the various combinations of args. Let time and threads be random. + # Run current version with write-no-sync txns. + ${test_env_vars|} ./test_random_abort 2>&1 + ${test_env_vars|} ./test_timestamp_abort 2>&1 + + # Current version with memory-based txns (MongoDB usage). + ${test_env_vars|} ./test_random_abort -m 2>&1 + ${test_env_vars|} ./test_timestamp_abort -m 2>&1 + + # V1 log compatibility mode with write-no-sync txns. + ${test_env_vars|} ./test_random_abort -C 2>&1 + ${test_env_vars|} ./test_timestamp_abort -C 2>&1 + + # V1 log compatibility mode with memory-based txns. + ${test_env_vars|} ./test_random_abort -C -m 2>&1 + ${test_env_vars|} ./test_timestamp_abort -C -m 2>&1 + + ${test_env_vars|} ./test_truncated_log ${truncated_log_args|} 2>&1 + + # Just let the system take a breath + sleep 10s + done "upload artifact": - command: archive.targz_pack params: @@ -191,6 +278,28 @@ functions: set -o verbose ./t ${checkpoint_args} 2>&1 + "checkpoint stress test": + command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + + export WIREDTIGER_CONFIG='checkpoint_sync=0,transaction_sync=(method=none)' + CMD='./t -h WT_TEST.$i.$t -t r -r 2 -W 3 -n 1000000 -k 1000000 -C "cache_size=100MB"' + + for i in $(seq ${times|1}); do + for t in $(seq ${no_of_procs|1}); do + eval nohup $CMD > nohup.out.$i.$t 2>&1 & + done + + for t in $(seq ${no_of_procs|1}); do + wait -n || exit $? + done + done + pre: - func: "cleanup" post: @@ -283,7 +392,7 @@ tasks: posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - func: "make check all" vars: - test_env_vars: MSAN_OPTIONS=abort_on_error=1:disable_coredump=0 MSAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + test_env_vars: MSAN_OPTIONS=abort_on_error=1:disable_coredump=0 MSAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer TESTUTIL_SLOW_MACHINE=1 - name: make-check-asan-test depends_on: @@ -632,14 +741,7 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} $(pwd)/test/csuite/test_truncated_log 2>&1 + - func: "truncated log test" - name: csuite-wt1965-col-efficiency-test tags: ["pull_request"] @@ -1337,7 +1439,7 @@ tasks: commands: - func: "fetch artifacts" - func: "compile wiredtiger" - - func: "test format" + - func: "format test" vars: times: 10 config: ../../../test/format/CONFIG.endian @@ -1399,7 +1501,7 @@ tasks: commands: - func: "fetch artifacts" - func: "compile wiredtiger" - - func: "test format" + - func: "format test" vars: times: 10 config: ../../../test/format/CONFIG.endian @@ -1493,7 +1595,7 @@ tasks: commands: - func: "fetch artifacts" - func: "compile wiredtiger" - - func: "test format" + - func: "format test" vars: times: 3 config: ../../../test/format/CONFIG.stress @@ -1507,7 +1609,7 @@ tasks: vars: dependent_task: compile-linux-no-ftruncate - func: "compile wiredtiger no linux ftruncate" - - func: "test format" + - func: "format test" vars: times: 3 @@ -1564,37 +1666,37 @@ tasks: - func: "unit test" vars: unit_test_args: -v 2 --long - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=snappy logging=1 logging_compression=snappy logging_prealloc=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row alter=1 backups=1 compaction=1 data_extend=1 prepare=1 rebalance=1 salvage=1 statistics=1 statistics_server=1 verify=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row direct_io=1 firstfit=1 internal_key_truncation=1 - - func: "test format" + - func: "format test" vars: extra_args: leak_memory=0 mmap=1 file_type=row checkpoints=0 in_memory=1 reverse=1 truncate=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=zlib huffman_key=1 huffman_value=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row isolation=random transaction_timestamps=0 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row data_source=lsm bloom=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=var compression=snappy checksum=uncompressed dictionary=1 repeat_data_pct=10 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=lz4 prefix_compression=1 leaf_page_max=9 internal_page_max=9 key_min=256 value_min=256 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=var leaf_page_max=9 internal_page_max=9 value_min=256 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=fix - command: shell.exec @@ -1623,7 +1725,7 @@ tasks: vars: posix_configure_flags: --enable-python --with-spinlock=gcc --enable-strict - func: "make check all" - - func: "test format" + - func: "format test" vars: times: 3 - func: "unit test" @@ -1635,7 +1737,7 @@ tasks: vars: posix_configure_flags: --enable-python --with-spinlock=pthread_adaptive --enable-strict - func: "make check all" - - func: "test format" + - func: "format test" vars: times: 3 - func: "unit test" @@ -1747,20 +1849,13 @@ tasks: random_abort_args: -t 40 # truncated-log - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix/test/csuite/" - script: | - set -o errexit - set -o verbose - - ./test_truncated_log + - func: "truncated log test" # format test - - func: "test format" + - func: "format test" vars: extra_args: file_type=fix - - func: "test format" + - func: "format test" vars: extra_args: file_type=row @@ -1783,6 +1878,145 @@ tasks: set -o verbose ./time_shift_test.sh /usr/local/lib/faketime/libfaketimeMT.so.1 0-1 2>&1 + + - name: format-stress-sanitizer-test + #set a 25 hours (25*60*60 = 90000 seconds) timeout + exec_timeout_secs: 90000 + commands: + - func: "get project" + - func: "compile wiredtiger address sanitizer" + - func: "format test script" + vars: + test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + # run for 24 hours ( 24 * 60 = 1440 minutes), don't stop at failed tests, use default config + format_test_script_args: -t 1440 + + - name: format-stress-sanitizer-smoke-test + #set a 7 hours timeout + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger address sanitizer" + - func: "format test script" + # to emulate the original Jenkins job's test coverage, we are running the smoke test 16 times + # run smoke tests, don't stop at failed tests, use default config + vars: + test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + format_test_script_args: -S + times: 16 + + - name: recovery-stress-test + #set a 25 hours timeout + exec_timeout_secs: 90000 + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib + - func: "recovery stress test script" + vars: + # Repeat this script enough times to make this task run for 24 hours + # At the time of writing this script, one call to underlying scripts takes about ~15 mins to finish in worst case. + # We are giving an extra ~20% room for vairance in execution time. + times: 80 + + # This is special task until lz4 issues are resolved for zSeries distros + - name: recovery-stress-test-without-lz4 + #set a 25 hours timeout + exec_timeout_secs: 90000 + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=snappy,zlib + - func: "recovery stress test script" + vars: + # Repeat this script enough times to make this task run for 24 hours + # At the time of writing this script, one call to underlying scripts takes about 8 mins to finish in worst case. + # We are giving an extra ~20% room for vairance in execution time. + times: 120 + + - name: split-stress-test + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + configure_env_vars: CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" + - command: shell.exec + params: + working_dir: "wiredtiger/bench/workgen/runner" + script: | + set -o errexit + set -o verbose + for i in {1..10}; do ${python_binary|python} split_stress.py; done + + - name: format-stress-test + # Set 25 hours timeout + exec_timeout_secs: 90000 + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + #run for 24 hours ( 24 * 60 = 1440 minutes), use default config + format_test_script_args: -b "SEGFAULT_SIGNALS=all catchsegv ./t" -t 1440 + + - name: format-stress-smoke-test + # Set 7 hours timeout + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + # to emulate the original Jenkins job's test coverage, we are running the smoke test 16 times + # run smoke tests, use default config (-S) + format_test_script_args: -b "SEGFAULT_SIGNALS=all catchsegv ./t" -S + times: 16 + + - name: checkpoint-stress-test + commands: + - command: timeout.update + params: + # set 24 hour timeout for the task and command + exec_timeout_secs: 86400 + timeout_secs: 86400 + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "checkpoint stress test" + vars: + # No of times to run the loop + times: 2 + # No of processes to run in the background + no_of_procs: 10 + + # The task name is ppc-zseries because this task will be used in both buildVariants + - name: format-stress-ppc-zseries-test + # Set 2.5 hours timeout (60 * 60 * 2.5) + exec_timeout_secs: 9000 + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + # Make sure we dump core on failure + format_test_setting: ulimit -c unlimited + #run for 2 hours ( 2 * 60 = 120 minutes), use default config + format_test_script_args: -b "SEGFAULT_SIGNALS=all catchsegv ./t" -t 120 + + - name: format-wtperf-test + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/bench/wtperf" + script: | + set -o errexit + set -o verbose + + cp ../../../bench/wtperf/split_heavy.wtperf . + ./wtperf -O ./split_heavy.wtperf -o verbose=2 buildvariants: - name: ubuntu1804 @@ -1813,6 +2047,13 @@ buildvariants: - name: wtperf-test - name: ftruncate-test - name: long-test + - name: recovery-stress-test + - name: format-stress-sanitizer-test + - name: format-stress-sanitizer-smoke-test + - name: split-stress-test + - name: format-stress-test + - name: format-stress-smoke-test + - name: checkpoint-stress-test - name: ubuntu1804-python3 display_name: Ubuntu 18.04 (Python3) @@ -1952,6 +2193,9 @@ buildvariants: tasks: - name: compile - name: unit-test + - name: format-stress-ppc-zseries-test + - name: format-stress-smoke-test + - name: format-wtperf-test - name: ubuntu1804-zseries display_name: Ubuntu 18.04 zSeries @@ -1965,3 +2209,5 @@ buildvariants: tasks: - name: compile - name: unit-test + - name: recovery-stress-test-without-lz4 + - name: split-stress-test diff --git a/src/third_party/wiredtiger/test/format/Makefile.am b/src/third_party/wiredtiger/test/format/Makefile.am index 2d9bbf21eb8..bff2986f25e 100644 --- a/src/third_party/wiredtiger/test/format/Makefile.am +++ b/src/third_party/wiredtiger/test/format/Makefile.am @@ -4,7 +4,7 @@ AM_CPPFLAGS +=-I$(top_srcdir)/test/utility noinst_PROGRAMS = t t_SOURCES =\ - backup.c bulk.c compact.c config.c lrt.c ops.c random.c rebalance.c \ + backup.c bulk.c compact.c config.c ops.c random.c rebalance.c \ salvage.c snap.c t.c util.c wts.c t_LDADD = $(top_builddir)/test/utility/libtest_util.la diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 8cec1318efc..9af28f668cd 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -39,7 +39,6 @@ static bool config_fix(void); static void config_in_memory(void); static void config_in_memory_reset(void); static int config_is_perm(const char *); -static void config_lrt(void); static void config_lsm_reset(void); static void config_map_checkpoint(const char *, u_int *); static void config_map_checksum(const char *, u_int *); @@ -176,7 +175,6 @@ config_setup(void) config_compression("compression"); config_compression("logging_compression"); config_encryption(); - config_lrt(); /* Configuration based on the configuration already chosen. */ config_pct(); @@ -238,7 +236,7 @@ config_setup(void) static void config_cache(void) { - uint32_t required; + uint32_t required, workers; /* Page sizes are powers-of-two for bad historic reasons. */ g.intl_page_max = 1U << g.c_intl_page_max; @@ -269,7 +267,8 @@ config_cache(void) * This code is what dramatically increases the cache size when there are lots of threads, it * grows the cache to several megabytes per thread. */ - g.c_cache = WT_MAX(g.c_cache, 2 * g.c_threads * g.c_memory_page_max); + workers = g.c_threads + (g.c_random_cursor ? 1 : 0); + g.c_cache = WT_MAX(g.c_cache, 2 * workers * g.c_memory_page_max); /* * Ensure cache size sanity for LSM runs. An LSM tree open requires 3 @@ -281,7 +280,7 @@ config_cache(void) */ if (DATASOURCE("lsm")) { required = WT_LSM_TREE_MINIMUM_SIZE( - g.c_chunk_size * WT_MEGABYTE, g.c_threads * g.c_merge_max, g.c_threads * g.leaf_page_max); + g.c_chunk_size * WT_MEGABYTE, workers * g.c_merge_max, workers * g.leaf_page_max); required = (required + (WT_MEGABYTE - 1)) / WT_MEGABYTE; if (g.c_cache < required) g.c_cache = required; @@ -451,12 +450,7 @@ config_encryption(void) static bool config_fix(void) { - /* - * Fixed-length column stores don't support the lookaside table (so, no long running - * transactions), or modify operations. - */ - if (config_is_perm("long_running_txn")) - return (false); + /* Fixed-length column stores don't support the lookaside table, so no modify operations. */ if (config_is_perm("modify_pct")) return (false); return (true); @@ -573,25 +567,6 @@ config_lsm_reset(void) } /* - * config_lrt -- - * Long-running transaction configuration. - */ -static void -config_lrt(void) -{ - /* - * WiredTiger doesn't support a lookaside file for fixed-length column stores. - */ - if (g.type == FIX && g.c_long_running_txn) { - if (config_is_perm("long_running_txn")) - testutil_die(EINVAL, - "long_running_txn not supported with fixed-length " - "column store"); - config_single("long_running_txn=off", false); - } -} - -/* * config_pct -- * Configure operation percentages. */ diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 492d5124a1c..4c9587d5146 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -190,9 +190,6 @@ static CONFIG c[] = {{"abort", "if timed run should drop core", /* 0% */ {"logging_prealloc", "if log file pre-allocation configured", /* 50% */ C_BOOL, 50, 0, 0, &g.c_logging_prealloc, NULL}, - {"long_running_txn", "if a long-running transaction configured", /* 0% */ - C_BOOL, 0, 0, 0, &g.c_long_running_txn, NULL}, - {"lsm_worker_threads", "the number of LSM worker threads", 0x0, 3, 4, 20, &g.c_lsm_worker_threads, NULL}, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 38755e47564..a1fa5d425b9 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -169,7 +169,6 @@ typedef struct { char *c_logging_compression; uint32_t c_logging_file_max; uint32_t c_logging_prealloc; - uint32_t c_long_running_txn; uint32_t c_lsm_worker_threads; uint32_t c_memory_page_max; uint32_t c_merge_max; @@ -347,7 +346,6 @@ void key_gen_init(WT_ITEM *); void key_gen_insert(WT_RAND_STATE *, WT_ITEM *, uint64_t); void key_gen_teardown(WT_ITEM *); void key_init(void); -WT_THREAD_RET lrt(void *); WT_THREAD_RET random_kv(void *); void path_setup(const char *); int read_row_worker(WT_CURSOR *, uint64_t, WT_ITEM *, WT_ITEM *, bool); diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index 9bdc55a4d69..37607d6b8df 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -18,9 +18,10 @@ trap 'onintr' 2 usage() { echo "usage: $0 [-aFSv] [-c config] " - echo " [-h home] [-j parallel-jobs] [-n total-jobs] [-t minutes] [format-configuration]" + echo " [-b format-binary] [-h home] [-j parallel-jobs] [-n total-jobs] [-t minutes] [format-configuration]" echo echo " -a abort/recovery testing (defaults to off)" + echo " -b binary format binary (defaults to "./t")" echo " -c config format configuration file (defaults to CONFIG.stress)" echo " -F quit on first failure (defaults to off)" echo " -h home run directory (defaults to .)" @@ -47,9 +48,6 @@ smoke_list=( "$smoke_base_1 file_type=row huffman_key=1 huffman_value=1" "$smoke_base_1 file_type=var huffman_key=1 huffman_value=1" - # Abort/recovery test. - "$smoke_base_1 file_type=row abort=1" - # LSM "$smoke_base_1 file_type=row data_source=lsm" @@ -74,12 +72,16 @@ parallel_jobs=8 smoke_test=0 total_jobs=0 verbose=0 +format_binary="./t" while :; do case "$1" in -a) abort_test=1 shift ;; + -b) + format_binary="$2" + shift ; shift ;; -c) config="$2" shift ; shift ;; @@ -155,16 +157,17 @@ cd $(dirname $0) || exit 1 # local. [[ $config_found -eq 0 ]] && [[ -f "$config" ]] && config="$PWD/$config" -# Find the format binary. Builds are normally in the WiredTiger source tree, in which case it's -# in the same directory as format.sh, else it's in the build_posix tree. If the build is in the -# build_posix tree, move there, we have to run in the directory where the format binary lives -# because the format binary "knows" the wt utility is two directory levels above it. -format_binary="./t" -[[ -x $format_binary ]] || { +# Find the last part of format_binary, which is format binary file. Builds are normally in the +# WiredTiger source tree, in which case it's in the same directory as format.sh, else it's in +# the build_posix tree. If the build is in the build_posix tree, move there, we have to run in +# the directory where the format binary lives because the format binary "knows" the wt utility +# is two directory levels above it. + +[[ -x ${format_binary##* } ]] || { build_posix_directory="../../build_posix/test/format" [[ ! -d $build_posix_directory ]] || cd $build_posix_directory || exit 1 - [[ -x $format_binary ]] || { - echo "$name: format program \"$format_binary\" not found" + [[ -x ${format_binary##* } ]] || { + echo "$name: format program \"${format_binary##* }\" not found" exit 1 } } diff --git a/src/third_party/wiredtiger/test/format/lrt.c b/src/third_party/wiredtiger/test/format/lrt.c deleted file mode 100644 index 472a8a0d877..00000000000 --- a/src/third_party/wiredtiger/test/format/lrt.c +++ /dev/null @@ -1,175 +0,0 @@ -/*- - * Public Domain 2014-2019 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "format.h" - -/* - * lrt -- - * Start a long-running transaction. - */ -WT_THREAD_RET -lrt(void *arg) -{ - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_ITEM key, value; - WT_SESSION *session; - size_t buf_len, buf_size; - uint64_t keyno, saved_keyno; - uint8_t bitfield; - u_int period; - int pinned, ret; - void *buf; - - (void)(arg); /* Unused parameter */ - - saved_keyno = 0; /* [-Werror=maybe-uninitialized] */ - - key_gen_init(&key); - val_gen_init(&value); - - buf = NULL; - buf_len = buf_size = 0; - - /* Open a session and cursor. */ - conn = g.wts_conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); - - for (pinned = 0;;) { - if (pinned) { - /* Re-read the record at the end of the table. */ - while ((ret = read_row_worker(cursor, saved_keyno, &key, &value, false)) == WT_ROLLBACK) - ; - if (ret != 0) - testutil_die(ret, "read_row_worker %" PRIu64, saved_keyno); - - /* Compare the previous value with the current one. */ - if (g.type == FIX) { - ret = cursor->get_value(cursor, &bitfield); - value.data = &bitfield; - value.size = 1; - } else - ret = cursor->get_value(cursor, &value); - if (ret != 0) - testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); - - if (buf_size != value.size || memcmp(buf, value.data, value.size) != 0) - testutil_die(0, "mismatched start/stop values"); - - /* End the transaction. */ - testutil_check(session->commit_transaction(session, NULL)); - - /* Reset the cursor, releasing our pin. */ - testutil_check(cursor->reset(cursor)); - pinned = 0; - } else { - /* - * Test named snapshots: create a snapshot, wait to give the transaction state time to - * move forward, then start a transaction with the named snapshot, drop it, then commit - * the transaction. This exercises most of the named snapshot logic under load. - */ - testutil_check(session->snapshot(session, "name=test")); - __wt_sleep(1, 0); - wiredtiger_begin_transaction(session, "snapshot=test"); - testutil_check(session->snapshot(session, "drop=(all)")); - testutil_check(session->commit_transaction(session, NULL)); - - /* - * Begin transaction: without an explicit transaction, the snapshot is only kept around - * while a cursor is positioned. As soon as the cursor loses its position a new snapshot - * will be allocated. - */ - while ( - (ret = session->begin_transaction(session, "snapshot=snapshot")) == WT_CACHE_FULL) - ; - testutil_check(ret); - - /* Read a record at the end of the table. */ - do { - saved_keyno = mmrand(NULL, (u_int)(g.key_cnt - g.key_cnt / 10), (u_int)g.key_cnt); - while ( - (ret = read_row_worker(cursor, saved_keyno, &key, &value, false)) == WT_ROLLBACK) - ; - } while (ret == WT_NOTFOUND); - if (ret != 0) - testutil_die(ret, "read_row_worker %" PRIu64, saved_keyno); - - /* Copy the cursor's value. */ - if (g.type == FIX) { - ret = cursor->get_value(cursor, &bitfield); - value.data = &bitfield; - value.size = 1; - } else - ret = cursor->get_value(cursor, &value); - if (ret != 0) - testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); - if (buf_len < value.size) - buf = drealloc(buf, buf_len = value.size); - memcpy(buf, value.data, buf_size = value.size); - - /* - * Move the cursor to an early record in the table, hopefully allowing the page with the - * record just retrieved to be evicted from memory. - */ - do { - keyno = mmrand(NULL, 1, (u_int)g.key_cnt / 5); - while ((ret = read_row_worker(cursor, keyno, &key, &value, false)) == WT_ROLLBACK) - ; - } while (ret == WT_NOTFOUND); - if (ret != 0) - testutil_die(ret, "read_row_worker %" PRIu64, keyno); - - pinned = 1; - } - - /* Sleep for some number of seconds. */ - period = mmrand(NULL, 1, 10); - - /* Sleep for short periods so we don't make the run wait. */ - while (period > 0 && !g.workers_finished) { - --period; - __wt_sleep(1, 0); - } - if (g.workers_finished) - break; - } - - testutil_check(session->close(session, NULL)); - - key_gen_teardown(&key); - val_gen_teardown(&value); - free(buf); - - return (WT_THREAD_RET_VALUE); -} diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 2b4ae05cc5f..669cb7a484e 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -126,7 +126,7 @@ wts_ops(bool lastrun) TINFO *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, lrt_tid, random_tid; + wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, random_tid; wt_thread_t timestamp_tid; int64_t fourths, quit_fourths, thread_ops; uint32_t i; @@ -139,7 +139,6 @@ wts_ops(bool lastrun) memset(&backup_tid, 0, sizeof(backup_tid)); memset(&checkpoint_tid, 0, sizeof(checkpoint_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); - memset(&lrt_tid, 0, sizeof(lrt_tid)); memset(&random_tid, 0, sizeof(random_tid)); memset(×tamp_tid, 0, sizeof(timestamp_tid)); @@ -209,7 +208,7 @@ wts_ops(bool lastrun) } /* - * If a multi-threaded run, start optional backup, compaction and long-running reader threads. + * If a multi-threaded run, start optional special-purpose threads. */ if (g.c_alter) testutil_check(__wt_thread_create(NULL, &alter_tid, alter, NULL)); @@ -219,8 +218,6 @@ wts_ops(bool lastrun) testutil_check(__wt_thread_create(NULL, &checkpoint_tid, checkpoint, NULL)); if (g.c_compact) testutil_check(__wt_thread_create(NULL, &compact_tid, compact, NULL)); - if (!SINGLETHREADED && g.c_long_running_txn) - testutil_check(__wt_thread_create(NULL, &lrt_tid, lrt, NULL)); if (g.c_random_cursor) testutil_check(__wt_thread_create(NULL, &random_tid, random_kv, NULL)); if (g.c_txn_timestamps) @@ -293,7 +290,7 @@ wts_ops(bool lastrun) } } - /* Wait for the other threads. */ + /* Wait for the special-purpose threads. */ g.workers_finished = true; if (g.c_alter) testutil_check(__wt_thread_join(NULL, &alter_tid)); @@ -303,8 +300,6 @@ wts_ops(bool lastrun) testutil_check(__wt_thread_join(NULL, &checkpoint_tid)); if (g.c_compact) testutil_check(__wt_thread_join(NULL, &compact_tid)); - if (!SINGLETHREADED && g.c_long_running_txn) - testutil_check(__wt_thread_join(NULL, &lrt_tid)); if (g.c_random_cursor) testutil_check(__wt_thread_join(NULL, &random_tid)); if (g.c_txn_timestamps) @@ -842,7 +837,7 @@ ops(void *arg) READ_OP_FAILED(true); break; case REMOVE: - remove_instead_of_truncate: +remove_instead_of_truncate: switch (g.type) { case ROW: ret = row_remove(tinfo, cursor, positioned); @@ -930,7 +925,7 @@ ops(void *arg) WRITE_OP_FAILED(false); break; case UPDATE: - update_instead_of_chosen_op: +update_instead_of_chosen_op: ++tinfo->update; switch (g.type) { case ROW: @@ -1250,7 +1245,7 @@ nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) } else if (tinfo->keyno > keyno || (!record_gaps && keyno != tinfo->keyno + 1)) goto order_error_col; if (0) { - order_error_col: +order_error_col: testutil_die( 0, "%s returned %" PRIu64 " then %" PRIu64, which, tinfo->keyno, keyno); } @@ -1284,7 +1279,7 @@ nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) goto order_error_row; } if (0) { - order_error_row: +order_error_row: testutil_die(0, "%s returned {%.*s} then {%.*s}", which, (int)tinfo->key->size, (char *)tinfo->key->data, (int)key.size, (char *)key.data); } diff --git a/src/third_party/wiredtiger/test/format/snap.c b/src/third_party/wiredtiger/test/format/snap.c index 15df14b71dc..b304faa7376 100644 --- a/src/third_party/wiredtiger/test/format/snap.c +++ b/src/third_party/wiredtiger/test/format/snap.c @@ -397,6 +397,12 @@ snap_repeat_txn(WT_CURSOR *cursor, TINFO *tinfo) if (current->opid != tinfo->opid) break; + /* + * The transaction is not yet resolved, so the rules are as if the transaction has + * committed. Note we are NOT checking if reads are repeatable based on the chosen + * timestamp. This is because we expect snapshot isolation to work even in the presence of + * other threads of control committing in our past, until the transaction resolves. + */ if (snap_repeat_ok_commit(tinfo, current)) WT_RET(snap_verify(cursor, tinfo, current)); } diff --git a/src/third_party/wiredtiger/test/suite/test_backup06.py b/src/third_party/wiredtiger/test/suite/test_backup06.py index d4efba4c6f0..d9db39d241d 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup06.py +++ b/src/third_party/wiredtiger/test/suite/test_backup06.py @@ -34,6 +34,12 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet +try: + # Windows does not getrlimit/setrlimit so we must catch the resource + # module load. + import resource +except: + None # test_backup06.py # Test that opening a backup cursor does not open file handles. @@ -80,11 +86,21 @@ class test_backup06(wttest.WiredTigerTestCase, suite_subprocess): # Test that the open handle count does not change. def test_cursor_open_handles(self): + if os.name == "nt": + self.skipTest('Unix specific test skipped on Windows') + + limits = resource.getrlimit(resource.RLIMIT_NOFILE) + if limits[0] < 1024: + new = (1024, limits[1]) + resource.setrlimit(resource.RLIMIT_NOFILE, new) self.populate_many() # Close and reopen the connection so the populate dhandles are # not in the list. self.reopen_conn() + new = (limits[0], limits[1]) + resource.setrlimit(resource.RLIMIT_NOFILE, new) + # Confirm that opening a backup cursor does not open # file handles. stat_cursor = self.session.open_cursor('statistics:', None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_backup10.py b/src/third_party/wiredtiger/test/suite/test_backup10.py index 3682db7cf01..b8806c439dc 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup10.py +++ b/src/third_party/wiredtiger/test/suite/test_backup10.py @@ -38,7 +38,6 @@ from wtscenario import make_scenarios class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): dir='backup.dir' # Backup directory name logmax="100K" - newuri="table:newtable" uri="table:test" nops=100 @@ -95,7 +94,6 @@ class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): if ret != 0: break newfile = bkup_c.get_key() - self.assertNotEqual(newfile, self.newuri) sz = os.path.getsize(newfile) self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) shutil.copy(newfile, self.dir) @@ -130,6 +128,7 @@ class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): diff = dup_set.difference(orig_set) self.assertEqual(len(diff), 1) self.assertTrue(log3 in dup_set) + self.assertFalse(log3 in orig_set) # Test a few error cases now. # - We cannot make multiple duplcate backup cursors. @@ -148,7 +147,7 @@ class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): dupc.close() # Test we must use the log target. - msg = "/must be for logs/" + msg = "/must be for /" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.assertEquals(self.session.open_cursor(None, bkup_c, None), 0), msg) diff --git a/src/third_party/wiredtiger/test/suite/test_backup11.py b/src/third_party/wiredtiger/test/suite/test_backup11.py new file mode 100644 index 00000000000..f1622e9d2bc --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup11.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +import os, shutil +from helper import compare_files +from suite_subprocess import suite_subprocess +from wtdataset import simple_key +from wtscenario import make_scenarios + +# test_backup11.py +# Test cursor backup with a duplicate backup cursor. +class test_backup11(wttest.WiredTigerTestCase, suite_subprocess): + dir='backup.dir' # Backup directory name + logmax="100K" + uri="table:test" + nops=100 + + pfx = 'test_backup' + + # ('archiving', dict(archive='true')), + # ('not-archiving', dict(archive='false')), + scenarios = make_scenarios([ + ('archiving', dict(archive='true')), + ]) + + # Create a large cache, otherwise this test runs quite slowly. + def conn_config(self): + return 'cache_size=1G,log=(archive=%s,' % self.archive + \ + 'enabled,file_max=%s)' % self.logmax + + def add_data(self): + log2 = "WiredTigerLog.0000000002" + log3 = "WiredTigerLog.0000000003" + + self.session.create(self.uri, "key_format=S,value_format=S") + + # Insert small amounts of data at a time stopping after we + # cross into log file 2. + loop = 0 + c = self.session.open_cursor(self.uri) + while not os.path.exists(log2): + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + self.session.checkpoint() + c.close() + return loop + + def test_backup11(self): + + loop = self.add_data() + + # Open up the backup cursor. This causes a new log file to be created. + # That log file is not part of the list returned. This is a full backup + # primary cursor with incremental configured. + os.mkdir(self.dir) + config = 'incremental=(enabled,this_id="ID1")' + bkup_c = self.session.open_cursor('backup:', None, config) + + # Add some data that will appear in log file 3. + c = self.session.open_cursor(self.uri) + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + c.close() + self.session.log_flush('sync=on') + self.session.checkpoint() + + # Now copy the files returned by the backup cursor. + orig_logs = [] + while True: + ret = bkup_c.next() + if ret != 0: + break + newfile = bkup_c.get_key() + sz = os.path.getsize(newfile) + self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + if "WiredTigerLog" in newfile: + orig_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + + # Now open a duplicate backup cursor. + # We *can* use a log target duplicate on an incremental primary backup so that + # a backup process can get all the log files that occur while that primary cursor + # is open. + config = 'target=("log:")' + dupc = self.session.open_cursor(None, bkup_c, config) + dup_logs = [] + while True: + ret = dupc.next() + if ret != 0: + break + newfile = dupc.get_key() + self.assertTrue("WiredTigerLog" in newfile) + sz = os.path.getsize(newfile) + if (newfile not in orig_logs): + self.pr('DUP: Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + # Record all log files returned for later verification. + dup_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + dupc.close() + bkup_c.close() + + # Add more data + c = self.session.open_cursor(self.uri) + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + c.close() + self.session.log_flush('sync=on') + self.session.checkpoint() + + # Test a few error cases now. + # - Incremental filename must be on duplicate, not primary. + # - An incremental duplicate must have an incremental primary. + # - We cannot make multiple incremental duplcate backup cursors. + # - We cannot duplicate the duplicate backup cursor. + # - We cannot mix block incremental with a log target on the same duplicate. + # - Incremental ids must be on primary, not duplicate. + # - Force stop must be on primary, not duplicate. + + # - Incremental filename must be on duplicate, not primary. + # Test this first because we currently do not have a primary open. + config = 'incremental=(file=test.wt)' + msg = "/file name can only be specified on a duplicate/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor('backup:', + None, config), 0), msg) + + # Open a non-incremental full backup cursor. + # - An incremental duplicate must have an incremental primary. + bkup_c = self.session.open_cursor('backup:', None, None) + msg = "/must have an incremental primary/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + bkup_c.close() + + config = 'incremental=(src_id="ID1",this_id="ID2")' + bkup_c = self.session.open_cursor('backup:', None, config) + + self.pr("Opened backup for error testing") + + # Now test all the error cases with an incremental primary open. + # - We cannot make multiple incremental duplcate backup cursors. + # - We cannot duplicate the duplicate backup cursor. + config = 'incremental=(file=test.wt)' + dupc = self.session.open_cursor(None, bkup_c, config) + msg = "/already a duplicate backup cursor open/" + self.pr("Test multiple dups") + self.pr("=========") + # Test multiple duplicate backup cursors. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + # Test duplicate of duplicate backup cursor. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + dupc, config), 0), msg) + dupc.close() + + # - A duplicate cursor must specify incremental or log target. + self.pr("Test dup and file target") + self.pr("=========") + msg = "/cannot be used for/" + config = 'target=("file:test.wt")' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + + # - We cannot mix block incremental with a log target on the same duplicate. + self.pr("Test mixed targets") + self.pr("=========") + config = 'incremental=(file=test.wt),target=("log:")' + msg = "/incremental backup incompatible/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + + # - Incremental ids must be on primary, not duplicate. + self.pr("Test ids on dups") + self.pr("=========") + config = 'incremental=(src_id="ID1")' + msg = "/specified on a primary/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + config = 'incremental=(this_id="ID1")' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + + # - Force stop must be on primary, not duplicate. + #self.pr("Test force stop") + #self.pr("=========") + #config = 'incremental=(force_stop=true)' + #print "config is " + config + #self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + # lambda:self.assertEquals(self.session.open_cursor(None, + # bkup_c, config), 0), msg) + + bkup_c.close() + + # After the full backup, open and recover the backup database. + #backup_conn = self.wiredtiger_open(self.dir) + #backup_conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_backup12.py b/src/third_party/wiredtiger/test/suite/test_backup12.py new file mode 100644 index 00000000000..6726164d038 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup12.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +import os, shutil +from helper import compare_files +from suite_subprocess import suite_subprocess +from wtdataset import simple_key +from wtscenario import make_scenarios + +# test_backup12.py +# Test cursor backup with a block-based incremental cursor. +class test_backup12(wttest.WiredTigerTestCase, suite_subprocess): + dir='backup.dir' # Backup directory name + logmax="100K" + uri="table:test" + nops=100 + + pfx = 'test_backup' + + # ('archiving', dict(archive='true')), + # ('not-archiving', dict(archive='false')), + scenarios = make_scenarios([ + ('archiving', dict(archive='true')), + ]) + + # Create a large cache, otherwise this test runs quite slowly. + def conn_config(self): + return 'cache_size=1G,log=(archive=%s,' % self.archive + \ + 'enabled,file_max=%s)' % self.logmax + + def add_data(self): + log2 = "WiredTigerLog.0000000002" + log3 = "WiredTigerLog.0000000003" + + self.session.create(self.uri, "key_format=S,value_format=S") + + # Insert small amounts of data at a time stopping after we + # cross into log file 2. + loop = 0 + c = self.session.open_cursor(self.uri) + while not os.path.exists(log2): + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + self.session.checkpoint() + c.close() + return loop + + def test_backup12(self): + + loop = self.add_data() + + # Open up the backup cursor. This causes a new log file to be created. + # That log file is not part of the list returned. This is a full backup + # primary cursor with incremental configured. + os.mkdir(self.dir) + config = 'incremental=(enabled,this_id="ID1")' + bkup_c = self.session.open_cursor('backup:', None, config) + + # Add some data that will appear in log file 3. + c = self.session.open_cursor(self.uri) + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + c.close() + self.session.log_flush('sync=on') + self.session.checkpoint() + + # Now copy the files returned by the backup cursor. + orig_logs = [] + while True: + ret = bkup_c.next() + if ret != 0: + break + newfile = bkup_c.get_key() + sz = os.path.getsize(newfile) + self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + if "WiredTigerLog" in newfile: + orig_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + + # Now open a duplicate backup cursor. + # We *can* use a log target duplicate on an incremental primary backup so that + # a backup process can get all the log files that occur while that primary cursor + # is open. + config = 'target=("log:")' + dupc = self.session.open_cursor(None, bkup_c, config) + dup_logs = [] + while True: + ret = dupc.next() + if ret != 0: + break + newfile = dupc.get_key() + self.assertTrue("WiredTigerLog" in newfile) + sz = os.path.getsize(newfile) + if (newfile not in orig_logs): + self.pr('DUP: Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + # Record all log files returned for later verification. + dup_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + dupc.close() + bkup_c.close() + + # Add more data. + c = self.session.open_cursor(self.uri) + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + c.close() + self.session.log_flush('sync=on') + self.session.checkpoint() + + # Now do an incremental backup. + config = 'incremental=(src_id="ID1",this_id="ID2")' + bkup_c = self.session.open_cursor('backup:', None, config) + self.pr('Open backup cursor ID1') + while True: + ret = bkup_c.next() + if ret != 0: + break + newfile = bkup_c.get_key() + config = 'incremental=(file=' + newfile + ')' + self.pr('Open incremental cursor with ' + config) + dup_cnt = 0 + dupc = self.session.open_cursor(None, bkup_c, config) + while True: + ret = dupc.next() + if ret != 0: + break + incrlist = dupc.get_keys() + offset = incrlist[0] + size = incrlist[1] + curtype = incrlist[2] + self.assertEqual(offset, 0) + # For now assert WT_BACKUP_FILE (which is 1). + self.assertEqual(curtype, 1) + dup_cnt += 1 + dupc.close() + self.assertEqual(dup_cnt, 1) + self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + bkup_c.close() + + # After the full backup, open and recover the backup database. + backup_conn = self.wiredtiger_open(self.dir) + backup_conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode06.py b/src/third_party/wiredtiger/test/suite/test_debug_mode06.py new file mode 100644 index 00000000000..355102fcebc --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode06.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wttest, wiredtiger + +# test_debug_mode06.py +# Test the debug mode settings. Test slow_checkpoint use (from WT-4981). +# Note: testing timing will make results unreliable so we won't do that. +class test_debug_mode06(wttest.WiredTigerTestCase): + conn_config = 'log=(enabled=true),debug_mode=(slow_checkpoint=true),statistics=(all)' + uri = 'file:test_debug_mode06' + + # Insert some data to ensure setting/unsetting the flag does not + # break existing functionality + def insert_data(self, assert_time=0): + self.session.create(self.uri, 'key_format=s,value_format=s') + self.cursor = self.session.open_cursor(self.uri, None) + self.cursor['key'] = 'value' + self.cursor.close() + self.session.checkpoint() + + # Validate checkpoint time if asked for. + if (assert_time > 0): + stat_cur = self.session.open_cursor('statistics:', None, None) + checkpoint_time = int(stat_cur[wiredtiger.stat.conn.txn_checkpoint_time_recent][2]) + self.assertTrue(checkpoint_time >= assert_time) + stat_cur.close() + + # Make flag works when set. + def test_slow_checkpoints(self): + # Ensure the checkpoint takes at least 10ms (the delay we have set). + self.insert_data(10) + + # Make sure the flag can be 'turned off' as well. + def test_slow_checkpoints_off(self): + conn_reconfig = 'debug_mode=(slow_checkpoint=false)' + self.conn.reconfigure(conn_reconfig) + self.insert_data() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode07.py b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py new file mode 100644 index 00000000000..f4a34307579 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wttest, wiredtiger + +# test_debug_mode07.py +# Test the debug mode settings. Test realloc_exact use (from WT-4919). +class test_debug_mode07(wttest.WiredTigerTestCase): + conn_config = 'log=(enabled=true),debug_mode=(realloc_exact=true),statistics=(all)' + uri = 'file:test_debug_mode07' + + # Insert some data to ensure setting/unsetting the flag does not + # break existing functionality. Also call checkpoint because it + # causes the realloc function to be called numerous times. + def insert_data(self): + self.session.create(self.uri, 'key_format=s,value_format=s') + self.cursor = self.session.open_cursor(self.uri, None) + self.cursor['key'] = 'value' + self.cursor.close() + self.session.checkpoint() + + # Make flag works when set. + def test_realloc_exact(self): + self.insert_data() + + # Make sure the flag can be 'turned off' as well. + def test_realloc_exact_off(self): + conn_reconfig = 'debug_mode=(realloc_exact=false)' + self.conn.reconfigure(conn_reconfig) + self.insert_data() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp09.py b/src/third_party/wiredtiger/test/suite/test_timestamp09.py index 1734efecd0e..e1db60bbd44 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp09.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp09.py @@ -62,7 +62,6 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(2)), '/older than the first commit timestamp/') - c[2] = 2 self.session.rollback_transaction() self.session.begin_transaction() @@ -87,7 +86,6 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(2)), '/less than the oldest timestamp/') - c[2] = 2 self.session.rollback_transaction() self.session.begin_transaction() @@ -137,7 +135,6 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(5)), '/less than the stable timestamp/') - c[5] = 5 self.session.rollback_transaction() self.session.begin_transaction() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 4f97ac2aff7..668b53a7334 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -160,7 +160,7 @@ class TestSuiteConnection(object): class ExtensionList(list): skipIfMissing = False def extension(self, dirname, name, extarg=None): - if name != None and name != 'none': + if name and name != 'none': ext = '' if extarg == None else '=' + extarg self.append(dirname + '/' + name + ext) diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index 3442e8edcec..734026683c9 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -148,6 +148,18 @@ typedef struct { } while (0) /* + * error_sys_check -- + * Complain and quit if a function call fails. A special name because it appears in the + * documentation. Allow any non-negative values. + */ +#define error_sys_check(call) \ + do { \ + int __r; \ + if ((__r = (int)(call)) < 0 && __r != ENOTSUP) \ + testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call); \ + } while (0) + +/* * error_check -- * Complain and quit if a function call fails. A special name because it appears in the * documentation. Ignore ENOTSUP to allow library calls which might not be included in any |