diff options
| author | Ramon Fernandez <ramon@mongodb.com> | 2016-11-07 17:02:46 -0500 |
|---|---|---|
| committer | Ramon Fernandez <ramon@mongodb.com> | 2016-11-07 17:02:46 -0500 |
| commit | 7c626977cf052314d09648d1061e07c44e964264 (patch) | |
| tree | 645b63e4b6cd43879c66b22a85a3b93f7c168923 | |
| parent | 415100327abaf0e2265f86d30640bc3a1ae2a372 (diff) | |
| download | mongo-7c626977cf052314d09648d1061e07c44e964264.tar.gz | |
Import wiredtiger: 2e2d5fe23badac038944bdc6639eb62d77136f1a from branch mongodb-3.2
ref: b11ed312ce..2e2d5fe23b
for: 3.2.11
SERVER-26753 Minor speed regression (13%) and 'choppy' performance in 3.4 vs 3.2
WT-2415 Add option for join cursors to return false positives from a Bloom filter
WT-2945 Occasional hang running reconfigure fuzz test
WT-2947 test suite populate functions need refactoring
WT-2953 Test format failure - stuck cache with 1/42MB dirty
WT-2961 Add a version drop-down to the web version of the docs
WT-2964 WTPERF parallel-pop-btree hung in populate phase
WT-2968 backup cursor opens every file
WT-2971 Add documentation for raw-compression to the tune_compression page in WT docs
WT-2975 Reduce the memory allocated for statistics
WT-2991 Coverity issue 1365198
WT-2998 Add error messages to error returns that might be confusing.
WT-2999 Memory leak in join WT_CURSOR with custom extractor
WT-3001 WT_EXTENSION_API references are named inconsistently.
WT-3003 log generation should not auto generate the record and op types
WT-3005 Add top-level .gitignore file.
WT-3007 Remove duplicated word in comment
WT-3008 Run wtperf stress testing against all maintained branches
WT-3009 Test format hung/aborted with small cache (sub 10mB)
144 files changed, 2855 insertions, 1819 deletions
diff --git a/src/third_party/wiredtiger/.gitignore b/src/third_party/wiredtiger/.gitignore index e69de29bb2d..bef47daabf9 100644 --- a/src/third_party/wiredtiger/.gitignore +++ b/src/third_party/wiredtiger/.gitignore @@ -0,0 +1,123 @@ +# Editor files +*.swp + +# Build directories +.deps/ +.libs/ + +# Build files +*.dll +*.exe +*.exp +*.la +*.lib +*.lo +*.o +*.obj +*.pdb +*.pyc +.sconf_temp +.sconsign.dblite +.dirstamp +/Makefile.am +/aclocal.m4 +/build_posix/aclocal/libtool.m4 +/build_posix/aclocal/ltoptions.m4 +/build_posix/aclocal/ltsugar.m4 +/build_posix/aclocal/ltversion.m4 +/build_posix/aclocal/lt~obsolete.m4 +/build_posix/config.hin +/build_posix/gnu-support/ +/config.log +/config.status +/configure +/configure.ac +/libtool/ +/stamp-h1 +/wiredtiger.h +/wiredtiger.pc +/wiredtiger_config.h +/wiredtiger_ext.h +/wt +Makefile +Makefile.in +build_posix/config.log +build_posix/config.status +build_posix/libtool +build_posix/stamp-h1 +build_posix/wiredtiger.h +build_posix/wiredtiger.pc +build_posix/wiredtiger_config.h +build_posix/wiredtiger_ext.h +build_posix/wt +tags + +# Documentation +/docs/ +/src/docs/changelog.md +/src/docs/doxygen.log + +# Tests +WT_HOME/ +WT_TEST/ + +# Python +/lang/python/_wiredtiger.so +/lang/python/wiredtiger/__init__.py +/lang/python/wiredtiger/_wiredtiger.pyd +/lang/python/wiredtiger.py +/lang/python/wiredtiger_wrap.c +_wiredtiger.pyd + +# Binaries +**/bench/wtperf/wtperf +**/examples/c/ex_access +**/examples/c/ex_all +**/examples/c/ex_async +**/examples/c/ex_backup +**/examples/c/ex_call_center +**/examples/c/ex_config_parse +**/examples/c/ex_cursor +**/examples/c/ex_data_source +**/examples/c/ex_encrypt +**/examples/c/ex_event_handler +**/examples/c/ex_extending +**/examples/c/ex_extractor +**/examples/c/ex_file_system +**/examples/c/ex_hello +**/examples/c/ex_log +**/examples/c/ex_pack +**/examples/c/ex_process +**/examples/c/ex_schema +**/examples/c/ex_scope +**/examples/c/ex_stat +**/examples/c/ex_sync +**/examples/c/ex_thread +**/test/bloom/t +**/test/checkpoint/t +**/test/csuite/test_wt1965_col_efficiency +**/test/csuite/test_wt2246_col_append +**/test/csuite/test_wt2323_join_visibility +**/test/csuite/test_wt2403_lsm_workload +**/test/csuite/test_wt2447_join_main_table +**/test/csuite/test_wt2535_insert_race +**/test/csuite/test_wt2592_join_schema +**/test/csuite/test_wt2695_checksum +**/test/csuite/test_wt2719_reconfig +**/test/csuite/test_wt2834_join_bloom_fix +**/test/csuite/test_wt2853_perf +**/test/cursor_order/cursor_order +**/test/fops/t +**/test/format/s_dumpcmp +**/test/format/t +**/test/huge/t +**/test/manydbs/t +**/test/packing/intpack-test +**/test/packing/intpack-test2 +**/test/packing/intpack-test3 +**/test/packing/packing-test +**/test/readonly/t +**/test/recovery/random-abort +**/test/recovery/truncated-log +**/test/salvage/t +**/test/thread/t diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/btree-split-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf index deb8c70d12f..deb8c70d12f 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/btree-split-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/shared-cache-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/stress/shared-cache-stress.wtperf index 87d14f4f5c1..87d14f4f5c1 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/shared-cache-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/stress/shared-cache-stress.wtperf diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 7affc58a217..22d06c380ae 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -869,6 +869,10 @@ methods = { Config('bloom_bit_count', '16', r''' the number of bits used per item for the bloom filter''', min='2', max='1000'), + Config('bloom_false_positives', 'false', r''' + return all values that pass the bloom filter, without eliminating + any false positives''', + type='boolean'), Config('bloom_hash_count', '8', r''' the number of hash values per item for the bloom filter''', min='2', max='100'), diff --git a/src/third_party/wiredtiger/dist/log.py b/src/third_party/wiredtiger/dist/log.py index 8743dd3a71c..ac35d884eac 100644 --- a/src/third_party/wiredtiger/dist/log.py +++ b/src/third_party/wiredtiger/dist/log.py @@ -98,34 +98,6 @@ def printf_line(f, optype, i, ishex): return ifbegin + body + ifend ##################################################################### -# Update log.h with #defines for types -##################################################################### -log_defines = ( - ''.join('/*! %s */\n#define\t%s\t%d\n' % (r.desc, r.macro_name(), i) - for i, r in enumerate(log_data.rectypes)) + - ''.join('/*! %s */\n#define\t%s\t%d\n' % (r.desc, r.macro_name(), i) - for i, r in enumerate(log_data.optypes,start=1)) -) - -tfile = open(tmp_file, 'w') -skip = 0 -for line in open('../src/include/wiredtiger.in', 'r'): - if skip: - if 'Log record declarations: END' in line: - tfile.write('/*\n' + line) - skip = 0 - else: - tfile.write(line) - if 'Log record declarations: BEGIN' in line: - skip = 1 - tfile.write(' */\n') - tfile.write('/*! invalid operation */\n') - tfile.write('#define\tWT_LOGOP_INVALID\t0\n') - tfile.write(log_defines) -tfile.close() -compare_srcfile(tmp_file, '../src/include/wiredtiger.in') - -##################################################################### # Create log_auto.c with handlers for each record / operation type. ##################################################################### f='../src/log/log_auto.c' @@ -270,11 +242,13 @@ __wt_logop_%(name)s_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, %(arg_decls)s) { +\tWT_DECL_RET; \tconst char *fmt = WT_UNCHECKED_STRING(%(fmt)s); \tuint32_t optype, size; -\tWT_RET(__wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, -\t &optype, &size%(arg_names)s)); +\tif ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, +\t &optype, &size%(arg_names)s)) != 0) +\t\tWT_RET_MSG(session, ret, "logop_%(name)s: unpack failure"); \tWT_ASSERT(session, optype == %(macro)s); \t*pp += size; diff --git a/src/third_party/wiredtiger/dist/log_data.py b/src/third_party/wiredtiger/dist/log_data.py index 8fd1a5787fe..bbfb51708bf 100644 --- a/src/third_party/wiredtiger/dist/log_data.py +++ b/src/third_party/wiredtiger/dist/log_data.py @@ -18,6 +18,11 @@ class LogRecordType: def prname(self): return '__logrec_print_' + self.name +# +# If you add a new record type you must also add its record type value in +# src/include/wiredtiger.in. The values cannot be generated because they must +# never change after they're written in a log file. +# rectypes = [ # A database-wide checkpoint. LogRecordType('checkpoint', 'checkpoint', [ @@ -46,6 +51,11 @@ class LogOperationType: def macro_name(self): return 'WT_LOGOP_%s' % self.name.upper() +# +# If you add a new operation type you must also add its type value in +# src/include/wiredtiger.in. The values cannot be generated because they must +# never change after they're written in a log file. +# optypes = [ LogOperationType('col_put', 'column put', [('uint32', 'fileid'), ('recno', 'recno'), ('item', 'value')]), diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index b4f449fa093..f4332257193 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -183,7 +183,8 @@ EOF clean=0 additional_languages=1 -filter="|sed '/PROJECT_NUMBER/s,=.*,=\"Version $WIREDTIGER_VERSION\",'" +filter="|sed '/PROJECT_BRIEF/s,=.*,=\"$WIREDTIGER_VERSION\",'" +filter="$filter| sed '/PROJECT_NUMBER/s,=.*,=\"Version $WIREDTIGER_VERSION\",'" while : do case "$1" in -a) # Build from scratch @@ -200,6 +201,13 @@ while : -t) # Include the TODO list filter="$filter| sed '/GENERATE_TODOLIST/s,=.*,=YES,'" shift;; + -v) # Override the version with <slug> <version string> + filter="$filter| sed '/HTML_HEADER/s,=.*,= style/header-web.html,'" + shift + filter="$filter| sed '/PROJECT_BRIEF/s,=.*,=\"$1\",'" + shift + filter="$filter| sed '/PROJECT_NUMBER/s,=.*,=\"$1\",'" + shift;; *) break;; esac diff --git a/src/third_party/wiredtiger/dist/s_errno b/src/third_party/wiredtiger/dist/s_errno new file mode 100644 index 00000000000..9981842e458 --- /dev/null +++ b/src/third_party/wiredtiger/dist/s_errno @@ -0,0 +1,73 @@ +#! /bin/sh + +# Complain about code that returns a system error value without an associated +# verbose message. +# +# This script is a kluge and isn't run by default. + +t=__wt.$$ +trap 'rm -f $t' 0 1 2 3 13 15 + +cd .. + +# Strip out a list of errors that will be flagged, but are OK. +error_ok() +{ + sed -e '/ERET(/d' \ + -e '/WT_ERR_MSG(/d' \ + -e '/WT_ERR_TEST(/d' \ + -e '/WT_PANIC_ERR(/d' \ + -e '/WT_PANIC_RET(/d' \ + -e '/WT_RET_MSG(/d' \ + -e '/\/intpack.i:.*EINVAL/d'\ + -e '/\/intpack.i:.*ENOMEM/d'\ + -e '/\/pack_impl.c:.*EINVAL/d'\ + -e '/\/pack_impl.c:.*ENOMEM/d'\ + -e '/\/pack_stream.c:.*ENOMEM/d'\ + -e '/\/packing.i:.*EINVAL/d'\ + -e '/__config_err(/d' \ + -e '/__wt_block_panic(/d' \ + -e '/__wt_err(/d' \ + -e '/__wt_errx(/d' \ + -e '/csv_error(/d' \ + -e '/nop_error(/d' \ + -e '/rotn_error(/d' \ + -e '/zlib_error(/d' +} + +# Loop through source files. +for f in `find ext src -name '*.[ci]'`; do + if expr "$f" : 'ext/datasources/helium/helium.c' > /dev/null; then + continue + fi + if expr "$f" : 'src/os_win/os_winerr.c' > /dev/null; then + continue + fi + if expr "$f" : 'src/utilities/.*' > /dev/null; then + continue + fi + + # Strip include files, + # then use the C preprocessor to strip comments, + # then turn each file into a single line, + # then chunk the file by semicolons, + # then search for explicit error returns (ignoring EBUSY), + # then prepend the file name to each line, + # then skip known calls that include verbose messages. + sed '/^#include/d' $f | + ${CC:-cc} -E - | + tr -s '\012' ' ' | tr ';' '\012' | + egrep -w 'EPERM|ENOENT|ESRCH|EINTR|EIO|ENXIO|E2BIG|ENOEXEC|EBADF|ECHILD|EDEADLK|ENOMEM|EACCES|EFAULT|ENOTBLK|EEXIST|EXDEV|ENODEV|ENOTDIR|EISDIR|EINVAL|ENFILE|EMFILE|ENOTTY|ETXTBSY|EFBIG|ENOSPC|ESPIPE|EROFS|EMLINK|EPIPE|EDOM|ERANGE|EAGAIN|EWOULDBLOCK|EINPROGRESS|EALREADY|ENOTSOCK|EDESTADDRREQ|EMSGSIZE|EPROTOTYPE|ENOPROTOOPT|EPROTONOSUPPORT|ESOCKTNOSUPPORT|EOPNOTSUPP|ENOTSUP|EPFNOSUPPORT|EAFNOSUPPORT|EADDRINUSE|EADDRNOTAVAIL|ENETDOWN|ENETUNREACH|ENETRESET|ECONNABORTED|ECONNRESET|ENOBUFS|EISCONN|ENOTCONN|ESHUTDOWN|ETOOMANYREFS|ETIMEDOUT|ECONNREFUSED|ELOOP|ENAMETOOLONG|EHOSTDOWN|EHOSTUNREACH|ENOTEMPTY|EPROCLIM|EUSERS|EDQUOT|ESTALE|EREMOTE|EBADRPC|ERPCMISMATCH|EPROGUNAVAIL|EPROGMISMATCH|EPROCUNAVAIL|ENOLCK|ENOSYS|EFTYPE|EAUTH|ENEEDAUTH|EIDRM|ENOMSG|EOVERFLOW|ECANCELED|EILSEQ|ENOATTR|EDOOFUS|EBADMSG|EMULTIHOP|ENOLINK|EPROTO|ENOTCAPABLE|ECAPMODE|ENOTRECOVERABLE|EOWNERDEAD|ELAST|ERESTART|EJUSTRETURN|ENOIOCTL|EDIRIOCTL' | + sed -e 's/^[ ]*//' \ + -e "s;^;$f: ;" | + error_ok +done > $t + +test -s $t && { + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo 'Unexpected error usage.' + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + cat $t + exit 1 +} +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 7cf96aec399..65c74b61995 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -842,6 +842,7 @@ localtime logf logmgr lognum +logop logread logrec logsize diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index e5e9f97c0b7..b9c3f5e0ef7 100644 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -32,6 +32,7 @@ func_ok() -e '/int __bm_stat$/d' \ -e '/int __checkpoint_presync$/d' \ -e '/int __compact_uri_analyze$/d' \ + -e '/int __config_parser_close$/d' \ -e '/int __curlog_reset$/d' \ -e '/int __handle_close_default$/d' \ -e '/int __handle_progress_default$/d' \ @@ -63,6 +64,7 @@ func_ok() -e '/int __wt_stat_join_desc$/d' \ -e '/int __wt_win_directory_list_free$/d' \ -e '/int bdb_compare_reverse$/d' \ + -e '/int csv_error$/d' \ -e '/int csv_terminate$/d' \ -e '/int demo_file_close$/d' \ -e '/int demo_file_lock$/d' \ @@ -81,6 +83,7 @@ func_ok() -e '/int main$/d' \ -e '/int nop_decompress$/d' \ -e '/int nop_decrypt$/d' \ + -e '/int nop_error$/d' \ -e '/int nop_pre_size$/d' \ -e '/int nop_sizing$/d' \ -e '/int nop_terminate$/d' \ @@ -90,14 +93,14 @@ func_ok() -e '/int rotn_error$/d' \ -e '/int rotn_sizing$/d' \ -e '/int rotn_terminate$/d' \ + -e '/int snappy_pre_size$/d' \ + -e '/int snappy_terminate$/d' \ -e '/int uri2name$/d' \ -e '/int usage$/d' \ -e '/int util_err$/d' \ -e '/int wiredtiger_extension_init$/d' \ -e '/int wiredtiger_extension_terminate$/d' \ -e '/int wiredtiger_pack_close$/d' \ - -e '/int snappy_pre_size$/d' \ - -e '/int snappy_terminate$/d' \ -e '/int zlib_error$/d' \ -e '/int zlib_terminate$/d' \ -e '/int zstd_error$/d' \ diff --git a/src/third_party/wiredtiger/dist/stat.py b/src/third_party/wiredtiger/dist/stat.py index e42585c1b8c..d04a0514c49 100644 --- a/src/third_party/wiredtiger/dist/stat.py +++ b/src/third_party/wiredtiger/dist/stat.py @@ -138,15 +138,27 @@ __wt_stat_''' + name + '_init_single(WT_' + name.upper() + '''_STATS *stats) if handle != None: f.write(''' -void -__wt_stat_''' + name + '_init(' + handle + ''' *handle) +int +__wt_stat_''' + name + '''_init( + WT_SESSION_IMPL *session, ''' + handle + ''' *handle) { \tint i; +\tWT_RET(__wt_calloc(session, (size_t)WT_COUNTER_SLOTS, +\t sizeof(*handle->stat_array), &handle->stat_array)); + \tfor (i = 0; i < WT_COUNTER_SLOTS; ++i) { \t\thandle->stats[i] = &handle->stat_array[i]; \t\t__wt_stat_''' + name + '''_init_single(handle->stats[i]); \t} +\treturn (0); +} + +void +__wt_stat_''' + name + '''_discard( + WT_SESSION_IMPL *session, ''' + handle + ''' *handle) +{ +\t__wt_free(session, handle->stat_array); } ''') diff --git a/src/third_party/wiredtiger/ext/collators/revint/revint_collator.c b/src/third_party/wiredtiger/ext/collators/revint/revint_collator.c index b8ebbdc8585..cfad3989adb 100644 --- a/src/third_party/wiredtiger/ext/collators/revint/revint_collator.c +++ b/src/third_party/wiredtiger/ext/collators/revint/revint_collator.c @@ -52,14 +52,14 @@ revint_compare(WT_COLLATOR *collator, WT_SESSION *session, const WT_ITEM *k1, const WT_ITEM *k2, int *cmp) { const REVINT_COLLATOR *revint_collator; - WT_EXTENSION_API *wtapi; + WT_EXTENSION_API *wt_api; WT_PACK_STREAM *pstream; int64_t i1, i2, p1, p2; int ret; i1 = i2 = p1 = p2 = 0; revint_collator = (const REVINT_COLLATOR *)collator; - wtapi = revint_collator->wt_api; + wt_api = revint_collator->wt_api; /* * All indices using this collator have an integer key, and the @@ -79,25 +79,25 @@ revint_compare(WT_COLLATOR *collator, * To keep this code simple, we do not reverse the ordering * when comparing primary keys. */ - if ((ret = wtapi->unpack_start( - wtapi, session, "ii", k1->data, k1->size, &pstream)) != 0 || - (ret = wtapi->unpack_int(wtapi, pstream, &i1)) != 0) + if ((ret = wt_api->unpack_start( + wt_api, session, "ii", k1->data, k1->size, &pstream)) != 0 || + (ret = wt_api->unpack_int(wt_api, pstream, &i1)) != 0) return (ret); - if ((ret = wtapi->unpack_int(wtapi, pstream, &p1)) != 0) + if ((ret = wt_api->unpack_int(wt_api, pstream, &p1)) != 0) /* A missing primary key is OK and sorts first. */ p1 = INT64_MIN; - if ((ret = wtapi->pack_close(wtapi, pstream, NULL)) != 0) + if ((ret = wt_api->pack_close(wt_api, pstream, NULL)) != 0) return (ret); /* Unpack the second pair of numbers. */ - if ((ret = wtapi->unpack_start( - wtapi, session, "ii", k2->data, k2->size, &pstream)) != 0 || - (ret = wtapi->unpack_int(wtapi, pstream, &i2)) != 0) + if ((ret = wt_api->unpack_start( + wt_api, session, "ii", k2->data, k2->size, &pstream)) != 0 || + (ret = wt_api->unpack_int(wt_api, pstream, &i2)) != 0) return (ret); - if ((ret = wtapi->unpack_int(wtapi, pstream, &p2)) != 0) + if ((ret = wt_api->unpack_int(wt_api, pstream, &p2)) != 0) /* A missing primary key is OK and sorts first. */ p2 = INT64_MIN; - if ((ret = wtapi->pack_close(wtapi, pstream, NULL)) != 0) + if ((ret = wt_api->pack_close(wt_api, pstream, NULL)) != 0) return (ret); /* sorting is reversed */ diff --git a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c index ef20503df0a..3665ec48b9a 100644 --- a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c @@ -371,8 +371,7 @@ retry: /* If we reached our maximum reserve, quit. */ ret = 0; } -err: if (zs != NULL && - (tret = deflateEnd(zs)) != Z_OK && tret != Z_DATA_ERROR) +err: if ((tret = deflateEnd(zs)) != Z_OK && tret != Z_DATA_ERROR) ret = zlib_error(compressor, session, "deflateEnd", tret); if (last_zs != NULL && (tret = deflateEnd(last_zs)) != Z_OK && tret != Z_DATA_ERROR) @@ -393,19 +392,27 @@ err: if (zs != NULL && #if 0 /* Decompress the result and confirm it matches the original source. */ if (ret == 0 && last_slot > 0) { + WT_EXTENSION_API *wt_api; void *decomp; size_t result_len; + wt_api = ((ZLIB_COMPRESSOR *)compressor)->wt_api; + if ((decomp = zalloc( - &opaque, 1, (uint32_t)best_zs->total_in + 100)) == NULL) + &opaque, 1, (uint32_t)best_zs->total_in + 100)) == NULL) { + (void)wt_api->err_printf(wt_api, session, + "zlib_compress_raw: zalloc failure"); return (ENOMEM); + } if ((ret = zlib_decompress( compressor, session, dst, (size_t)best_zs->total_out, decomp, (size_t)best_zs->total_in + 100, &result_len)) == 0) - if (memcmp(src, decomp, result_len) != 0) - ret = zlib_error(compressor, session, - "deflate compare with original source", - Z_DATA_ERROR); + if (memcmp(src, decomp, result_len) != 0) { + (void)wt_api->err_printf(wt_api, session, + "zlib_compress_raw: " + "deflate compare with original source"); + return (WT_ERROR); + } zfree(&opaque, decomp); } #endif @@ -478,7 +485,7 @@ zlib_init_config( { WT_CONFIG_ITEM k, v; WT_CONFIG_PARSER *config_parser; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret, zlib_level; /* If configured as a built-in, there's no configuration argument. */ @@ -489,18 +496,19 @@ zlib_init_config( * Zlib compression engine allows applications to specify a compression * level; review the configuration. */ - wtext = connection->get_extension_api(connection); - if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) { - (void)wtext->err_printf(wtext, NULL, + wt_api = connection->get_extension_api(connection); + if ((ret = + wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { + (void)wt_api->err_printf(wt_api, NULL, "WT_EXTENSION_API.config_get: zlib configure: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); return (ret); } - if ((ret = wtext->config_parser_open( - wtext, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wtext->err_printf(wtext, NULL, + if ((ret = wt_api->config_parser_open( + wt_api, NULL, v.str, v.len, &config_parser)) != 0) { + (void)wt_api->err_printf(wt_api, NULL, "WT_EXTENSION_API.config_parser_open: zlib configure: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); return (ret); } while ((ret = config_parser->next(config_parser, &k, &v)) == 0) @@ -511,7 +519,7 @@ zlib_init_config( */ zlib_level = (int)v.val; if (zlib_level < 0 || zlib_level > 9) { - (void)wtext->err_printf(wtext, NULL, + (void)wt_api->err_printf(wt_api, NULL, "WT_CONFIG_PARSER.next: zlib configure: " "unsupported compression level %d", zlib_level); @@ -521,15 +529,15 @@ zlib_init_config( continue; } if (ret != WT_NOTFOUND) { - (void)wtext->err_printf(wtext, NULL, + (void)wt_api->err_printf(wt_api, NULL, "WT_CONFIG_PARSER.next: zlib configure: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); return (ret); } if ((ret = config_parser->close(config_parser)) != 0) { - (void)wtext->err_printf(wtext, NULL, + (void)wt_api->err_printf(wt_api, NULL, "WT_CONFIG_PARSER.close: zlib configure: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); return (ret); } return (0); diff --git a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c index 3d0447248b6..a459b01d60a 100644 --- a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c @@ -234,7 +234,7 @@ zstd_init_config( { WT_CONFIG_ITEM k, v; WT_CONFIG_PARSER *config_parser; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret; /* If configured as a built-in, there's no configuration argument. */ @@ -245,18 +245,19 @@ zstd_init_config( * Zstd compression engine allows applications to specify a compression * level; review the configuration. */ - wtext = connection->get_extension_api(connection); - if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) { - (void)wtext->err_printf(wtext, NULL, + wt_api = connection->get_extension_api(connection); + if ((ret = + wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { + (void)wt_api->err_printf(wt_api, NULL, "WT_EXTENSION_API.config_get: zstd configure: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); return (ret); } - if ((ret = wtext->config_parser_open( - wtext, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wtext->err_printf(wtext, NULL, + if ((ret = wt_api->config_parser_open( + wt_api, NULL, v.str, v.len, &config_parser)) != 0) { + (void)wt_api->err_printf(wt_api, NULL, "WT_EXTENSION_API.config_parser_open: zstd configure: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); return (ret); } while ((ret = config_parser->next(config_parser, &k, &v)) == 0) @@ -266,15 +267,15 @@ zstd_init_config( continue; } if (ret != WT_NOTFOUND) { - (void)wtext->err_printf(wtext, NULL, + (void)wt_api->err_printf(wt_api, NULL, "WT_CONFIG_PARSER.next: zstd configure: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); return (ret); } if ((ret = config_parser->close(config_parser)) != 0) { - (void)wtext->err_printf(wtext, NULL, + (void)wt_api->err_printf(wt_api, NULL, "WT_CONFIG_PARSER.close: zstd configure: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); return (ret); } return (0); diff --git a/src/third_party/wiredtiger/ext/datasources/helium/helium.c b/src/third_party/wiredtiger/ext/datasources/helium/helium.c index e934d61ddf6..473c569f0cc 100644 --- a/src/third_party/wiredtiger/ext/datasources/helium/helium.c +++ b/src/third_party/wiredtiger/ext/datasources/helium/helium.c @@ -80,22 +80,22 @@ static int verbose = 0; /* Verbose messages */ } \ } while (0) #undef ERET -#define ERET(wtext, session, v, ...) do { \ +#define ERET(wt_api, session, v, ...) do { \ (void) \ - wtext->err_printf(wtext, session, "helium: " __VA_ARGS__); \ + wt_api->err_printf(wt_api, session, "helium: " __VA_ARGS__);\ ESET(v); \ return (ret); \ } while (0) #undef EMSG -#define EMSG(wtext, session, v, ...) do { \ +#define EMSG(wt_api, session, v, ...) do { \ (void) \ - wtext->err_printf(wtext, session, "helium: " __VA_ARGS__); \ + wt_api->err_printf(wt_api, session, "helium: " __VA_ARGS__);\ ESET(v); \ } while (0) #undef EMSG_ERR -#define EMSG_ERR(wtext, session, v, ...) do { \ +#define EMSG_ERR(wt_api, session, v, ...) do { \ (void) \ - wtext->err_printf(wtext, session, "helium: " __VA_ARGS__); \ + wt_api->err_printf(wt_api, session, "helium: " __VA_ARGS__);\ ESET(v); \ goto err; \ } while (0) @@ -104,10 +104,10 @@ static int verbose = 0; /* Verbose messages */ #undef VERBOSE_L2 #define VERBOSE_L2 2 #undef VMSG -#define VMSG(wtext, session, v, ...) do { \ +#define VMSG(wt_api, session, v, ...) do { \ if (verbose >= v) \ - (void)wtext-> \ - msg_printf(wtext, session, "helium: " __VA_ARGS__); \ + (void)wt_api-> \ + msg_printf(wt_api, session, "helium: " __VA_ARGS__);\ } while (0) /* @@ -183,7 +183,7 @@ typedef struct __he_source { */ WT_TXN_NOTIFY txn_notify; /* Transaction commit handler */ - WT_EXTENSION_API *wtext; /* Extension functions */ + WT_EXTENSION_API *wt_api; /* Extension functions */ char *name; /* Unique WiredTiger name */ char *device; /* Unique Helium volume name */ @@ -231,7 +231,7 @@ typedef struct __he_source { typedef struct __data_source { WT_DATA_SOURCE wtds; /* Must come first */ - WT_EXTENSION_API *wtext; /* Extension functions */ + WT_EXTENSION_API *wt_api; /* Extension functions */ pthread_rwlock_t global_lock; /* Global lock */ int lockinit; /* Lock created */ @@ -269,7 +269,7 @@ typedef struct __cache_record { typedef struct __cursor { WT_CURSOR wtcursor; /* Must come first */ - WT_EXTENSION_API *wtext; /* Extension functions */ + WT_EXTENSION_API *wt_api; /* Extension functions */ WT_SOURCE *ws; /* Underlying source */ @@ -345,12 +345,13 @@ os_errno(void) * Initialize a lock. */ static int -lock_init(WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) +lock_init( + WT_EXTENSION_API *wt_api, WT_SESSION *session, pthread_rwlock_t *lockp) { int ret = 0; if ((ret = pthread_rwlock_init(lockp, NULL)) != 0) - ERET(wtext, session, WT_PANIC, + ERET(wt_api, session, WT_PANIC, "pthread_rwlock_init: %s", strerror(ret)); return (0); } @@ -361,12 +362,12 @@ lock_init(WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) */ static int lock_destroy( - WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) + WT_EXTENSION_API *wt_api, WT_SESSION *session, pthread_rwlock_t *lockp) { int ret = 0; if ((ret = pthread_rwlock_destroy(lockp)) != 0) - ERET(wtext, session, WT_PANIC, + ERET(wt_api, session, WT_PANIC, "pthread_rwlock_destroy: %s", strerror(ret)); return (0); } @@ -376,12 +377,13 @@ lock_destroy( * Acquire a write lock. */ static inline int -writelock(WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) +writelock( + WT_EXTENSION_API *wt_api, WT_SESSION *session, pthread_rwlock_t *lockp) { int ret = 0; if ((ret = pthread_rwlock_wrlock(lockp)) != 0) - ERET(wtext, session, WT_PANIC, + ERET(wt_api, session, WT_PANIC, "pthread_rwlock_wrlock: %s", strerror(ret)); return (0); } @@ -391,12 +393,12 @@ writelock(WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) * Release a lock. */ static inline int -unlock(WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) +unlock(WT_EXTENSION_API *wt_api, WT_SESSION *session, pthread_rwlock_t *lockp) { int ret = 0; if ((ret = pthread_rwlock_unlock(lockp)) != 0) - ERET(wtext, session, WT_PANIC, + ERET(wt_api, session, WT_PANIC, "pthread_rwlock_unlock: %s", strerror(ret)); return (0); } @@ -425,7 +427,7 @@ helium_dump_kv(const char *pfx, uint8_t *p, size_t len, FILE *fp) * Dump the records in a Helium store. */ static int -helium_dump(WT_EXTENSION_API *wtext, he_t he, const char *tag) +helium_dump(WT_EXTENSION_API *wt_api, he_t he, const char *tag) { HE_ITEM *r, _r; uint8_t k[4 * 1024], v[4 * 1024]; @@ -440,7 +442,7 @@ helium_dump(WT_EXTENSION_API *wtext, he_t he, const char *tag) while ((ret = he_next(he, r, (size_t)0, sizeof(v))) == 0) { #if 0 uint64_t recno; - if ((ret = wtext->struct_unpack(wtext, + if ((ret = wt_api->struct_unpack(wt_api, NULL, r->key, r->key_len, "r", &recno)) != 0) return (ret); fprintf(stderr, "K: %" PRIu64, recno); @@ -462,13 +464,13 @@ helium_dump(WT_EXTENSION_API *wtext, he_t he, const char *tag) */ static int helium_stats( - WT_EXTENSION_API *wtext, WT_SESSION *session, he_t he, const char *tag) + WT_EXTENSION_API *wt_api, WT_SESSION *session, he_t he, const char *tag) { HE_STATS stats; int ret = 0; if ((ret = he_stats(he, &stats)) != 0) - ERET(wtext, session, ret, "he_stats: %s", he_strerror(ret)); + ERET(wt_api, session, ret, "he_stats: %s", he_strerror(ret)); fprintf(stderr, "== %s\n", tag); fprintf(stderr, "name=%s\n", stats.name); fprintf(stderr, "deleted_items=%" PRIu64 "\n", stats.deleted_items); @@ -490,14 +492,14 @@ helium_call(WT_CURSOR *wtcursor, const char *fname, { CURSOR *cursor; HE_ITEM *r; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; int ret = 0; char *p; session = wtcursor->session; cursor = (CURSOR *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; r = &cursor->record; r->val = cursor->v; @@ -506,7 +508,7 @@ restart: if ((ret = f(he, r, (size_t)0, cursor->mem_len)) != 0) { if (ret == HE_ERR_ITEM_NOT_FOUND) return (WT_NOTFOUND); - ERET(wtext, session, ret, "%s: %s", fname, he_strerror(ret)); + ERET(wt_api, session, ret, "%s: %s", fname, he_strerror(ret)); } /* @@ -537,7 +539,7 @@ restart: if ((ret = he_lookup(he, r, (size_t)0, cursor->mem_len)) != 0) { if (ret == HE_ERR_ITEM_NOT_FOUND) goto restart; - ERET(wtext, + ERET(wt_api, session, ret, "he_lookup: %s", he_strerror(ret)); } } @@ -549,7 +551,7 @@ restart: * Resolve a transaction. */ static int -txn_state_set(WT_EXTENSION_API *wtext, +txn_state_set(WT_EXTENSION_API *wt_api, WT_SESSION *session, HELIUM_SOURCE *hs, uint64_t txnid, int commit) { HE_ITEM txn; @@ -571,10 +573,10 @@ txn_state_set(WT_EXTENSION_API *wtext, txn.val_len = sizeof(val); if ((ret = he_update(hs->he_txn, &txn)) != 0) - ERET(wtext, session, ret, "he_update: %s", he_strerror(ret)); + ERET(wt_api, session, ret, "he_update: %s", he_strerror(ret)); if (commit && (ret = he_commit(hs->he_txn)) != 0) - ERET(wtext, session, ret, "he_commit: %s", he_strerror(ret)); + ERET(wt_api, session, ret, "he_commit: %s", he_strerror(ret)); return (0); } @@ -589,7 +591,7 @@ txn_notify(WT_TXN_NOTIFY *handler, HELIUM_SOURCE *hs; hs = (HELIUM_SOURCE *)handler; - return (txn_state_set(hs->wtext, session, hs, txnid, committed)); + return (txn_state_set(hs->wt_api, session, hs, txnid, committed)); } /* @@ -627,7 +629,7 @@ cache_value_append(WT_CURSOR *wtcursor, int remove_op) { CURSOR *cursor; HE_ITEM *r; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; uint64_t txnid; size_t len; @@ -636,7 +638,7 @@ cache_value_append(WT_CURSOR *wtcursor, int remove_op) session = wtcursor->session; cursor = (CURSOR *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; r = &cursor->record; @@ -662,7 +664,7 @@ cache_value_append(WT_CURSOR *wtcursor, int remove_op) } /* Get the transaction ID. */ - txnid = wtext->transaction_id(wtext, session); + txnid = wt_api->transaction_id(wt_api, session); /* Update the number of records in this value. */ if (cursor->len == 0) { @@ -787,17 +789,17 @@ cache_value_update_check(WT_CURSOR *wtcursor) { CACHE_RECORD *cp; CURSOR *cursor; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; u_int i; session = wtcursor->session; cursor = (CURSOR *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; /* Only interesting for snapshot isolation. */ - if (wtext-> - transaction_isolation_level(wtext, session) != WT_TXN_ISO_SNAPSHOT) + if (wt_api-> + transaction_isolation_level(wt_api, session) != WT_TXN_ISO_SNAPSHOT) return (0); /* @@ -806,7 +808,7 @@ cache_value_update_check(WT_CURSOR *wtcursor) */ for (i = 0, cp = cursor->cache; i < cursor->cache_entries; ++i, ++cp) if (!cache_value_aborted(wtcursor, cp) && - !wtext->transaction_visible(wtext, session, cp->txnid)) + !wt_api->transaction_visible(wt_api, session, cp->txnid)) return (WT_ROLLBACK); return (0); } @@ -821,7 +823,7 @@ cache_value_visible(WT_CURSOR *wtcursor, CACHE_RECORD **cpp) { CACHE_RECORD *cp; CURSOR *cursor; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; u_int i; @@ -829,7 +831,7 @@ cache_value_visible(WT_CURSOR *wtcursor, CACHE_RECORD **cpp) session = wtcursor->session; cursor = (CURSOR *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; /* * We want the most recent cache entry update; the cache entries are @@ -839,7 +841,7 @@ cache_value_visible(WT_CURSOR *wtcursor, CACHE_RECORD **cpp) for (i = 0; i < cursor->cache_entries; ++i) { --cp; if (!cache_value_aborted(wtcursor, cp) && - wtext->transaction_visible(wtext, session, cp->txnid)) { + wt_api->transaction_visible(wt_api, session, cp->txnid)) { *cpp = cp; return (1); } @@ -970,11 +972,11 @@ cache_value_txnmin(WT_CURSOR *wtcursor, uint64_t *txnminp) * Common error when a WiredTiger key is too large. */ static int -key_max_err(WT_EXTENSION_API *wtext, WT_SESSION *session, size_t len) +key_max_err(WT_EXTENSION_API *wt_api, WT_SESSION *session, size_t len) { int ret = 0; - ERET(wtext, session, EINVAL, + ERET(wt_api, session, EINVAL, "key length (%zu bytes) larger than the maximum Helium " "key length of %d bytes", len, HE_MAX_KEY_LEN); @@ -989,7 +991,7 @@ copyin_key(WT_CURSOR *wtcursor, int allocate_key) { CURSOR *cursor; HE_ITEM *r; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; WT_SOURCE *ws; size_t size; @@ -998,7 +1000,7 @@ copyin_key(WT_CURSOR *wtcursor, int allocate_key) session = wtcursor->session; cursor = (CURSOR *)wtcursor; ws = cursor->ws; - wtext = cursor->wtext; + wt_api = cursor->wt_api; r = &cursor->record; if (ws->config_recno) { @@ -1019,23 +1021,23 @@ copyin_key(WT_CURSOR *wtcursor, int allocate_key) * not quite right. */ if (allocate_key && cursor->config_append) { - if ((ret = writelock(wtext, session, &ws->lock)) != 0) + if ((ret = writelock(wt_api, session, &ws->lock)) != 0) return (ret); wtcursor->recno = ++ws->append_recno; - if ((ret = unlock(wtext, session, &ws->lock)) != 0) + if ((ret = unlock(wt_api, session, &ws->lock)) != 0) return (ret); } else if (wtcursor->recno > ws->append_recno) { - if ((ret = writelock(wtext, session, &ws->lock)) != 0) + if ((ret = writelock(wt_api, session, &ws->lock)) != 0) return (ret); if (wtcursor->recno > ws->append_recno) ws->append_recno = wtcursor->recno; - if ((ret = unlock(wtext, session, &ws->lock)) != 0) + if ((ret = unlock(wt_api, session, &ws->lock)) != 0) return (ret); } - if ((ret = wtext->struct_size(wtext, session, + if ((ret = wt_api->struct_size(wt_api, session, &size, "r", wtcursor->recno)) != 0 || - (ret = wtext->struct_pack(wtext, session, + (ret = wt_api->struct_pack(wt_api, session, r->key, HE_MAX_KEY_LEN, "r", wtcursor->recno)) != 0) return (ret); r->key_len = size; @@ -1043,7 +1045,7 @@ copyin_key(WT_CURSOR *wtcursor, int allocate_key) /* I'm not sure this test is necessary, but it's cheap. */ if (wtcursor->key.size > HE_MAX_KEY_LEN) return ( - key_max_err(wtext, session, wtcursor->key.size)); + key_max_err(wt_api, session, wtcursor->key.size)); /* * A set cursor key might reference application memory, which @@ -1068,19 +1070,19 @@ copyout_key(WT_CURSOR *wtcursor) { CURSOR *cursor; HE_ITEM *r; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; WT_SOURCE *ws; int ret = 0; session = wtcursor->session; cursor = (CURSOR *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; ws = cursor->ws; r = &cursor->record; if (ws->config_recno) { - if ((ret = wtext->struct_unpack(wtext, + if ((ret = wt_api->struct_unpack(wt_api, session, r->key, r->key_len, "r", &wtcursor->recno)) != 0) return (ret); } else { @@ -1122,7 +1124,7 @@ nextprev(WT_CURSOR *wtcursor, const char *fname, CACHE_RECORD *cp; CURSOR *cursor; HE_ITEM *r; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_ITEM a, b; WT_SESSION *session; WT_SOURCE *ws; @@ -1132,7 +1134,7 @@ nextprev(WT_CURSOR *wtcursor, const char *fname, session = wtcursor->session; cursor = (CURSOR *)wtcursor; ws = cursor->ws; - wtext = cursor->wtext; + wt_api = cursor->wt_api; r = &cursor->record; cache_rm = 0; @@ -1243,8 +1245,8 @@ cache_clean: a.size = (uint32_t)r->key_len; b.data = cursor->t2.v; /* b is the cache */ b.size = (uint32_t)cursor->t2.len; - if ((ret = wtext->collate( - wtext, session, NULL, &a, &b, &cmp)) != 0) + if ((ret = wt_api->collate( + wt_api, session, NULL, &a, &b, &cmp)) != 0) return (ret); if (f == he_next) { @@ -1423,14 +1425,14 @@ helium_cursor_insert(WT_CURSOR *wtcursor) CURSOR *cursor; HE_ITEM *r; HELIUM_SOURCE *hs; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; WT_SOURCE *ws; int ret = 0; session = wtcursor->session; cursor = (CURSOR *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; ws = cursor->ws; hs = ws->hs; r = &cursor->record; @@ -1439,14 +1441,14 @@ helium_cursor_insert(WT_CURSOR *wtcursor) if ((ret = copyin_key(wtcursor, 1)) != 0) return (ret); - VMSG(wtext, session, VERBOSE_L2, + VMSG(wt_api, session, VERBOSE_L2, "I %.*s.%.*s", (int)r->key_len, r->key, (int)r->val_len, r->val); /* Clear the value, assume we're adding the first cache entry. */ cursor->len = 0; /* Updates are read-modify-writes, lock the underlying cache. */ - if ((ret = writelock(wtext, session, &ws->lock)) != 0) + if ((ret = writelock(wt_api, session, &ws->lock)) != 0) return (ret); /* Read the record from the cache store. */ @@ -1502,19 +1504,19 @@ helium_cursor_insert(WT_CURSOR *wtcursor) if ((ret = cache_value_append(wtcursor, 0)) != 0) goto err; if ((ret = he_update(ws->he_cache, r)) != 0) - EMSG(wtext, session, ret, "he_update: %s", he_strerror(ret)); + EMSG(wt_api, session, ret, "he_update: %s", he_strerror(ret)); /* Update the state while still holding the lock. */ if (ws->he_cache_inuse == 0) ws->he_cache_inuse = 1; /* Discard the lock. */ -err: ESET(unlock(wtext, session, &ws->lock)); +err: ESET(unlock(wt_api, session, &ws->lock)); /* If successful, request notification at transaction resolution. */ if (ret == 0) - ESET( - wtext->transaction_notify(wtext, session, &hs->txn_notify)); + ESET(wt_api->transaction_notify( + wt_api, session, &hs->txn_notify)); return (ret); } @@ -1530,14 +1532,14 @@ update(WT_CURSOR *wtcursor, int remove_op) CURSOR *cursor; HE_ITEM *r; HELIUM_SOURCE *hs; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; WT_SOURCE *ws; int ret = 0; session = wtcursor->session; cursor = (CURSOR *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; ws = cursor->ws; hs = ws->hs; r = &cursor->record; @@ -1546,7 +1548,7 @@ update(WT_CURSOR *wtcursor, int remove_op) if ((ret = copyin_key(wtcursor, 0)) != 0) return (ret); - VMSG(wtext, session, VERBOSE_L2, + VMSG(wt_api, session, VERBOSE_L2, "%c %.*s.%.*s", remove_op ? 'R' : 'U', (int)r->key_len, r->key, (int)r->val_len, r->val); @@ -1555,7 +1557,7 @@ update(WT_CURSOR *wtcursor, int remove_op) cursor->len = 0; /* Updates are read-modify-writes, lock the underlying cache. */ - if ((ret = writelock(wtext, session, &ws->lock)) != 0) + if ((ret = writelock(wt_api, session, &ws->lock)) != 0) return (ret); /* Read the record from the cache store. */ @@ -1614,19 +1616,19 @@ update(WT_CURSOR *wtcursor, int remove_op) /* Push the record into the cache. */ if ((ret = he_update(ws->he_cache, r)) != 0) - EMSG(wtext, session, ret, "he_update: %s", he_strerror(ret)); + EMSG(wt_api, session, ret, "he_update: %s", he_strerror(ret)); /* Update the state while still holding the lock. */ if (ws->he_cache_inuse == 0) ws->he_cache_inuse = 1; /* Discard the lock. */ -err: ESET(unlock(wtext, session, &ws->lock)); +err: ESET(unlock(wt_api, session, &ws->lock)); /* If successful, request notification at transaction resolution. */ if (ret == 0) - ESET( - wtext->transaction_notify(wtext, session, &hs->txn_notify)); + ESET(wt_api->transaction_notify( + wt_api, session, &hs->txn_notify)); return (ret); } @@ -1674,19 +1676,19 @@ static int helium_cursor_close(WT_CURSOR *wtcursor) { CURSOR *cursor; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; WT_SOURCE *ws; int ret = 0; session = wtcursor->session; cursor = (CURSOR *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; ws = cursor->ws; - if ((ret = writelock(wtext, session, &ws->lock)) == 0) { + if ((ret = writelock(wt_api, session, &ws->lock)) == 0) { --ws->ref; - ret = unlock(wtext, session, &ws->lock); + ret = unlock(wt_api, session, &ws->lock); } cursor_destroy(cursor); @@ -1702,13 +1704,13 @@ ws_source_name(WT_DATA_SOURCE *wtds, WT_SESSION *session, const char *uri, const char *suffix, char **pp) { DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; size_t len; int ret = 0; const char *p; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* * Create the store's name. Application URIs are "helium:device/name"; @@ -1717,7 +1719,7 @@ ws_source_name(WT_DATA_SOURCE *wtds, * and add an optional suffix. */ if (!prefix_match(uri, "helium:") || (p = strchr(uri, '/')) == NULL) - ERET(wtext, session, EINVAL, "%s: illegal Helium URI", uri); + ERET(wt_api, session, EINVAL, "%s: illegal Helium URI", uri); ++p; len = strlen(WT_NAME_PREFIX) + @@ -1734,7 +1736,7 @@ ws_source_name(WT_DATA_SOURCE *wtds, * Close a WT_SOURCE reference. */ static int -ws_source_close(WT_EXTENSION_API *wtext, WT_SESSION *session, WT_SOURCE *ws) +ws_source_close(WT_EXTENSION_API *wt_api, WT_SESSION *session, WT_SOURCE *ws) { int ret = 0, tret; @@ -1743,29 +1745,29 @@ ws_source_close(WT_EXTENSION_API *wtext, WT_SESSION *session, WT_SOURCE *ws) * WiredTiger prevent it, so we don't do anything more than warn. */ if (ws->ref != 0) - EMSG(wtext, session, WT_ERROR, + EMSG(wt_api, session, WT_ERROR, "%s: open object with %u open cursors being closed", ws->uri, ws->ref); if (ws->he != NULL) { if ((tret = he_commit(ws->he)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "he_commit: %s: %s", ws->uri, he_strerror(tret)); if ((tret = he_close(ws->he)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "he_close: %s: %s", ws->uri, he_strerror(tret)); ws->he = NULL; } if (ws->he_cache != NULL) { if ((tret = he_close(ws->he_cache)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "he_close: %s(cache): %s", ws->uri, he_strerror(tret)); ws->he_cache = NULL; } if (ws->lockinit) - ESET(lock_destroy(wtext, session, &ws->lock)); + ESET(lock_destroy(wt_api, session, &ws->lock)); free(ws->uri); OVERWRITE_AND_FREE(ws); @@ -1783,7 +1785,7 @@ ws_source_open_object(WT_DATA_SOURCE *wtds, WT_SESSION *session, const char *uri, const char *suffix, int flags, he_t *hep) { DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; he_t he; char *p; int ret = 0; @@ -1791,16 +1793,16 @@ ws_source_open_object(WT_DATA_SOURCE *wtds, WT_SESSION *session, *hep = NULL; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; p = NULL; /* Open the underlying Helium object. */ if ((ret = ws_source_name(wtds, session, uri, suffix, &p)) != 0) return (ret); - VMSG(wtext, session, VERBOSE_L1, "open %s/%s", hs->name, p); + VMSG(wt_api, session, VERBOSE_L1, "open %s/%s", hs->name, p); if ((he = he_open(hs->device, p, flags, NULL)) == NULL) { ret = os_errno(); - EMSG(wtext, session, ret, + EMSG(wt_api, session, ret, "he_open: %s/%s: %s", hs->name, p, he_strerror(ret)); } *hep = he; @@ -1824,7 +1826,7 @@ ws_source_open(WT_DATA_SOURCE *wtds, WT_SESSION *session, DATA_SOURCE *ds; HELIUM_SOURCE *hs; WT_CONFIG_ITEM a; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SOURCE *ws; size_t len; int oflags, ret = 0; @@ -1833,7 +1835,7 @@ ws_source_open(WT_DATA_SOURCE *wtds, WT_SESSION *session, *refp = NULL; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; ws = NULL; /* @@ -1844,7 +1846,7 @@ ws_source_open(WT_DATA_SOURCE *wtds, WT_SESSION *session, goto bad_name; p = uri + strlen("helium:"); if (p[0] == '/' || (t = strchr(p, '/')) == NULL || t[1] == '\0') -bad_name: ERET(wtext, session, EINVAL, "%s: illegal name format", uri); +bad_name: ERET(wt_api, session, EINVAL, "%s: illegal name format", uri); len = (size_t)(t - p); /* Find a matching Helium device. */ @@ -1852,14 +1854,14 @@ bad_name: ERET(wtext, session, EINVAL, "%s: illegal name format", uri); if (string_match(hs->name, p, len)) break; if (hs == NULL) - ERET(wtext, NULL, + ERET(wt_api, NULL, EINVAL, "%s: no matching Helium store found", uri); /* * We're about to walk the Helium device's list of files, acquire the * global lock. */ - if ((ret = writelock(wtext, session, &ds->global_lock)) != 0) + if ((ret = writelock(wt_api, session, &ds->global_lock)) != 0) return (ret); /* @@ -1872,13 +1874,13 @@ bad_name: ERET(wtext, session, EINVAL, "%s: illegal name format", uri); /* Check to see if the object is busy. */ if (ws->ref != 0 && (flags & WS_SOURCE_OPEN_BUSY)) { ret = EBUSY; - ESET(unlock(wtext, session, &ds->global_lock)); + ESET(unlock(wt_api, session, &ds->global_lock)); return (ret); } /* Swap the global lock for an object lock. */ if (!(flags & WS_SOURCE_OPEN_GLOBAL)) { - ret = writelock(wtext, session, &ws->lock); - ESET(unlock(wtext, session, &ds->global_lock)); + ret = writelock(wt_api, session, &ws->lock); + ESET(unlock(wt_api, session, &ds->global_lock)); if (ret != 0) return (ret); } @@ -1892,7 +1894,7 @@ bad_name: ERET(wtext, session, EINVAL, "%s: illegal name format", uri); ret = os_errno(); goto err; } - if ((ret = lock_init(wtext, session, &ws->lock)) != 0) + if ((ret = lock_init(wt_api, session, &ws->lock)) != 0) goto err; ws->lockinit = 1; ws->hs = hs; @@ -1907,13 +1909,13 @@ bad_name: ERET(wtext, session, EINVAL, "%s: illegal name format", uri); * handles attempts to create existing objects. */ oflags = HE_O_CREATE; - if ((ret = wtext->config_get(wtext, + if ((ret = wt_api->config_get(wt_api, session, config, "helium_o_truncate", &a)) == 0 && a.val != 0) oflags |= HE_O_TRUNCATE; if (ret != 0 && ret != WT_NOTFOUND) - EMSG_ERR(wtext, session, ret, + EMSG_ERR(wt_api, session, ret, "helium_o_truncate configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); if ((ret = ws_source_open_object( wtds, session, hs, uri, NULL, oflags, &ws->he)) != 0) @@ -1922,12 +1924,12 @@ bad_name: ERET(wtext, session, EINVAL, "%s: illegal name format", uri); wtds, session, hs, uri, WT_NAME_CACHE, oflags, &ws->he_cache)) != 0) goto err; if ((ret = he_commit(ws->he)) != 0) - EMSG_ERR(wtext, session, ret, + EMSG_ERR(wt_api, session, ret, "he_commit: %s", he_strerror(ret)); /* Optionally trade the global lock for the object lock. */ if (!(flags & WS_SOURCE_OPEN_GLOBAL) && - (ret = writelock(wtext, session, &ws->lock)) != 0) + (ret = writelock(wt_api, session, &ws->lock)) != 0) goto err; /* Insert the new entry at the head of the list. */ @@ -1939,7 +1941,7 @@ bad_name: ERET(wtext, session, EINVAL, "%s: illegal name format", uri); if (0) { err: if (ws != NULL) - ESET(ws_source_close(wtext, session, ws)); + ESET(ws_source_close(wt_api, session, ws)); } /* @@ -1947,7 +1949,7 @@ err: if (ws != NULL) * release the global lock. */ if (!(flags & WS_SOURCE_OPEN_GLOBAL) || ret != 0) - ESET(unlock(wtext, session, &ds->global_lock)); + ESET(unlock(wt_api, session, &ds->global_lock)); return (ret); } @@ -1961,12 +1963,12 @@ master_uri_get(WT_DATA_SOURCE *wtds, WT_SESSION *session, const char *uri, char **valuep) { DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; - return (wtext->metadata_search(wtext, session, uri, valuep)); + return (wt_api->metadata_search(wt_api, session, uri, valuep)); } /* @@ -1977,12 +1979,12 @@ static int master_uri_drop(WT_DATA_SOURCE *wtds, WT_SESSION *session, const char *uri) { DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; - return (wtext->metadata_remove(wtext, session, uri)); + return (wt_api->metadata_remove(wt_api, session, uri)); } /* @@ -1994,25 +1996,26 @@ master_uri_rename(WT_DATA_SOURCE *wtds, WT_SESSION *session, const char *uri, const char *newuri) { DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; char *value; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; value = NULL; /* Insert the record under a new name. */ if ((ret = master_uri_get(wtds, session, uri, &value)) != 0 || - (ret = wtext->metadata_insert(wtext, session, newuri, value)) != 0) + (ret = + wt_api->metadata_insert(wt_api, session, newuri, value)) != 0) goto err; /* * Remove the original record, and if that fails, attempt to remove * the new record. */ - if ((ret = wtext->metadata_remove(wtext, session, uri)) != 0) - (void)wtext->metadata_remove(wtext, session, newuri); + if ((ret = wt_api->metadata_remove(wt_api, session, uri)) != 0) + (void)wt_api->metadata_remove(wt_api, session, newuri); err: free((void *)value); return (ret); @@ -2028,53 +2031,53 @@ master_uri_set(WT_DATA_SOURCE *wtds, { DATA_SOURCE *ds; WT_CONFIG_ITEM a, b, c; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int exclusive, ret = 0; char value[1024]; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; exclusive = 0; if ((ret = - wtext->config_get(wtext, session, config, "exclusive", &a)) == 0) + wt_api->config_get(wt_api, session, config, "exclusive", &a)) == 0) exclusive = a.val != 0; else if (ret != WT_NOTFOUND) - ERET(wtext, session, ret, + ERET(wt_api, session, ret, "exclusive configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); /* Get the key/value format strings. */ - if ((ret = wtext->config_get( - wtext, session, config, "key_format", &a)) != 0) { + if ((ret = wt_api->config_get( + wt_api, session, config, "key_format", &a)) != 0) { if (ret == WT_NOTFOUND) { a.str = "u"; a.len = 1; } else - ERET(wtext, session, ret, + ERET(wt_api, session, ret, "key_format configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); } - if ((ret = wtext->config_get( - wtext, session, config, "value_format", &b)) != 0) { + if ((ret = wt_api->config_get( + wt_api, session, config, "value_format", &b)) != 0) { if (ret == WT_NOTFOUND) { b.str = "u"; b.len = 1; } else - ERET(wtext, session, ret, + ERET(wt_api, session, ret, "value_format configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); } /* Get the compression configuration. */ - if ((ret = wtext->config_get( - wtext, session, config, "helium_o_compress", &c)) != 0) { + if ((ret = wt_api->config_get( + wt_api, session, config, "helium_o_compress", &c)) != 0) { if (ret == WT_NOTFOUND) c.val = 0; else - ERET(wtext, session, ret, + ERET(wt_api, session, ret, "helium_o_compress configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); } /* @@ -2087,12 +2090,12 @@ master_uri_set(WT_DATA_SOURCE *wtds, "helium_o_compress=%d", WIREDTIGER_HELIUM_MAJOR, WIREDTIGER_HELIUM_MINOR, (int)a.len, a.str, (int)b.len, b.str, c.val ? 1 : 0); - if ((ret = wtext->metadata_insert(wtext, session, uri, value)) == 0) + if ((ret = wt_api->metadata_insert(wt_api, session, uri, value)) == 0) return (0); if (ret == WT_DUPLICATE_KEY) return (exclusive ? EEXIST : 0); - ERET(wtext, - session, ret, "%s: %s", uri, wtext->strerror(wtext, session, ret)); + ERET(wt_api, session, + ret, "%s: %s", uri, wt_api->strerror(wt_api, session, ret)); } /* @@ -2108,7 +2111,7 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, WT_CONFIG_ITEM v; WT_CONFIG_PARSER *config_parser; WT_CURSOR *wtcursor; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SOURCE *ws; int locked, own, ret, tret; char *value; @@ -2118,7 +2121,7 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, config_parser = NULL; cursor = NULL; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; ws = NULL; locked = 0; ret = tret = 0; @@ -2128,25 +2131,25 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, if ((cursor = calloc(1, sizeof(CURSOR))) == NULL) return (os_errno()); - if ((ret = wtext->config_get( /* Parse configuration */ - wtext, session, config, "append", &v)) != 0) - EMSG_ERR(wtext, session, ret, + if ((ret = wt_api->config_get( /* Parse configuration */ + wt_api, session, config, "append", &v)) != 0) + EMSG_ERR(wt_api, session, ret, "append configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); cursor->config_append = v.val != 0; - if ((ret = wtext->config_get( - wtext, session, config, "overwrite", &v)) != 0) - EMSG_ERR(wtext, session, ret, + if ((ret = wt_api->config_get( + wt_api, session, config, "overwrite", &v)) != 0) + EMSG_ERR(wt_api, session, ret, "overwrite configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); cursor->config_overwrite = v.val != 0; - if ((ret = wtext->collator_config( - wtext, session, uri, config, NULL, &own)) != 0) - EMSG_ERR(wtext, session, ret, + if ((ret = wt_api->collator_config( + wt_api, session, uri, config, NULL, &own)) != 0) + EMSG_ERR(wt_api, session, ret, "collator configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); /* Finish initializing the cursor. */ cursor->wtcursor.close = helium_cursor_close; @@ -2159,7 +2162,7 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, cursor->wtcursor.search_near = helium_cursor_search_near; cursor->wtcursor.update = helium_cursor_update; - cursor->wtext = wtext; + cursor->wt_api = wt_api; cursor->record.key = cursor->__key; if ((cursor->v = malloc(128)) == NULL) goto err; @@ -2179,31 +2182,31 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, if ((ret = master_uri_get(wtds, session, uri, &value)) != 0) goto err; - if ((ret = wtext->config_parser_open(wtext, + if ((ret = wt_api->config_parser_open(wt_api, session, value, strlen(value), &config_parser)) != 0) - EMSG_ERR(wtext, session, ret, + EMSG_ERR(wt_api, session, ret, "Configuration string parser: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); if ((ret = config_parser->get( config_parser, "key_format", &v)) != 0) - EMSG_ERR(wtext, session, ret, + EMSG_ERR(wt_api, session, ret, "key_format configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); ws->config_recno = v.len == 1 && v.str[0] == 'r'; if ((ret = config_parser->get( config_parser, "value_format", &v)) != 0) - EMSG_ERR(wtext, session, ret, + EMSG_ERR(wt_api, session, ret, "value_format configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); ws->config_bitfield = v.len == 2 && isdigit((u_char)v.str[0]) && v.str[1] == 't'; if ((ret = config_parser->get( config_parser, "helium_o_compress", &v)) != 0) - EMSG_ERR(wtext, session, ret, + EMSG_ERR(wt_api, session, ret, "helium_o_compress configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); ws->config_compress = v.val ? 1 : 0; /* @@ -2229,21 +2232,21 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, /* Increment the open reference count to pin the URI and unlock it. */ ++ws->ref; - if ((ret = unlock(wtext, session, &ws->lock)) != 0) + if ((ret = unlock(wt_api, session, &ws->lock)) != 0) goto err; *new_cursor = (WT_CURSOR *)cursor; if (0) { err: if (ws != NULL && locked) - ESET(unlock(wtext, session, &ws->lock)); + ESET(unlock(wt_api, session, &ws->lock)); cursor_destroy(cursor); } if (config_parser != NULL && (tret = config_parser->close(config_parser)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "WT_CONFIG_PARSER.close: %s", - wtext->strerror(wtext, session, tret)); + wt_api->strerror(wt_api, session, tret)); free((void *)value); return (ret); @@ -2258,12 +2261,12 @@ helium_session_create(WT_DATA_SOURCE *wtds, WT_SESSION *session, const char *uri, WT_CONFIG_ARG *config) { DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SOURCE *ws; int ret = 0; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* * Get a locked reference to the WiredTiger source, then immediately @@ -2271,7 +2274,7 @@ helium_session_create(WT_DATA_SOURCE *wtds, */ if ((ret = ws_source_open(wtds, session, uri, config, 0, &ws)) != 0) return (ret); - if ((ret = unlock(wtext, session, &ws->lock)) != 0) + if ((ret = unlock(wt_api, session, &ws->lock)) != 0) return (ret); /* @@ -2297,12 +2300,12 @@ helium_session_drop(WT_DATA_SOURCE *wtds, { DATA_SOURCE *ds; HELIUM_SOURCE *hs; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SOURCE **p, *ws; int ret = 0; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* * Get a locked reference to the data source: hold the global lock, @@ -2328,7 +2331,7 @@ helium_session_drop(WT_DATA_SOURCE *wtds, ws->he_cache = NULL; /* The handle is dead. */ /* Close the source, discarding the structure. */ - ESET(ws_source_close(wtext, session, ws)); + ESET(ws_source_close(wt_api, session, ws)); ws = NULL; /* Discard the metadata entry. */ @@ -2341,7 +2344,7 @@ helium_session_drop(WT_DATA_SOURCE *wtds, if (ret != 0) ret = WT_PANIC; - ESET(unlock(wtext, session, &ds->global_lock)); + ESET(unlock(wt_api, session, &ds->global_lock)); return (ret); } @@ -2354,13 +2357,13 @@ helium_session_rename(WT_DATA_SOURCE *wtds, WT_SESSION *session, const char *uri, const char *newuri, WT_CONFIG_ARG *config) { DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SOURCE *ws; int ret = 0; char *p; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* * Get a locked reference to the data source; hold the global lock, @@ -2401,7 +2404,7 @@ helium_session_rename(WT_DATA_SOURCE *wtds, WT_SESSION *session, if (ret != 0) ret = WT_PANIC; -err: ESET(unlock(wtext, session, &ds->global_lock)); +err: ESET(unlock(wt_api, session, &ds->global_lock)); return (ret); } @@ -2415,12 +2418,12 @@ helium_session_truncate(WT_DATA_SOURCE *wtds, WT_SESSION *session, const char *uri, WT_CONFIG_ARG *config) { DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SOURCE *ws; int ret = 0, tret; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* Get a locked reference to the WiredTiger source. */ if ((ret = ws_source_open(wtds, session, @@ -2429,13 +2432,13 @@ helium_session_truncate(WT_DATA_SOURCE *wtds, /* Truncate the underlying namespaces. */ if ((tret = he_truncate(ws->he)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "he_truncate: %s: %s", ws->uri, he_strerror(tret)); if ((tret = he_truncate(ws->he_cache)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "he_truncate: %s: %s", ws->uri, he_strerror(tret)); - ESET(unlock(wtext, session, &ws->lock)); + ESET(unlock(wt_api, session, &ws->lock)); return (ret); } @@ -2464,18 +2467,18 @@ helium_session_checkpoint( { DATA_SOURCE *ds; HELIUM_SOURCE *hs; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; (void)config; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* Flush all volumes. */ if ((hs = ds->hs_head) != NULL && (ret = he_commit(hs->he_volume)) != 0) - ERET(wtext, session, ret, + ERET(wt_api, session, ret, "he_commit: %s: %s", hs->device, he_strerror(ret)); return (0); @@ -2487,7 +2490,7 @@ helium_session_checkpoint( */ static int helium_source_close( - WT_EXTENSION_API *wtext, WT_SESSION *session, HELIUM_SOURCE *hs) + WT_EXTENSION_API *wt_api, WT_SESSION *session, HELIUM_SOURCE *hs) { WT_SOURCE *ws; int ret = 0, tret; @@ -2497,7 +2500,7 @@ helium_source_close( hs->cleaner_stop = 1; if ((tret = pthread_join(hs->cleaner_id, NULL)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "pthread_join: %s", strerror(tret)); hs->cleaner_id = 0; } @@ -2505,13 +2508,13 @@ helium_source_close( /* Close the underlying WiredTiger sources. */ while ((ws = hs->ws_head) != NULL) { hs->ws_head = ws->next; - ESET(ws_source_close(wtext, session, ws)); + ESET(ws_source_close(wt_api, session, ws)); } /* If the owner, close the database transaction store. */ if (hs->he_txn != NULL && hs->he_owner) { if ((tret = he_close(hs->he_txn)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "he_close: %s: %s: %s", hs->name, WT_NAME_TXN, he_strerror(tret)); hs->he_txn = NULL; @@ -2520,12 +2523,12 @@ helium_source_close( /* Flush and close the Helium source. */ if (hs->he_volume != NULL) { if ((tret = he_commit(hs->he_volume)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "he_commit: %s: %s", hs->device, he_strerror(tret)); if ((tret = he_close(hs->he_volume)) != 0) - EMSG(wtext, session, tret, + EMSG(wt_api, session, tret, "he_close: %s: %s: %s", hs->name, WT_NAME_INIT, he_strerror(tret)); hs->he_volume = NULL; @@ -2543,7 +2546,7 @@ helium_source_close( * Migrate information from the cache to the primary store. */ static int -cache_cleaner(WT_EXTENSION_API *wtext, +cache_cleaner(WT_EXTENSION_API *wt_api, WT_CURSOR *wtcursor, uint64_t oldest, uint64_t *txnminp) { CACHE_RECORD *cp; @@ -2612,7 +2615,7 @@ cache_cleaner(WT_EXTENSION_API *wtext, ret = 0; continue; } - ERET(wtext, NULL, ret, + ERET(wt_api, NULL, ret, "he_delete: %s", he_strerror(ret)); } else { r->val = cp->v; @@ -2628,7 +2631,7 @@ cache_cleaner(WT_EXTENSION_API *wtext, if (ret == 0) continue; - ERET(wtext, NULL, ret, + ERET(wt_api, NULL, ret, "he_update: %s", he_strerror(ret)); } } @@ -2636,7 +2639,7 @@ cache_cleaner(WT_EXTENSION_API *wtext, if (ret == WT_NOTFOUND) ret = 0; if (ret != 0) - ERET(wtext, NULL, ret, "he_next: %s", he_strerror(ret)); + ERET(wt_api, NULL, ret, "he_next: %s", he_strerror(ret)); /* * If we didn't move any keys from the cache to the primary, quit. It's @@ -2651,7 +2654,7 @@ cache_cleaner(WT_EXTENSION_API *wtext, * what Helium handle we commit, so we just commit one of them.) */ if ((ret = he_commit(ws->he)) != 0) - ERET(wtext, NULL, ret, "he_commit: %s", he_strerror(ret)); + ERET(wt_api, NULL, ret, "he_commit: %s", he_strerror(ret)); /* * If we're performing recovery, that's all we need to do, we're going @@ -2668,7 +2671,7 @@ cache_cleaner(WT_EXTENSION_API *wtext, * We're updating the cache, which requires a lock during normal * cleaning. */ - if ((ret = writelock(wtext, NULL, &ws->lock)) != 0) + if ((ret = writelock(wt_api, NULL, &ws->lock)) != 0) goto err; locked = 1; @@ -2682,7 +2685,7 @@ cache_cleaner(WT_EXTENSION_API *wtext, goto err; if (cache_value_visible_all(wtcursor, oldest)) { if ((ret = he_delete(ws->he_cache, r)) != 0) - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "he_delete: %s", he_strerror(ret)); continue; } @@ -2701,15 +2704,15 @@ cache_cleaner(WT_EXTENSION_API *wtext, } locked = 0; - if ((ret = unlock(wtext, NULL, &ws->lock)) != 0) + if ((ret = unlock(wt_api, NULL, &ws->lock)) != 0) goto err; if (ret == WT_NOTFOUND) ret = 0; if (ret != 0) - EMSG_ERR(wtext, NULL, ret, "he_next: %s", he_strerror(ret)); + EMSG_ERR(wt_api, NULL, ret, "he_next: %s", he_strerror(ret)); err: if (locked) - ESET(unlock(wtext, NULL, &ws->lock)); + ESET(unlock(wt_api, NULL, &ws->lock)); return (ret); } @@ -2723,12 +2726,12 @@ txn_cleaner(WT_CURSOR *wtcursor, he_t he_txn, uint64_t txnmin) { CURSOR *cursor; HE_ITEM *r; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; uint64_t txnid; int ret = 0; cursor = (CURSOR *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; r = &cursor->record; /* @@ -2739,13 +2742,13 @@ txn_cleaner(WT_CURSOR *wtcursor, he_t he_txn, uint64_t txnmin) (ret = helium_call(wtcursor, "he_next", he_txn, he_next)) == 0;) { memcpy(&txnid, r->key, sizeof(txnid)); if (txnid < txnmin && (ret = he_delete(he_txn, r)) != 0) - ERET(wtext, NULL, ret, + ERET(wt_api, NULL, ret, "he_delete: %s", he_strerror(ret)); } if (ret == WT_NOTFOUND) ret = 0; if (ret != 0) - ERET(wtext, NULL, ret, "he_next: %s", he_strerror(ret)); + ERET(wt_api, NULL, ret, "he_next: %s", he_strerror(ret)); return (0); } @@ -2755,7 +2758,7 @@ txn_cleaner(WT_CURSOR *wtcursor, he_t he_txn, uint64_t txnmin) * Fake up enough of a cursor to do Helium operations. */ static int -fake_cursor(WT_EXTENSION_API *wtext, WT_CURSOR **wtcursorp) +fake_cursor(WT_EXTENSION_API *wt_api, WT_CURSOR **wtcursorp) { CURSOR *cursor; WT_CURSOR *wtcursor; @@ -2765,7 +2768,7 @@ fake_cursor(WT_EXTENSION_API *wtext, WT_CURSOR **wtcursorp) */ if ((cursor = calloc(1, sizeof(CURSOR))) == NULL) return (os_errno()); - cursor->wtext = wtext; + cursor->wt_api = wt_api; cursor->record.key = cursor->__key; if ((cursor->v = malloc(128)) == NULL) { free(cursor); @@ -2796,7 +2799,7 @@ cache_cleaner_worker(void *arg) HELIUM_SOURCE *hs; HE_STATS stats; WT_CURSOR *wtcursor; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SOURCE *ws; uint64_t oldest, txnmin, txntmp; int cleaner_stop, delay, ret = 0; @@ -2804,10 +2807,10 @@ cache_cleaner_worker(void *arg) hs = (HELIUM_SOURCE *)arg; cursor = NULL; - wtext = hs->wtext; + wt_api = hs->wt_api; - if ((ret = fake_cursor(wtext, &wtcursor)) != 0) - EMSG_ERR(wtext, NULL, ret, "cleaner: %s", strerror(ret)); + if ((ret = fake_cursor(wt_api, &wtcursor)) != 0) + EMSG_ERR(wt_api, NULL, ret, "cleaner: %s", strerror(ret)); cursor = (CURSOR *)wtcursor; for (cleaner_stop = delay = 0; !cleaner_stop;) { @@ -2846,7 +2849,7 @@ cache_cleaner_worker(void *arg) #define CACHE_SIZE_TRIGGER (50 * 1048576) for (ws = hs->ws_head; ws != NULL; ws = ws->next) { if ((ret = he_stats(ws->he_cache, &stats)) != 0) - EMSG_ERR(wtext, NULL, + EMSG_ERR(wt_api, NULL, ret, "he_stats: %s", he_strerror(ret)); if (stats.size > CACHE_SIZE_TRIGGER) break; @@ -2862,7 +2865,7 @@ cache_cleaner_worker(void *arg) * transaction. Do this before doing anything else, avoiding * any race with creating new WT_SOURCE handles. */ - oldest = wtext->transaction_oldest(wtext); + oldest = wt_api->transaction_oldest(wt_api); /* * If any cache needs cleaning, clean them all, because we have @@ -2875,7 +2878,7 @@ cache_cleaner_worker(void *arg) for (ws = hs->ws_head; ws != NULL; ws = ws->next) { cursor->ws = ws; if ((ret = cache_cleaner( - wtext, wtcursor, oldest, &txntmp)) != 0) + wt_api, wtcursor, oldest, &txntmp)) != 0) goto err; if (txntmp < txnmin) txnmin = txntmp; @@ -2904,7 +2907,7 @@ err: cursor_destroy(cursor); * Parse the Helium configuration. */ static int -helium_config_read(WT_EXTENSION_API *wtext, WT_CONFIG_ITEM *config, +helium_config_read(WT_EXTENSION_API *wt_api, WT_CONFIG_ITEM *config, char **devicep, HE_ENV *envp, int *env_setp, int *flagsp) { WT_CONFIG_ITEM k, v; @@ -2915,11 +2918,11 @@ helium_config_read(WT_EXTENSION_API *wtext, WT_CONFIG_ITEM *config, *flagsp = 0; /* Traverse the configuration arguments list. */ - if ((ret = wtext->config_parser_open( - wtext, NULL, config->str, config->len, &config_parser)) != 0) - ERET(wtext, NULL, ret, + if ((ret = wt_api->config_parser_open( + wt_api, NULL, config->str, config->len, &config_parser)) != 0) + ERET(wt_api, NULL, ret, "WT_EXTENSION_API.config_parser_open: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { if (string_match("helium_devices", k.str, k.len)) { if ((*devicep = calloc(1, v.len + 1)) == NULL) @@ -2942,21 +2945,21 @@ helium_config_read(WT_EXTENSION_API *wtext, WT_CONFIG_ITEM *config, *flagsp |= HE_O_VOLUME_TRUNCATE; continue; } - EMSG_ERR(wtext, NULL, EINVAL, + EMSG_ERR(wt_api, NULL, EINVAL, "unknown configuration key value pair %.*s=%.*s", (int)k.len, k.str, (int)v.len, v.str); } if (ret == WT_NOTFOUND) ret = 0; if (ret != 0) - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "WT_CONFIG_PARSER.next: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); err: if ((tret = config_parser->close(config_parser)) != 0) - EMSG(wtext, NULL, tret, + EMSG(wt_api, NULL, tret, "WT_CONFIG_PARSER.close: %s", - wtext->strerror(wtext, NULL, tret)); + wt_api->strerror(wt_api, NULL, tret)); return (ret); } @@ -2970,13 +2973,13 @@ helium_source_open(DATA_SOURCE *ds, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) { struct he_env env; HELIUM_SOURCE *hs; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int env_set, flags, ret = 0; - wtext = ds->wtext; + wt_api = ds->wt_api; hs = NULL; - VMSG(wtext, NULL, VERBOSE_L1, "volume %.*s=%.*s", + VMSG(wt_api, NULL, VERBOSE_L1, "volume %.*s=%.*s", (int)k->len, k->str, (int)v->len, v->str); /* @@ -2987,7 +2990,7 @@ helium_source_open(DATA_SOURCE *ds, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) */ for (hs = ds->hs_head; hs != NULL; hs = hs->next) if (string_match(hs->name, k->str, k->len)) - ERET(wtext, NULL, + ERET(wt_api, NULL, EINVAL, "%s: device already open", hs->name); /* Allocate and initialize a new underlying Helium source object. */ @@ -2998,15 +3001,15 @@ helium_source_open(DATA_SOURCE *ds, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) } memcpy(hs->name, k->str, k->len); hs->txn_notify.notify = txn_notify; - hs->wtext = wtext; + hs->wt_api = wt_api; /* Read the configuration, require a device naming the Helium store. */ memset(&env, 0, sizeof(env)); if ((ret = helium_config_read( - wtext, v, &hs->device, &env, &env_set, &flags)) != 0) + wt_api, v, &hs->device, &env, &env_set, &flags)) != 0) goto err; if (hs->device == NULL) - EMSG_ERR(wtext, NULL, + EMSG_ERR(wt_api, NULL, EINVAL, "%s: no Helium volumes specified", hs->name); /* @@ -3019,7 +3022,7 @@ helium_source_open(DATA_SOURCE *ds, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) if ((hs->he_volume = he_open( hs->device, WT_NAME_INIT, flags, env_set ? &env : NULL)) == NULL) { ret = os_errno(); - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "he_open: %s: %s: %s", hs->name, WT_NAME_INIT, he_strerror(ret)); } @@ -3030,7 +3033,7 @@ helium_source_open(DATA_SOURCE *ds, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) if (0) { err: if (hs != NULL) - ESET(helium_source_close(wtext, NULL, hs)); + ESET(helium_source_close(wt_api, NULL, hs)); } return (ret); } @@ -3043,11 +3046,11 @@ static int helium_source_open_txn(DATA_SOURCE *ds) { HELIUM_SOURCE *hs, *hs_txn; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; he_t he_txn, t; int ret = 0; - wtext = ds->wtext; + wt_api = ds->wt_api; /* * The global txn namespace is per connection, it spans multiple Helium @@ -3063,7 +3066,7 @@ helium_source_open_txn(DATA_SOURCE *ds) if (hs_txn != NULL) { (void)he_close(t); (void)he_close(hs_txn); - ERET(wtext, NULL, WT_PANIC, + ERET(wt_api, NULL, WT_PANIC, "found multiple transaction stores, " "unable to proceed"); } @@ -3083,17 +3086,17 @@ helium_source_open_txn(DATA_SOURCE *ds) if ((he_txn = he_open( hs->device, WT_NAME_TXN, HE_O_CREATE, NULL)) == NULL) { ret = os_errno(); - ERET(wtext, NULL, ret, + ERET(wt_api, NULL, ret, "he_open: %s: %s: %s", hs->name, WT_NAME_TXN, he_strerror(ret)); } /* Push the change. */ if ((ret = he_commit(he_txn)) != 0) - ERET(wtext, NULL, ret, + ERET(wt_api, NULL, ret, "he_commit: %s", he_strerror(ret)); } - VMSG(wtext, NULL, VERBOSE_L1, "%s" "transactional store on %s", + VMSG(wt_api, NULL, VERBOSE_L1, "%s" "transactional store on %s", hs_txn == NULL ? "creating " : "", hs->name); /* Set the owner field, this Helium source has to be closed last. */ @@ -3117,7 +3120,7 @@ helium_source_recover_namespace(WT_DATA_SOURCE *wtds, CURSOR *cursor; DATA_SOURCE *ds; WT_CURSOR *wtcursor; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SOURCE *ws; size_t len; int ret = 0; @@ -3125,7 +3128,7 @@ helium_source_recover_namespace(WT_DATA_SOURCE *wtds, char *uri; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; cursor = NULL; ws = NULL; uri = NULL; @@ -3149,26 +3152,26 @@ helium_source_recover_namespace(WT_DATA_SOURCE *wtds, */ if ((ret = ws_source_open(wtds, NULL, uri, config, 0, &ws)) != 0) goto err; - if ((ret = unlock(wtext, NULL, &ws->lock)) != 0) + if ((ret = unlock(wt_api, NULL, &ws->lock)) != 0) goto err; /* Fake up a cursor. */ - if ((ret = fake_cursor(wtext, &wtcursor)) != 0) - EMSG_ERR(wtext, NULL, ret, "recovery: %s", strerror(ret)); + if ((ret = fake_cursor(wt_api, &wtcursor)) != 0) + EMSG_ERR(wt_api, NULL, ret, "recovery: %s", strerror(ret)); cursor = (CURSOR *)wtcursor; cursor->ws = ws; /* Process, then clear, the cache. */ - if ((ret = cache_cleaner(wtext, wtcursor, 0, NULL)) != 0) + if ((ret = cache_cleaner(wt_api, wtcursor, 0, NULL)) != 0) goto err; if ((ret = he_truncate(ws->he_cache)) != 0) - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "he_truncate: %s(cache): %s", ws->uri, he_strerror(ret)); /* Close the underlying WiredTiger sources. */ err: while ((ws = hs->ws_head) != NULL) { hs->ws_head = ws->next; - ESET(ws_source_close(wtext, NULL, ws)); + ESET(ws_source_close(wt_api, NULL, ws)); } cursor_destroy(cursor); @@ -3233,20 +3236,20 @@ helium_source_recover( { struct helium_namespace_cookie names; DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; u_int i; int ret = 0; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; memset(&names, 0, sizeof(names)); - VMSG(wtext, NULL, VERBOSE_L1, "recover %s", hs->name); + VMSG(wt_api, NULL, VERBOSE_L1, "recover %s", hs->name); /* Get a list of the cache/primary object pairs in the Helium source. */ if ((ret = he_enumerate( hs->device, helium_namespace_list, &names)) != 0) - ERET(wtext, NULL, ret, + ERET(wt_api, NULL, ret, "he_enumerate: %s: %s", hs->name, he_strerror(ret)); /* Recover the objects. */ @@ -3257,7 +3260,7 @@ helium_source_recover( /* Clear the transaction store. */ if ((ret = he_truncate(hs->he_txn)) != 0) - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "he_truncate: %s: %s: %s", hs->name, WT_NAME_TXN, he_strerror(ret)); @@ -3277,15 +3280,15 @@ helium_terminate(WT_DATA_SOURCE *wtds, WT_SESSION *session) { DATA_SOURCE *ds; HELIUM_SOURCE *hs, *last; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* Lock the system down. */ if (ds->lockinit) - ret = writelock(wtext, session, &ds->global_lock); + ret = writelock(wt_api, session, &ds->global_lock); /* * Close the Helium sources, close the Helium source that "owns" the @@ -3298,15 +3301,15 @@ helium_terminate(WT_DATA_SOURCE *wtds, WT_SESSION *session) last = hs; continue; } - ESET(helium_source_close(wtext, session, hs)); + ESET(helium_source_close(wt_api, session, hs)); } if (last != NULL) - ESET(helium_source_close(wtext, session, last)); + ESET(helium_source_close(wt_api, session, last)); /* Unlock and destroy the system. */ if (ds->lockinit) { - ESET(unlock(wtext, session, &ds->global_lock)); - ESET(lock_destroy(wtext, NULL, &ds->global_lock)); + ESET(unlock(wt_api, session, &ds->global_lock)); + ESET(lock_destroy(wt_api, NULL, &ds->global_lock)); } OVERWRITE_AND_FREE(ds); @@ -3347,24 +3350,24 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) HELIUM_SOURCE *hs; WT_CONFIG_ITEM k, v; WT_CONFIG_PARSER *config_parser; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int vmajor, vminor, ret = 0; const char **p; config_parser = NULL; ds = NULL; - wtext = connection->get_extension_api(connection); + wt_api = connection->get_extension_api(connection); /* Check the library version */ #if HE_VERSION_MAJOR != 2 || HE_VERSION_MINOR != 2 - ERET(wtext, NULL, EINVAL, + ERET(wt_api, NULL, EINVAL, "unsupported Levyx/Helium header file %d.%d, expected version 2.2", HE_VERSION_MAJOR, HE_VERSION_MINOR); #endif he_version(&vmajor, &vminor); if (vmajor != 2 || vminor != 2) - ERET(wtext, NULL, EINVAL, + ERET(wt_api, NULL, EINVAL, "unsupported Levyx/Helium library version %d.%d, expected " "version 2.2", vmajor, vminor); @@ -3372,23 +3375,23 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) if ((ds = calloc(1, sizeof(DATA_SOURCE))) == NULL) return (os_errno()); ds->wtds = wtds; - ds->wtext = wtext; - if ((ret = lock_init(wtext, NULL, &ds->global_lock)) != 0) + ds->wt_api = wt_api; + if ((ret = lock_init(wt_api, NULL, &ds->global_lock)) != 0) goto err; ds->lockinit = 1; /* Get the configuration string. */ - if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) - EMSG_ERR(wtext, NULL, ret, + if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) + EMSG_ERR(wt_api, NULL, ret, "WT_EXTENSION_API.config_get: config: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); /* Step through the list of Helium sources, opening each one. */ - if ((ret = wtext->config_parser_open( - wtext, NULL, v.str, v.len, &config_parser)) != 0) - EMSG_ERR(wtext, NULL, ret, + if ((ret = wt_api->config_parser_open( + wt_api, NULL, v.str, v.len, &config_parser)) != 0) + EMSG_ERR(wt_api, NULL, ret, "WT_EXTENSION_API.config_parser_open: config: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { if (string_match("helium_verbose", k.str, k.len)) { verbose = v.val == 0 ? 0 : 1; @@ -3398,13 +3401,13 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) goto err; } if (ret != WT_NOTFOUND) - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "WT_CONFIG_PARSER.next: config: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); if ((ret = config_parser->close(config_parser)) != 0) - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "WT_CONFIG_PARSER.close: config: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); config_parser = NULL; /* Find and open the database transaction store. */ @@ -3420,7 +3423,7 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) for (hs = ds->hs_head; hs != NULL; hs = hs->next) if ((ret = pthread_create( &hs->cleaner_id, NULL, cache_cleaner_worker, hs)) != 0) - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "%s: pthread_create: cleaner thread: %s", hs->name, strerror(ret)); @@ -3428,17 +3431,17 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) for (p = session_create_opts; *p != NULL; ++p) if ((ret = connection->configure_method(connection, "WT_SESSION.create", "helium:", *p, "boolean", NULL)) != 0) - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "WT_CONNECTION.configure_method: session.create: " "%s: %s", - *p, wtext->strerror(wtext, NULL, ret)); + *p, wt_api->strerror(wt_api, NULL, ret)); /* Add the data source */ if ((ret = connection->add_data_source( connection, "helium:", (WT_DATA_SOURCE *)ds, NULL)) != 0) - EMSG_ERR(wtext, NULL, ret, + EMSG_ERR(wt_api, NULL, ret, "WT_CONNECTION.add_data_source: %s", - wtext->strerror(wtext, NULL, ret)); + wt_api->strerror(wt_api, NULL, ret)); return (0); err: if (ds != NULL) diff --git a/src/third_party/wiredtiger/ext/encryptors/nop/nop_encrypt.c b/src/third_party/wiredtiger/ext/encryptors/nop/nop_encrypt.c index eac7d8f4632..af65f397549 100644 --- a/src/third_party/wiredtiger/ext/encryptors/nop/nop_encrypt.c +++ b/src/third_party/wiredtiger/ext/encryptors/nop/nop_encrypt.c @@ -38,13 +38,29 @@ typedef struct { WT_ENCRYPTOR encryptor; /* Must come first */ - WT_EXTENSION_API *wt_api; /* Extension API */ + WT_EXTENSION_API *wt_api; /* Extension API */ - unsigned long nop_calls; /* Count of calls */ + unsigned long nop_calls; /* Count of calls */ } NOP_ENCRYPTOR; /*! [WT_ENCRYPTOR initialization structure] */ +/* + * nop_error -- + * Display an error from this module in a standard way. + */ +static int +nop_error( + NOP_ENCRYPTOR *encryptor, WT_SESSION *session, int err, const char *msg) +{ + WT_EXTENSION_API *wt_api; + + wt_api = encryptor->wt_api; + (void)wt_api->err_printf(wt_api, session, + "nop encryption: %s: %s", msg, wt_api->strerror(wt_api, NULL, err)); + return (err); +} + /*! [WT_ENCRYPTOR encrypt] */ /* * nop_encrypt -- @@ -63,7 +79,8 @@ nop_encrypt(WT_ENCRYPTOR *encryptor, WT_SESSION *session, ++nop_encryptor->nop_calls; /* Call count */ if (dst_len < src_len) - return (ENOMEM); + return (nop_error(nop_encryptor, session, + ENOMEM, "encrypt buffer not big enough")); memcpy(dst, src, src_len); *result_lenp = src_len; diff --git a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c index 4d614e2e409..559c8e6e33a 100644 --- a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c +++ b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c @@ -68,7 +68,7 @@ typedef struct { WT_ENCRYPTOR encryptor; /* Must come first */ - WT_EXTENSION_API *wtext; /* Extension API */ + WT_EXTENSION_API *wt_api; /* Extension API */ int rot_N; /* rotN value */ char *keyid; /* Saved keyid */ @@ -89,14 +89,15 @@ typedef struct { * Display an error from this module in a standard way. */ static int -rotn_error(ROTN_ENCRYPTOR *encryptor, WT_SESSION *session, int err, - const char *msg) +rotn_error( + ROTN_ENCRYPTOR *encryptor, WT_SESSION *session, int err, const char *msg) { - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; - wtext = encryptor->wtext; - (void)wtext->err_printf(wtext, session, - "rotn encryption: %s: %s", msg, wtext->strerror(wtext, NULL, err)); + wt_api = encryptor->wt_api; + (void)wt_api->err_printf(wt_api, session, + "rotn encryption: %s: %s", + msg, wt_api->strerror(wt_api, NULL, err)); return (err); } @@ -189,7 +190,8 @@ rotn_encrypt(WT_ENCRYPTOR *encryptor, WT_SESSION *session, (void)session; /* Unused */ if (dst_len < src_len + CHKSUM_LEN + IV_LEN) - return (ENOMEM); + return (rotn_error(rotn_encryptor, session, + ENOMEM, "encrypt buffer not big enough")); /* * !!! Most implementations would verify any needed @@ -308,7 +310,7 @@ rotn_customize(WT_ENCRYPTOR *encryptor, WT_SESSION *session, const ROTN_ENCRYPTOR *orig; ROTN_ENCRYPTOR *rotn_encryptor; WT_CONFIG_ITEM keyid, secret; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; size_t i, len; int ret, keyid_val; u_char base; @@ -317,7 +319,7 @@ rotn_customize(WT_ENCRYPTOR *encryptor, WT_SESSION *session, keyid_val = 0; orig = (const ROTN_ENCRYPTOR *)encryptor; - wtext = orig->wtext; + wt_api = orig->wt_api; if ((rotn_encryptor = calloc(1, sizeof(ROTN_ENCRYPTOR))) == NULL) return (errno); @@ -327,13 +329,14 @@ rotn_customize(WT_ENCRYPTOR *encryptor, WT_SESSION *session, /* * Stash the keyid from the configuration string. */ - if ((ret = wtext->config_get(wtext, session, encrypt_config, + if ((ret = wt_api->config_get(wt_api, session, encrypt_config, "keyid", &keyid)) == 0 && keyid.len != 0) { /* * In this demonstration, we expect keyid to be a number. */ if ((keyid_val = atoi(keyid.str)) < 0) { - ret = EINVAL; + ret = rotn_error(rotn_encryptor, + NULL, EINVAL, "rotn_customize: invalid keyid"); goto err; } if ((rotn_encryptor->keyid = malloc(keyid.len + 1)) == NULL) { @@ -349,7 +352,7 @@ rotn_customize(WT_ENCRYPTOR *encryptor, WT_SESSION *session, * We stash the secret key from the configuration string * and build some shift bytes to make encryption/decryption easy. */ - if ((ret = wtext->config_get(wtext, session, encrypt_config, + if ((ret = wt_api->config_get(wt_api, session, encrypt_config, "secretkey", &secret)) == 0 && secret.len != 0) { len = secret.len; if ((rotn_encryptor->secretkey = malloc(len + 1)) == NULL || @@ -364,7 +367,8 @@ rotn_customize(WT_ENCRYPTOR *encryptor, WT_SESSION *session, else if ('A' <= secret.str[i] && secret.str[i] <= 'Z') base = 'A'; else { - ret = EINVAL; + ret = rotn_error(rotn_encryptor, NULL, + EINVAL, "rotn_customize: invalid key"); goto err; } base -= (u_char)keyid_val; @@ -427,19 +431,19 @@ rotn_configure(ROTN_ENCRYPTOR *rotn_encryptor, WT_CONFIG_ARG *config) { WT_CONFIG_ITEM k, v; WT_CONFIG_PARSER *config_parser; - WT_EXTENSION_API *wtext; /* Extension API */ + WT_EXTENSION_API *wt_api; /* Extension API */ int ret, t_ret; - wtext = rotn_encryptor->wtext; + wt_api = rotn_encryptor->wt_api; /* Get the configuration string. */ - if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) + if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) return (rotn_error(rotn_encryptor, NULL, ret, "WT_EXTENSION_API.config_get")); /* Step through the list of configuration options. */ - if ((ret = wtext->config_parser_open( - wtext, NULL, v.str, v.len, &config_parser)) != 0) + if ((ret = wt_api->config_parser_open( + wt_api, NULL, v.str, v.len, &config_parser)) != 0) return (rotn_error(rotn_encryptor, NULL, ret, "WT_EXTENSION_API.config_parser_open")); @@ -492,7 +496,7 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) rotn_encryptor->encryptor.sizing = rotn_sizing; rotn_encryptor->encryptor.customize = rotn_customize; rotn_encryptor->encryptor.terminate = rotn_terminate; - rotn_encryptor->wtext = connection->get_extension_api(connection); + rotn_encryptor->wt_api = connection->get_extension_api(connection); if ((ret = rotn_configure(rotn_encryptor, config)) != 0) { free(rotn_encryptor); diff --git a/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c b/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c index 698a9d6fa96..e47ce6e2255 100644 --- a/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c +++ b/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c @@ -54,6 +54,22 @@ typedef struct { } CSV_EXTRACTOR; /* + * csv_error -- + * Display an error from this module in a standard way. + */ +static int +csv_error(const CSV_EXTRACTOR *csv_extractor, + WT_SESSION *session, int err, const char *msg) +{ + WT_EXTENSION_API *wt_api; + + wt_api = csv_extractor->wt_api; + (void)wt_api->err_printf(wt_api, session, + "csv extraction: %s: %s", msg, wt_api->strerror(wt_api, NULL, err)); + return (err); +} + +/* * csv_extract -- * WiredTiger CSV extraction. */ @@ -61,19 +77,19 @@ static int csv_extract(WT_EXTRACTOR *extractor, WT_SESSION *session, const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor) { - char *copy, *p, *pend, *valstr; const CSV_EXTRACTOR *csv_extractor; - int i, ret, val; + WT_EXTENSION_API *wt_api; size_t len; - WT_EXTENSION_API *wtapi; + int i, ret, val; + char *copy, *p, *pend, *valstr; (void)key; /* Unused parameters */ csv_extractor = (const CSV_EXTRACTOR *)extractor; - wtapi = csv_extractor->wt_api; + wt_api = csv_extractor->wt_api; /* Unpack the value. */ - if ((ret = wtapi->struct_unpack(wtapi, + if ((ret = wt_api->struct_unpack(wt_api, session, value->data, value->size, "S", &valstr)) != 0) return (ret); @@ -98,8 +114,10 @@ csv_extract(WT_EXTRACTOR *extractor, WT_SESSION *session, copy[len] = '\0'; if (csv_extractor->format_isnum) { if ((val = atoi(copy)) < 0) { + ret = csv_error(csv_extractor, + session, EINVAL, "invalid key value"); free(copy); - return (EINVAL); + return (ret); } result_cursor->set_key(result_cursor, val); } else @@ -125,29 +143,38 @@ csv_customize(WT_EXTRACTOR *extractor, WT_SESSION *session, CSV_EXTRACTOR *csv_extractor; WT_CONFIG_ITEM field, format; WT_CONFIG_PARSER *parser; - WT_EXTENSION_API *wtapi; - int ret; + WT_EXTENSION_API *wt_api; long field_num; + int ret; (void)session; /* Unused parameters */ (void)uri; /* Unused parameters */ orig = (const CSV_EXTRACTOR *)extractor; - wtapi = orig->wt_api; - if ((ret = wtapi->config_parser_open(wtapi, session, appcfg->str, + wt_api = orig->wt_api; + if ((ret = wt_api->config_parser_open(wt_api, session, appcfg->str, appcfg->len, &parser)) != 0) return (ret); if ((ret = parser->get(parser, "field", &field)) != 0 || (ret = parser->get(parser, "format", &format)) != 0) { - if (ret == WT_NOTFOUND) - return (EINVAL); + if (ret == WT_NOTFOUND) { + (void)wt_api->err_printf( + wt_api, session, "field or format not found"); + return (WT_NOTFOUND); + } return (ret); } field_num = strtol(field.str, NULL, 10); - if (field_num < 0 || field_num > INT_MAX) + if (field_num < 0 || field_num > INT_MAX) { + (void)wt_api->err_printf( + wt_api, session, "field: invalid format"); return (EINVAL); - if (format.len != 1 || (format.str[0] != 'S' && format.str[0] != 'i')) + } + if (format.len != 1 || (format.str[0] != 'S' && format.str[0] != 'i')) { + (void)wt_api->err_printf( + wt_api, session, "format: invalid format"); return (EINVAL); + } if ((csv_extractor = calloc(1, sizeof(CSV_EXTRACTOR))) == NULL) return (errno); diff --git a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c index be63b1d7e7d..866cd0663ce 100644 --- a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c +++ b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c @@ -57,13 +57,13 @@ * int ret; */ #undef ERET -#define ERET(wtext, session, v, ...) do { \ - (void)wtext->err_printf(wtext, session, __VA_ARGS__); \ +#define ERET(wt_api, session, v, ...) do { \ + (void)wt_api->err_printf(wt_api, session, __VA_ARGS__); \ return (v); \ } while (0) #undef ESET -#define ESET(wtext, session, v, ...) do { \ - (void)wtext->err_printf(wtext, session, __VA_ARGS__); \ +#define ESET(wt_api, session, v, ...) do { \ + (void)wt_api->err_printf(wt_api, session, __VA_ARGS__); \ ret = v; \ } while (0) #undef ETRET @@ -80,7 +80,7 @@ typedef struct __data_source DATA_SOURCE; typedef struct __cursor_source { WT_CURSOR wtcursor; /* Must come first */ - WT_EXTENSION_API *wtext; /* Extension functions */ + WT_EXTENSION_API *wt_api; /* Extension functions */ DATA_SOURCE *ds; /* Underlying Berkeley DB */ @@ -98,7 +98,7 @@ typedef struct __cursor_source { struct __data_source { WT_DATA_SOURCE wtds; /* Must come first */ - WT_EXTENSION_API *wtext; /* Extension functions */ + WT_EXTENSION_API *wt_api; /* Extension functions */ /* * We single thread all WT_SESSION methods and return EBUSY if a @@ -134,12 +134,12 @@ os_errno(void) */ static int lock_init( - WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) + WT_EXTENSION_API *wt_api, WT_SESSION *session, pthread_rwlock_t *lockp) { int ret = 0; if ((ret = pthread_rwlock_init(lockp, NULL)) != 0) - ERET(wtext, session, WT_PANIC, "lock init: %s", strerror(ret)); + ERET(wt_api, session, WT_PANIC, "lock init: %s", strerror(ret)); return (0); } @@ -149,12 +149,12 @@ lock_init( */ static int lock_destroy( - WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) + WT_EXTENSION_API *wt_api, WT_SESSION *session, pthread_rwlock_t *lockp) { int ret = 0; if ((ret = pthread_rwlock_destroy(lockp)) != 0) - ERET(wtext, + ERET(wt_api, session, WT_PANIC, "lock destroy: %s", strerror(ret)); return (0); } @@ -165,12 +165,12 @@ lock_destroy( */ static INLINE int writelock( - WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) + WT_EXTENSION_API *wt_api, WT_SESSION *session, pthread_rwlock_t *lockp) { int ret = 0; if ((ret = pthread_rwlock_wrlock(lockp)) != 0) - ERET(wtext, + ERET(wt_api, session, WT_PANIC, "write-lock: %s", strerror(ret)); return (0); } @@ -180,12 +180,12 @@ writelock( * Release an object's lock. */ static INLINE int -unlock(WT_EXTENSION_API *wtext, WT_SESSION *session, pthread_rwlock_t *lockp) +unlock(WT_EXTENSION_API *wt_api, WT_SESSION *session, pthread_rwlock_t *lockp) { int ret = 0; if ((ret = pthread_rwlock_unlock(lockp)) != 0) - ERET(wtext, session, WT_PANIC, "unlock: %s", strerror(ret)); + ERET(wt_api, session, WT_PANIC, "unlock: %s", strerror(ret)); return (0); } @@ -194,16 +194,16 @@ single_thread( WT_DATA_SOURCE *wtds, WT_SESSION *session, pthread_rwlock_t *lockp) { DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; - if ((ret = writelock(wtext, session, lockp)) != 0) + if ((ret = writelock(wt_api, session, lockp)) != 0) return (ret); if (ds->open_cursors != 0) { - if ((ret = unlock(wtext, session, lockp)) != 0) + if ((ret = unlock(wt_api, session, lockp)) != 0) return (ret); return (EBUSY); } @@ -211,13 +211,13 @@ single_thread( } static int -uri2name(WT_EXTENSION_API *wtext, +uri2name(WT_EXTENSION_API *wt_api, WT_SESSION *session, const char *uri, const char **namep) { const char *name; if ((name = strchr(uri, ':')) == NULL || *++name == '\0') - ERET(wtext, session, EINVAL, "unsupported object: %s", uri); + ERET(wt_api, session, EINVAL, "unsupported object: %s", uri); *namep = name; return (0); } @@ -226,15 +226,15 @@ static INLINE int recno_convert(WT_CURSOR *wtcursor, db_recno_t *recnop) { CURSOR_SOURCE *cursor; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; if (wtcursor->recno > UINT32_MAX) - ERET(wtext, + ERET(wt_api, session, ERANGE, "record number %" PRIuMAX ": %s", (uintmax_t)wtcursor->recno, strerror(ERANGE)); @@ -315,18 +315,18 @@ bdb_dump(WT_CURSOR *wtcursor, WT_SESSION *session, const char *tag) DB *db; DBC *dbc; DBT *key, *value; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; db = cursor->db; key = &cursor->key; value = &cursor->value; if ((ret = db->cursor(db, NULL, &dbc, 0)) != 0) - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "Db.cursor: %s", db_strerror(ret)); printf("==> %s\n", tag); while ((ret = dbc->get(dbc, key, value, DB_NEXT)) == 0) @@ -340,7 +340,7 @@ bdb_dump(WT_CURSOR *wtcursor, WT_SESSION *session, const char *tag) (int)value->size, (char *)value->data); if (ret != DB_NOTFOUND) - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); return (0); @@ -353,13 +353,13 @@ kvs_cursor_next(WT_CURSOR *wtcursor) CURSOR_SOURCE *cursor; DBC *dbc; DBT *key, *value; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; int ret = 0; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; dbc = cursor->dbc; key = &cursor->key; @@ -373,7 +373,7 @@ kvs_cursor_next(WT_CURSOR *wtcursor) if (ret == DB_NOTFOUND || ret == DB_KEYEMPTY) return (WT_NOTFOUND); - ERET(wtext, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); + ERET(wt_api, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); } static int @@ -382,13 +382,13 @@ kvs_cursor_prev(WT_CURSOR *wtcursor) CURSOR_SOURCE *cursor; DBC *dbc; DBT *key, *value; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; int ret = 0; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; dbc = cursor->dbc; key = &cursor->key; @@ -402,7 +402,7 @@ kvs_cursor_prev(WT_CURSOR *wtcursor) if (ret == DB_NOTFOUND || ret == DB_KEYEMPTY) return (WT_NOTFOUND); - ERET(wtext, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); + ERET(wt_api, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); } static int @@ -410,23 +410,23 @@ kvs_cursor_reset(WT_CURSOR *wtcursor) { CURSOR_SOURCE *cursor; DBC *dbc; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; int ret = 0; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; /* Close and re-open the Berkeley DB cursor */ if ((dbc = cursor->dbc) != NULL) { cursor->dbc = NULL; if ((ret = dbc->close(dbc)) != 0) - ERET(wtext, session, WT_ERROR, + ERET(wt_api, session, WT_ERROR, "DbCursor.close: %s", db_strerror(ret)); if ((ret = cursor->db->cursor(cursor->db, NULL, &dbc, 0)) != 0) - ERET(wtext, session, WT_ERROR, + ERET(wt_api, session, WT_ERROR, "Db.cursor: %s", db_strerror(ret)); cursor->dbc = dbc; } @@ -439,13 +439,13 @@ kvs_cursor_search(WT_CURSOR *wtcursor) CURSOR_SOURCE *cursor; DBC *dbc; DBT *key, *value; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; int ret = 0; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; dbc = cursor->dbc; key = &cursor->key; @@ -462,7 +462,7 @@ kvs_cursor_search(WT_CURSOR *wtcursor) if (ret == DB_NOTFOUND || ret == DB_KEYEMPTY) return (WT_NOTFOUND); - ERET(wtext, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); + ERET(wt_api, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); } static int @@ -471,14 +471,14 @@ kvs_cursor_search_near(WT_CURSOR *wtcursor, int *exact) CURSOR_SOURCE *cursor; DBC *dbc; DBT *key, *value; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; size_t len; int ret = 0; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; dbc = cursor->dbc; key = &cursor->key; @@ -529,7 +529,7 @@ retry: if ((ret = dbc->get(dbc, key, value, DB_SET_RANGE)) == 0) { if (ret == DB_NOTFOUND || ret == DB_KEYEMPTY) return (WT_NOTFOUND); - ERET(wtext, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); + ERET(wt_api, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); } static int @@ -539,13 +539,13 @@ kvs_cursor_insert(WT_CURSOR *wtcursor) DB *db; DBC *dbc; DBT *key, *value; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; int ret = 0; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; dbc = cursor->dbc; db = cursor->db; @@ -566,16 +566,16 @@ kvs_cursor_insert(WT_CURSOR *wtcursor) * number. */ if ((ret = db->put(db, NULL, key, value, DB_APPEND)) != 0) - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "Db.put: %s", db_strerror(ret)); wtcursor->recno = *(db_recno_t *)key->data; if ((ret = dbc->get(dbc, key, value, DB_SET)) != 0) - ERET(wtext, session, WT_ERROR, + ERET(wt_api, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); } else if (cursor->config_overwrite) { if ((ret = dbc->put(dbc, key, value, DB_KEYFIRST)) != 0) - ERET(wtext, session, WT_ERROR, + ERET(wt_api, session, WT_ERROR, "DbCursor.put: %s", db_strerror(ret)); } else { /* @@ -586,11 +586,11 @@ kvs_cursor_insert(WT_CURSOR *wtcursor) db->put(db, NULL, key, value, DB_NOOVERWRITE)) != 0) { if (ret == DB_KEYEXIST) return (WT_DUPLICATE_KEY); - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "Db.put: %s", db_strerror(ret)); } if ((ret = dbc->get(dbc, key, value, DB_SET)) != 0) - ERET(wtext, session, WT_ERROR, + ERET(wt_api, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); } @@ -603,13 +603,13 @@ kvs_cursor_update(WT_CURSOR *wtcursor) CURSOR_SOURCE *cursor; DBC *dbc; DBT *key, *value; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; int ret = 0; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; dbc = cursor->dbc; key = &cursor->key; @@ -620,7 +620,7 @@ kvs_cursor_update(WT_CURSOR *wtcursor) copyin_value(wtcursor); if ((ret = dbc->put(dbc, key, value, DB_KEYFIRST)) != 0) - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "DbCursor.put: %s", db_strerror(ret)); return (0); @@ -632,13 +632,13 @@ kvs_cursor_remove(WT_CURSOR *wtcursor) CURSOR_SOURCE *cursor; DBC *dbc; DBT *key, *value; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; int ret = 0; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; - wtext = cursor->wtext; + wt_api = cursor->wt_api; dbc = cursor->dbc; key = &cursor->key; @@ -660,11 +660,11 @@ kvs_cursor_remove(WT_CURSOR *wtcursor) if ((ret = dbc->get(dbc, key, value, DB_SET)) != 0) { if (ret == DB_NOTFOUND || ret == DB_KEYEMPTY) return (WT_NOTFOUND); - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret)); } if ((ret = dbc->del(dbc, 0)) != 0) - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "DbCursor.del: %s", db_strerror(ret)); return (0); @@ -677,32 +677,32 @@ kvs_cursor_close(WT_CURSOR *wtcursor) DATA_SOURCE *ds; DB *db; DBC *dbc; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; WT_SESSION *session; int ret = 0; session = wtcursor->session; cursor = (CURSOR_SOURCE *)wtcursor; ds = cursor->ds; - wtext = cursor->wtext; + wt_api = cursor->wt_api; dbc = cursor->dbc; cursor->dbc = NULL; if (dbc != NULL && (ret = dbc->close(dbc)) != 0) - ERET(wtext, session, WT_ERROR, + ERET(wt_api, session, WT_ERROR, "DbCursor.close: %s", db_strerror(ret)); db = cursor->db; cursor->db = NULL; if (db != NULL && (ret = db->close(db, 0)) != 0) - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "Db.close: %s", db_strerror(ret)); free(wtcursor); - if ((ret = writelock(wtext, session, &ds->rwlock)) != 0) + if ((ret = writelock(wt_api, session, &ds->rwlock)) != 0) return (ret); --ds->open_cursors; - if ((ret = unlock(wtext, session, &ds->rwlock)) != 0) + if ((ret = unlock(wt_api, session, &ds->rwlock)) != 0) return (ret); return (0); @@ -716,32 +716,32 @@ kvs_session_create(WT_DATA_SOURCE *wtds, DB *db; DBTYPE type; WT_CONFIG_ITEM v; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; const char *name; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* Get the object name */ - if ((ret = uri2name(wtext, session, uri, &name)) != 0) + if ((ret = uri2name(wt_api, session, uri, &name)) != 0) return (ret); /* Check key/value formats */ if ((ret = - wtext->config_get(wtext, session, config, "key_format", &v)) != 0) - ERET(wtext, session, ret, + wt_api->config_get(wt_api, session, config, "key_format", &v)) != 0) + ERET(wt_api, session, ret, "key_format configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); type = v.len == 1 && v.str[0] == 'r' ? DB_RECNO : DB_BTREE; /* Create the Berkeley DB table */ if ((ret = db_create(&db, ds->dbenv, 0)) != 0) - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "db_create: %s", db_strerror(ret)); if ((ret = db->open(db, NULL, name, NULL, type, DB_CREATE, 0)) != 0) - ERET(wtext, + ERET(wt_api, session, WT_ERROR, "Db.open: %s", uri, db_strerror(ret)); if ((ret = db->close(db, 0)) != 0) - ERET(wtext, session, WT_ERROR, "Db.close", db_strerror(ret)); + ERET(wt_api, session, WT_ERROR, "Db.close", db_strerror(ret)); return (0); } @@ -752,30 +752,30 @@ kvs_session_drop(WT_DATA_SOURCE *wtds, { DB *db; DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; const char *name; (void)config; /* Unused parameters */ ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* Get the object name */ - if ((ret = uri2name(wtext, session, uri, &name)) != 0) + if ((ret = uri2name(wt_api, session, uri, &name)) != 0) return (ret); if ((ret = single_thread(wtds, session, &ds->rwlock)) != 0) return (ret); if ((ret = db_create(&db, ds->dbenv, 0)) != 0) - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "db_create: %s", db_strerror(ret)); else if ((ret = db->remove(db, name, NULL, 0)) != 0) - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "Db.remove: %s", db_strerror(ret)); /* db handle is dead */ - ETRET(unlock(wtext, session, &ds->rwlock)); + ETRET(unlock(wt_api, session, &ds->rwlock)); return (ret); } @@ -787,77 +787,77 @@ kvs_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, DATA_SOURCE *ds; DB *db; WT_CONFIG_ITEM v; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int locked, ret; const char *name; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; locked = 0; /* Get the object name */ - if ((ret = uri2name(wtext, session, uri, &name)) != 0) + if ((ret = uri2name(wt_api, session, uri, &name)) != 0) return (ret); /* Allocate the cursor */ if ((cursor = calloc(1, sizeof(CURSOR_SOURCE))) == NULL) return (os_errno()); cursor->ds = (DATA_SOURCE *)wtds; - cursor->wtext = wtext; + cursor->wt_api = wt_api; /* Parse configuration */ - if ((ret = wtext->config_get( - wtext, session, config, "append", &v)) != 0) { - ESET(wtext, session, ret, + if ((ret = wt_api->config_get( + wt_api, session, config, "append", &v)) != 0) { + ESET(wt_api, session, ret, "append configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); goto err; } cursor->config_append = v.val != 0; - if ((ret = wtext->config_get( - wtext, session, config, "overwrite", &v)) != 0) { - ESET(wtext, session, ret, + if ((ret = wt_api->config_get( + wt_api, session, config, "overwrite", &v)) != 0) { + ESET(wt_api, session, ret, "overwrite configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); goto err; } cursor->config_overwrite = v.val != 0; - if ((ret = wtext->config_get( - wtext, session, config, "key_format", &v)) != 0) { - ESET(wtext, session, ret, + if ((ret = wt_api->config_get( + wt_api, session, config, "key_format", &v)) != 0) { + ESET(wt_api, session, ret, "key_format configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); goto err; } cursor->config_recno = v.len == 1 && v.str[0] == 'r'; - if ((ret = wtext->config_get( - wtext, session, config, "value_format", &v)) != 0) { - ESET(wtext, session, ret, + if ((ret = wt_api->config_get( + wt_api, session, config, "value_format", &v)) != 0) { + ESET(wt_api, session, ret, "value_format configuration: %s", - wtext->strerror(wtext, session, ret)); + wt_api->strerror(wt_api, session, ret)); goto err; } cursor->config_bitfield = v.len == 2 && isdigit((u_char)v.str[0]) && v.str[1] == 't'; - if ((ret = writelock(wtext, session, &ds->rwlock)) != 0) + if ((ret = writelock(wt_api, session, &ds->rwlock)) != 0) goto err; locked = 1; /* Open the Berkeley DB cursor */ if ((ret = db_create(&cursor->db, ds->dbenv, 0)) != 0) { - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "db_create: %s", db_strerror(ret)); goto err; } db = cursor->db; if ((ret = db->open(db, NULL, name, NULL, cursor->config_recno ? DB_RECNO : DB_BTREE, DB_CREATE, 0)) != 0) { - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "Db.open: %s", db_strerror(ret)); goto err; } if ((ret = db->cursor(db, NULL, &cursor->dbc, 0)) != 0) { - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "Db.cursor: %s", db_strerror(ret)); goto err; } @@ -882,7 +882,7 @@ err: free(cursor); } if (locked) - ETRET(unlock(wtext, session, &ds->rwlock)); + ETRET(unlock(wt_api, session, &ds->rwlock)); return (ret); } @@ -892,30 +892,30 @@ kvs_session_rename(WT_DATA_SOURCE *wtds, WT_SESSION *session, { DATA_SOURCE *ds; DB *db; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; const char *name; (void)config; /* Unused parameters */ ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* Get the object name */ - if ((ret = uri2name(wtext, session, uri, &name)) != 0) + if ((ret = uri2name(wt_api, session, uri, &name)) != 0) return (ret); if ((ret = single_thread(wtds, session, &ds->rwlock)) != 0) return (ret); if ((ret = db_create(&db, ds->dbenv, 0)) != 0) - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "db_create: %s", db_strerror(ret)); else if ((ret = db->rename(db, name, NULL, newname, 0)) != 0) - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "Db.rename: %s", db_strerror(ret)); /* db handle is dead */ - ETRET(unlock(wtext, session, &ds->rwlock)); + ETRET(unlock(wt_api, session, &ds->rwlock)); return (ret); } @@ -925,35 +925,35 @@ kvs_session_truncate(WT_DATA_SOURCE *wtds, { DATA_SOURCE *ds; DB *db; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int tret, ret = 0; const char *name; (void)config; /* Unused parameters */ ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* Get the object name */ - if ((ret = uri2name(wtext, session, uri, &name)) != 0) + if ((ret = uri2name(wt_api, session, uri, &name)) != 0) return (ret); if ((ret = single_thread(wtds, session, &ds->rwlock)) != 0) return (ret); if ((ret = db_create(&db, ds->dbenv, 0)) != 0) - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "db_create: %s", db_strerror(ret)); else { if ((ret = db->open(db, NULL, name, NULL, DB_UNKNOWN, DB_TRUNCATE, 0)) != 0) - ESET(wtext, session, WT_ERROR, + ESET(wt_api, session, WT_ERROR, "Db.open: %s", db_strerror(ret)); if ((tret = db->close(db, 0)) != 0) - ESET(wtext, session, WT_ERROR, + ESET(wt_api, session, WT_ERROR, "Db.close: %s", db_strerror(tret)); } - ETRET(unlock(wtext, session, &ds->rwlock)); + ETRET(unlock(wt_api, session, &ds->rwlock)); return (ret); } @@ -963,30 +963,30 @@ kvs_session_verify(WT_DATA_SOURCE *wtds, { DATA_SOURCE *ds; DB *db; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; const char *name; (void)config; /* Unused parameters */ ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; /* Get the object name */ - if ((ret = uri2name(wtext, session, uri, &name)) != 0) + if ((ret = uri2name(wt_api, session, uri, &name)) != 0) return (ret); if ((ret = single_thread(wtds, session, &ds->rwlock)) != 0) return (ret); if ((ret = db_create(&db, ds->dbenv, 0)) != 0) - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "db_create: %s", db_strerror(ret)); else if ((ret = db->verify(db, name, NULL, NULL, 0)) != 0) - ESET(wtext, session, WT_ERROR, + ESET(wt_api, session, WT_ERROR, "Db.verify: %s: %s", uri, db_strerror(ret)); /* db handle is dead */ - ETRET(unlock(wtext, session, &ds->rwlock)); + ETRET(unlock(wt_api, session, &ds->rwlock)); return (ret); } @@ -995,18 +995,18 @@ kvs_terminate(WT_DATA_SOURCE *wtds, WT_SESSION *session) { DB_ENV *dbenv; DATA_SOURCE *ds; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; int ret = 0; ds = (DATA_SOURCE *)wtds; - wtext = ds->wtext; + wt_api = ds->wt_api; dbenv = ds->dbenv; if (dbenv != NULL && (ret = dbenv->close(dbenv, 0)) != 0) - ESET(wtext, + ESET(wt_api, session, WT_ERROR, "DbEnv.close: %s", db_strerror(ret)); - ETRET(lock_destroy(wtext, session, &ds->rwlock)); + ETRET(lock_destroy(wt_api, session, &ds->rwlock)); return (ret); } @@ -1033,7 +1033,7 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) }; DATA_SOURCE *ds; DB_ENV *dbenv; - WT_EXTENSION_API *wtext; + WT_EXTENSION_API *wt_api; size_t len; int ret = 0; const char *home; @@ -1045,21 +1045,21 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) dbenv = NULL; path = NULL; /* Acquire the extension API */ - wtext = connection->get_extension_api(connection); + wt_api = connection->get_extension_api(connection); /* Allocate the local data-source structure. */ if ((ds = calloc(1, sizeof(DATA_SOURCE))) == NULL) return (os_errno()); - ds->wtext = wtext; + ds->wt_api = wt_api; /* Configure the global lock */ - if ((ret = lock_init(wtext, NULL, &ds->rwlock)) != 0) + if ((ret = lock_init(wt_api, NULL, &ds->rwlock)) != 0) goto err; ds->wtds = wtds; /* Configure the methods */ /* Berkeley DB environment */ if ((ret = db_env_create(&dbenv, 0)) != 0) { - ESET(wtext, + ESET(wt_api, NULL, WT_ERROR, "db_env_create: %s", db_strerror(ret)); goto err; } @@ -1073,7 +1073,8 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) (void)snprintf(path, len, "%s/KVS", home); if ((ret = dbenv->open(dbenv, path, DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL | DB_PRIVATE, 0)) != 0) { - ESET(wtext, NULL, WT_ERROR, "DbEnv.open: %s", db_strerror(ret)); + ESET(wt_api, + NULL, WT_ERROR, "DbEnv.open: %s", db_strerror(ret)); goto err; } ds->dbenv = dbenv; @@ -1081,7 +1082,7 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) if ((ret = /* Add the data source */ connection->add_data_source( connection, "kvsbdb:", (WT_DATA_SOURCE *)ds, NULL)) != 0) { - ESET(wtext, NULL, ret, "WT_CONNECTION.add_data_source"); + ESET(wt_api, NULL, ret, "WT_CONNECTION.add_data_source"); goto err; } diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c index bac4dc01bae..54bcb7cd26c 100644 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ b/src/third_party/wiredtiger/src/async/async_api.c @@ -144,7 +144,7 @@ retry: */ if (op == NULL || op->state != WT_ASYNCOP_FREE) { WT_STAT_CONN_INCR(session, async_full); - WT_RET(EBUSY); + return (EBUSY); } /* * Set the state of this op handle as READY for the user to use. @@ -232,7 +232,7 @@ __async_start(WT_SESSION_IMPL *session) uint32_t i, session_flags; conn = S2C(session); - conn->async_cfg = 1; + conn->async_cfg = true; /* * Async is on, allocate the WT_ASYNC structure and initialize the ops. */ @@ -339,16 +339,16 @@ __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) * 2. If async is off, and the user wants it on, start it. * 3. If not a toggle and async is off, we're done. */ - if (conn->async_cfg > 0 && !run) { + if (conn->async_cfg && !run) { /* Case 1 */ WT_TRET(__wt_async_flush(session)); ret = __wt_async_destroy(session); - conn->async_cfg = 0; + conn->async_cfg = false; return (ret); - } else if (conn->async_cfg == 0 && run) + } else if (!conn->async_cfg && run) /* Case 2 */ return (__async_start(session)); - else if (conn->async_cfg == 0) + else if (!conn->async_cfg) /* Case 3 */ return (0); @@ -599,7 +599,8 @@ __wt_async_new_op(WT_SESSION_IMPL *session, const char *uri, conn = S2C(session); if (!conn->async_cfg) - return (ENOTSUP); + WT_RET_MSG( + session, ENOTSUP, "Asynchronous operations not configured"); op = NULL; WT_ERR(__async_new_op_alloc(session, uri, config, &op)); diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index b02fea8707d..26acc8c560f 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -344,7 +344,8 @@ __block_off_remove(WT_SESSION_IMPL *session, WT_BLOCK *block, __block_size_srch(el->sz, ext->size, sstack); szp = *sstack[0]; if (szp == NULL || szp->size != ext->size) - return (EINVAL); + WT_PANIC_RET(session, EINVAL, + "extent not found in by-size list during remove"); __block_off_srch(szp->off, off, astack, true); ext = *astack[0]; if (ext == NULL || ext->off != off) diff --git a/src/third_party/wiredtiger/src/bloom/bloom.c b/src/third_party/wiredtiger/src/bloom/bloom.c index 3afada05dda..dde08d8a982 100644 --- a/src/third_party/wiredtiger/src/bloom/bloom.c +++ b/src/third_party/wiredtiger/src/bloom/bloom.c @@ -65,7 +65,9 @@ __bloom_setup( WT_BLOOM *bloom, uint64_t n, uint64_t m, uint32_t factor, uint32_t k) { if (k < 2) - return (EINVAL); + WT_RET_MSG(bloom->session, EINVAL, + "bloom filter hash values to be set/tested must be " + "greater than 2"); bloom->k = k; bloom->factor = factor; @@ -342,7 +344,12 @@ __wt_bloom_intersection(WT_BLOOM *bloom, WT_BLOOM *other) if (bloom->k != other->k || bloom->factor != other->factor || bloom->m != other->m || bloom->n != other->n) - return (EINVAL); + WT_RET_MSG(bloom->session, EINVAL, + "bloom filter intersection configuration mismatch: (" + "%" PRIu32 "/%" PRIu32 ", %" PRIu32 "/%" PRIu32 ", " + "%" PRIu64 "/%" PRIu64 ", %" PRIu64 "/%" PRIu64 ")", + bloom->k, other->k, bloom->factor, other->factor, + bloom->m, other->m, bloom->n, other->n); nbytes = __bitstr_size(bloom->m); for (i = 0; i < nbytes; i++) diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 41ae457b0fe..0454ea7c20e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -37,21 +37,20 @@ __cursor_size_chk(WT_SESSION_IMPL *session, WT_ITEM *kv) if (kv->size <= WT_GIGABYTE) return (0); - /* - * There are two checks: what we are willing to store in the tree, and - * what the block manager can actually write. - */ + /* Check what we are willing to store in the tree. */ if (kv->size > WT_BTREE_MAX_OBJECT_SIZE) - ret = EINVAL; - else { - size = kv->size; - ret = bm->write_size(bm, session, &size); - } - if (ret != 0) - WT_RET_MSG(session, ret, + WT_RET_MSG(session, EINVAL, "item size of %" WT_SIZET_FMT " exceeds the maximum " - "supported size", + "supported WiredTiger size of %d", + kv->size, WT_BTREE_MAX_OBJECT_SIZE); + + /* Check what the block manager can actually write. */ + size = kv->size; + if ((ret = bm->write_size(bm, session, &size)) != 0) + WT_RET_MSG(session, ret, + "item size of %" WT_SIZET_FMT " refused by block manager", kv->size); + return (0); } @@ -858,7 +857,8 @@ __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) * value from a column-store, if there were any reason to do so. */ if (btree->type != BTREE_ROW) - WT_RET(ENOTSUP); + WT_RET_MSG(session, ENOTSUP, + "WT_CURSOR.next_random only supported by row-store tables"); WT_STAT_CONN_INCR(session, cursor_next); WT_STAT_DATA_INCR(session, cursor_next); diff --git a/src/third_party/wiredtiger/src/config/config_api.c b/src/third_party/wiredtiger/src/config/config_api.c index 8820b81a410..76a51903588 100644 --- a/src/third_party/wiredtiger/src/config/config_api.c +++ b/src/third_party/wiredtiger/src/config/config_api.c @@ -19,9 +19,6 @@ __config_parser_close(WT_CONFIG_PARSER *wt_config_parser) config_parser = (WT_CONFIG_PARSER_IMPL *)wt_config_parser; - if (config_parser == NULL) - return (EINVAL); - __wt_free(config_parser->session, config_parser); return (0); } @@ -38,9 +35,6 @@ __config_parser_get(WT_CONFIG_PARSER *wt_config_parser, config_parser = (WT_CONFIG_PARSER_IMPL *)wt_config_parser; - if (config_parser == NULL) - return (EINVAL); - return (__wt_config_subgets(config_parser->session, &config_parser->config_item, key, cval)); } @@ -57,9 +51,6 @@ __config_parser_next(WT_CONFIG_PARSER *wt_config_parser, config_parser = (WT_CONFIG_PARSER_IMPL *)wt_config_parser; - if (config_parser == NULL) - return (EINVAL); - return (__wt_config_next(&config_parser->config, key, cval)); } diff --git a/src/third_party/wiredtiger/src/config/config_check.c b/src/third_party/wiredtiger/src/config/config_check.c index d0aa09a96c1..2f372651cb9 100644 --- a/src/third_party/wiredtiger/src/config/config_check.c +++ b/src/third_party/wiredtiger/src/config/config_check.c @@ -128,8 +128,7 @@ config_check(WT_SESSION_IMPL *session, badtype = false; } else WT_RET_MSG(session, EINVAL, - "unknown configuration type: '%s'", - checks[i].type); + "unknown configuration type: '%s'", checks[i].type); if (badtype) WT_RET_MSG(session, EINVAL, diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 018cc7a8ac4..d57bc418c93 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -299,6 +299,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_drop[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_join[] = { { "bloom_bit_count", "int", NULL, "min=2,max=1000", NULL, 0 }, + { "bloom_false_positives", "boolean", NULL, NULL, NULL, 0 }, { "bloom_hash_count", "int", NULL, "min=2,max=100", NULL, 0 }, { "compare", "string", NULL, "choices=[\"eq\",\"ge\",\"gt\",\"le\",\"lt\"]", @@ -1090,9 +1091,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_SESSION_drop, 4 }, { "WT_SESSION.join", - "bloom_bit_count=16,bloom_hash_count=8,compare=\"eq\",count=," - "operation=\"and\",strategy=", - confchk_WT_SESSION_join, 6 + "bloom_bit_count=16,bloom_false_positives=false," + "bloom_hash_count=8,compare=\"eq\",count=,operation=\"and\"," + "strategy=", + confchk_WT_SESSION_join, 7 }, { "WT_SESSION.log_flush", "sync=on", diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 04c29e957a3..a6e0f57f02e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1451,7 +1451,7 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_FH *fh; size_t len; wt_off_t size; - bool bytelock, exist, is_create; + bool bytelock, exist, is_create, match; char buf[256]; conn = S2C(session); @@ -1477,13 +1477,14 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) * won't open a database in multiple threads, but we don't want to have * it fail the first time, but succeed the second. */ + match = false; TAILQ_FOREACH(t, &__wt_process.connqh, q) if (t->home != NULL && t != conn && strcmp(t->home, conn->home) == 0) { - ret = EBUSY; + match = true; break; } - if (ret != 0) + if (match) WT_ERR_MSG(session, EBUSY, "WiredTiger database is already being managed by another " "thread in this process"); diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index ec850c793cc..ac72e330b67 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -20,6 +20,7 @@ __conn_dhandle_destroy(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle) __wt_free(session, dhandle->checkpoint); __wt_free(session, dhandle->handle); __wt_spin_destroy(session, &dhandle->close_lock); + __wt_stat_dsrc_discard(session, dhandle); __wt_overwrite_and_free(session, dhandle); } @@ -54,8 +55,6 @@ __conn_dhandle_alloc(WT_SESSION_IMPL *session, WT_ERR(__wt_spin_init( session, &dhandle->close_lock, "data handle close")); - __wt_stat_dsrc_init(dhandle); - if (strcmp(uri, WT_METAFILE_URI) == 0) F_SET(dhandle, WT_DHANDLE_IS_METADATA); @@ -310,16 +309,15 @@ __wt_conn_btree_open( WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_CLOSING)); /* - * If the handle is already open, it has to be closed so it can - * be reopened with a new configuration. + * If the handle is already open, it has to be closed so it can be + * reopened with a new configuration. * - * This call can return EBUSY if there's an update in the - * object that's not yet globally visible. That's not a - * problem because it can only happen when we're switching from - * a normal handle to a "special" one, so we're returning EBUSY - * to an attempt to verify or do other special operations. The - * reverse won't happen because when the handle from a verify - * or other special operation is closed, there won't be updates + * This call can return EBUSY if there's an update in the object that's + * not yet globally visible. That's not a problem because it can only + * happen when we're switching from a normal handle to a "special" one, + * so we're returning EBUSY to an attempt to verify or do other special + * operations. The reverse won't happen because when the handle from a + * verify or other special operation is closed, there won't be updates * in the tree that can block the close. */ if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) @@ -332,6 +330,16 @@ __wt_conn_btree_open( /* Set any special flags on the handle. */ F_SET(btree, LF_MASK(WT_BTREE_SPECIAL_FLAGS)); + /* + * Allocate data-source statistics memory. We don't allocate that memory + * when allocating the data-handle because not all data handles need + * statistics (for example, handles used for checkpoint locking). If we + * are reopening the handle, then it may already have statistics memory, + * check to avoid the leak. + */ + if (dhandle->stat_array == NULL) + WT_ERR(__wt_stat_dsrc_init(session, dhandle)); + WT_ERR(__wt_btree_open(session, cfg)); /* @@ -554,8 +562,7 @@ __wt_conn_dhandle_discard_single( dhandle = session->dhandle; - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) || - (final && F_ISSET(dhandle, WT_DHANDLE_DEAD))) { + if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { tret = __wt_conn_btree_sync_and_close(session, final, force); if (final && tret != 0) { __wt_err(session, tret, diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 5104624523b..3571cc60115 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -48,7 +48,7 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_conn_config_init(session)); /* Statistics. */ - __wt_stat_connection_init(conn); + WT_RET(__wt_stat_connection_init(session, conn)); /* Spinlocks. */ WT_RET(__wt_spin_init(session, &conn->api_lock, "api")); @@ -161,6 +161,7 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_free(session, conn->home); __wt_free(session, conn->error_prefix); __wt_free(session, conn->sessions); + __wt_stat_connection_discard(session, conn); __wt_free(NULL, conn); return (ret); diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 34743034877..8198b3a1a02 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -87,14 +87,16 @@ __logmgr_config( if (reconfig && ((enabled && !FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) || (!enabled && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)))) - return (EINVAL); + WT_RET_MSG(session, EINVAL, + "log manager reconfigure: enabled mismatch with existing " + "setting"); /* Logging is incompatible with in-memory */ if (enabled) { WT_RET(__wt_config_gets(session, cfg, "in_memory", &cval)); if (cval.val != 0) WT_RET_MSG(session, EINVAL, - "In memory configuration incompatible with " + "In-memory configuration incompatible with " "log=(enabled=true)"); } @@ -137,8 +139,7 @@ __logmgr_config( if (!reconfig) { WT_RET(__wt_config_gets(session, cfg, "log.file_max", &cval)); conn->log_file_max = (wt_off_t)cval.val; - WT_STAT_CONN_SET(session, - log_max_filesize, conn->log_file_max); + WT_STAT_CONN_SET(session, log_max_filesize, conn->log_file_max); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index 0715a035807..3bcdfd7ecb1 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -469,12 +469,12 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) } /* - * __wt_statlog_log_one -- - * Log a set of statistics into the configured statistics log. Requires - * that the server is not currently running. + * __statlog_on_close -- + * Log a set of statistics at close. Requires the server is not currently + * running. */ -int -__wt_statlog_log_one(WT_SESSION_IMPL *session) +static int +__statlog_on_close(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -643,7 +643,7 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) /* Log a set of statistics on shutdown if configured. */ if (is_close) - WT_TRET(__wt_statlog_log_one(session)); + WT_TRET(__statlog_on_close(session)); /* Discard all configuration information. */ WT_TRET(__stat_config_discard(session)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 3a3ff7de92b..3585082644f 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -31,13 +31,13 @@ __curbackup_next(WT_CURSOR *cursor) cb = (WT_CURSOR_BACKUP *)cursor; CURSOR_API_CALL(cursor, session, next, NULL); - if (cb->list == NULL || cb->list[cb->next].name == NULL) { + if (cb->list == NULL || cb->list[cb->next] == NULL) { F_CLR(cursor, WT_CURSTD_KEY_SET); WT_ERR(WT_NOTFOUND); } - cb->iface.key.data = cb->list[cb->next].name; - cb->iface.key.size = strlen(cb->list[cb->next].name) + 1; + cb->iface.key.data = cb->list[cb->next]; + cb->iface.key.size = strlen(cb->list[cb->next]) + 1; ++cb->next; F_SET(cursor, WT_CURSTD_KEY_INT); @@ -226,9 +226,13 @@ __backup_start( * holds the lock until it's finished the checkpoint, otherwise we * could start a hot backup that would race with an already-started * checkpoint. + * + * We are holding the checkpoint and schema locks so schema operations + * will not see the backup file list until it is complete and valid. */ __wt_writelock(session, conn->hot_backup_lock); conn->hot_backup = true; + conn->hot_backup_list = NULL; __wt_writeunlock(session, conn->hot_backup_lock); /* We're the lock holder, we own cleanup. */ @@ -293,37 +297,15 @@ err: /* Close the hot backup file. */ if (ret == 0) { WT_ASSERT(session, dest != NULL); WT_TRET(__wt_fs_rename(session, WT_BACKUP_TMP, dest, false)); + __wt_writelock(session, conn->hot_backup_lock); + conn->hot_backup_list = cb->list; + __wt_writeunlock(session, conn->hot_backup_lock); } return (ret); } /* - * __backup_cleanup_handles -- - * Release and free all btree handles held by the backup. - */ -static int -__backup_cleanup_handles(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) -{ - WT_CURSOR_BACKUP_ENTRY *p; - WT_DECL_RET; - - if (cb->list == NULL) - return (0); - - /* Release the handles, free the file names, free the list itself. */ - for (p = cb->list; p->name != NULL; ++p) { - if (p->handle != NULL) - WT_WITH_DHANDLE(session, p->handle, - WT_TRET(__wt_session_release_btree(session))); - __wt_free(session, p->name); - } - - __wt_free(session, cb->list); - return (ret); -} - -/* * __backup_stop -- * Stop a backup. */ @@ -332,11 +314,19 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; + int i; conn = S2C(session); - /* Release all btree handles held by the backup. */ - WT_TRET(__backup_cleanup_handles(session, cb)); + /* Release all btree names held by the backup. */ + __wt_writelock(session, conn->hot_backup_lock); + conn->hot_backup_list = NULL; + __wt_writeunlock(session, conn->hot_backup_lock); + if (cb->list != NULL) { + for (i = 0; cb->list[i] != NULL; ++i) + __wt_free(session, cb->list[i]); + __wt_free(session, cb->list); + } /* Remove any backup specific file. */ WT_TRET(__wt_backup_file_remove(session)); @@ -513,40 +503,23 @@ static int __backup_list_append( WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *uri) { - WT_CURSOR_BACKUP_ENTRY *p; - WT_DATA_HANDLE *old_dhandle; - WT_DECL_RET; + char **p; const char *name; /* Leave a NULL at the end to mark the end of the list. */ WT_RET(__wt_realloc_def(session, &cb->list_allocated, cb->list_next + 2, &cb->list)); p = &cb->list[cb->list_next]; - p[0].name = p[1].name = NULL; - p[0].handle = p[1].handle = NULL; + p[0] = p[1] = NULL; name = uri; /* - * If it's a file in the database, get a handle for the underlying - * object (this handle blocks schema level operations, for example - * WT_SESSION.drop or an LSM file discard after level merging). - * - * If the handle is busy (e.g., it is being bulk-loaded), silently skip - * it. We have a special fake checkpoint in the metadata, and recovery - * will recreate an empty file. + * If it's a file in the database we need to remove the prefix. */ - if (WT_PREFIX_MATCH(uri, "file:")) { + if (WT_PREFIX_MATCH(uri, "file:")) name += strlen("file:"); - old_dhandle = session->dhandle; - ret = __wt_session_get_btree(session, uri, NULL, NULL, 0); - p->handle = session->dhandle; - session->dhandle = old_dhandle; - if (ret != 0) - return (ret == EBUSY ? 0 : ret); - } - /* * !!! * Assumes metadata file entries map one-to-one to physical files. @@ -556,7 +529,7 @@ __backup_list_append( * that for now, that block manager might not even support physical * copying of files by applications. */ - WT_RET(__wt_strdup(session, name, &p->name)); + WT_RET(__wt_strdup(session, name, p)); ++cb->list_next; return (0); diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index ea742cac435..eb5e15ae5c3 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -38,7 +38,9 @@ __curindex_set_value(WT_CURSOR *cursor, ...) WT_SESSION_IMPL *session; JOINABLE_CURSOR_API_CALL(cursor, session, set_value, NULL); - ret = ENOTSUP; + WT_ERR_MSG(session, ENOTSUP, + "WT_CURSOR.set_value not supported for index cursors"); + err: cursor->saved_err = ret; F_CLR(cursor, WT_CURSTD_VALUE_SET); API_END(session, ret); diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 087411febda..806436ebf38 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -38,8 +38,8 @@ __wt_curjoin_joined(WT_CURSOR *cursor) WT_SESSION_IMPL *session; session = (WT_SESSION_IMPL *)cursor->session; - __wt_errx(session, "cursor is being used in a join"); - return (ENOTSUP); + + WT_RET_MSG(session, ENOTSUP, "cursor is being used in a join"); } /* @@ -613,8 +613,8 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, if (entry->bloom != NULL) { /* * If the item is not in the Bloom filter, we return - * immediately, otherwise, we still need to check the long - * way, since it may be a false positive. + * immediately, otherwise, we still may need to check the + * long way, since it may be a false positive. * * If we don't own the Bloom filter, we must be sharing one * in a previous entry. So the shared filter has already @@ -623,6 +623,8 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, */ if (F_ISSET(entry, WT_CURJOIN_ENTRY_OWN_BLOOM)) WT_ERR(__wt_bloom_inmem_get(entry->bloom, key)); + if (F_ISSET(entry, WT_CURJOIN_ENTRY_FALSE_POSITIVES)) + return (0); bloom_found = true; } if (entry->subjoin != NULL) { @@ -671,6 +673,8 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, extract_cursor.entry = entry; WT_ERR(idx->extractor->extract(idx->extractor, &session->iface, key, &v, &extract_cursor.iface)); + __wt_buf_free(session, &extract_cursor.iface.key); + __wt_buf_free(session, &extract_cursor.iface.value); if (!extract_cursor.ismember) WT_ERR(WT_NOTFOUND); } else @@ -1302,11 +1306,15 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_JOIN, iface) == 0); + if (owner != NULL) + WT_RET_MSG(session, EINVAL, + "unable to initialize a join cursor with existing owner"); + if (!WT_PREFIX_SKIP(uri, "join:")) - return (EINVAL); + return (__wt_unexpected_object_type(session, uri, "join:")); tablename = uri; if (!WT_PREFIX_SKIP(tablename, "table:")) - return (EINVAL); + return (__wt_unexpected_object_type(session, uri, "table:")); columns = strchr(tablename, '('); if (columns == NULL) @@ -1334,9 +1342,6 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, WT_ERR(__wt_strdup(session, columns, &cjoin->projection)); } - if (owner != NULL) - WT_ERR(EINVAL); - WT_ERR(__wt_cursor_init(cursor, uri, owner, cfg, cursorp)); if (0) { @@ -1442,6 +1447,11 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_RET_MSG(session, EINVAL, "join has incompatible strategy " "values for the same index"); + if (LF_MASK(WT_CURJOIN_ENTRY_FALSE_POSITIVES) != + F_MASK(entry, WT_CURJOIN_ENTRY_FALSE_POSITIVES)) + WT_RET_MSG(session, EINVAL, + "join has incompatible bloom_false_positives " + "values for the same index"); /* * Check against other comparisons (we call them endpoints) diff --git a/src/third_party/wiredtiger/src/cursor/cur_json.c b/src/third_party/wiredtiger/src/cursor/cur_json.c index bb24e88cba5..093ec3c59ac 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_json.c +++ b/src/third_party/wiredtiger/src/cursor/cur_json.c @@ -492,11 +492,9 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, uc = (const u_char *)src; if (__wt_hex2byte(&uc[1], &ignored) || - __wt_hex2byte(&uc[3], &ignored)) { - __wt_errx(session, + __wt_hex2byte(&uc[3], &ignored)) + WT_RET_MSG(session, EINVAL, "invalid Unicode within JSON string"); - return (-1); - } src += 4; } backslash = false; @@ -569,7 +567,10 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, } *toklen = (size_t)(src - *tokstart); *toktype = result; - return (result < 0 ? EINVAL : 0); + + if (result < 0) + WT_RET_MSG(session, EINVAL, "illegal token in JSON"); + return (0); } /* @@ -606,24 +607,20 @@ __wt_json_tokname(int toktype) static int json_string_arg(WT_SESSION_IMPL *session, const char **jstr, WT_ITEM *item) { - WT_DECL_RET; int tok; const char *tokstart; - WT_RET(__wt_json_token((WT_SESSION *)session, *jstr, &tok, &tokstart, - &item->size)); + WT_RET(__wt_json_token( + (WT_SESSION *)session, *jstr, &tok, &tokstart, &item->size)); if (tok == 's') { *jstr = tokstart + item->size; /* The tokenizer includes the '"' chars */ item->data = tokstart + 1; item->size -= 2; - ret = 0; - } else { - __wt_errx(session, "expected JSON <string>, got %s", - __wt_json_tokname(tok)); - ret = EINVAL; - } - return (ret); + } else + WT_RET_MSG(session, EINVAL, + "expected JSON <string>, got %s", __wt_json_tokname(tok)); + return (0); } /* @@ -648,11 +645,9 @@ json_int_arg(WT_SESSION_IMPL *session, const char **jstr, int64_t *ip) WT_RET_MSG(session, EINVAL, "JSON <int> extraneous input"); *jstr = tokstart + toksize; - } else { - __wt_errx(session, "expected JSON <int>, got %s", - __wt_json_tokname(tok)); - return (EINVAL); - } + } else + WT_RET_MSG(session, EINVAL, + "expected JSON <int>, got %s", __wt_json_tokname(tok)); return (0); } @@ -678,22 +673,20 @@ json_uint_arg(WT_SESSION_IMPL *session, const char **jstr, uint64_t *up) WT_RET_MSG(session, EINVAL, "JSON <int> extraneous input"); *jstr = tokstart + toksize; - } else { - __wt_errx(session, "expected unsigned JSON <int>, got %s", + } else + WT_RET_MSG(session, EINVAL, + "expected unsigned JSON <int>, got %s", __wt_json_tokname(tok)); - return (EINVAL); - } return (0); } #define JSON_EXPECT_TOKEN_GET(session, jstr, tokval, start, sz) do { \ int __tok; \ WT_RET(__wt_json_token((WT_SESSION *)session, jstr, &__tok, &start, &sz));\ - if (__tok != tokval) { \ - __wt_errx(session, "expected JSON %s, got %s", \ + if (__tok != tokval) \ + WT_RET_MSG(session, EINVAL, \ + "expected JSON %s, got %s", \ __wt_json_tokname(tokval), __wt_json_tokname(__tok)); \ - return (EINVAL); \ - } \ jstr = start + sz; \ } while (0) @@ -782,11 +775,10 @@ __json_pack_size( JSON_EXPECT_TOKEN_GET(session, jstr, 's', tokstart, toksize); WT_RET(__pack_name_next(&packname, &name)); if (toksize - 2 != name.len || - strncmp(tokstart + 1, name.str, toksize - 2) != 0) { - __wt_errx(session, "JSON expected %s name: \"%.*s\"", + strncmp(tokstart + 1, name.str, toksize - 2) != 0) + WT_RET_MSG(session, EINVAL, + "JSON expected %s name: \"%.*s\"", iskey ? "key" : "value", (int)name.len, name.str); - return (EINVAL); - } JSON_EXPECT_TOKEN(session, jstr, ':'); WT_PACK_JSON_GET(session, pv, jstr); total += __pack_size(session, &pv); @@ -861,9 +853,8 @@ __wt_json_strlen(const char *src, size_t srclen) /* * __wt_json_strncpy -- - * Copy bytes of string in JSON format to a destination, - * up to dstlen bytes. If dstlen is greater than the needed size, - * the result if zero padded. + * Copy bytes of string in JSON format to a destination, up to dstlen + * bytes. If dstlen is greater than the needed size, the result if zero padded. */ int __wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, @@ -884,18 +875,16 @@ __wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, if ((ch = *src++) == '\\') switch (ch = *src++) { case 'u': - if (__wt_hex2byte((const u_char *)src, &hi)) - return (EINVAL); - src += 2; - if (__wt_hex2byte((const u_char *)src, &lo)) - return (EINVAL); - src += 2; - if (hi != 0) { - __wt_errx(NULL, "Unicode \"%6.6s\"" - " byte out of range in JSON", + if (__wt_hex2byte((const u_char *)src, &hi) || + __wt_hex2byte((const u_char *)src + 2, &lo)) + WT_RET_MSG(session, EINVAL, + "invalid Unicode within JSON string"); + src += 4; + if (hi != 0) + WT_RET_MSG(session, EINVAL, + "Unicode \"%6.6s\" byte out of " + "range in JSON", src - 6); - return (EINVAL); - } *dst++ = (char)lo; break; case 'f': @@ -920,7 +909,8 @@ __wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, *dst++ = ch; } if (src != srcend) - return (ENOMEM); + WT_RET_MSG(session, + ENOMEM, "JSON string copy destination buffer too small"); *pdst = dst; while (dst < dstend) *dst++ = '\0'; diff --git a/src/third_party/wiredtiger/src/cursor/cur_metadata.c b/src/third_party/wiredtiger/src/cursor/cur_metadata.c index fd00acdf0ab..10e2fdf28be 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_metadata.c +++ b/src/third_party/wiredtiger/src/cursor/cur_metadata.c @@ -48,7 +48,10 @@ __schema_source_config(WT_SESSION_IMPL *session, WT_ERR(__wt_buf_fmt(session, buf, "%.*s", (int)cval.len, cval.str)); srch->set_key(srch, buf->data); if ((ret = srch->search(srch)) == WT_NOTFOUND) - WT_ERR(EINVAL); + WT_ERR_MSG(session, EINVAL, + "metadata information for source configuration \"%s\" " + "not found", + (char *)buf->data); WT_ERR(ret); WT_ERR(srch->get_value(srch, &v)); WT_ERR(__wt_strdup(session, v, result)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index b36416debe1..5fde64c74ca 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -383,7 +383,8 @@ __curstat_file_init(WT_SESSION_IMPL *session, if (F_ISSET(cst, WT_STAT_TYPE_SIZE)) { filename = uri; if (!WT_PREFIX_SKIP(filename, "file:")) - return (EINVAL); + return ( + __wt_unexpected_object_type(session, uri, "file:")); __wt_stat_dsrc_init_single(&cst->u.dsrc_stats); WT_RET(__wt_block_manager_named_size(session, filename, &size)); cst->u.dsrc_stats.block_size = size; diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 6543d54e90f..502d93639a8 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -892,7 +892,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, tablename = uri; if (!WT_PREFIX_SKIP(tablename, "table:")) - return (EINVAL); + return (__wt_unexpected_object_type(session, uri, "table:")); columns = strchr(tablename, '('); if (columns == NULL) size = strlen(tablename); diff --git a/src/third_party/wiredtiger/src/docs/file-formats.dox b/src/third_party/wiredtiger/src/docs/file-formats.dox index 8346024953a..d8990aca7a6 100644 --- a/src/third_party/wiredtiger/src/docs/file-formats.dox +++ b/src/third_party/wiredtiger/src/docs/file-formats.dox @@ -110,7 +110,7 @@ considered. (See @subpage_single huffman for details.) compressing blocks of the backing object's file. The cost is additional CPU and memory use when reading and writing pages to disk. Note the additional CPU cost of block compression can be high, and should be -considered. (See @ref compression for details.) +considered. (See @x_ref compression_formats for details.) Block compression is disabled by default. @@ -146,7 +146,7 @@ Huffman encoding can be high, and should be considered. compressing blocks of the backing object's file. The cost is additional CPU and memory use when reading and writing pages to disk. Note the additional CPU cost of block compression can be high, and should be -considered. (See @ref compression for details.) +considered. (See @x_ref compression_formats for details.) Block compression is disabled by default. @@ -157,7 +157,7 @@ compression: block compression. compressing blocks of the backing object's file. The cost is additional CPU and memory use when reading and writing pages to disk. Note the additional CPU cost of block compression can be high, and should be -considered. (See @ref compression for details.) +considered. (See @x_ref compression_formats for details.) Block compression is disabled by default. diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 4b1337f84b8..2413cbc93fb 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -346,6 +346,7 @@ nolock nolocking nommap nop +noraw nosql nosync notgranted diff --git a/src/third_party/wiredtiger/src/docs/style/header-web.html b/src/third_party/wiredtiger/src/docs/style/header-web.html new file mode 100644 index 00000000000..63b22bb763e --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/style/header-web.html @@ -0,0 +1,73 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> +<head> +<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> +<meta http-equiv="X-UA-Compatible" content="IE=9"/> +<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME--> +<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME--> +<link href="$relpath$tabs.css" rel="stylesheet" type="text/css"/> +<script type="text/javascript" src="$relpath$jquery.js"></script> +<script type="text/javascript" src="$relpath$dynsections.js"></script> +$treeview +$search +$mathjax +<link href="$relpath$$stylesheet" rel="stylesheet" type="text/css" /> +$extrastylesheet +</head> +<body> +<div id="top"><!-- do not remove this div, it is closed by doxygen! --> + +<!--BEGIN TITLEAREA--> +<div id="titlearea"> +<table cellspacing="0" cellpadding="0"> + <tbody> + <tr style="height: 56px;"> + <!--BEGIN PROJECT_LOGO--> + <td id="projectlogo"><a href="http://wiredtiger.com/"><img alt="Logo" src="$relpath$$projectlogo" alt="WiredTiger" /></a></td> + <!--END PROJECT_LOGO--> + <!--BEGIN PROJECT_NAME--> + <td style="padding-left: 0.5em;"> + <div id="projectname"> + <!--BEGIN PROJECT_NUMBER--><span id="projectnumber"><!--$projectnumber--></span><!--END PROJECT_NUMBER--> + <!--BEGIN PROJECT_BRIEF--> + <div id="projectbrief"> + + <script type="text/javascript"> + window.addEventListener("load",function(){ + document.getElementById("version_select").options.namedItem("$projectbrief").selected=true; + },false); + </script> + + <select id="version_select" onchange="window.location=window.location.href.replace('$projectbrief',this.value);"> + <option name="$projectbrief" value="$projectbrief">$projectnumber</option> + <option name="stable" value="stable">Latest stable release</option> + <option name="develop" value="develop">Current develop branch</option> + <option name="mongodb-3.2" value="mongodb-3.2">MongoDB 3.2</option> + <option name="mongodb-3.4" value="mongodb-3.4">MongoDB 3.4</option> + </select> + </div><!--END PROJECT_BRIEF--> + </div> + </td> + <!--END PROJECT_NAME--> + <!--BEGIN !PROJECT_NAME--> + <!--BEGIN PROJECT_BRIEF--> + <td style="padding-left: 0.5em;"> + <div id="projectbrief">$projectbrief</div> + </td> + <!--END PROJECT_BRIEF--> + <!--END !PROJECT_NAME--> + <!--BEGIN DISABLE_INDEX--> + <!--BEGIN SEARCHENGINE--> + <td>$searchbox</td> + <!--END SEARCHENGINE--> + <!--END DISABLE_INDEX--> + </tr> + </tbody> +</table> +</div> +<div class="banner"> + <a href="https://github.com/wiredtiger/wiredtiger">Fork me on GitHub</a> + <a class="last" href="http://groups.google.com/group/wiredtiger-users">Join my user group</a> +</div> +<!--END TITLEAREA--> +<!-- end header part --> diff --git a/src/third_party/wiredtiger/src/docs/style/header.html b/src/third_party/wiredtiger/src/docs/style/header.html index 0c045a750d1..422268b7a47 100644 --- a/src/third_party/wiredtiger/src/docs/style/header.html +++ b/src/third_party/wiredtiger/src/docs/style/header.html @@ -30,13 +30,13 @@ $extrastylesheet <div id="projectname"> <!--BEGIN PROJECT_NUMBER--> <span id="projectnumber">$projectnumber</span><!--END PROJECT_NUMBER--> </div> - <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF--> + <!--BEGIN PROJECT_BRIEF--><div id="projectbrief"><!-- $projectbrief --></div><!--END PROJECT_BRIEF--> </td> <!--END PROJECT_NAME--> <!--BEGIN !PROJECT_NAME--> <!--BEGIN PROJECT_BRIEF--> <td style="padding-left: 0.5em;"> - <div id="projectbrief">$projectbrief</div> + <div id="projectbrief"><!-- $projectbrief --></div> </td> <!--END PROJECT_BRIEF--> <!--END !PROJECT_NAME--> diff --git a/src/third_party/wiredtiger/src/docs/tools/doxfilter.py b/src/third_party/wiredtiger/src/docs/tools/doxfilter.py index b2d5f857df1..f1c3308c689 100755 --- a/src/third_party/wiredtiger/src/docs/tools/doxfilter.py +++ b/src/third_party/wiredtiger/src/docs/tools/doxfilter.py @@ -98,6 +98,9 @@ def process_lang(lang, lines): subpage_pat = re.compile(r'@subpage\s+(\w*)') subpage_rep = r'@subpage \1' + lang_suffix exref_pat = re.compile(r'@ex_ref{ex_([^.]*)[.]c}') + # Add some ability to have non-language references + x_ref_pat = re.compile(r'@x_ref\s+(\w*)') + x_ref_rep = r'@ref \1' if lang == 'c': exref_rep = r'@ex_ref{ex_\1' + lang_ext + '}' else: @@ -118,6 +121,7 @@ def process_lang(lang, lines): line = re.sub(snip_pat, snip_rep, line) line = re.sub(mpage_pat, mpage_rep, line) line = re.sub(subpage_pat, subpage_rep, line) + line = re.sub(x_ref_pat, x_ref_rep, line) if '@m_if' in line: m = re.search(mif_pat, line) if not m: diff --git a/src/third_party/wiredtiger/src/docs/tune-compression.dox b/src/third_party/wiredtiger/src/docs/tune-compression.dox index bb675337a0d..8db2151aa76 100644 --- a/src/third_party/wiredtiger/src/docs/tune-compression.dox +++ b/src/third_party/wiredtiger/src/docs/tune-compression.dox @@ -2,7 +2,7 @@ WiredTiger includes a number of optional compression techniques. Configuring compression generally decreases on-disk and in-memory resource requirements -and the amount of I/O, and increases CPU cost when rows are read and written. +and the amount of I/O, and increases CPU cost when data are read and written. Configuring compression may change application throughput. For example, in applications using solid-state drives (where I/O is less expensive), @@ -19,7 +19,44 @@ An example of turning on row-store or column-store dictionary compression: @snippet ex_all.c Configure dictionary compression on +@section compression_formats Block Compression Formats +WiredTiger provides two methods of compressing your data when using block +compression: the raw and noraw methods. These methods change how WiredTiger +works to fit data into the blocks that are stored on disk. + +@subsection noraw_compression Noraw Compression +Noraw compression is the traditional compression model where a fixed +amount of data is given to the compression system, then turned into a +compressed block of data. The amount of data chosen to compress is the +data needed to fill the uncompressed block. Thus when compressed, the block will +be smaller than the normal data size and the sizes written to disk will often +vary depending on how compressible the data being stored is. Algorithms +using noraw compression include zlib-noraw, lz4-noraw and snappy. + +@subsection raw_compression Raw Compression +WiredTiger's raw compression takes advantage of compressors that provide a +streaming compression API. Using the streaming API WiredTiger will try to fit +as much data as possible into one block. This means that blocks created +with raw compression should be of similar size. Using a streaming compression +method should also make for less overhead in compression, as the setup and +initial work for compressing is done fewer times compared to the amount of +data stored. Algorithms using raw compression include zlib, lz4. + +@subsection to_raw_or_noraw Choosing between Raw and Noraw Compression +When looking at which compression method to use the biggest consideration is +that raw compression will normally provide higher compression levels while +using more CPU for compression. + +An additional consideration is that raw compression may provide a performance +advantage in workloads where data is accessed sequentially. That is because +more data is generally packed into each block on disk. Conversely, noraw +compression may perform better for workloads with random access patterns +because each block will tend to be smaller and require less work to read and +decompress. + See @ref file_formats_compression for more information on available compression techniques. +See @ref compression for information on how to configure and enable compression. + */ diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 6c99f3a13dc..909fe813f98 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -910,23 +910,6 @@ __evict_lru_walk(WT_SESSION_IMPL *session) WT_RET_NOTFOUND_OK(ret); /* - * If we found no pages at all during the walk, something is wrong. - * Be more aggressive next time. - * - * Continue on to sort the queue, in case there are pages left from a - * previous walk. - */ - if (ret == WT_NOTFOUND) { - if (F_ISSET(cache, - WT_CACHE_EVICT_CLEAN_HARD | WT_CACHE_EVICT_DIRTY_HARD)) - cache->evict_aggressive_score = WT_MIN( - cache->evict_aggressive_score + WT_EVICT_SCORE_BUMP, - WT_EVICT_SCORE_MAX); - WT_STAT_CONN_SET(session, cache_eviction_aggressive_set, - cache->evict_aggressive_score); - } - - /* * If the queue we are filling is empty, pages are being requested * faster than they are being queued. */ @@ -1296,7 +1279,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, WT_PAGE_MODIFY *mod; WT_REF *ref; WT_TXN_GLOBAL *txn_global; - uint64_t btree_inuse, bytes_per_slot, cache_inuse; + uint64_t btree_inuse, bytes_per_slot, cache_inuse, min_pages; uint64_t pages_seen, pages_queued, refs_walked; uint32_t remaining_slots, total_slots, walk_flags; uint32_t target_pages_clean, target_pages_dirty, target_pages; @@ -1369,6 +1352,16 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, FLD_SET(walk_flags, WT_READ_PREV); /* + * Examine at least a reasonable number of pages before deciding + * whether to give up. When we are only looking for dirty pages, + * search the tree for longer. + */ + min_pages = 10 * target_pages; + if (F_ISSET(cache, WT_CACHE_EVICT_DIRTY) && + !F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) + min_pages *= 10; + + /* * Get some more eviction candidate pages. * * !!! Take care terminating this loop. @@ -1390,9 +1383,10 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, * no good eviction candidates can be found. Abandon the walk * if we get into that situation. */ - give_up = !__wt_cache_aggressive(session) && pages_seen > 100 && + give_up = !__wt_cache_aggressive(session) && + pages_seen > min_pages && (pages_queued == 0 || (pages_seen / pages_queued) > - (10 * total_slots / target_pages)); + (min_pages / target_pages)); if (give_up) break; @@ -1453,15 +1447,15 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, if (modified && !F_ISSET(cache, WT_CACHE_EVICT_DIRTY)) continue; - /* If eviction gets aggressive, anything else is fair game. */ - if (__wt_cache_aggressive(session)) - goto fast; - /* Limit internal pages to 50% of the total. */ if (WT_PAGE_IS_INTERNAL(page) && internal_pages >= (int)(evict - start) / 2) continue; + /* If eviction gets aggressive, anything else is fair game. */ + if (__wt_cache_aggressive(session)) + goto fast; + /* * If the oldest transaction hasn't changed since the last time * this page was written, it's unlikely we can make progress. diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index daf2eb158c1..e48189d50ea 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1425,9 +1425,11 @@ __wt_page_swap_func( * page not having been released, and that's not the case. */ if (LF_ISSET(WT_READ_NOTFOUND_OK) && ret == WT_NOTFOUND) - return (EINVAL); + WT_RET_MSG(session, + EINVAL, "page-release WT_NOTFOUND error mapped to EINVAL"); if (LF_ISSET(WT_READ_RESTART_OK) && ret == WT_RESTART) - return (EINVAL); + WT_RET_MSG(session, + EINVAL, "page-release WT_RESTART error mapped to EINVAL"); return (ret); } diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index d7c3bf69686..7d3d07a6abc 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -269,7 +269,8 @@ struct __wt_connection_impl { WT_TXN_GLOBAL txn_global; /* Global transaction state */ WT_RWLOCK *hot_backup_lock; /* Hot backup serialization */ - bool hot_backup; + bool hot_backup; /* Hot backup in progress */ + char **hot_backup_list; /* Hot backup file list */ WT_SESSION_IMPL *ckpt_session; /* Checkpoint thread session */ wt_thread_t ckpt_tid; /* Checkpoint thread */ @@ -289,10 +290,10 @@ struct __wt_connection_impl { /* Connection statistics */ WT_CONNECTION_STATS *stats[WT_COUNTER_SLOTS]; - WT_CONNECTION_STATS stat_array[WT_COUNTER_SLOTS]; + WT_CONNECTION_STATS *stat_array; WT_ASYNC *async; /* Async structure */ - int async_cfg; /* Global async configuration */ + bool async_cfg; /* Global async configuration */ uint32_t async_size; /* Async op array size */ uint32_t async_workers; /* Number of async workers */ diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index e322a53a65d..d522abc2a56 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -59,10 +59,6 @@ 0 /* uint32_t flags */ \ } -struct __wt_cursor_backup_entry { - char *name; /* File name */ - WT_DATA_HANDLE *handle; /* Handle */ -}; struct __wt_cursor_backup { WT_CURSOR iface; @@ -70,7 +66,7 @@ struct __wt_cursor_backup { WT_FSTREAM *bfs; /* Backup file stream */ uint32_t maxid; /* Maximum log file ID seen */ - WT_CURSOR_BACKUP_ENTRY *list; /* List of files to be copied. */ + char **list; /* List of files to be copied. */ size_t list_allocated; size_t list_next; @@ -365,9 +361,11 @@ struct __wt_cursor_join_entry { uint32_t bloom_hash_count; /* hash functions in bloom */ uint64_t count; /* approx number of matches */ -#define WT_CURJOIN_ENTRY_BLOOM 0x01 /* use a bloom filter */ -#define WT_CURJOIN_ENTRY_DISJUNCTION 0x02 /* endpoints are or-ed */ -#define WT_CURJOIN_ENTRY_OWN_BLOOM 0x04 /* this entry owns the bloom */ +#define WT_CURJOIN_ENTRY_BLOOM 0x01 /* use a bloom filter */ +#define WT_CURJOIN_ENTRY_DISJUNCTION 0x02 /* endpoints are or-ed */ +#define WT_CURJOIN_ENTRY_FALSE_POSITIVES 0x04 /* after bloom filter do not + * filter false positives */ +#define WT_CURJOIN_ENTRY_OWN_BLOOM 0x08 /* this entry owns the bloom */ uint8_t flags; WT_CURSOR_JOIN_ENDPOINT *ends; /* reference endpoints */ diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 9a11594c893..001eb268f86 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -75,7 +75,7 @@ struct __wt_data_handle { /* Data-source statistics */ WT_DSRC_STATS *stats[WT_COUNTER_SLOTS]; - WT_DSRC_STATS stat_array[WT_COUNTER_SLOTS]; + WT_DSRC_STATS *stat_array; /* Flags values over 0xff are reserved for WT_BTREE_* */ #define WT_DHANDLE_DEAD 0x01 /* Dead, awaiting discard */ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 79e6405e148..6234f2f6bc5 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -270,7 +270,6 @@ extern int __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) WT_ extern int __wt_connection_close(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); -extern int __wt_statlog_log_one(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_sweep_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -584,6 +583,7 @@ extern int __wt_curstat_table_init(WT_SESSION_IMPL *session, const char *uri, co extern int __wt_schema_truncate( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_range_truncate(WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_range_truncate( WT_SESSION_IMPL *session, WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const char *name); extern int __wt_str_name_check(WT_SESSION_IMPL *session, const char *str) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_name_check(WT_SESSION_IMPL *session, const char *str, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -638,6 +638,7 @@ extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_unexpected_object_type( WT_SESSION_IMPL *session, const char *uri, const char *expect) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_library_init(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_breakpoint(void); extern void __wt_attach(WT_SESSION_IMPL *session); @@ -666,6 +667,7 @@ extern int __wt_huffman_encode(WT_SESSION_IMPL *session, void *huffman_arg, cons extern int __wt_huffman_decode(WT_SESSION_IMPL *session, void *huffman_arg, const uint8_t *from_arg, size_t from_len, WT_ITEM *to_buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_rwlock_alloc( WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); extern void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); extern void __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -697,14 +699,16 @@ extern void *__wt_ext_scr_alloc( WT_EXTENSION_API *wt_api, WT_SESSION *wt_sessio extern void __wt_ext_scr_free(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, void *p); extern int __wt_stat_dsrc_desc(WT_CURSOR_STAT *cst, int slot, const char **p) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_stat_dsrc_init_single(WT_DSRC_STATS *stats); -extern void __wt_stat_dsrc_init(WT_DATA_HANDLE *handle); +extern int __wt_stat_dsrc_init( WT_SESSION_IMPL *session, WT_DATA_HANDLE *handle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_stat_dsrc_discard( WT_SESSION_IMPL *session, WT_DATA_HANDLE *handle); extern void __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats); extern void __wt_stat_dsrc_clear_all(WT_DSRC_STATS **stats); extern void __wt_stat_dsrc_aggregate_single( WT_DSRC_STATS *from, WT_DSRC_STATS *to); extern void __wt_stat_dsrc_aggregate( WT_DSRC_STATS **from, WT_DSRC_STATS *to); extern int __wt_stat_connection_desc(WT_CURSOR_STAT *cst, int slot, const char **p) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_stat_connection_init_single(WT_CONNECTION_STATS *stats); -extern void __wt_stat_connection_init(WT_CONNECTION_IMPL *handle); +extern int __wt_stat_connection_init( WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *handle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_stat_connection_discard( WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *handle); extern void __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats); extern void __wt_stat_connection_clear_all(WT_CONNECTION_STATS **stats); extern void __wt_stat_connection_aggregate( WT_CONNECTION_STATS **from, WT_CONNECTION_STATS *to); @@ -743,7 +747,7 @@ extern int __wt_ext_transaction_visible( WT_EXTENSION_API *wt_api, WT_SESSION *w extern void __wt_txn_op_free(WT_SESSION_IMPL *session, WT_TXN_OP *op); extern int __wt_txn_log_op(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_log_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_checkpoint_logread( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, WT_LSN *ckpt_lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_checkpoint_logread(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, WT_LSN *ckpt_lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_checkpoint_log( WT_SESSION_IMPL *session, bool full, uint32_t flags, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_truncate_log( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_truncate_end(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index 63c79dbc72e..6d4a167a8e5 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -29,9 +29,8 @@ #define WT_LAS_URI "file:WiredTigerLAS.wt" /* Lookaside table URI*/ /* - * Pre computed hash for the metadata file. Used to optimize comparisons - * against the metafile URI. The validity is checked on connection open - * when diagnostic is enabled. + * Optimize comparisons against the metafile URI, flag handles that reference + * the metadata file. */ #define WT_IS_METADATA(session, dh) \ F_ISSET((dh), WT_DHANDLE_IS_METADATA) diff --git a/src/third_party/wiredtiger/src/include/mutex.h b/src/third_party/wiredtiger/src/include/mutex.h index b736d6ee9fb..84f015d6b67 100644 --- a/src/third_party/wiredtiger/src/include/mutex.h +++ b/src/third_party/wiredtiger/src/include/mutex.h @@ -43,7 +43,7 @@ typedef union { /* Read/write lock */ uint16_t writers; /* Now serving for writers */ uint16_t readers; /* Now serving for readers */ uint16_t next; /* Next available ticket number */ - uint16_t __notused; /* Padding */ + uint16_t writers_active;/* Count of active writers */ } s; } wt_rwlock_t; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index d0b0b60585a..e53414fc0c9 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -225,21 +225,24 @@ __wt_stats_clear(void *stats_arg, int slot) * necessary until everything is converted to using data-source handles. */ #define WT_STAT_DATA_DECRV(session, fld, value) do { \ - if ((session)->dhandle != NULL) \ + if ((session)->dhandle != NULL && \ + (session)->dhandle->stat_array != NULL) \ WT_STAT_DECRV( \ session, (session)->dhandle->stats, fld, value); \ } while (0) #define WT_STAT_DATA_DECR(session, fld) \ WT_STAT_DATA_DECRV(session, fld, 1) #define WT_STAT_DATA_INCRV(session, fld, value) do { \ - if ((session)->dhandle != NULL) \ + if ((session)->dhandle != NULL && \ + (session)->dhandle->stat_array != NULL) \ WT_STAT_INCRV( \ session, (session)->dhandle->stats, fld, value); \ } while (0) #define WT_STAT_DATA_INCR(session, fld) \ WT_STAT_DATA_INCRV(session, fld, 1) #define WT_STAT_DATA_SET(session, fld, value) do { \ - if ((session)->dhandle != NULL) \ + if ((session)->dhandle != NULL && \ + (session)->dhandle->stat_array != NULL) \ WT_STAT_SET( \ session, (session)->dhandle->stats, fld, value); \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index b6185b4ead6..665e8eaf4b0 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1284,6 +1284,9 @@ struct __wt_session { * @configstart{WT_SESSION.join, see dist/api_data.py} * @config{bloom_bit_count, the number of bits used per item for the * bloom filter., an integer between 2 and 1000; default \c 16.} + * @config{bloom_false_positives, return all values that pass the bloom + * filter\, without eliminating any false positives., a boolean flag; + * default \c false.} * @config{bloom_hash_count, the number of hash values per item for the * bloom filter., an integer between 2 and 100; default \c 8.} * @config{compare, modifies the set of items to be returned so that the @@ -4197,13 +4200,48 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); /*! @} */ -/******************************************* - * Statistic reference. - *******************************************/ /*! * @addtogroup wt * @{ */ + +/*! + * @name Log record and operation types + * @anchor log_types + * @{ + */ +/* + * NOTE: The values of these record types and operations must + * never change because they're written into the log. Append + * any new records or operations to the appropriate set. + */ +/*! checkpoint */ +#define WT_LOGREC_CHECKPOINT 0 +/*! transaction commit */ +#define WT_LOGREC_COMMIT 1 +/*! file sync */ +#define WT_LOGREC_FILE_SYNC 2 +/*! message */ +#define WT_LOGREC_MESSAGE 3 +/*! invalid operation */ +#define WT_LOGOP_INVALID 0 +/*! column put */ +#define WT_LOGOP_COL_PUT 1 +/*! column remove */ +#define WT_LOGOP_COL_REMOVE 2 +/*! column truncate */ +#define WT_LOGOP_COL_TRUNCATE 3 +/*! row put */ +#define WT_LOGOP_ROW_PUT 4 +/*! row remove */ +#define WT_LOGOP_ROW_REMOVE 5 +/*! row truncate */ +#define WT_LOGOP_ROW_TRUNCATE 6 +/*! @} */ + +/******************************************* + * Statistic reference. + *******************************************/ /* * DO NOT EDIT: automatically built by dist/api_stat.py. * Statistics section: BEGIN @@ -5067,42 +5105,6 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * Statistics section: END * DO NOT EDIT: automatically built by dist/api_stat.py. */ -/*! - * @name Log record and operation types - * @anchor log_types - * @{ - */ -/* - * DO NOT EDIT: automatically built by dist/log.py. - * Log record declarations: BEGIN - */ -/*! invalid operation */ -#define WT_LOGOP_INVALID 0 -/*! checkpoint */ -#define WT_LOGREC_CHECKPOINT 0 -/*! transaction commit */ -#define WT_LOGREC_COMMIT 1 -/*! file sync */ -#define WT_LOGREC_FILE_SYNC 2 -/*! message */ -#define WT_LOGREC_MESSAGE 3 -/*! column put */ -#define WT_LOGOP_COL_PUT 1 -/*! column remove */ -#define WT_LOGOP_COL_REMOVE 2 -/*! column truncate */ -#define WT_LOGOP_COL_TRUNCATE 3 -/*! row put */ -#define WT_LOGOP_ROW_PUT 4 -/*! row remove */ -#define WT_LOGOP_ROW_REMOVE 5 -/*! row truncate */ -#define WT_LOGOP_ROW_TRUNCATE 6 -/* - * Log record declarations: END - * DO NOT EDIT: automatically built by dist/log.py. - */ -/*! @} */ /*! @} */ #undef __F diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index d354757c592..dc3e3a0b6c0 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -126,8 +126,6 @@ struct __wt_connection_stats; typedef struct __wt_connection_stats WT_CONNECTION_STATS; struct __wt_cursor_backup; typedef struct __wt_cursor_backup WT_CURSOR_BACKUP; -struct __wt_cursor_backup_entry; - typedef struct __wt_cursor_backup_entry WT_CURSOR_BACKUP_ENTRY; struct __wt_cursor_btree; typedef struct __wt_cursor_btree WT_CURSOR_BTREE; struct __wt_cursor_bulk; diff --git a/src/third_party/wiredtiger/src/log/log_auto.c b/src/third_party/wiredtiger/src/log/log_auto.c index 34bccd0ede4..c2b38184405 100644 --- a/src/third_party/wiredtiger/src/log/log_auto.c +++ b/src/third_party/wiredtiger/src/log/log_auto.c @@ -119,11 +119,13 @@ __wt_logop_col_put_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, uint64_t *recnop, WT_ITEM *valuep) { + WT_DECL_RET; const char *fmt = WT_UNCHECKED_STRING(IIIru); uint32_t optype, size; - WT_RET(__wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, - &optype, &size, fileidp, recnop, valuep)); + if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, + &optype, &size, fileidp, recnop, valuep)) != 0) + WT_RET_MSG(session, ret, "logop_col_put: unpack failure"); WT_ASSERT(session, optype == WT_LOGOP_COL_PUT); *pp += size; @@ -192,11 +194,13 @@ __wt_logop_col_remove_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, uint64_t *recnop) { + WT_DECL_RET; const char *fmt = WT_UNCHECKED_STRING(IIIr); uint32_t optype, size; - WT_RET(__wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, - &optype, &size, fileidp, recnop)); + if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, + &optype, &size, fileidp, recnop)) != 0) + WT_RET_MSG(session, ret, "logop_col_remove: unpack failure"); WT_ASSERT(session, optype == WT_LOGOP_COL_REMOVE); *pp += size; @@ -252,11 +256,13 @@ __wt_logop_col_truncate_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, uint64_t *startp, uint64_t *stopp) { + WT_DECL_RET; const char *fmt = WT_UNCHECKED_STRING(IIIrr); uint32_t optype, size; - WT_RET(__wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, - &optype, &size, fileidp, startp, stopp)); + if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, + &optype, &size, fileidp, startp, stopp)) != 0) + WT_RET_MSG(session, ret, "logop_col_truncate: unpack failure"); WT_ASSERT(session, optype == WT_LOGOP_COL_TRUNCATE); *pp += size; @@ -315,11 +321,13 @@ __wt_logop_row_put_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, WT_ITEM *keyp, WT_ITEM *valuep) { + WT_DECL_RET; const char *fmt = WT_UNCHECKED_STRING(IIIuu); uint32_t optype, size; - WT_RET(__wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, - &optype, &size, fileidp, keyp, valuep)); + if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, + &optype, &size, fileidp, keyp, valuep)) != 0) + WT_RET_MSG(session, ret, "logop_row_put: unpack failure"); WT_ASSERT(session, optype == WT_LOGOP_ROW_PUT); *pp += size; @@ -394,11 +402,13 @@ __wt_logop_row_remove_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, WT_ITEM *keyp) { + WT_DECL_RET; const char *fmt = WT_UNCHECKED_STRING(IIIu); uint32_t optype, size; - WT_RET(__wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, - &optype, &size, fileidp, keyp)); + if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, + &optype, &size, fileidp, keyp)) != 0) + WT_RET_MSG(session, ret, "logop_row_remove: unpack failure"); WT_ASSERT(session, optype == WT_LOGOP_ROW_REMOVE); *pp += size; @@ -464,11 +474,13 @@ __wt_logop_row_truncate_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, WT_ITEM *startp, WT_ITEM *stopp, uint32_t *modep) { + WT_DECL_RET; const char *fmt = WT_UNCHECKED_STRING(IIIuuI); uint32_t optype, size; - WT_RET(__wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, - &optype, &size, fileidp, startp, stopp, modep)); + if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, + &optype, &size, fileidp, startp, stopp, modep)) != 0) + WT_RET_MSG(session, ret, "logop_row_truncate: unpack failure"); WT_ASSERT(session, optype == WT_LOGOP_ROW_TRUNCATE); *pp += size; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index 067c527a21a..c20673563d9 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1678,7 +1678,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, lsm_tree = NULL; if (!WT_PREFIX_MATCH(uri, "lsm:")) - return (EINVAL); + return (__wt_unexpected_object_type(session, uri, "lsm:")); if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) WT_RET_MSG(session, EINVAL, diff --git a/src/third_party/wiredtiger/src/lsm/lsm_manager.c b/src/third_party/wiredtiger/src/lsm/lsm_manager.c index 0a5f4fdd8b5..cbd83a5cd30 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_manager.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_manager.c @@ -55,6 +55,7 @@ __lsm_general_worker_start(WT_SESSION_IMPL *session) * as many worker threads as are required to keep up with demand. */ WT_ASSERT(session, manager->lsm_workers > 0); + WT_ASSERT(session, manager->lsm_workers < manager->lsm_workers_max); for (; manager->lsm_workers < manager->lsm_workers_max; manager->lsm_workers++) { worker_args = @@ -116,17 +117,18 @@ __lsm_stop_workers(WT_SESSION_IMPL *session) { WT_LSM_MANAGER *manager; WT_LSM_WORKER_ARGS *worker_args; - uint32_t i; manager = &S2C(session)->lsm_manager; /* - * Start at the end of the list of threads and stop them until we - * have the desired number. We want to keep all active threads - * packed at the front of the worker array. + * Start at the end of the list of threads and stop them until we have + * the desired number. We want to keep all active threads packed at the + * front of the worker array. */ - WT_ASSERT(session, manager->lsm_workers != 0); - for (i = manager->lsm_workers - 1; i >= manager->lsm_workers_max; i--) { - worker_args = &manager->lsm_worker_cookies[i]; + WT_ASSERT(session, manager->lsm_workers > manager->lsm_workers_max); + for (; manager->lsm_workers > manager->lsm_workers_max; + manager->lsm_workers--) { + worker_args = + &manager->lsm_worker_cookies[manager->lsm_workers - 1]; /* * Clear this worker's flag so it stops. */ @@ -136,7 +138,6 @@ __lsm_stop_workers(WT_SESSION_IMPL *session) worker_args->tid = 0; worker_args->type = 0; worker_args->flags = 0; - manager->lsm_workers--; /* * We do not clear the session because they are allocated * statically when the connection was opened. diff --git a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c index 842bb6eeec9..d4ee532a2af 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c @@ -103,27 +103,30 @@ void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) { WT_DECL_RET; - bool locked; - - locked = false; __wt_verbose(session, WT_VERB_MUTEX, "signal %s", cond->name); - /* Fast path if already signalled. */ - if (cond->waiters == -1) - return; + /* + * Our callers are often setting flags to cause a thread to exit. Add + * a barrier to ensure the flags are seen by the threads. + */ + WT_WRITE_BARRIER(); - if (cond->waiters > 0 || !__wt_atomic_casi32(&cond->waiters, 0, -1)) { - WT_ERR(pthread_mutex_lock(&cond->mtx)); - locked = true; - WT_ERR(pthread_cond_broadcast(&cond->cond)); - } + /* + * Fast path if we are in (or can enter), a state where the next waiter + * will return immediately as already signaled. + */ + if (cond->waiters == -1 || + (cond->waiters == 0 && __wt_atomic_casi32(&cond->waiters, 0, -1))) + return; -err: if (locked) - WT_TRET(pthread_mutex_unlock(&cond->mtx)); + WT_ERR(pthread_mutex_lock(&cond->mtx)); + ret = pthread_cond_broadcast(&cond->cond); + WT_TRET(pthread_mutex_unlock(&cond->mtx)); if (ret == 0) return; +err: WT_PANIC_MSG(session, ret, "pthread_cond_broadcast: %s", cond->name); } diff --git a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c index 41acec3ab4c..79c62ccd7f2 100644 --- a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c @@ -123,22 +123,23 @@ __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) __wt_verbose(session, WT_VERB_MUTEX, "signal %s", cond->name); - /* Fast path if already signalled. */ - if (cond->waiters == -1) - return; - - if (cond->waiters > 0 || !__wt_atomic_casi32(&cond->waiters, 0, -1)) { - EnterCriticalSection(&cond->mtx); - locked = true; - WakeAllConditionVariable(&cond->cond); - } + /* + * Our callers are often setting flags to cause a thread to exit. Add + * a barrier to ensure the flags are seen by the threads. + */ + WT_WRITE_BARRIER(); - if (locked) - LeaveCriticalSection(&cond->mtx); - if (ret == 0) + /* + * Fast path if we are in (or can enter), a state where the next waiter + * will return immediately as already signaled. + */ + if (cond->waiters == -1 || + (cond->waiters == 0 && __wt_atomic_casi32(&cond->waiters, 0, -1))) return; - WT_PANIC_MSG(session, ret, "WakeAllConditionVariable: %s", cond->name); + EnterCriticalSection(&cond->mtx); + WakeAllConditionVariable(&cond->cond); + LeaveCriticalSection(&cond->mtx); } /* @@ -149,7 +150,6 @@ int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) { WT_CONDVAR *cond; - WT_DECL_RET; cond = *condp; if (cond == NULL) @@ -159,5 +159,5 @@ __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) DeleteCriticalSection(&cond->mtx); __wt_free(session, *condp); - return (ret); + return (0); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 810f3fd976b..fe288beed15 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -547,8 +547,11 @@ __rec_write_status(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) btree = S2BT(session); mod = page->modify; - /* Check for a lookaside table and checkpoint collision. */ - if (__rec_las_checkpoint_test(session, r)) + /* + * If we have used the lookaside table, check for a lookaside table and + * checkpoint collision. + */ + if (r->cache_write_lookaside && __rec_las_checkpoint_test(session, r)) return (EBUSY); /* diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index ed88e687a19..020d5e72c13 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -70,7 +70,7 @@ __create_file(WT_SESSION_IMPL *session, filename = uri; if (!WT_PREFIX_SKIP(filename, "file:")) - WT_RET_MSG(session, EINVAL, "Expected a 'file:' URI: %s", uri); + return (__wt_unexpected_object_type(session, uri, "file:")); /* Check if the file already exists. */ if (!is_metadata && (ret = @@ -193,7 +193,8 @@ __create_colgroup(WT_SESSION_IMPL *session, tablename = name; if (!WT_PREFIX_SKIP(tablename, "colgroup:")) - return (EINVAL); + return ( + __wt_unexpected_object_type(session, name, "colgroup:")); cgname = strchr(tablename, ':'); if (cgname != NULL) { tlen = (size_t)(cgname - tablename); @@ -384,7 +385,7 @@ __create_index(WT_SESSION_IMPL *session, tablename = name; if (!WT_PREFIX_SKIP(tablename, "index:")) - return (EINVAL); + return (__wt_unexpected_object_type(session, name, "index:")); idxname = strchr(tablename, ':'); if (idxname == NULL) WT_RET_MSG(session, EINVAL, "Invalid index name, " @@ -570,7 +571,7 @@ __create_table(WT_SESSION_IMPL *session, tablename = name; if (!WT_PREFIX_SKIP(tablename, "table:")) - return (EINVAL); + return (__wt_unexpected_object_type(session, name, "table:")); if ((ret = __wt_schema_get_table(session, tablename, strlen(tablename), false, &table)) == 0) { diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index ead8cc45c62..65c955cf1e9 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -26,8 +26,9 @@ __drop_file( filename = uri; if (!WT_PREFIX_SKIP(filename, "file:")) - return (EINVAL); + return (__wt_unexpected_object_type(session, uri, "file:")); + WT_RET(__wt_schema_backup_check(session, filename)); /* Close all btree handles associated with this file. */ WT_WITH_HANDLE_LIST_LOCK(session, ret = __wt_conn_dhandle_close_all(session, uri, force)); diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index bc92c882117..f512482c162 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -24,11 +24,14 @@ __rename_file( newvalue = oldvalue = NULL; filename = uri; + if (!WT_PREFIX_SKIP(filename, "file:")) + return (__wt_unexpected_object_type(session, uri, "file:")); newfile = newuri; - if (!WT_PREFIX_SKIP(filename, "file:") || - !WT_PREFIX_SKIP(newfile, "file:")) - return (EINVAL); + if (!WT_PREFIX_SKIP(newfile, "file:")) + return (__wt_unexpected_object_type(session, newuri, "file:")); + WT_RET(__wt_schema_backup_check(session, filename)); + WT_RET(__wt_schema_backup_check(session, newfile)); /* Close any btree handles in the file. */ WT_WITH_HANDLE_LIST_LOCK(session, ret = __wt_conn_dhandle_close_all(session, uri, false)); diff --git a/src/third_party/wiredtiger/src/schema/schema_util.c b/src/third_party/wiredtiger/src/schema/schema_util.c index d1c84dc8d85..433224a868e 100644 --- a/src/third_party/wiredtiger/src/schema/schema_util.c +++ b/src/third_party/wiredtiger/src/schema/schema_util.c @@ -9,6 +9,45 @@ #include "wt_internal.h" /* + * __wt_schema_backup_check -- + * Check if a backup cursor is open and give an error if the schema + * operation will conflict. This is called after the schema operations + * have taken the schema lock so no hot backup cursor can be created until + * this is done. + */ +int +__wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + int i; + char **backup_list; + + conn = S2C(session); + if (!conn->hot_backup) + return (0); + __wt_readlock(session, conn->hot_backup_lock); + /* + * There is a window at the end of a backup where the list has been + * cleared from the connection but the flag is still set. It is safe + * to drop at that point. + */ + if (!conn->hot_backup || + (backup_list = conn->hot_backup_list) == NULL) { + __wt_readunlock(session, conn->hot_backup_lock); + return (0); + } + for (i = 0; backup_list[i] != NULL; ++i) { + if (strcmp(backup_list[i], name) == 0) { + ret = EBUSY; + break; + } + } + __wt_readunlock(session, conn->hot_backup_lock); + return (ret); +} + +/* * __wt_schema_get_source -- * Find a matching data source or report an error. */ diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index f594450db74..3afea383b08 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -920,8 +920,7 @@ __session_join(WT_SESSION *wt_session, WT_CURSOR *join_cursor, "table for join cursor does not match table for " "ref_cursor"); if (F_ISSET(ref_cursor, WT_CURSTD_JOINED)) - WT_ERR_MSG(session, EINVAL, - "cursor already used in a join"); + WT_ERR_MSG(session, EINVAL, "cursor already used in a join"); /* "ge" is the default */ range = WT_CURJOIN_END_GT | WT_CURJOIN_END_EQ; @@ -937,7 +936,9 @@ __session_join(WT_SESSION *wt_session, WT_CURSOR *join_cursor, else if (WT_STRING_MATCH("eq", cval.str, cval.len)) range = WT_CURJOIN_END_EQ; else if (!WT_STRING_MATCH("ge", cval.str, cval.len)) - WT_ERR(EINVAL); + WT_ERR_MSG(session, EINVAL, + "compare=%.*s not supported", + (int)cval.len, cval.str); } WT_ERR(__wt_config_gets(session, cfg, "count", &cval)); if (cval.len != 0) @@ -948,12 +949,13 @@ __session_join(WT_SESSION *wt_session, WT_CURSOR *join_cursor, if (WT_STRING_MATCH("bloom", cval.str, cval.len)) LF_SET(WT_CURJOIN_ENTRY_BLOOM); else if (!WT_STRING_MATCH("default", cval.str, cval.len)) - WT_ERR(EINVAL); + WT_ERR_MSG(session, EINVAL, + "strategy=%.*s not supported", + (int)cval.len, cval.str); } WT_ERR(__wt_config_gets(session, cfg, "bloom_bit_count", &cval)); if ((uint64_t)cval.val > UINT32_MAX) - WT_ERR_MSG(session, EINVAL, - "bloom_bit_count: value too large"); + WT_ERR_MSG(session, EINVAL, "bloom_bit_count: value too large"); bloom_bit_count = (uint32_t)cval.val; WT_ERR(__wt_config_gets(session, cfg, "bloom_hash_count", &cval)); if ((uint64_t)cval.val > UINT32_MAX) @@ -963,6 +965,10 @@ __session_join(WT_SESSION *wt_session, WT_CURSOR *join_cursor, if (LF_ISSET(WT_CURJOIN_ENTRY_BLOOM) && count == 0) WT_ERR_MSG(session, EINVAL, "count must be nonzero when strategy=bloom"); + WT_ERR(__wt_config_gets_def( + session, cfg, "bloom_false_positives", 0, &cval)); + if (cval.val != 0) + LF_SET(WT_CURJOIN_ENTRY_FALSE_POSITIVES); WT_ERR(__wt_config_gets(session, cfg, "operation", &cval)); if (cval.len != 0 && WT_STRING_MATCH("or", cval.str, cval.len)) @@ -1007,7 +1013,9 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, salvage, config, cfg); if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR(ENOTSUP); + WT_ERR_MSG(session, ENOTSUP, + "WT_SESSION.salvage not supported for in-memory " + "configurations"); /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, ret, @@ -1311,7 +1319,9 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, verify, config, cfg); if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR(ENOTSUP); + WT_ERR_MSG(session, ENOTSUP, + "WT_SESSION.verify not supported for in-memory " + "configurations"); /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, ret, @@ -1556,7 +1566,9 @@ __session_checkpoint(WT_SESSION *wt_session, const char *config) SESSION_API_CALL(session, checkpoint, config, cfg); if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR(ENOTSUP); + WT_ERR_MSG(session, ENOTSUP, + "WT_SESSION.checkpoint not supported for in-memory " + "configurations"); /* * Checkpoints require a snapshot to write a transactionally consistent diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 3ecbab1cbe9..e48225c2c13 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -562,3 +562,16 @@ __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_RET_MSG(session, ENOTSUP, "unknown object type: %s", uri); } + +/* + * __wt_unexpected_object_type -- + * Print a standard error message when given an unexpected object type. + */ +int +__wt_unexpected_object_type( + WT_SESSION_IMPL *session, const char *uri, const char *expect) + WT_GCC_FUNC_ATTRIBUTE((cold)) +{ + WT_RET_MSG(session, + EINVAL, "uri %s doesn't match expected \"%s\"", uri, expect); +} diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index 46b63ac6129..b1978dbe97a 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -121,13 +121,11 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp return (0); } - __wt_errx(session, - "session %p: hazard pointer table full", (void *)session); #ifdef HAVE_DIAGNOSTIC __hazard_dump(session); #endif - - return (ENOMEM); + WT_RET_MSG(session, ENOMEM, + "session %p: hazard pointer table full", (void *)session); } /* diff --git a/src/third_party/wiredtiger/src/support/huffman.c b/src/third_party/wiredtiger/src/support/huffman.c index 3ec282915ca..afc785b39a9 100644 --- a/src/third_party/wiredtiger/src/support/huffman.c +++ b/src/third_party/wiredtiger/src/support/huffman.c @@ -823,7 +823,8 @@ __wt_huffman_decode(WT_SESSION_IMPL *session, void *huffman_arg, * where that's not true. */ if (from_len_bits < len) /* corrupted */ - WT_ERR(EINVAL); + WT_ERR_MSG(session, EINVAL, + "huffman decompression detected input corruption"); from_len_bits -= len; WT_ASSERT(session, diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c index fefa346ac14..ea18f556257 100644 --- a/src/third_party/wiredtiger/src/support/mtx_rw.c +++ b/src/third_party/wiredtiger/src/support/mtx_rw.c @@ -167,6 +167,32 @@ __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) } /* + * __wt_readlock_spin -- + * Spin to get a read lock: only yield the CPU if the lock is held + * exclusive. + */ +void +__wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +{ + wt_rwlock_t *l; + + l = &rwlock->rwlock; + + /* + * Try to get the lock in a single operation if it is available to + * readers. This avoids the situation where multiple readers arrive + * concurrently and have to line up in order to enter the lock. For + * read-heavy workloads it can make a significant difference. + */ + while (__wt_try_readlock(session, rwlock) != 0) { + if (l->s.writers_active > 0) + __wt_yield(); + else + WT_PAUSE(); + } +} + +/* * __wt_readlock -- * Get a shared lock. */ @@ -192,17 +218,14 @@ __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) for (pause_cnt = 0; ticket != l->s.readers;) { /* * We failed to get the lock; pause before retrying and if we've - * paused enough, sleep so we don't burn CPU to no purpose. This + * paused enough, yield so we don't burn CPU to no purpose. This * situation happens if there are more threads than cores in the * system and we're thrashing on shared resources. - * - * Don't sleep long when waiting on a read lock, hopefully we're - * waiting on another read thread to increment the reader count. */ if (++pause_cnt < WT_THOUSAND) WT_PAUSE(); else - __wt_sleep(0, 10); + __wt_yield(); } /* @@ -264,6 +287,7 @@ __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) /* The replacement lock value is a result of allocating a new ticket. */ ++new.s.next; + ++new.s.writers_active; return (__wt_atomic_cas64(&l->u, old.u, new.u) ? 0 : EBUSY); } @@ -288,6 +312,7 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * lock. */ ticket = __wt_atomic_fetch_add16(&l->s.next, 1); + (void)__wt_atomic_add16(&l->s.writers_active, 1); for (pause_cnt = 0; ticket != l->s.writers;) { /* * We failed to get the lock; pause before retrying and if we've @@ -319,14 +344,15 @@ __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) WT_UNUSED(session); + l = &rwlock->rwlock; + (void)__wt_atomic_sub16(&l->s.writers_active, 1); + /* * Ensure that all updates made while the lock was held are visible to * the next thread to acquire the lock. */ WT_WRITE_BARRIER(); - l = &rwlock->rwlock; - new = *l; /* diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 6e8e218a0db..5acd9fc713f 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -138,15 +138,27 @@ __wt_stat_dsrc_init_single(WT_DSRC_STATS *stats) memset(stats, 0, sizeof(*stats)); } -void -__wt_stat_dsrc_init(WT_DATA_HANDLE *handle) +int +__wt_stat_dsrc_init( + WT_SESSION_IMPL *session, WT_DATA_HANDLE *handle) { int i; + WT_RET(__wt_calloc(session, (size_t)WT_COUNTER_SLOTS, + sizeof(*handle->stat_array), &handle->stat_array)); + for (i = 0; i < WT_COUNTER_SLOTS; ++i) { handle->stats[i] = &handle->stat_array[i]; __wt_stat_dsrc_init_single(handle->stats[i]); } + return (0); +} + +void +__wt_stat_dsrc_discard( + WT_SESSION_IMPL *session, WT_DATA_HANDLE *handle) +{ + __wt_free(session, handle->stat_array); } void @@ -860,15 +872,27 @@ __wt_stat_connection_init_single(WT_CONNECTION_STATS *stats) memset(stats, 0, sizeof(*stats)); } -void -__wt_stat_connection_init(WT_CONNECTION_IMPL *handle) +int +__wt_stat_connection_init( + WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *handle) { int i; + WT_RET(__wt_calloc(session, (size_t)WT_COUNTER_SLOTS, + sizeof(*handle->stat_array), &handle->stat_array)); + for (i = 0; i < WT_COUNTER_SLOTS; ++i) { handle->stats[i] = &handle->stat_array[i]; __wt_stat_connection_init_single(handle->stats[i]); } + return (0); +} + +void +__wt_stat_connection_discard( + WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *handle) +{ + __wt_free(session, handle->stat_array); } void diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index d60ea73c660..1c47f94bb60 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -112,7 +112,6 @@ int __wt_txn_get_snapshot(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; - WT_DECL_RET; WT_TXN *txn; WT_TXN_GLOBAL *txn_global; WT_TXN_STATE *s, *txn_state; @@ -126,14 +125,8 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) txn_state = WT_SESSION_TXN_STATE(session); n = 0; - /* - * Spin waiting for the lock: the sleeps in our blocking readlock - * implementation are too slow for scanning the transaction table. - */ - while ((ret = - __wt_try_readlock(session, txn_global->scan_rwlock)) == EBUSY) - WT_PAUSE(); - WT_RET(ret); + /* We're going to scan the table: wait for the lock. */ + __wt_readlock_spin(session, txn_global->scan_rwlock); current_id = pinned_id = txn_global->current; prev_oldest_id = txn_global->oldest_id; @@ -292,7 +285,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) /* First do a read-only scan. */ if (wait) - __wt_readlock(session, txn_global->scan_rwlock); + __wt_readlock_spin(session, txn_global->scan_rwlock); else if ((ret = __wt_try_readlock(session, txn_global->scan_rwlock)) != 0) return (ret == EBUSY ? 0 : ret); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 0557e6ce60c..1efd0200a3d 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -562,7 +562,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) /* * Do a pass over the configuration arguments and figure out what kind - * kind of checkpoint this is. + * of checkpoint this is. */ WT_RET(__checkpoint_apply_all(session, cfg, NULL, &full)); diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index f9dd9bee807..f1b78879d76 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -262,20 +262,20 @@ err: __wt_logrec_free(session, &logrec); * Read a log record for a checkpoint operation. */ int -__wt_txn_checkpoint_logread( - WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, - WT_LSN *ckpt_lsn) +__wt_txn_checkpoint_logread(WT_SESSION_IMPL *session, + const uint8_t **pp, const uint8_t *end, WT_LSN *ckpt_lsn) { - WT_ITEM ckpt_snapshot; + WT_DECL_RET; + WT_ITEM ckpt_snapshot_unused; uint32_t ckpt_file, ckpt_offset; - u_int ckpt_nsnapshot; + u_int ckpt_nsnapshot_unused; const char *fmt = WT_UNCHECKED_STRING(IIIU); - WT_RET(__wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, + if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, &ckpt_file, &ckpt_offset, - &ckpt_nsnapshot, &ckpt_snapshot)); - WT_UNUSED(ckpt_nsnapshot); - WT_UNUSED(ckpt_snapshot); + &ckpt_nsnapshot_unused, &ckpt_snapshot_unused)) != 0) + WT_RET_MSG(session, + ret, "txn_checkpoint_logread: unpack failure"); WT_SET_LSN(ckpt_lsn, ckpt_file, ckpt_offset); *pp = end; return (0); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index ae21e58d9b6..65811aa3bf4 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -231,9 +231,12 @@ __txn_op_apply( /* Reset the cursor so it doesn't block eviction. */ if (cursor != NULL) WT_ERR(cursor->reset(cursor)); + return (0); -err: if (ret != 0) - __wt_err(session, ret, "Operation failed during recovery"); +err: __wt_err(session, ret, + "operation apply failed during recovery: operation type %d " + "at LSN %" PRIu32 "/%" PRIu32, + optype, lsnp->l.file, lsnp->l.offset); return (ret); } @@ -263,12 +266,14 @@ __txn_log_recover(WT_SESSION_IMPL *session, WT_ITEM *logrec, WT_LSN *lsnp, WT_LSN *next_lsnp, void *cookie, int firstrecord) { + WT_DECL_RET; WT_RECOVERY *r; - const uint8_t *end, *p; - uint64_t txnid; + uint64_t txnid_unused; uint32_t rectype; + const uint8_t *end, *p; WT_UNUSED(next_lsnp); + r = cookie; p = WT_LOG_SKIP_HEADER(logrec->data); end = (const uint8_t *)logrec->data + logrec->size; @@ -285,8 +290,10 @@ __txn_log_recover(WT_SESSION_IMPL *session, break; case WT_LOGREC_COMMIT: - WT_RET(__wt_vunpack_uint(&p, WT_PTRDIFF(end, p), &txnid)); - WT_UNUSED(txnid); + if ((ret = __wt_vunpack_uint( + &p, WT_PTRDIFF(end, p), &txnid_unused)) != 0) + WT_RET_MSG( + session, ret, "txn_log_recover: unpack failure"); WT_RET(__txn_commit_apply(r, lsnp, &p, end)); break; } diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index 8f1714237b9..a96492c1e71 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -37,6 +37,9 @@ noinst_PROGRAMS += test_wt2834_join_bloom_fix test_wt2853_perf_SOURCES = wt2853_perf/main.c noinst_PROGRAMS += test_wt2853_perf +test_wt2999_join_extractor_SOURCES = wt2999_join_extractor/main.c +noinst_PROGRAMS += test_wt2999_join_extractor + # Run this during a "make check" smoke test. TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) diff --git a/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c b/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c new file mode 100644 index 00000000000..646a7077af1 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c @@ -0,0 +1,168 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-2999 + * + * Test case description: Create a table that stores ~4K size blobs; + * two indices are defined using a pair of custom extractors + * that pull the first and second 32-bit integers from the blob. + * A simple join is created using the two indices, and iterated. + * + * Failure mode: When a custom extractor is used with cursor + * joins, there are memory leaks at the point where the extractor + * sets the key. + */ +static int +custom_extract1(WT_EXTRACTOR *extractor, WT_SESSION *session, + const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor) +{ + WT_ITEM item; + int32_t v1; + + (void)extractor; + (void)key; + testutil_check(wiredtiger_struct_unpack( + session, value->data, value->size, "u", &item)); + + v1 = ((int*)item.data)[0]; + item.data = &v1; + item.size = sizeof(v1); + + result_cursor->set_key(result_cursor, &item); + return (result_cursor->insert(result_cursor)); +} + +static int +custom_extract2(WT_EXTRACTOR *extractor, WT_SESSION *session, + const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor) +{ + WT_ITEM item; + int32_t v2; + + (void)extractor; + (void)key; + testutil_check(wiredtiger_struct_unpack( + session, value->data, value->size, "u", &item)); + + v2 = ((int*)item.data)[1]; + item.data = &v2; + item.size = sizeof(v2); + + result_cursor->set_key(result_cursor, &item); + return (result_cursor->insert(result_cursor)); +} + +static WT_EXTRACTOR custom_extractor1 = { custom_extract1, NULL, NULL }; +static WT_EXTRACTOR custom_extractor2 = { custom_extract2, NULL, NULL }; + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor1, *cursor2, *jcursor; + WT_ITEM k, v; + WT_SESSION *session; + int i, ret; + int32_t key, val[2]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", &conn)); + opts->conn = conn; + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + testutil_check(conn->add_extractor(conn, "custom_extractor1", + &custom_extractor1, NULL)); + testutil_check(conn->add_extractor(conn, "custom_extractor2", + &custom_extractor2, NULL)); + + testutil_check(session->create(session, + "table:main", "key_format=u,value_format=u,columns=(k,v)")); + testutil_check(session->create(session, + "index:main:index1", "key_format=u,extractor=custom_extractor1")); + testutil_check(session->create(session, + "index:main:index2", "key_format=u,extractor=custom_extractor2")); + + testutil_check(session->open_cursor(session, "table:main", NULL, NULL, + &cursor1)); + + v.data = val; + v.size = sizeof(val); + k.data = &key; + k.size = sizeof(key); + + key = 10; + val[0] = 20; + val[1] = 30; + for (i = 0; i < 100000; ++i) { + key += i; + val[0] += i; val[1] += i; + cursor1->set_key(cursor1, &k); + cursor1->set_value(cursor1, &v); + testutil_check(cursor1->insert(cursor1)); + } + + testutil_check(cursor1->close(cursor1)); + + testutil_check(session->open_cursor(session, "index:main:index1", NULL, + NULL, &cursor1)); + key = 20; + cursor1->set_key(cursor1, &k); + testutil_check(cursor1->search(cursor1)); + + testutil_check(session->open_cursor(session, "index:main:index2", NULL, + NULL, &cursor2)); + key = 30; + cursor2->set_key(cursor2, &k); + testutil_check(cursor2->search(cursor2)); + + testutil_check(session->open_cursor(session, "join:table:main", NULL, + NULL, &jcursor)); + testutil_check(session->join(session, jcursor, cursor1, "compare=gt")); + testutil_check(session->join(session, jcursor, cursor2, "compare=gt")); + + while ((ret = jcursor->next(jcursor)) == 0) //leak + ; + testutil_assert(ret == WT_NOTFOUND); + + testutil_check(jcursor->close(jcursor)); + testutil_check(cursor1->close(cursor1)); + testutil_check(cursor2->close(cursor2)); + + testutil_check(opts->conn->close(opts->conn, NULL)); + opts->conn = NULL; + testutil_cleanup(opts); + + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 23fdbce156c..da234ce53c7 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -157,8 +157,7 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) g.c_lsm_worker_threads); if (DATASOURCE("lsm") || g.c_cache < 20) { - p += snprintf(p, REMAIN(p, end), - ",eviction_dirty_target=80,eviction_dirty_trigger=95"); + p += snprintf(p, REMAIN(p, end), ",eviction_dirty_trigger=95"); } /* Eviction worker configuration. */ diff --git a/src/third_party/wiredtiger/test/suite/helper.py b/src/third_party/wiredtiger/test/suite/helper.py index 9f34b566b3c..d1f41f05e8b 100644 --- a/src/third_party/wiredtiger/test/suite/helper.py +++ b/src/third_party/wiredtiger/test/suite/helper.py @@ -30,6 +30,8 @@ import glob, os, shutil, string, subprocess import wiredtiger +from wtdataset import SimpleDataSet, SimpleIndexDataSet, ComplexDataSet + # python has a filecmp.cmp function, but different versions of python approach # file comparison differently. To make sure we get byte for byte comparison, # we define it here. @@ -121,188 +123,3 @@ def copy_wiredtiger_home(olddir, newdir, aligned=True): cmd_list = ['dd', inpf, outf, 'bs=300'] a = subprocess.Popen(cmd_list) a.wait() - -# create a simple_populate or complex_populate key -def key_populate(cursor, i): - key_format = cursor.key_format - if key_format == 'i' or key_format == 'r' or key_format == 'u': - return i - elif key_format == 'S': - return str('%015d' % i) - else: - raise AssertionError( - 'key_populate: object has unexpected format: ' + key_format) - -# create a simple_populate value -def value_populate(cursor, i): - value_format = cursor.value_format - if value_format == 'i' or value_format == 'r' or value_format == 'u': - return i - elif value_format == 'S': - return str(i) + ': abcdefghijklmnopqrstuvwxyz' - elif value_format == '8t': - value = ( - 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, - 0xb7, 0xb8, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf) - return value[i % len(value)] - else: - raise AssertionError( - 'value_populate: object has unexpected format: ' + value_format) - -# population of a simple object -# uri: object -# config: prefix of the session.create configuration string (defaults -# to string value formats) -# rows: entries to insert -def simple_populate(self, uri, config, rows): - self.pr('simple_populate: ' + uri + ' with ' + str(rows) + ' rows') - self.session.create(uri, 'value_format=S,' + config) - cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) - cursor.close() - -def simple_populate_check_cursor(self, cursor, rows): - i = 0 - for key,val in cursor: - i += 1 - self.assertEqual(key, key_populate(cursor, i)) - if cursor.value_format == '8t' and val == 0: # deleted - continue - self.assertEqual(val, value_populate(cursor, i)) - self.assertEqual(i, rows) - -def simple_populate_check(self, uri, rows): - self.pr('simple_populate_check: ' + uri) - cursor = self.session.open_cursor(uri, None) - simple_populate_check_cursor(self, cursor, rows) - cursor.close() - -# population of a simple object, with a single index -# uri: object -# config: prefix of the session.create configuration string (defaults -# to string value formats) -# rows: entries to insert -def simple_index_populate(self, uri, config, rows): - self.pr('simple_index_populate: ' + uri + ' with ' + str(rows) + ' rows') - self.session.create(uri, 'value_format=S,columns=(key0,value0),' + config) - indxname = 'index:' + uri.split(":")[1] - self.session.create(indxname + ':index1', 'columns=(value0,key0)') - cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) - cursor.close() - -def simple_index_populate_check_cursor(self, cursor, rows): - i = 0 - for key,val in cursor: - i += 1 - self.assertEqual(key, key_populate(cursor, i)) - if cursor.value_format == '8t' and val == 0: # deleted - continue - self.assertEqual(val, value_populate(cursor, i)) - self.assertEqual(i, rows) - -def simple_index_populate_check(self, uri, rows): - self.pr('simple_index_populate_check: ' + uri) - - # Check values in the main table. - cursor = self.session.open_cursor(uri, None) - simple_index_populate_check_cursor(self, cursor, rows) - - # Check values in the index. - indxname = 'index:' + uri.split(":")[1] - idxcursor = self.session.open_cursor(indxname + ':index1') - for i in range(1, rows + 1): - k = key_populate(cursor, i) - v = value_populate(cursor, i) - ik = (v,k) # The index key is columns=(v,k). - self.assertEqual(v, idxcursor[ik]) - idxcursor.close() - cursor.close() - -# Return the value stored in a complex object. -def complex_value_populate(cursor, i): - return [str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%26], - i, - str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%23], - str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%18]] - -# Return the number of column groups used -def complex_populate_colgroup_count(): - return 6 - -# Return the number of indices used -def complex_populate_index_count(): - return 6 - -# population of a complex object -# uri: object -# config: prefix of the session.create configuration string -# rows: entries to insert -def complex_populate(self, uri, config, rows): - complex_populate_type(self, uri, config, '', rows, '') -def complex_populate_cgconfig(self, uri, config, rows): - complex_populate_type(self, uri, config, config, rows, '') -def complex_populate_lsm(self, uri, config, rows): - complex_populate_type(self, uri, config, '', rows, 'type=lsm') -def complex_populate_cgconfig_lsm(self, uri, config, rows): - complex_populate_type(self, uri, config, config, rows, 'type=lsm') -def complex_populate_type(self, uri, config, cgconfig, rows, type): - self.session.create(uri, - config + ',value_format=SiSS,' + - 'columns=(record,column2,column3,column4,column5),' + - 'colgroups=(cgroup1,cgroup2,cgroup3,cgroup4,cgroup5,cgroup6)') - - cgname = 'colgroup:' + uri.split(":")[1] - cgcfg = ',' + cgconfig + ',' + type - self.session.create(cgname + ':cgroup1', 'columns=(column2)' + ',' + cgcfg) - self.session.create(cgname + ':cgroup2', 'columns=(column3)' + ',' + cgcfg) - self.session.create(cgname + ':cgroup3', 'columns=(column4)' + ',' + cgcfg) - self.session.create( - cgname + ':cgroup4', 'columns=(column2,column3)' + ',' + cgcfg) - self.session.create( - cgname + ':cgroup5', 'columns=(column3,column4)' + ',' + cgcfg) - self.session.create( - cgname + ':cgroup6', 'columns=(column2,column4,column5)' + ',' + cgcfg) - indxname = 'index:' + uri.split(":")[1] - self.session.create(indxname + ':indx1', 'columns=(column2)' + ',' + cgcfg) - self.session.create(indxname + ':indx2', 'columns=(column3)' + ',' + cgcfg) - self.session.create(indxname + ':indx3', 'columns=(column4)' + ',' + cgcfg) - self.session.create( - indxname + ':indx4', 'columns=(column2,column4)' + ',' + cgcfg) - cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = \ - tuple(complex_value_populate(cursor, i)) - cursor.close() - # add some indices after populating - self.session.create( - indxname + ':indx5', 'columns=(column3,column5)' + ',' + cgcfg) - self.session.create( - indxname + ':indx6', 'columns=(column3,column5,column4)' + ',' + cgcfg) - -def complex_populate_colgroup_name(self, uri, i): - return 'colgroup:' + uri.split(":")[1] + ':cgroup' + str(i + 1) - -def complex_populate_index_name(self, uri, i): - return 'index:' + uri.split(":")[1] + ':indx' + str(i + 1) - -def complex_populate_check_cursor(self, cursor, rows): - i = 0 - for key, s1, i2, s3, s4 in cursor: - i += 1 - self.assertEqual(key, key_populate(cursor, i)) - v = complex_value_populate(cursor, i) - self.assertEqual(s1, v[0]) - self.assertEqual(i2, v[1]) - self.assertEqual(s3, v[2]) - self.assertEqual(s4, v[3]) - self.assertEqual(i, rows) - -def complex_populate_check(self, uri, rows): - self.pr('complex_populate_check: ' + uri) - cursor = self.session.open_cursor(uri, None) - complex_populate_check_cursor(self, cursor, rows) - cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_async03.py b/src/third_party/wiredtiger/test/suite/test_async03.py index 98851885f70..4859360924a 100644 --- a/src/third_party/wiredtiger/test/suite/test_async03.py +++ b/src/third_party/wiredtiger/test/suite/test_async03.py @@ -54,12 +54,11 @@ class test_async03(wttest.WiredTigerTestCase): tablearg = 'table:' + self.table_name1 self.session.create(tablearg, 'key_format=S,value_format=S') - # Populate table with async inserts, callback checks - # to ensure key/value is correct. callback = Callback() - self.assertRaises(wiredtiger.WiredTigerError, - lambda: self.conn.async_new_op(tablearg, None, callback)) + msg = '/Asynchronous operations not configured/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.async_new_op(tablearg, None, callback), msg) self.conn.async_flush() diff --git a/src/third_party/wiredtiger/test/suite/test_backup01.py b/src/third_party/wiredtiger/test/suite/test_backup01.py index 92d39514440..4e98b6d8e77 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup01.py +++ b/src/third_party/wiredtiger/test/suite/test_backup01.py @@ -32,8 +32,8 @@ import shutil import string from suite_subprocess import suite_subprocess import wiredtiger, wttest -from helper import compare_files,\ - complex_populate, complex_populate_lsm, simple_populate +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet +from helper import compare_files # test_backup.py # Utilities: wt backup @@ -43,14 +43,14 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): pfx = 'test_backup' objs = [ - ( 'file:' + pfx + '.1', simple_populate, 0), - ( 'file:' + pfx + '.2', simple_populate, 0), - ('table:' + pfx + '.3', simple_populate, 0), - ('table:' + pfx + '.4', simple_populate, 0), - ('table:' + pfx + '.5', complex_populate, 0), - ('table:' + pfx + '.6', complex_populate, 0), - ('table:' + pfx + '.7', complex_populate_lsm, 1), - ('table:' + pfx + '.8', complex_populate_lsm, 1), + ( 'file:' + pfx + '.1', SimpleDataSet, 0), + ( 'file:' + pfx + '.2', SimpleDataSet, 0), + ('table:' + pfx + '.3', SimpleDataSet, 0), + ('table:' + pfx + '.4', SimpleDataSet, 0), + ('table:' + pfx + '.5', ComplexDataSet, 0), + ('table:' + pfx + '.6', ComplexDataSet, 0), + ('table:' + pfx + '.7', ComplexLSMDataSet, 1), + ('table:' + pfx + '.8', ComplexLSMDataSet, 1), ] # Populate a set of objects. @@ -59,7 +59,7 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): if i[2]: if skiplsm: continue - i[1](self, i[0], 'key_format=S', 100) + i[1](self, i[0], 100).populate() # Compare the original and backed-up files using the wt dump command. def compare(self, uri): diff --git a/src/third_party/wiredtiger/test/suite/test_backup02.py b/src/third_party/wiredtiger/test/suite/test_backup02.py index 398d55abd7a..d4089273be0 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup02.py +++ b/src/third_party/wiredtiger/test/suite/test_backup02.py @@ -28,7 +28,6 @@ import Queue import threading, time, wiredtiger, wttest -from helper import key_populate, simple_populate from wtthread import backup_thread, checkpoint_thread, op_thread from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_backup03.py b/src/third_party/wiredtiger/test/suite/test_backup03.py index ba7ce60f519..73d05f0b0a1 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup03.py +++ b/src/third_party/wiredtiger/test/suite/test_backup03.py @@ -27,11 +27,11 @@ # OTHER DEALINGS IN THE SOFTWARE. import glob, os, shutil, string +import wiredtiger, wttest +from helper import compare_files from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios -import wiredtiger, wttest -from helper import compare_files,\ - complex_populate, complex_populate_lsm, simple_populate # test_backup03.py # Utilities: wt backup @@ -50,10 +50,10 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): # to the backup to confirm the backup is correct. pfx = 'test_backup' objs = [ # Objects - ('table:' + pfx + '.1', simple_populate, 0), - ( 'lsm:' + pfx + '.2', simple_populate, 1), - ('table:' + pfx + '.3', complex_populate, 2), - ('table:' + pfx + '.4', complex_populate_lsm, 3), + ('table:' + pfx + '.1', SimpleDataSet, 0), + ( 'lsm:' + pfx + '.2', SimpleDataSet, 1), + ('table:' + pfx + '.3', ComplexDataSet, 2), + ('table:' + pfx + '.4', ComplexLSMDataSet, 3), ] list = [ ( 'backup_1', dict(big=0,list=[0])), # Target objects individually @@ -85,7 +85,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): rows = 200000 # Big object else: rows = 1000 # Small object - i[1](self, i[0], 'key_format=S', rows) + i[1](self, i[0], rows).populate() # Backup needs a checkpoint self.session.checkpoint(None) diff --git a/src/third_party/wiredtiger/test/suite/test_backup04.py b/src/third_party/wiredtiger/test/suite/test_backup04.py index dd4dbc1d60f..919649fed57 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup04.py +++ b/src/third_party/wiredtiger/test/suite/test_backup04.py @@ -29,10 +29,11 @@ import Queue import threading, time, wiredtiger, wttest import glob, os, shutil +from helper import compare_files from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, simple_key from wtscenario import make_scenarios from wtthread import op_thread -from helper import compare_files, key_populate # test_backup04.py # Utilities: wt backup @@ -67,14 +68,14 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): self.pr('populate: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = str(i) + ':' + 'a' * dsize + cursor[simple_key(cursor, i)] = str(i) + ':' + 'a' * dsize cursor.close() def update(self, uri, dsize, upd, rows): self.pr('update: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = str(i) + ':' + upd * dsize + cursor[simple_key(cursor, i)] = str(i) + ':' + upd * dsize cursor.close() # Compare the original and backed-up files using the wt dump command. diff --git a/src/third_party/wiredtiger/test/suite/test_backup06.py b/src/third_party/wiredtiger/test/suite/test_backup06.py new file mode 100644 index 00000000000..9f7a247f2b9 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup06.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. + +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import glob +import os +import shutil +import string +from suite_subprocess import suite_subprocess +import wiredtiger, wttest +from wiredtiger import stat +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet + +# test_backup06.py +# Test that opening a backup cursor does not open file handles. +class test_backup06(wttest.WiredTigerTestCase, suite_subprocess): + conn_config = 'statistics=(fast)' + # This will create several hundred tables. + num_table_sets = 10 + pfx='test_backup' + + # We try to do some schema operations. Have some well + # known names. + schema_uri = 'file:schema_test' + rename_uri = 'file:new_test' + trename_uri = 'table:new_test' + + fobjs = [ + ( 'file:' + pfx + '.1', SimpleDataSet), + ( 'file:' + pfx + '.2', SimpleDataSet), + ] + tobjs = [ + ('table:' + pfx + '.3', SimpleDataSet), + ('table:' + pfx + '.4', SimpleDataSet), + ('table:' + pfx + '.5', ComplexDataSet), + ('table:' + pfx + '.6', ComplexDataSet), + ('table:' + pfx + '.7', ComplexLSMDataSet), + ('table:' + pfx + '.8', ComplexLSMDataSet), + ] + + # Populate a set of objects. + def populate_many(self): + for t in range(0, self.num_table_sets): + for i in self.fobjs: + uri = i[0] + "." + str(t) + i[1](self, uri, 10).populate() + for i in self.tobjs: + uri = i[0] + "." + str(t) + i[1](self, uri, 10).populate() + + def populate(self): + for i in self.fobjs: + i[1](self, i[0], 100).populate() + for i in self.tobjs: + i[1](self, i[0], 100).populate() + + # Test that the open handle count does not change. + def test_cursor_open_handles(self): + self.populate_many() + # Close and reopen the connection so the populate dhandles are + # not in the list. + self.reopen_conn() + + # Confirm that opening a backup cursor does not open + # file handles. + stat_cursor = self.session.open_cursor('statistics:', None, None) + dh_before = stat_cursor[stat.conn.dh_conn_handle_count][2] + stat_cursor.close() + cursor = self.session.open_cursor('backup:', None, None) + stat_cursor = self.session.open_cursor('statistics:', None, None) + dh_after = stat_cursor[stat.conn.dh_conn_handle_count][2] + stat_cursor.close() + if (dh_before != dh_after): + print "Dhandles open before backup open: " + str(dh_before) + print "Dhandles open after backup open: " + str(dh_after) + self.assertEqual(dh_after == dh_before, True) + cursor.close() + + def test_cursor_schema_protect(self): + schema_uri = 'file:schema_test' + rename_uri = 'file:new_test' + trename_uri = 'table:new_test' + + # + # Set up a number of tables. Close and reopen the connection so that + # we do not have open dhandles. Then we want to open a backup cursor + # testing both with and without the configuration setting. + # We want to confirm that we open data handles when using schema + # protection and we do not open the data handles when set to false. + # We also want to make sure we detect and get an error when set to + # false. When set to true the open handles protect against schema + # operations. + self.populate() + cursor = self.session.open_cursor('backup:', None, None) + # Check that we can create. + self.session.create(schema_uri, None) + for i in self.fobjs: + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.drop(i[0], None)) + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.rename(i[0], rename_uri)) + for i in self.tobjs: + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.drop(i[0], None)) + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.rename(i[0], trename_uri)) + cursor.close() + + # Test cursor reset runs through the list twice. + def test_cursor_reset(self): + self.populate() + cursor = self.session.open_cursor('backup:', None, None) + i = 0 + while True: + ret = cursor.next() + if ret != 0: + break + i += 1 + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + total = i * 2 + cursor.reset() + while True: + ret = cursor.next() + if ret != 0: + break + i += 1 + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + self.assertEqual(i, total) + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug004.py b/src/third_party/wiredtiger/test/suite/test_bug004.py index 316d15d8080..464cc57e272 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug004.py +++ b/src/third_party/wiredtiger/test/suite/test_bug004.py @@ -30,7 +30,7 @@ # Regression tests. import wiredtiger, wttest -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value # Check to make sure we see the right versions of overflow keys and values # when they are deleted in reconciliation without having been instantiated @@ -50,8 +50,8 @@ class test_bug004(wttest.WiredTigerTestCase): self.session.create(self.uri, self.config) c1 = self.session.open_cursor(self.uri, None) for i in range(1, self.nentries): - c1[key_populate(c1, i) + 'abcdef' * 100] = \ - value_populate(c1, i) + 'abcdef' * 100 + c1[simple_key(c1, i) + 'abcdef' * 100] = \ + simple_value(c1, i) + 'abcdef' * 100 c1.close() # Verify the object, force it to disk, and verify the on-disk version. @@ -69,9 +69,9 @@ class test_bug004(wttest.WiredTigerTestCase): # currently do -- that's unlikely to change, but is a problem for the # test going forward.) c1 = self.session.open_cursor(self.uri, None) - c1.set_key(key_populate(c1, self.nentries - 5) + 'abcdef' * 100) + c1.set_key(simple_key(c1, self.nentries - 5) + 'abcdef' * 100) c2 = self.session.open_cursor(self.uri, None) - c2.set_key(key_populate(c2, self.nentries + 5) + 'abcdef' * 100) + c2.set_key(simple_key(c2, self.nentries + 5) + 'abcdef' * 100) self.session.truncate(None, c1, c2, None) c1.close() c2.close() @@ -81,14 +81,14 @@ class test_bug004(wttest.WiredTigerTestCase): # Use the snapshot cursor to retrieve the old key/value pairs c1 = tmp_session.open_cursor(self.uri, None) - c1.set_key(key_populate(c1, 1) + 'abcdef' * 100) + c1.set_key(simple_key(c1, 1) + 'abcdef' * 100) c1.search() for i in range(2, self.nentries): c1.next() self.assertEquals( - c1.get_key(), key_populate(c1, i) + 'abcdef' * 100) + c1.get_key(), simple_key(c1, i) + 'abcdef' * 100) self.assertEquals( - c1.get_value(), value_populate(c1, i) + 'abcdef' * 100) + c1.get_value(), simple_value(c1, i) + 'abcdef' * 100) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug005.py b/src/third_party/wiredtiger/test/suite/test_bug005.py index 6437b329096..69df175ae67 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug005.py +++ b/src/third_party/wiredtiger/test/suite/test_bug005.py @@ -30,7 +30,7 @@ # Regression tests. import wiredtiger, wttest -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value # Check that verify works when the file has additional data after the last # checkpoint. @@ -43,7 +43,7 @@ class test_bug005(wttest.WiredTigerTestCase): self.session.create(self.uri, 'value_format=S,key_format=S') cursor = self.session.open_cursor(self.uri, None) for i in range(1, 1000): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) cursor.close() # Verify the object, force it to disk, and verify the on-disk version. diff --git a/src/third_party/wiredtiger/test/suite/test_bug006.py b/src/third_party/wiredtiger/test/suite/test_bug006.py index e01333f59de..c0f6055f720 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug006.py +++ b/src/third_party/wiredtiger/test/suite/test_bug006.py @@ -30,7 +30,7 @@ # Regression tests. import wiredtiger, wttest -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios # Check that verify and salvage both raise exceptions if there is an open @@ -47,7 +47,7 @@ class test_bug006(wttest.WiredTigerTestCase): self.session.create(uri, 'value_format=S,key_format=S') cursor = self.session.open_cursor(uri, None) for i in range(1, 1000): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Table operations should fail, the cursor is open. self.assertRaises( diff --git a/src/third_party/wiredtiger/test/suite/test_bug008.py b/src/third_party/wiredtiger/test/suite/test_bug008.py index 8e787ae14d1..c54c92fc864 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug008.py +++ b/src/third_party/wiredtiger/test/suite/test_bug008.py @@ -30,7 +30,7 @@ # Regression tests. import wiredtiger, wttest -from helper import simple_populate, key_populate, value_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # Test search/search-near operations, including invisible values and keys @@ -38,23 +38,25 @@ from wtscenario import make_scenarios class test_bug008(wttest.WiredTigerTestCase): uri = 'file:test_bug008' # This is a btree layer test. scenarios = make_scenarios([ - ('fix', dict(fmt='key_format=r,value_format=8t', empty=1, colvar=0)), - ('row', dict(fmt='key_format=S', empty=0, colvar=0)), - ('var', dict(fmt='key_format=r', empty=0, colvar=1)) + ('fix', dict(key_format='r', value_format='8t', empty=1, colvar=0)), + ('row', dict(key_format='S', value_format='S', empty=0, colvar=0)), + ('var', dict(key_format='r', value_format='S', empty=0, colvar=1)) ]) # Verify cursor search and search-near operations in an empty table. def test_search_empty(self): # Create the object and open a cursor. - self.session.create(self.uri, self.fmt) + ds = SimpleDataSet(self, self.uri, 0, key_format=self.key_format, + value_format=self.value_format) + ds.create() cursor = self.session.open_cursor(self.uri, None) # Search for a record past the end of the table, which should fail. - cursor.set_key(key_populate(cursor, 100)) + cursor.set_key(ds.key(100)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Search-near for a record past the end of the table, which should fail. - cursor.set_key(key_populate(cursor, 100)) + cursor.set_key(ds.key(100)) self.assertEqual(cursor.search_near(), wiredtiger.WT_NOTFOUND) # Verify cursor search and search-near operations at and past the end of @@ -62,35 +64,37 @@ class test_bug008(wttest.WiredTigerTestCase): def test_search_eot(self): # Populate the tree and reopen the connection, forcing it to disk # and moving the records to an on-page format. - simple_populate(self, self.uri, self.fmt, 100) + ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format, + value_format=self.value_format) + ds.populate() self.reopen_conn() # Open a cursor. cursor = self.session.open_cursor(self.uri, None) # Search for a record at the end of the table, which should succeed. - cursor.set_key(key_populate(cursor, 100)) + cursor.set_key(ds.key(100)) self.assertEqual(cursor.search(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + self.assertEqual(cursor.get_key(), ds.key(100)) + self.assertEqual(cursor.get_value(), ds.value(100)) # Search-near for a record at the end of the table, which should # succeed, returning the last record. - cursor.set_key(key_populate(cursor, 100)) + cursor.set_key(ds.key(100)) self.assertEqual(cursor.search_near(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + self.assertEqual(cursor.get_key(), ds.key(100)) + self.assertEqual(cursor.get_value(), ds.value(100)) # Search for a record past the end of the table, which should fail. - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Search-near for a record past the end of the table, which should # succeed, returning the last record. - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) self.assertEqual(cursor.search_near(), -1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + self.assertEqual(cursor.get_key(), ds.key(100)) + self.assertEqual(cursor.get_value(), ds.value(100)) # Verify cursor search-near operations before and after a set of # column-store duplicates. @@ -99,18 +103,20 @@ class test_bug008(wttest.WiredTigerTestCase): return # Populate the tree. - simple_populate(self, self.uri, self.fmt, 105) + ds = SimpleDataSet(self, self.uri, 105, key_format=self.key_format, + value_format=self.value_format) + ds.populate() # Set up deleted records before and after a set of duplicate records, # and make sure search/search-near returns the correct record. cursor = self.session.open_cursor(self.uri, None) for i in range(20, 100): - cursor[key_populate(cursor, i)] = '=== IDENTICAL VALUE ===' + cursor[ds.key(i)] = '=== IDENTICAL VALUE ===' for i in range(15, 25): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.remove(), 0) for i in range(95, 106): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.remove(), 0) cursor.close() @@ -123,26 +129,28 @@ class test_bug008(wttest.WiredTigerTestCase): # Search-near for a record in the deleted set before the duplicate set, # which should succeed, returning the first record in the duplicate set. - cursor.set_key(key_populate(cursor, 18)) + cursor.set_key(ds.key(18)) self.assertEqual(cursor.search_near(), 1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 25)) + self.assertEqual(cursor.get_key(), ds.key(25)) # Search-near for a record in the deleted set after the duplicate set, # which should succeed, returning the last record in the duplicate set. - cursor.set_key(key_populate(cursor, 98)) + cursor.set_key(ds.key(98)) self.assertEqual(cursor.search_near(), -1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 94)) + self.assertEqual(cursor.get_key(), ds.key(94)) # Verify cursor search and search-near operations on a file with a set of # on-page visible records, and a set of insert-list invisible records. def test_search_invisible_one(self): # Populate the tree. - simple_populate(self, self.uri, self.fmt, 100) + ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format, + value_format=self.value_format) + ds.populate() # Delete a range of records. for i in range(5, 10): cursor = self.session.open_cursor(self.uri, None) - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.remove(), 0) # Reopen the connection, forcing it to disk and moving the records to @@ -155,11 +163,11 @@ class test_bug008(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor = self.session.open_cursor(self.uri, None) for i in range(5, 10): - cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) + cursor[ds.key(i)] = ds.value(i + 1000) for i in range(30, 40): - cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) + cursor[ds.key(i)] = ds.value(i + 1000) for i in range(100, 140): - cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) + cursor[ds.key(i)] = ds.value(i + 1000) # Open a separate session and cursor. s = self.conn.open_session() @@ -168,7 +176,7 @@ class test_bug008(wttest.WiredTigerTestCase): # Search for an existing record in the deleted range, should not find # it. for i in range(5, 10): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) if self.empty: # Fixed-length column-store rows always exist. self.assertEqual(cursor.search(), 0) @@ -180,13 +188,13 @@ class test_bug008(wttest.WiredTigerTestCase): # Search for an existing record in the updated range, should see the # original value. for i in range(30, 40): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.search(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + self.assertEqual(cursor.get_key(), ds.key(i)) # Search for a added record, should not find it. for i in range(120, 130): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past @@ -203,7 +211,7 @@ class test_bug008(wttest.WiredTigerTestCase): # the next largest record. (This depends on the implementation behavior # which currently includes a bias to prefix search.) for i in range(5, 10): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) if self.empty: # Fixed-length column-store rows always exist. self.assertEqual(cursor.search_near(), 0) @@ -211,19 +219,19 @@ class test_bug008(wttest.WiredTigerTestCase): self.assertEqual(cursor.get_value(), 0) else: self.assertEqual(cursor.search_near(), 1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 10)) + self.assertEqual(cursor.get_key(), ds.key(10)) # Search-near for an existing record in the updated range, should see # the original value. for i in range(30, 40): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.search_near(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + self.assertEqual(cursor.get_key(), ds.key(i)) # Search-near for an added record, should find the previous largest # record. for i in range(120, 130): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past @@ -234,7 +242,7 @@ class test_bug008(wttest.WiredTigerTestCase): self.assertEqual(cursor.get_value(), 0) else: self.assertEqual(cursor.search_near(), -1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_key(), ds.key(100)) # Verify cursor search and search-near operations on a file with a set of # on-page visible records, a set of insert-list visible records, and a set @@ -246,27 +254,29 @@ class test_bug008(wttest.WiredTigerTestCase): def test_search_invisible_two(self): # Populate the tree and reopen the connection, forcing it to disk # and moving the records to an on-page format. - simple_populate(self, self.uri, self.fmt, 100) + ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format, + value_format=self.value_format) + ds.populate() self.reopen_conn() # Add some additional visible records. cursor = self.session.open_cursor(self.uri, None) for i in range(100, 120): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) cursor.close() # Begin a transaction, and add some additional records. self.session.begin_transaction() cursor = self.session.open_cursor(self.uri, None) for i in range(120, 140): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) # Open a separate session and cursor. s = self.conn.open_session() cursor = s.open_cursor(self.uri, None) # Search for an invisible record. - cursor.set_key(key_populate(cursor, 130)) + cursor.set_key(ds.key(130)) if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past @@ -281,7 +291,7 @@ class test_bug008(wttest.WiredTigerTestCase): # Search-near for an invisible record, which should succeed, returning # the last visible record. - cursor.set_key(key_populate(cursor, 130)) + cursor.set_key(ds.key(130)) cursor.search_near() if self.empty: # Invisible updates to fixed-length column-store objects are @@ -294,8 +304,8 @@ class test_bug008(wttest.WiredTigerTestCase): else: # Otherwise, we should find the closest record for which we can see # the value. - self.assertEqual(cursor.get_key(), key_populate(cursor, 119)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 119)) + self.assertEqual(cursor.get_key(), ds.key(119)) + self.assertEqual(cursor.get_value(), ds.value(119)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug011.py b/src/third_party/wiredtiger/test/suite/test_bug011.py index fceb7a22ddb..29bb08ec2e5 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug011.py +++ b/src/third_party/wiredtiger/test/suite/test_bug011.py @@ -26,8 +26,8 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import helper, random, wiredtiger, wttest -from helper import simple_populate +import random, wiredtiger, wttest +from wtdataset import SimpleDataSet # test_bug011.py # Eviction working on more files than there are hazard pointers. @@ -46,11 +46,13 @@ class test_bug011(wttest.WiredTigerTestCase): def test_eviction(self): cursors = [] + datasets = [] for i in range(0, self.ntables): this_uri = 'table:%s-%03d' % (self.table_name, i) - simple_populate(self, this_uri, - 'key_format=S,allocation_size=1KB,leaf_page_max=1KB', - self.nrows) + ds = SimpleDataSet(self, this_uri, self.nrows, + config='allocation_size=1KB,leaf_page_max=1KB') + ds.populate() + datasets.append(ds) # Switch over to on-disk trees with multiple leaf pages self.reopen_conn() @@ -63,8 +65,7 @@ class test_bug011(wttest.WiredTigerTestCase): # Make use of the cache. for i in range(0, self.nops): for i in range(0, self.ntables): - cursors[i].set_key(helper.key_populate(cursors[i], - random.randint(0, self.nrows - 1))) + cursors[i].set_key(ds.key(random.randint(0, self.nrows - 1))) cursors[i].search() cursors[i].reset() diff --git a/src/third_party/wiredtiger/test/suite/test_bug012.py b/src/third_party/wiredtiger/test/suite/test_bug012.py index a28834f36d2..91f49d14b3f 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug012.py +++ b/src/third_party/wiredtiger/test/suite/test_bug012.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import complex_populate +from wtdataset import ComplexDataSet # test_bug012.py class test_bug012(wttest.WiredTigerTestCase): @@ -71,10 +71,11 @@ class test_bug012(wttest.WiredTigerTestCase): # Test that we detect illegal extractors. # - # This test is a little fragile, we're depending on complex_populate to do + # This test is a little fragile, we're depending on ComplexDataSet to do # the heavy-lifting, so if that function changes, this could break. def test_illegal_extractor(self): - complex_populate(self, 'table:A', 'key_format=S', 10) + ds = ComplexDataSet(self, 'table:A', 10) + ds.populate() msg = '/unknown extractor/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.create('index:A:xyzzy', diff --git a/src/third_party/wiredtiger/test/suite/test_bug014.py b/src/third_party/wiredtiger/test/suite/test_bug014.py index c951cb45293..1dee933e839 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug014.py +++ b/src/third_party/wiredtiger/test/suite/test_bug014.py @@ -27,7 +27,8 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import copy_wiredtiger_home, key_populate, simple_populate +from wtdataset import SimpleDataSet +from helper import copy_wiredtiger_home # test_bug014.py # JIRA WT-2115: fast-delete pages can be incorrectly lost due to a crash. @@ -35,8 +36,9 @@ class test_bug014(wttest.WiredTigerTestCase): def test_bug014(self): # Populate a table with 1000 keys on small pages. uri = 'table:test_bug014' - simple_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 1000) + ds = SimpleDataSet(self, uri, 1000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() # Reopen it so we can fast-delete pages. self.reopen_conn() @@ -44,9 +46,9 @@ class test_bug014(wttest.WiredTigerTestCase): # Truncate a chunk of the key/value pairs inside a transaction. self.session.begin_transaction(None) start = self.session.open_cursor(uri, None) - start.set_key(key_populate(start, 250)) + start.set_key(ds.key(250)) end = self.session.open_cursor(uri, None) - end.set_key(key_populate(end, 500)) + end.set_key(ds.key(500)) self.session.truncate(None, start, end, None) start.close() end.close() @@ -66,7 +68,7 @@ class test_bug014(wttest.WiredTigerTestCase): # Confirm all of the records are there. for i in range(1, 1001): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.search(), 0) conn.close() diff --git a/src/third_party/wiredtiger/test/suite/test_bug015.py b/src/third_party/wiredtiger/test/suite/test_bug015.py index 7892f66759d..68cca49688f 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug015.py +++ b/src/third_party/wiredtiger/test/suite/test_bug015.py @@ -27,7 +27,6 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import copy_wiredtiger_home, key_populate, simple_populate # test_bug015.py # JIRA WT-2162: index drop in a certain order triggers NULL pointer deref diff --git a/src/third_party/wiredtiger/test/suite/test_bulk01.py b/src/third_party/wiredtiger/test/suite/test_bulk01.py index addebe80647..8d5b6a04385 100644 --- a/src/third_party/wiredtiger/test/suite/test_bulk01.py +++ b/src/third_party/wiredtiger/test/suite/test_bulk01.py @@ -31,7 +31,7 @@ # import wiredtiger, wttest -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios # Smoke test bulk-load. @@ -61,7 +61,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") for i in range(1, 1000): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Test a bulk-load triggers variable-length column-store RLE correctly. def test_bulk_load_var_rle(self): @@ -76,7 +76,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") for i in range(1, 1000): - cursor[key_populate(cursor, i)] = value_populate(cursor, i/7) + cursor[simple_key(cursor, i)] = simple_value(cursor, i/7) # Test a bulk-load variable-length column-store append ignores any key. def test_bulk_load_var_append(self): @@ -88,13 +88,13 @@ class test_bulk_load(wttest.WiredTigerTestCase): 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk,append") for i in range(1, 1000): - cursor[key_populate(cursor, 37)] = value_populate(cursor, i) + cursor[simple_key(cursor, 37)] = simple_value(cursor, i) cursor.close() cursor = self.session.open_cursor(uri, None, None) for i in range(1, 1000): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(simple_key(cursor, i)) cursor.search() - self.assertEqual(cursor.get_value(), value_populate(cursor, i)) + self.assertEqual(cursor.get_value(), simple_value(cursor, i)) # Test that column-store bulk-load handles skipped records correctly. def test_bulk_load_col_delete(self): @@ -107,21 +107,21 @@ class test_bulk_load(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, "bulk") for i in range(1, 1000): if i % 7 == 0: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Ensure we create all the missing records. i = i + 1 - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) cursor.close() cursor = self.session.open_cursor(uri, None, None) # Verify all the records are there, in their proper state. for i in range(1, 1000): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(simple_key(cursor, i)) if i % 7 == 0: cursor.search() - self.assertEqual(cursor.get_value(), value_populate(cursor, i)) + self.assertEqual(cursor.get_value(), simple_value(cursor, i)) elif cursor.value_format == '8t': cursor.search() self.assertEqual(cursor.get_value(), 0) @@ -139,17 +139,17 @@ class test_bulk_load(wttest.WiredTigerTestCase): 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") for i in range(1, 10): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # A big record -- if it's not efficient, we'll just hang. big = 18446744073709551606 - cursor[key_populate(cursor, big)] = value_populate(cursor, big) + cursor[simple_key(cursor, big)] = simple_value(cursor, big) cursor.close() cursor = self.session.open_cursor(uri, None, None) - cursor.set_key(key_populate(cursor, big)) + cursor.set_key(simple_key(cursor, big)) cursor.search() - self.assertEqual(cursor.get_value(), value_populate(cursor, big)) + self.assertEqual(cursor.get_value(), simple_value(cursor, big)) # Test that bulk-load out-of-order fails. def test_bulk_load_order_check(self): @@ -157,16 +157,16 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - cursor[key_populate(cursor, 10)] = value_populate(cursor, 10) + cursor[simple_key(cursor, 10)] = simple_value(cursor, 10) for i in [1, 9, 10]: - cursor.set_key(key_populate(cursor, 1)) - cursor.set_value(value_populate(cursor, 1)) + cursor.set_key(simple_key(cursor, 1)) + cursor.set_value(simple_value(cursor, 1)) msg = '/than previously inserted key/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: cursor.insert(), msg) - cursor[key_populate(cursor, 11)] = value_populate(cursor, 11) + cursor[simple_key(cursor, 11)] = simple_value(cursor, 11) # Test that row-store bulk-load out-of-order can succeed. def test_bulk_load_row_order_nocheck(self): @@ -181,8 +181,8 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk,skip_sort_check") - cursor[key_populate(cursor, 10)] = value_populate(cursor, 10) - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[simple_key(cursor, 10)] = simple_value(cursor, 10) + cursor[simple_key(cursor, 1)] = simple_value(cursor, 1) if not wiredtiger.diagnostic_build(): self.skipTest('requires a diagnostic build') @@ -197,7 +197,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): uri = self.type + self.name self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None) - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[simple_key(cursor, 1)] = simple_value(cursor, 1) # Close the insert cursor, else we'll get EBUSY. cursor.close() msg = '/bulk-load is only supported on newly created objects/' @@ -209,7 +209,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): uri = self.type + self.name self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None) - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[simple_key(cursor, 1)] = simple_value(cursor, 1) # Don't close the insert cursor, we want EBUSY. self.assertRaises(wiredtiger.WiredTigerError, lambda: self.session.open_cursor(uri, None, "bulk")) diff --git a/src/third_party/wiredtiger/test/suite/test_bulk02.py b/src/third_party/wiredtiger/test/suite/test_bulk02.py index c959098179c..fb9240e91e7 100644 --- a/src/third_party/wiredtiger/test/suite/test_bulk02.py +++ b/src/third_party/wiredtiger/test/suite/test_bulk02.py @@ -30,10 +30,11 @@ # Bulk-load testing. import shutil, os -from helper import confirm_empty, key_populate, value_populate +import wiredtiger, wttest +from helper import confirm_empty from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios -import wiredtiger, wttest # test_bulkload_checkpoint # Test bulk-load with checkpoints. @@ -56,7 +57,7 @@ class test_bulkload_checkpoint(wttest.WiredTigerTestCase, suite_subprocess): self.session.create(self.uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(self.uri, None, 'bulk') for i in range(1, 10): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Checkpoint a few times (to test the drop code). for i in range(1, 5): @@ -110,7 +111,7 @@ class test_bulkload_backup(wttest.WiredTigerTestCase, suite_subprocess): self.session.create(self.uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(self.uri, None, 'bulk') for i in range(1, 10): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Test without a checkpoint, with an unnamed checkpoint, with a named # checkpoint. diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py index 013cda96e4e..c0d004db78d 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import key_populate, complex_populate_lsm, simple_populate +from wtdataset import SimpleDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_checkpoint01.py @@ -145,7 +145,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase): # Check that you cannot open a checkpoint that doesn't exist. def test_checkpoint_dne(self): - simple_populate(self, self.uri, 'key_format=' + self.fmt, 100) + SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate() self.assertRaises(wiredtiger.WiredTigerError, lambda: self.session.open_cursor( self.uri, None, "checkpoint=checkpoint-1")) @@ -155,7 +155,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase): # Check that you can open checkpoints more than once. def test_checkpoint_multiple_open(self): - simple_populate(self, self.uri, 'key_format=' + self.fmt, 100) + SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate() self.session.checkpoint("name=checkpoint-1") c1 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1") c2 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1") @@ -172,7 +172,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase): # Check that you cannot drop a checkpoint if it's in use. def test_checkpoint_inuse(self): - simple_populate(self, self.uri, 'key_format=' + self.fmt, 100) + SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate() self.session.checkpoint("name=checkpoint-1") self.session.checkpoint("name=checkpoint-2") self.session.checkpoint("name=checkpoint-3") @@ -208,44 +208,49 @@ class test_checkpoint_target(wttest.WiredTigerTestCase): ('table', dict(uri='table:checkpoint',fmt='S')) ]) - def update(self, uri, value): + def update(self, uri, ds, value): cursor = self.session.open_cursor(uri, None, "overwrite") - cursor[key_populate(cursor, 10)] = value + cursor[ds.key(10)] = value cursor.close() - def check(self, uri, value): + def check(self, uri, ds, value): cursor = self.session.open_cursor(uri, None, "checkpoint=checkpoint-1") - self.assertEquals(cursor[key_populate(cursor, 10)], value) + self.assertEquals(cursor[ds.key(10)], value) cursor.close() def test_checkpoint_target(self): # Create 3 objects, change one record to an easily recognizable string. uri = self.uri + '1' - simple_populate(self, uri, 'key_format=' + self.fmt, 100) - self.update(uri, 'ORIGINAL') + ds1 = SimpleDataSet(self, uri, 100, key_format=self.fmt) + ds1.populate() + self.update(uri, ds1, 'ORIGINAL') + uri = self.uri + '2' - simple_populate(self, uri, 'key_format=' + self.fmt, 100) - self.update(uri, 'ORIGINAL') + ds2 = SimpleDataSet(self, uri, 100, key_format=self.fmt) + ds2.populate() + self.update(uri, ds2, 'ORIGINAL') + uri = self.uri + '3' - simple_populate(self, uri, 'key_format=' + self.fmt, 100) - self.update(uri, 'ORIGINAL') + ds3 = SimpleDataSet(self, uri, 100, key_format=self.fmt) + ds3.populate() + self.update(uri, ds3, 'ORIGINAL') # Checkpoint all three objects. self.session.checkpoint("name=checkpoint-1") # Update all 3 objects, then checkpoint two of the objects with the # same checkpoint name. - self.update(self.uri + '1', 'UPDATE') - self.update(self.uri + '2', 'UPDATE') - self.update(self.uri + '3', 'UPDATE') + self.update(self.uri + '1', ds1, 'UPDATE') + self.update(self.uri + '2', ds2, 'UPDATE') + self.update(self.uri + '3', ds3, 'UPDATE') target = 'target=("' + self.uri + '1"' + ',"' + self.uri + '2")' self.session.checkpoint("name=checkpoint-1," + target) # Confirm the checkpoint has the old value in objects that weren't # checkpointed, and the new value in objects that were checkpointed. - self.check(self.uri + '1', 'UPDATE') - self.check(self.uri + '2', 'UPDATE') - self.check(self.uri + '3', 'ORIGINAL') + self.check(self.uri + '1', ds1, 'UPDATE') + self.check(self.uri + '2', ds2, 'UPDATE') + self.check(self.uri + '3', ds3, 'ORIGINAL') # Check that you can't write checkpoint cursors. class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): @@ -257,10 +262,11 @@ class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): ]) def test_checkpoint_cursor_update(self): - simple_populate(self, self.uri, 'key_format=' + self.fmt, 100) + ds = SimpleDataSet(self, self.uri, 100, key_format=self.fmt) + ds.populate() self.session.checkpoint("name=ckpt") cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") - cursor.set_key(key_populate(cursor, 10)) + cursor.set_key(ds.key(10)) cursor.set_value("XXX") msg = "/Unsupported cursor/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, @@ -284,12 +290,13 @@ class test_checkpoint_last(wttest.WiredTigerTestCase): # value. Repeat this action, we want to be sure the engine gets the # latest checkpoint information each time. uri = self.uri - simple_populate(self, uri, 'key_format=' + self.fmt, 100) + ds = SimpleDataSet(self, uri, 100, key_format=self.fmt) + ds.populate() for value in ('FIRST', 'SECOND', 'THIRD', 'FOURTH', 'FIFTH'): # Update the object. cursor = self.session.open_cursor(uri, None, "overwrite") - cursor[key_populate(cursor, 10)] = value + cursor[ds.key(10)] = value cursor.close() # Checkpoint the object. @@ -298,14 +305,15 @@ class test_checkpoint_last(wttest.WiredTigerTestCase): # Verify the "last" checkpoint sees the correct value. cursor = self.session.open_cursor( uri, None, "checkpoint=WiredTigerCheckpoint") - self.assertEquals(cursor[key_populate(cursor, 10)], value) + self.assertEquals(cursor[ds.key(10)], value) # Don't close the checkpoint cursor, we want it to remain open until # the test completes. # Check we can't use the reserved name as an application checkpoint name. class test_checkpoint_illegal_name(wttest.WiredTigerTestCase): def test_checkpoint_illegal_name(self): - simple_populate(self, "file:checkpoint", 'key_format=S', 100) + ds = SimpleDataSet(self, "file:checkpoint", 100, key_format='S') + ds.populate() msg = '/the checkpoint name.*is reserved/' for conf in ( 'name=WiredTigerCheckpoint', @@ -329,8 +337,8 @@ class test_checkpoint_illegal_name(wttest.WiredTigerTestCase): # Check we can't name checkpoints that include LSM tables. class test_checkpoint_lsm_name(wttest.WiredTigerTestCase): def test_checkpoint_lsm_name(self): - complex_populate_lsm(self, - "table:checkpoint", 'type=lsm,key_format=S', 1000) + ds = ComplexLSMDataSet(self, "table:checkpoint", 1000) + ds.populate() msg = '/object does not support named checkpoints/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.checkpoint("name=ckpt"), msg) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py index 0d166544472..b5d20fb73b1 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py @@ -28,7 +28,6 @@ import Queue import threading, time, wiredtiger, wttest -from helper import key_populate, simple_populate from wtthread import checkpoint_thread, op_thread from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_colgap.py b/src/third_party/wiredtiger/test/suite/test_colgap.py index dbe748ae58f..91df0fd6c1c 100644 --- a/src/third_party/wiredtiger/test/suite/test_colgap.py +++ b/src/third_party/wiredtiger/test/suite/test_colgap.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import simple_populate, key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios # test_colgap.py @@ -61,14 +61,16 @@ class test_column_store_gap(wttest.WiredTigerTestCase): # namespace. If this runs in less-than-glacial time, it's working. def test_column_store_gap(self): uri = 'table:gap' - simple_populate(self, uri, 'key_format=r,value_format=S', 0) + # Initially just create tables. + ds = SimpleDataSet(self, uri, 0, key_format='r') + ds.populate() cursor = self.session.open_cursor(uri, None, None) self.nentries = 0 # Create a column-store table with large gaps in the name-space. v = [ 1000, 2000000000000, 30000000000000 ] for i in v: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) self.nentries += 1 # In-memory cursor forward, backward. @@ -84,7 +86,9 @@ class test_column_store_gap(wttest.WiredTigerTestCase): def test_column_store_gap_traverse(self): uri = 'table:gap' - simple_populate(self, uri, 'key_format=r,value_format=S', 0) + # Initially just create tables. + ds = SimpleDataSet(self, uri, 0, key_format='r') + ds.populate() cursor = self.session.open_cursor(uri, None, None) self.nentries = 0 @@ -92,7 +96,7 @@ class test_column_store_gap(wttest.WiredTigerTestCase): # important, we just want some gaps. v = [ 1000, 1001, 2000, 2001] for i in v: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) self.nentries += 1 # In-memory cursor forward, backward. @@ -111,7 +115,7 @@ class test_column_store_gap(wttest.WiredTigerTestCase): # so the traversal walks to them. v2 = [ 1500, 1501 ] for i in v2: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) self.nentries += 1 # Tell the validation what to expect. @@ -168,7 +172,7 @@ class test_colmax(wttest.WiredTigerTestCase): # Optionaly make the big record the only record in the table. if not self.single: for i in range(1, 723): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Confirm searching past the end of the table works. if not self.bulk: @@ -176,7 +180,7 @@ class test_colmax(wttest.WiredTigerTestCase): self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Insert the big record. - cursor[key_populate(cursor, recno)] = value_populate(cursor, recno) + cursor[simple_key(cursor, recno)] = simple_value(cursor, recno) # Optionally flush to disk; re-open the cursor as necessary. if self.bulk or self.reopen: @@ -189,18 +193,18 @@ class test_colmax(wttest.WiredTigerTestCase): # Search for the large record. cursor.set_key(recno) self.assertEqual(cursor.search(), 0) - self.assertEqual(cursor.get_value(), value_populate(cursor, recno)) + self.assertEqual(cursor.get_value(), simple_value(cursor, recno)) # Update it. - cursor[key_populate(cursor, recno)] = value_populate(cursor, 37) + cursor[simple_key(cursor, recno)] = simple_value(cursor, 37) cursor.set_key(recno) self.assertEqual(cursor.search(), 0) - self.assertEqual(cursor.get_value(), value_populate(cursor, 37)) + self.assertEqual(cursor.get_value(), simple_value(cursor, 37)) # Remove it. cursor.set_key(recno) self.assertEqual(cursor.remove(), 0) - cursor.set_key(key_populate(cursor, recno)) + cursor.set_key(simple_key(cursor, recno)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_compact01.py b/src/third_party/wiredtiger/test/suite/test_compact01.py index 8da1a0df4da..56ab6d39076 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact01.py +++ b/src/third_party/wiredtiger/test/suite/test_compact01.py @@ -27,8 +27,8 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import complex_populate, simple_populate, key_populate from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, ComplexDataSet from wiredtiger import stat from wtscenario import make_scenarios @@ -39,14 +39,14 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): # Use a small page size because we want to create lots of pages. config = 'allocation_size=512,' +\ - 'leaf_page_max=512,value_format=S,key_format=S' + 'leaf_page_max=512,key_format=S' nentries = 50000 # The table is a complex object, give it roughly 5 pages per underlying # file. types = [ - ('file', dict(type='file:', pop=simple_populate, maxpages=5)), - ('table', dict(type='table:', pop=complex_populate, maxpages=50)) + ('file', dict(type='file:', dataset=SimpleDataSet, maxpages=5)), + ('table', dict(type='table:', dataset=ComplexDataSet, maxpages=50)) ] compact = [ ('method', dict(utility=0,reopen=0)), @@ -64,7 +64,8 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): def test_compact(self): # Populate an object uri = self.type + self.name - self.pop(self, uri, self.config, self.nentries - 1) + ds = self.dataset(self, uri, self.nentries - 1, config=self.config) + ds.populate() # Reopen the connection to force the object to disk. self.reopen_conn() @@ -76,9 +77,9 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): # Remove most of the object. c1 = self.session.open_cursor(uri, None) - c1.set_key(key_populate(c1, 5)) + c1.set_key(ds.key(5)) c2 = self.session.open_cursor(uri, None) - c2.set_key(key_populate(c2, self.nentries - 5)) + c2.set_key(ds.key(self.nentries - 5)) self.session.truncate(None, c1, c2, None) c1.close() c2.close() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor06.py b/src/third_party/wiredtiger/test/suite/test_cursor06.py index 3a6240bc6c7..117e29b0605 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor06.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor06.py @@ -27,8 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import key_populate, value_populate, simple_populate -from helper import complex_value_populate, complex_populate +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_cursor06.py @@ -36,33 +35,26 @@ from wtscenario import make_scenarios class test_cursor06(wttest.WiredTigerTestCase): name = 'reconfigure' scenarios = make_scenarios([ - ('file-r', dict(type='file:', config='key_format=r', complex=0)), - ('file-S', dict(type='file:', config='key_format=S', complex=0)), - ('lsm-S', dict(type='lsm:', config='key_format=S', complex=0)), - ('table-r', - dict(type='table:', config='key_format=r', complex=0)), - ('table-S', - dict(type='table:', config='key_format=S', complex=0)), - ('table-r-complex', - dict(type='table:', config='key_format=r', complex=1)), - ('table-S-complex', - dict(type='table:', config='key_format=S', complex=1)), - ('table-S-complex-lsm', - dict(type='table:', config='key_format=S,type=lsm', complex=1)), + ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), + ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), + ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), + ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r-complex', dict(type='table:', keyfmt='r', + dataset=ComplexDataSet)), + ('table-S-complex', dict(type='table:', keyfmt='S', + dataset=ComplexDataSet)), + ('table-S-complex-lsm', dict(type='table:', keyfmt='S', + dataset=ComplexLSMDataSet)), ]) def populate(self, uri): - if self.complex: - complex_populate(self, uri, self.config, 100) - else: - simple_populate(self, uri, self.config, 100) + self.ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + self.ds.populate() def set_kv(self, cursor): - cursor.set_key(key_populate(cursor, 10)) - if self.complex: - cursor.set_value(tuple(complex_value_populate(cursor, 10))) - else: - cursor.set_value(value_populate(cursor, 10)) + cursor.set_key(self.ds.key(10)) + cursor.set_value(self.ds.value(10)) def test_reconfigure_overwrite(self): uri = self.type + self.name diff --git a/src/third_party/wiredtiger/test/suite/test_cursor09.py b/src/third_party/wiredtiger/test/suite/test_cursor09.py index a05caea4f1f..9a1fc06b617 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor09.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor09.py @@ -27,27 +27,24 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import key_populate, value_populate, simple_populate -from helper import complex_populate, complex_value_populate +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_cursor09.py # JIRA WT-2217: insert resets key/value "set". class test_cursor09(wttest.WiredTigerTestCase): scenarios = make_scenarios([ - ('file-r', dict(type='file:', config='key_format=r', complex=0)), - ('file-S', dict(type='file:', config='key_format=S', complex=0)), - ('lsm-S', dict(type='lsm:', config='key_format=S', complex=0)), - ('table-r', - dict(type='table:', config='key_format=r', complex=0)), - ('table-S', - dict(type='table:', config='key_format=S', complex=0)), - ('table-r-complex', - dict(type='table:', config='key_format=r', complex=1)), - ('table-S-complex', - dict(type='table:', config='key_format=S', complex=1)), - ('table-S-complex-lsm', - dict(type='table:', config='key_format=S,type=lsm', complex=1)), + ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), + ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), + ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), + ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r-complex', dict(type='table:', keyfmt='r', + dataset=ComplexDataSet)), + ('table-S-complex', dict(type='table:', keyfmt='S', + dataset=ComplexDataSet)), + ('table-S-complex-lsm', dict(type='table:', keyfmt='S', + dataset=ComplexLSMDataSet)), ]) # WT_CURSOR.insert doesn't leave the cursor positioned, verify any @@ -55,15 +52,11 @@ class test_cursor09(wttest.WiredTigerTestCase): def test_cursor09(self): uri = self.type + 'cursor09' - if self.complex: - complex_populate(self, uri, self.config, 100) - else: - simple_populate(self, uri, self.config, 100) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds.populate() cursor = self.session.open_cursor(uri, None, None) - cursor[key_populate(cursor, 10)] = \ - tuple(complex_value_populate(cursor, 10)) if self.complex \ - else value_populate(cursor, 10) + cursor[ds.key(10)] = ds.value(10) msg = '/requires key be set/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, cursor.search, msg) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_compare.py b/src/third_party/wiredtiger/test/suite/test_cursor_compare.py index 4610a2f30a9..c0feb1d4867 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_compare.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_compare.py @@ -27,8 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest, exceptions -from helper import complex_populate, simple_populate, key_populate -from helper import complex_populate_index_name +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # Test cursor comparisons. @@ -36,9 +35,9 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): name = 'test_compare' types = [ - ('file', dict(type='file:', config='')), - ('lsm', dict(type='table:', config=',type=lsm')), - ('table', dict(type='table:', config='')) + ('file', dict(type='file:', dataset=SimpleDataSet)), + ('lsm', dict(type='table:', dataset=ComplexLSMDataSet)), + ('table', dict(type='table:', dataset=ComplexDataSet)) ] keyfmt = [ ('integer', dict(keyfmt='i')), @@ -52,28 +51,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): uriX = self.type + 'compareX' # Build the object. + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt) + ds.populate() + dsX.populate() if self.type == 'file:': - simple_populate( - self, uri, 'key_format=' + self.keyfmt + self.config, 100) - simple_populate( - self, uriX, 'key_format=' + self.keyfmt + self.config, 100) ix0_0 = None ix0_1 = None ix1_0 = None ixX_0 = None else: - complex_populate( - self, uri, 'key_format=' + self.keyfmt + self.config, 100) - complex_populate( - self, uriX, 'key_format=' + self.keyfmt + self.config, 100) - ix0_0 = self.session.open_cursor( - complex_populate_index_name(self, uri, 0), None) - ix0_1 = self.session.open_cursor( - complex_populate_index_name(self, uri, 0), None) - ix1_0 = self.session.open_cursor( - complex_populate_index_name(self, uri, 1), None) - ixX_0 = self.session.open_cursor( - complex_populate_index_name(self, uriX, 0), None) + ix0_0 = self.session.open_cursor(ds.index_name(0), None) + ix0_1 = self.session.open_cursor(ds.index_name(0), None) + ix1_0 = self.session.open_cursor(ds.index_name(1), None) + ixX_0 = self.session.open_cursor(dsX.index_name(0), None) ix0_0.next() ix0_1.next() ix1_0.next() @@ -90,17 +81,17 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: c2.compare(c1), msg) # Test cursors before they're positioned. - c1.set_key(key_populate(c1, 10)) - c2.set_key(key_populate(c2, 20)) + c1.set_key(ds.key(10)) + c2.set_key(ds.key(20)) self.assertGreater(c2.compare(c1), 0) self.assertLess(c1.compare(c2), 0) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertEqual(c1.compare(c2), 0) self.assertEqual(c2.compare(c1), 0) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) - cX.set_key(key_populate(cX, 10)) + cX.set_key(dsX.key(10)) msg = '/must reference the same object/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: cX.compare(c1), msg) @@ -125,20 +116,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: ix0_0.compare(ix1_0), msg) # Test cursors after they're positioned (shouldn't matter for compare). - c1.set_key(key_populate(c1, 10)) + c1.set_key(ds.key(10)) self.assertEqual(c1.search(), 0) - c2.set_key(key_populate(c2, 20)) + c2.set_key(ds.key(20)) self.assertEqual(c2.search(), 0) self.assertGreater(c2.compare(c1), 0) self.assertLess(c1.compare(c2), 0) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertEqual(c2.search(), 0) self.assertEqual(c1.compare(c2), 0) self.assertEqual(c2.compare(c1), 0) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) - cX.set_key(key_populate(cX, 10)) + cX.set_key(dsX.key(10)) self.assertEqual(cX.search(), 0) msg = '/must reference the same object/' self.assertRaisesWithMessage( @@ -149,28 +140,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): uriX = self.type + 'compareX' # Build the object. + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt) + ds.populate() + dsX.populate() if self.type == 'file:': - simple_populate( - self, uri, 'key_format=' + self.keyfmt + self.config, 100) - simple_populate( - self, uriX, 'key_format=' + self.keyfmt + self.config, 100) ix0_0 = None ix0_1 = None ix1_0 = None ixX_0 = None else: - complex_populate( - self, uri, 'key_format=' + self.keyfmt + self.config, 100) - complex_populate( - self, uriX, 'key_format=' + self.keyfmt + self.config, 100) - ix0_0 = self.session.open_cursor( - complex_populate_index_name(self, uri, 0), None) - ix0_1 = self.session.open_cursor( - complex_populate_index_name(self, uri, 0), None) - ix1_0 = self.session.open_cursor( - complex_populate_index_name(self, uri, 1), None) - ixX_0 = self.session.open_cursor( - complex_populate_index_name(self, uriX, 0), None) + ix0_0 = self.session.open_cursor(ds.index_name(0), None) + ix0_1 = self.session.open_cursor(ds.index_name(0), None) + ix1_0 = self.session.open_cursor(ds.index_name(1), None) + ixX_0 = self.session.open_cursor(dsX.index_name(0), None) ix0_0.next() ix0_1.next() ix1_0.next() @@ -187,17 +170,17 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: c2.equals(c1), msg) # Test cursors before they're positioned. - c1.set_key(key_populate(c1, 10)) - c2.set_key(key_populate(c2, 20)) + c1.set_key(ds.key(10)) + c2.set_key(ds.key(20)) self.assertFalse(c1.equals(c2)) self.assertFalse(c2.equals(c1)) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertTrue(c1.equals(c2)) self.assertTrue(c2.equals(c1)) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) - cX.set_key(key_populate(cX, 10)) + cX.set_key(dsX.key(10)) msg = '/must reference the same object/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: cX.equals(c1), msg) @@ -222,20 +205,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): # Test cursors after they're positioned (internally, it's a different # search path if keys are positioned in the tree). - c1.set_key(key_populate(c1, 10)) + c1.set_key(ds.key(10)) self.assertEqual(c1.search(), 0) - c2.set_key(key_populate(c2, 20)) + c2.set_key(ds.key(20)) self.assertEqual(c2.search(), 0) self.assertFalse(c1.equals(c2)) self.assertFalse(c2.equals(c1)) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertEqual(c2.search(), 0) self.assertTrue(c1.equals(c2)) self.assertTrue(c2.equals(c1)) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) - cX.set_key(key_populate(cX, 10)) + cX.set_key(dsX.key(10)) self.assertEqual(cX.search(), 0) msg = '/must reference the same object/' self.assertRaisesWithMessage( diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_pin.py b/src/third_party/wiredtiger/test/suite/test_cursor_pin.py index b6119a93897..cb7045c7e41 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_pin.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_pin.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import simple_populate, key_populate, value_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_cursor_pin.py @@ -36,7 +36,7 @@ from wtscenario import make_scenarios class test_cursor_pin(wttest.WiredTigerTestCase): uri = 'file:cursor_pin' nentries = 10000 - config = 'allocation_size=512,leaf_page_max=512,value_format=S,key_format=' + config = 'allocation_size=512,leaf_page_max=512' scenarios = make_scenarios([ ('recno', dict(keyfmt='r')), ('string', dict(keyfmt='S')), @@ -45,72 +45,75 @@ class test_cursor_pin(wttest.WiredTigerTestCase): # Create a multi-page file, confirm that a simple search to the local # page works, followed by a search to a different page. def test_smoke(self): - simple_populate(self, - self.uri, self.config + self.keyfmt, self.nentries) + ds = SimpleDataSet(self, self.uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() self.reopen_conn() c = self.session.open_cursor(self.uri, None) - c.set_key(key_populate(c, 100)) + c.set_key(ds.key(100)) self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, 100)) - c.set_key(key_populate(c, 101)) + self.assertEqual(c.get_value(), ds.value(100)) + c.set_key(ds.key(101)) self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, 101)) - c.set_key(key_populate(c, 9999)) + self.assertEqual(c.get_value(), ds.value(101)) + c.set_key(ds.key(9999)) self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, 9999)) + self.assertEqual(c.get_value(), ds.value(9999)) # Forward check. - def forward(self, c, max, notfound): + def forward(self, c, ds, max, notfound): for i in range(1, max + 1): - c.set_key(key_populate(c, i)) + c.set_key(ds.key(i)) if i in notfound: self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND) else: self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, i)) + self.assertEqual(c.get_value(), ds.value(i)) # Backward check. - def backward(self, c, max, notfound): + def backward(self, c, ds, max, notfound): for i in range(max, 0, -1): - c.set_key(key_populate(c, i)) + c.set_key(ds.key(i)) if i in notfound: self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND) else: self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, i)) + self.assertEqual(c.get_value(), ds.value(i)) # Create a multi-page file, search backward, forward to check page # boundaries. def test_basic(self): - simple_populate(self, - self.uri, self.config + self.keyfmt, self.nentries) + ds = SimpleDataSet(self, self.uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() self.reopen_conn() c = self.session.open_cursor(self.uri, None) - self.forward(c, self.nentries, []) - self.backward(c, self.nentries, []) + self.forward(c, ds, self.nentries, []) + self.backward(c, ds, self.nentries, []) # Create a multi-page file with a big chunk of missing space in the # middle (to exercise column-store searches). def test_missing(self): - simple_populate(self, - self.uri, self.config + self.keyfmt, self.nentries) + ds = SimpleDataSet(self, self.uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() c = self.session.open_cursor(self.uri, None) for i in range(self.nentries + 3000, self.nentries + 5001): - c[key_populate(c, i)] = value_populate(c, i) + c[ds.key(i)] = ds.value(i) self.reopen_conn() c = self.session.open_cursor(self.uri, None) - self.forward(c, self.nentries + 5000, + self.forward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 3000))) - self.backward(c, self.nentries + 5000, + self.backward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 3000))) # Insert into the empty space so we test searching inserted items. for i in range(self.nentries + 1000, self.nentries + 2001): - c[key_populate(c, i)] = value_populate(c, i) - self.forward(c, self.nentries + 5000, + c[ds.key(i)] = ds.value(i) + self.forward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 1000) +\ range(self.nentries + 2001, self.nentries + 3000))) - self.backward(c, self.nentries + 5000, + self.backward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 1000) +\ range(self.nentries + 2001, self.nentries + 3000))) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random.py b/src/third_party/wiredtiger/test/suite/test_cursor_random.py index 205b9c3c9c5..3bda6dc9946 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random.py @@ -27,16 +27,15 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import complex_populate, simple_populate -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, ComplexDataSet, simple_key, simple_value from wtscenario import make_scenarios # test_cursor_random.py # Cursor next_random operations class test_cursor_random(wttest.WiredTigerTestCase): types = [ - ('file', dict(type='file:random')), - ('table', dict(type='table:random')) + ('file', dict(type='file:random', dataset=SimpleDataSet)), + ('table', dict(type='table:random', dataset=ComplexDataSet)) ] config = [ ('sample', dict(config='next_random=true,next_random_sample_size=35')), @@ -89,12 +88,9 @@ class test_cursor_random(wttest.WiredTigerTestCase): # where the values are in an insert list. def test_cursor_random_multiple_insert_records(self): uri = self.type - if uri.startswith('file:'): - simple_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 100) - else: - complex_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 100) + ds = self.dataset(self, uri, 100, + config='allocation_size=512,leaf_page_max=512') + ds.populate() # In a insert list, next_random always selects the middle key/value # pair, all we can do is confirm cursor.next works. @@ -105,12 +101,9 @@ class test_cursor_random(wttest.WiredTigerTestCase): # where the values are in a disk format page. def cursor_random_multiple_page_records(self, reopen): uri = self.type - if uri.startswith('file:'): - simple_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 10000) - else: - complex_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 10000) + ds = self.dataset(self, uri, 10000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() # Optionally close the connection so everything is forced to disk, # insert lists are an entirely different path in the code. @@ -168,7 +161,7 @@ class test_cursor_random_invisible(wttest.WiredTigerTestCase): # Start a transaction. self.session.begin_transaction() for i in range(1, 100): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Open another session, the updates won't yet be visible, we shouldn't # find anything at all. @@ -182,19 +175,19 @@ class test_cursor_random_invisible(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None) # Insert a single leading record. - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[simple_key(cursor, 1)] = simple_value(cursor, 1) # Start a transaction. self.session.begin_transaction() for i in range(2, 100): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Open another session, the updates won't yet be visible, we should # return the only possible record. s = self.conn.open_session() cursor = s.open_cursor(uri, None, self.config) self.assertEquals(cursor.next(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, 1)) + self.assertEqual(cursor.get_key(), simple_key(cursor, 1)) def test_cursor_random_invisible_before(self): uri = self.type @@ -202,19 +195,19 @@ class test_cursor_random_invisible(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None) # Insert a single leading record. - cursor[key_populate(cursor, 99)] = value_populate(cursor, 99) + cursor[simple_key(cursor, 99)] = simple_value(cursor, 99) # Start a transaction. self.session.begin_transaction() for i in range(2, 100): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Open another session, the updates won't yet be visible, we should # return the only possible record. s = self.conn.open_session() cursor = s.open_cursor(uri, None, self.config) self.assertEquals(cursor.next(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, 99)) + self.assertEqual(cursor.get_key(), simple_key(cursor, 99)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random02.py b/src/third_party/wiredtiger/test/suite/test_cursor_random02.py index 195480d703b..d18d8efd94d 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random02.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random02.py @@ -27,8 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import complex_populate, simple_populate -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_cursor_random02.py @@ -56,12 +55,14 @@ class test_cursor_random02(wttest.WiredTigerTestCase): def test_cursor_random_reasonable_distribution(self): uri = self.type num_entries = self.records - config = 'key_format=S' if uri == 'table:random': - config = 'leaf_page_max=100MB,' + config + config = 'leaf_page_max=100MB' + else: + config = '' # Set the leaf-page-max value, otherwise the page might split. - simple_populate(self, uri, config, num_entries) + ds = SimpleDataSet(self, uri, num_entries, config=config) + ds.populate() # Setup an array to track which keys are seen visitedKeys = [0] * (num_entries + 1) # Setup a counter to see when we find a sequential key diff --git a/src/third_party/wiredtiger/test/suite/test_drop.py b/src/third_party/wiredtiger/test/suite/test_drop.py index a3e80214295..e241c05aa68 100644 --- a/src/third_party/wiredtiger/test/suite/test_drop.py +++ b/src/third_party/wiredtiger/test/suite/test_drop.py @@ -28,8 +28,8 @@ import os, time import wiredtiger, wttest -from helper import confirm_does_not_exist, complex_populate, \ - complex_populate_index_name, simple_populate +from helper import confirm_does_not_exist +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # test_drop.py @@ -45,9 +45,10 @@ class test_drop(wttest.WiredTigerTestCase): ]) # Populate an object, remove it and confirm it no longer exists. - def drop(self, populate, with_cursor, reopen, drop_index): + def drop(self, dataset, with_cursor, reopen, drop_index): uri = self.uri + self.name - populate(self, uri, 'key_format=S' + self.extra_config, 10) + ds = dataset(self, uri, 10, config=self.extra_config) + ds.populate() # Open cursors should cause failure. if with_cursor: @@ -60,7 +61,7 @@ class test_drop(wttest.WiredTigerTestCase): self.reopen_conn() if drop_index: - drop_uri = complex_populate_index_name(self, uri, 0) + drop_uri = ds.index_name(0) else: drop_uri = uri self.session.drop(drop_uri, None) @@ -73,7 +74,7 @@ class test_drop(wttest.WiredTigerTestCase): # case has no indices. for with_cursor in [False, True]: for reopen in [False, True]: - self.drop(simple_populate, with_cursor, reopen, False) + self.drop(SimpleDataSet, with_cursor, reopen, False) # A complex, multi-file table object. # Try all test combinations. @@ -81,7 +82,7 @@ class test_drop(wttest.WiredTigerTestCase): for with_cursor in [False, True]: for reopen in [False, True]: for drop_index in [False, True]: - self.drop(complex_populate, with_cursor, + self.drop(ComplexDataSet, with_cursor, reopen, drop_index) # Test drop of a non-existent object: force succeeds, without force fails. diff --git a/src/third_party/wiredtiger/test/suite/test_drop02.py b/src/third_party/wiredtiger/test/suite/test_drop02.py index 677ba3866b2..017aa64e312 100644 --- a/src/third_party/wiredtiger/test/suite/test_drop02.py +++ b/src/third_party/wiredtiger/test/suite/test_drop02.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_drop02.py # Test dropping an LSM tree on first open. There was a bug where this @@ -38,7 +38,8 @@ class test_drop02(wttest.WiredTigerTestCase): # Populate an object, remove it and confirm it no longer exists. def test_drop(self): uri = 'lsm:' + self.name - simple_populate(self, uri, 'key_format=S', 100000) + ds = SimpleDataSet(self, uri, 100000) + ds.populate() self.reopen_conn() self.session.drop(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_dump.py b/src/third_party/wiredtiger/test/suite/test_dump.py index 280d5870359..f6a83c32489 100644 --- a/src/third_party/wiredtiger/test/suite/test_dump.py +++ b/src/third_party/wiredtiger/test/suite/test_dump.py @@ -28,19 +28,18 @@ import os, shutil import wiredtiger, wttest -from helper import \ - complex_populate, complex_populate_check, \ - simple_populate, simple_populate_check, \ - simple_index_populate, simple_index_populate_check + from suite_subprocess import suite_subprocess from wtscenario import make_scenarios +from wtdataset import SimpleDataSet, SimpleIndexDataSet, SimpleLSMDataSet, \ + ComplexDataSet, ComplexLSMDataSet # test_dump.py # Utilities: wt dump # Test the dump utility (I'm not testing the dump cursors, that's what the # utility uses underneath). class test_dump(wttest.WiredTigerTestCase, suite_subprocess): - dir='dump.dir' # Backup directory name + dir = 'dump.dir' # Backup directory name name = 'test_dump' name2 = 'test_dumpb' @@ -56,30 +55,20 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): ('string', dict(keyfmt='S')) ] types = [ - ('file', dict(uri='file:', config='', lsm=False, - populate=simple_populate, - populate_check=simple_populate_check)), - ('lsm', dict(uri='lsm:', config='', lsm=True, - populate=simple_populate, - populate_check=simple_populate_check)), - ('table-simple', dict(uri='table:', config='', lsm=False, - populate=simple_populate, - populate_check=simple_populate_check)), - ('table-index', dict(uri='table:', config='', lsm=False, - populate=simple_index_populate, - populate_check=simple_index_populate_check)), - ('table-simple-lsm', dict(uri='table:', config='type=lsm', lsm=True, - populate=simple_populate, - populate_check=simple_populate_check)), - ('table-complex', dict(uri='table:', config='', lsm=False, - populate=complex_populate, - populate_check=complex_populate_check)), - ('table-complex-lsm', dict(uri='table:', config='type=lsm', lsm=True, - populate=complex_populate, - populate_check=complex_populate_check)) + ('file', dict(uri='file:', dataset=SimpleDataSet)), + ('lsm', dict(uri='lsm:', dataset=SimpleDataSet)), + ('table-simple', dict(uri='table:', dataset=SimpleDataSet)), + ('table-index', dict(uri='table:', dataset=SimpleIndexDataSet)), + ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)), + ('table-complex', dict(uri='table:', dataset=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)), ] scenarios = make_scenarios(types, keyfmt, dumpfmt) + def skip(self): + return (self.dataset.is_lsm() or self.uri == 'lsm:') and \ + self.keyfmt == 'r' + # Extract the values lines from the dump output. def value_lines(self, fname): # mode: @@ -108,14 +97,14 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Dump, re-load and do a content comparison. def test_dump(self): # LSM and column-store isn't a valid combination. - if self.lsm and self.keyfmt == 'r': + if self.skip(): return # Create the object. uri = self.uri + self.name uri2 = self.uri + self.name2 - self.populate(self, uri, - self.config + ',key_format=' + self.keyfmt, self.nentries) + pop = self.dataset(self, uri, self.nentries, key_format=self.keyfmt) + pop.populate() # Dump the object. os.mkdir(self.dir) @@ -136,14 +125,14 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Check the object's contents self.reopen_conn(self.dir) - self.populate_check(self, uri, self.nentries) + pop.check() # Re-load the object again in the original directory. self.reopen_conn('.') self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) # Check the contents, they shouldn't have changed. - self.populate_check(self, uri, self.nentries) + pop.check() # Re-load the object again, but confirm -n (no overwrite) fails. self.runWt(['-h', self.dir, 'load', '-n', '-f', 'dump.out'], @@ -151,7 +140,7 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): self.check_non_empty_file('errfile.out') # If there are indices, dump one of them and check the output. - if self.populate == complex_populate: + if self.dataset == ComplexDataSet: indexuri = 'index:' + self.name + ':indx1' hexopt = ['-x'] if self.hex == 1 else [] self.runWt(['-h', self.dir, 'dump'] + hexopt + [indexuri], @@ -166,7 +155,8 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Check the contents in the new table. self.reopen_conn(self.dir) - self.populate_check(self, uri2, self.nentries) + pop = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt) + pop.check() if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_dupc.py b/src/third_party/wiredtiger/test/suite/test_dupc.py index 12b18f1ba79..c0cf6acc75e 100644 --- a/src/third_party/wiredtiger/test/suite/test_dupc.py +++ b/src/third_party/wiredtiger/test/suite/test_dupc.py @@ -32,7 +32,7 @@ import os, time import wiredtiger, wttest -from helper import complex_populate, key_populate, simple_populate +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # Test session.open_cursor with cursor duplication. @@ -40,8 +40,6 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): name = 'test_dupc' nentries = 1000 - config = 'key_format=' - scenarios = make_scenarios([ ('file-r', dict(uri='file:', fmt='r')), ('file-S', dict(uri='file:', fmt='S')), @@ -51,7 +49,7 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): # Iterate through an object, duplicate the cursor and checking that it # matches the original and is set to the same record. - def iterate(self, uri): + def iterate(self, uri, ds): cursor = self.session.open_cursor(uri, None, None) next = 0 while True: @@ -59,10 +57,10 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): if nextret != 0: break next += 1 - self.assertEqual(cursor.get_key(), key_populate(cursor, next)) + self.assertEqual(cursor.get_key(), ds.key(next)) dupc = self.session.open_cursor(None, cursor, None) self.assertEqual(cursor.compare(dupc), 0) - self.assertEqual(dupc.get_key(), key_populate(dupc, next)) + self.assertEqual(dupc.get_key(), ds.key(next)) cursor.close() cursor = dupc self.assertEqual(next, self.nentries) @@ -73,14 +71,16 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): uri = self.uri + self.name # A simple, one-file file or table object. - simple_populate(self, uri, self.config + self.fmt, self.nentries) - self.iterate(uri) + ds = SimpleDataSet(self, uri, self.nentries, key_format=self.fmt) + ds.populate() + self.iterate(uri, ds) self.session.drop(uri, None) # A complex, multi-file table object. if self.uri == "table:": - complex_populate(self, uri, self.config + self.fmt, self.nentries) - self.iterate(uri) + ds = ComplexDataSet(self, uri, self.nentries, key_format=self.fmt) + ds.populate() + self.iterate(uri, ds) self.session.drop(uri, None) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_empty.py b/src/third_party/wiredtiger/test/suite/test_empty.py index 9fe88107412..578bec618c9 100644 --- a/src/third_party/wiredtiger/test/suite/test_empty.py +++ b/src/third_party/wiredtiger/test/suite/test_empty.py @@ -28,7 +28,7 @@ import os import wiredtiger, wttest -from helper import key_populate +from wtdataset import simple_key from wtscenario import make_scenarios # test_empty.py @@ -64,7 +64,7 @@ class test_empty(wttest.WiredTigerTestCase): # Add a few records to the object and remove them. cursor = self.session.open_cursor(uri, None, None) for i in range(1,5): - key = key_populate(cursor, i) + key = simple_key(cursor, i) cursor[key] = "XXX" del cursor[key] diff --git a/src/third_party/wiredtiger/test/suite/test_inmem01.py b/src/third_party/wiredtiger/test/suite/test_inmem01.py index c6ae7ff6c4b..1af43bbd9d9 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem01.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem01.py @@ -28,8 +28,7 @@ import wiredtiger, wttest from time import sleep -from helper import simple_populate, simple_populate_check -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_inmem01.py @@ -41,54 +40,60 @@ class test_inmem01(wttest.WiredTigerTestCase): table_config = ',memory_page_max=32k,leaf_page_max=4k' scenarios = make_scenarios([ - ('col', dict(fmt='key_format=r,value_format=S')), - ('fix', dict(fmt='key_format=r,value_format=8t')), - ('row', dict(fmt='key_format=S,value_format=S')) + ('col', dict(keyfmt='r', valuefmt='S')), + ('fix', dict(keyfmt='r', valuefmt='8t')), + ('row', dict(keyfmt='S', valuefmt='S')), ]) # Smoke-test in-memory configurations, add a small amount of data and # ensure it's visible. def test_insert(self): - config = self.fmt + self.table_config - simple_populate(self, self.uri, config, 1000) - simple_populate_check(self, self.uri, 1000) + ds = SimpleDataSet(self, self.uri, 1000, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + ds.populate() + ds.check() # Add more data than fits into the configured cache and verify it fails. def test_insert_over_capacity(self): - config = self.fmt + self.table_config msg = '/WT_CACHE_FULL.*/' + ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:simple_populate(self, self.uri, config, 10000000), msg) + lambda:ds.populate(), msg) # Figure out the last key we successfully inserted, and check all # previous inserts are still there. cursor = self.session.open_cursor(self.uri, None) cursor.prev() last_key = int(cursor.get_key()) - simple_populate_check(self, self.uri, last_key) + ds = SimpleDataSet(self, self.uri, last_key, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + ds.check() # Fill the cache with data, remove some data, ensure more data can be # inserted (after a reasonable amount of time for space to be reclaimed). def test_insert_over_delete(self): - config = self.fmt + self.table_config msg = '/WT_CACHE_FULL.*/' + ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:simple_populate(self, self.uri, config, 10000000), msg) + lambda:ds.populate(), msg) # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. cursor = self.session.open_cursor(self.uri, None) for i in range(1, 100): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) cursor.remove() # Run queries after adding, removing and re-inserting data. # Try out keeping a cursor open while adding new data. def test_insert_over_delete_replace(self): - config = self.fmt + self.table_config msg = '/WT_CACHE_FULL.*/' + ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:simple_populate(self, self.uri, config, 10000000), msg) + lambda:ds.populate(), msg) cursor = self.session.open_cursor(self.uri, None) cursor.prev() @@ -98,7 +103,7 @@ class test_inmem01(wttest.WiredTigerTestCase): # the configured cache, verify removes succeed. cursor = self.session.open_cursor(self.uri, None) for i in range(1, last_key / 4, 1): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) cursor.remove() cursor.reset() @@ -106,7 +111,7 @@ class test_inmem01(wttest.WiredTigerTestCase): inserted = False for i in range(1, 1000): try: - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[ds.key(1)] = ds.value(1) except wiredtiger.WiredTigerError: cursor.reset() sleep(1) diff --git a/src/third_party/wiredtiger/test/suite/test_inmem02.py b/src/third_party/wiredtiger/test/suite/test_inmem02.py index 9eb8330b2a3..b5e07fea967 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem02.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem02.py @@ -28,9 +28,7 @@ import wiredtiger, wttest from time import sleep -from helper import simple_populate, simple_populate_check -from helper import key_populate, value_populate -from wtscenario import make_scenarios +from wtdataset import SimpleDataSet # test_inmem02.py # Test in-memory with ignore-cache-size setting. @@ -38,7 +36,7 @@ class test_inmem02(wttest.WiredTigerTestCase): uri = 'table:inmem02' conn_config = \ 'cache_size=3MB,file_manager=(close_idle_time=0),in_memory=true' - table_config = 'key_format=S,value_format=S,memory_page_max=32k,leaf_page_max=4k' + table_config = 'memory_page_max=32k,leaf_page_max=4k' # Add more data than fits into the configured cache and verify it fails. def test_insert_over_allowed(self): @@ -50,9 +48,9 @@ class test_inmem02(wttest.WiredTigerTestCase): # Populate a table with enough data to fill the cache. msg = '/WT_CACHE_FULL.*/' + ds = SimpleDataSet(self, self.uri, 10000000, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:simple_populate( - self, self.uri, self.table_config, 10000000), msg) + lambda:ds.populate(), msg) # Add some content to the new table cursor = self.session.open_cursor(self.uri + '_over', None) diff --git a/src/third_party/wiredtiger/test/suite/test_join09.py b/src/third_party/wiredtiger/test/suite/test_join09.py new file mode 100644 index 00000000000..d48353b1580 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_join09.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +import wiredtiger, wttest, run +from wtscenario import make_scenarios + +# test_join09.py +# Join bloom filters with false positives +class test_join09(wttest.WiredTigerTestCase): + nentries = 1000 + + bloomscen = [ + ('nobloom', dict(false_positives=False, config='')), + ('bloom1000', dict(false_positives=False, + config='strategy=bloom,count=1000')), + ('bloom10fp', dict(false_positives=True, + config='strategy=bloom,count=10,bloom_false_positives=true')) + ] + + scenarios = make_scenarios(bloomscen) + + def gen_values(self, i): + s = str(i) # 345 => "345" + f = s[0:1] + s[0:1] + s[0:1] # 345 => "333" + return [s, f] + + def populate(self, s, gen_values): + c = s.open_cursor('table:join09', None, None) + for i in range(0, self.nentries): + c.set_key(i) + c.set_value(*gen_values(i)) + c.insert() + c.close() + + # Common function for testing the most basic functionality + # of joins + def test_join(self): + self.session.create('table:join09', + 'columns=(k,v0,v1),key_format=i,value_format=SS') + self.session.create('index:join09:index0','columns=(v0)') + self.session.create('index:join09:index1','columns=(v1)') + + self.populate(self.session, self.gen_values) + + jc = self.session.open_cursor('join:table:join09', None, None) + c0 = self.session.open_cursor('index:join09:index0', None, None) + c0.set_key('520') + self.assertEquals(0, c0.search()) + self.session.join(jc, c0, 'compare=ge') + + joinconfig = 'compare=eq,' + self.config + c1 = self.session.open_cursor('index:join09:index1', None, None) + c1.set_key('555') + self.assertEquals(0, c1.search()) + self.session.join(jc, c1, joinconfig) + + mbr = set(range(520,600)) | set(range(53,60)) + + fp_count = 0 + while jc.next() == 0: + [k] = jc.get_keys() + [v0,v1] = jc.get_values() + self.assertEquals(self.gen_values(k), [v0, v1]) + if not k in mbr: + # With false positives, we can see extra values + if self.false_positives: + fp_count += 1 + continue + self.tty('**** ERROR: result ' + str(k) + ' is not in: ' + + str(mbr)) + self.assertTrue(k in mbr) + mbr.remove(k) + + if len(mbr) != 0: + self.tty('**** ERROR: did not see these: ' + str(mbr)) + self.assertEquals(0, len(mbr)) + + # Turning on false positives does not guarantee we'll see extra + # values, but we've configured our test with a low count to + # make sure it happens. + if self.false_positives: + self.assertTrue(fp_count > 0) + jc.close() + c1.close() + c0.close() + self.session.drop('table:join09') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_jsondump01.py b/src/third_party/wiredtiger/test/suite/test_jsondump01.py index dc8027c2115..c7fa9cdf397 100644 --- a/src/third_party/wiredtiger/test/suite/test_jsondump01.py +++ b/src/third_party/wiredtiger/test/suite/test_jsondump01.py @@ -28,11 +28,9 @@ import os, json import wiredtiger, wttest -from helper import \ - complex_populate, complex_populate_check, complex_populate_check_cursor,\ - simple_populate, simple_populate_check, simple_populate_check_cursor, \ - simple_index_populate, simple_index_populate_check, \ - simple_index_populate_check_cursor, compare_files +from wtdataset import SimpleDataSet, SimpleLSMDataSet, SimpleIndexDataSet, \ + ComplexDataSet, ComplexLSMDataSet +from helper import compare_files from suite_subprocess import suite_subprocess from wtscenario import make_scenarios @@ -40,7 +38,8 @@ from wtscenario import make_scenarios # It emulates a WT cursor well enough for the *_check_cursor methods. # They just need an iterable object. class FakeCursor: - def __init__(self, keyfmt, valuefmt, rows): + def __init__(self, uri, keyfmt, valuefmt, rows): + self.uri = uri self.key_format = keyfmt self.value_format = valuefmt self.rows = rows @@ -79,47 +78,30 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess): ('string', dict(keyfmt='S')) ] types = [ - ('file', dict(uri='file:', config='', lsm=False, - populate=simple_populate, - populate_check=simple_populate_check, - populate_check_cursor=simple_populate_check_cursor)), - ('lsm', dict(uri='lsm:', config='', lsm=True, - populate=simple_populate, - populate_check=simple_populate_check, - populate_check_cursor=simple_populate_check_cursor)), - ('table-simple', dict(uri='table:', config='', lsm=False, - populate=simple_populate, - populate_check=simple_populate_check, - populate_check_cursor=simple_populate_check_cursor)), - ('table-index', dict(uri='table:', config='', lsm=False, - populate=simple_index_populate, - populate_check=simple_index_populate_check, - populate_check_cursor=simple_index_populate_check_cursor)), - ('table-simple-lsm', dict(uri='table:', config='type=lsm', lsm=True, - populate=simple_populate, - populate_check=simple_populate_check, - populate_check_cursor=simple_populate_check_cursor)), - ('table-complex', dict(uri='table:', config='', lsm=False, - populate=complex_populate, - populate_check=complex_populate_check, - populate_check_cursor=complex_populate_check_cursor)), - ('table-complex-lsm', dict(uri='table:', config='type=lsm', lsm=True, - populate=complex_populate, - populate_check=complex_populate_check, - populate_check_cursor=complex_populate_check_cursor)) + ('file', dict(uri='file:', dataset=SimpleDataSet)), + ('lsm', dict(uri='lsm:', dataset=SimpleDataSet)), + ('table-simple', dict(uri='table:', dataset=SimpleDataSet)), + ('table-index', dict(uri='table:', dataset=SimpleIndexDataSet)), + ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)), + ('table-complex', dict(uri='table:', dataset=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(types, keyfmt) + def skip(self): + return (self.dataset.is_lsm() or self.uri == 'lsm:') and \ + self.keyfmt == 'r' + # Dump using util, re-load using python's JSON, and do a content comparison. def test_jsondump_util(self): # LSM and column-store isn't a valid combination. - if self.lsm and self.keyfmt == 'r': + if self.skip(): return # Create the object. uri = self.uri + self.name - self.populate(self, uri, self.config + ',key_format=' + self.keyfmt, - self.nentries) + ds = self.dataset(self, uri, self.nentries, key_format=self.keyfmt) + ds.populate() # Dump the object. self.runWt(['dump', '-j', uri], outfilename='jsondump.out') @@ -141,20 +123,21 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess): # check the contents of the data we read. # we only use a wt cursor to get the key_format/value_format. cursor = self.session.open_cursor(uri, None) - fake = FakeCursor(cursor.key_format, cursor.value_format, data) + fake = FakeCursor(uri, cursor.key_format, cursor.value_format, data) cursor.close() - self.populate_check_cursor(self, fake, self.nentries) + ds.check_cursor(fake) # Dump using util, re-load using python's JSON, and do a content comparison. def test_jsonload_util(self): # LSM and column-store isn't a valid combination. - if self.lsm and self.keyfmt == 'r': + if self.skip(): return # Create the object. uri = self.uri + self.name uri2 = self.uri + self.name2 - self.populate(self, uri, 'key_format=' + self.keyfmt, self.nentries) + ds = self.dataset(self, uri, self.nentries, key_format=self.keyfmt) + ds.populate() # Dump the object. self.runWt(['dump', '-j', uri], outfilename='jsondump.out') @@ -165,7 +148,9 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess): self.runWt(loadcmd) # Check the contents of the data we read. - self.populate_check(self, uri2, self.nentries) + # We use the dataset only for checking. + ds2 = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt) + ds2.check() # Reload into the original uri, and dump into another file. self.session.drop(uri, None) @@ -175,7 +160,7 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess): # Compare the two outputs, and check the content again. compare_files(self, 'jsondump.out', 'jsondump2.out') - self.populate_check(self, uri, self.nentries) + ds.check() if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_lsm01.py b/src/third_party/wiredtiger/test/suite/test_lsm01.py index f6cee20e896..b44df4bae14 100644 --- a/src/third_party/wiredtiger/test/suite/test_lsm01.py +++ b/src/third_party/wiredtiger/test/suite/test_lsm01.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wtscenario, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_lsm01.py # Test LSM tree configuration options. @@ -77,7 +77,7 @@ class test_lsm01(wttest.WiredTigerTestCase): args += ')' # Close the LSM configuration option group self.verbose(3, 'Test LSM with config: ' + args + ' count: ' + str(self.nrecs)) - simple_populate(self, self.uri, args, self.nrecs) + SimpleDataSet(self, self.uri, self.nrecs).populate() # TODO: Adding an explicit drop here can cause deadlocks, if a merge # is still happening. See issue #349. diff --git a/src/third_party/wiredtiger/test/suite/test_lsm02.py b/src/third_party/wiredtiger/test/suite/test_lsm02.py index 85067665017..e9628139a97 100644 --- a/src/third_party/wiredtiger/test/suite/test_lsm02.py +++ b/src/third_party/wiredtiger/test/suite/test_lsm02.py @@ -27,7 +27,6 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wtscenario, wttest -from helper import simple_populate # test_lsm02.py # Test LSM schema level operations diff --git a/src/third_party/wiredtiger/test/suite/test_lsm03.py b/src/third_party/wiredtiger/test/suite/test_lsm03.py index 448d864c646..d916db415da 100644 --- a/src/third_party/wiredtiger/test/suite/test_lsm03.py +++ b/src/third_party/wiredtiger/test/suite/test_lsm03.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wtscenario, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_lsm03.py # Check to make sure that LSM schema operations don't get EBUSY when @@ -37,13 +37,14 @@ class test_lsm03(wttest.WiredTigerTestCase): # Use small pages so we generate some internal layout # Setup LSM so multiple chunks are present - config = 'key_format=S,allocation_size=512,internal_page_max=512' + \ + config = 'allocation_size=512,internal_page_max=512' + \ ',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)' # Populate an object then drop it. def test_lsm_drop_active(self): uri = 'lsm:' + self.name - simple_populate(self, uri, self.config, 10000) + ds = SimpleDataSet(self, uri, 10000, config=self.config) + ds.populate() # Force to disk self.reopen_conn() @@ -55,6 +56,7 @@ class test_lsm03(wttest.WiredTigerTestCase): cursor.close() # Add enough records that a merge should be running - simple_populate(self, uri, self.config, 50000) + ds = SimpleDataSet(self, uri, 50000, config=self.config) + ds.populate() # The drop should succeed even when LSM work units are active self.session.drop(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap01.py b/src/third_party/wiredtiger/test/suite/test_nsnap01.py index 7e8951750f8..4d5555277fe 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap01.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap01.py @@ -30,7 +30,7 @@ # Named snapshots: basic API from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wiredtiger, wttest class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess): @@ -53,7 +53,7 @@ class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess): def test_named_snapshots(self): # Populate a table end = start = 0 - simple_populate(self, self.uri, 'key_format=i', 0) + SimpleDataSet(self, self.uri, 0, key_format='i').populate() # Now run a workload: # every iteration: diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap02.py b/src/third_party/wiredtiger/test/suite/test_nsnap02.py index 510c9d421ef..ed1c96ebe50 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap02.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap02.py @@ -30,7 +30,7 @@ # Named snapshots: Combinations of dropping snapshots from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wiredtiger, wttest class test_nsnap02(wttest.WiredTigerTestCase, suite_subprocess): @@ -63,7 +63,7 @@ class test_nsnap02(wttest.WiredTigerTestCase, suite_subprocess): def create_snapshots(self): # Populate a table end = start = 0 - simple_populate(self, self.uri, 'key_format=i', 0) + SimpleDataSet(self, self.uri, 0, key_format='i').populate() # Create a set of snapshots # Each snapshot has a bunch of new data diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap03.py b/src/third_party/wiredtiger/test/suite/test_nsnap03.py index 3986c0c1a0a..6964fb914c3 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap03.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap03.py @@ -30,7 +30,7 @@ # Named snapshots: Access and create from multiple sessions from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wiredtiger, wttest class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess): @@ -55,7 +55,7 @@ class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess): def test_named_snapshots(self): # Populate a table end = start = 0 - simple_populate(self, self.uri, 'key_format=i', 0) + SimpleDataSet(self, self.uri, 0, key_format='i').populate() # Now run a workload: # every iteration: diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap04.py b/src/third_party/wiredtiger/test/suite/test_nsnap04.py index f9ef26b5600..60901dd2ee3 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap04.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap04.py @@ -29,9 +29,9 @@ # test_nsnap04.py # Named snapshots: Create snapshot from running transaction -from suite_subprocess import suite_subprocess -from helper import simple_populate import wiredtiger, wttest +from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_nsnap04' @@ -52,7 +52,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): def test_named_snapshots(self): # Populate a table end = start = 0 - simple_populate(self, self.uri, 'key_format=i', 0) + SimpleDataSet(self, self.uri, 0, key_format='i').populate() snapshots = [] c = self.session.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_overwrite.py b/src/third_party/wiredtiger/test/suite/test_overwrite.py index b7af1612243..4739abaa578 100644 --- a/src/third_party/wiredtiger/test/suite/test_overwrite.py +++ b/src/third_party/wiredtiger/test/suite/test_overwrite.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import key_populate, simple_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_overwrite.py @@ -35,22 +35,23 @@ from wtscenario import make_scenarios class test_overwrite(wttest.WiredTigerTestCase): name = 'overwrite' scenarios = make_scenarios([ - ('file-r', dict(type='file:',keyfmt='r')), - ('file-S', dict(type='file:',keyfmt='S')), - ('lsm-S', dict(type='lsm:',keyfmt='S')), - ('table-r', dict(type='table:',keyfmt='r')), - ('table-S', dict(type='table:',keyfmt='S')), + ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), + ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), + ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), + ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), ]) # Confirm a cursor configured with/without overwrite correctly handles # non-existent records during insert, remove and update operations. def test_overwrite_insert(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds.populate() # Insert of an existing record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) cursor.set_value('XXXXXXXXXX') self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) @@ -60,78 +61,80 @@ class test_overwrite(wttest.WiredTigerTestCase): # failure modes are for non-existent records, and you cannot duplicate # cursor pointing to non-existent records. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) dupc = self.session.open_cursor(None, cursor, "overwrite=true") dupc.set_value('XXXXXXXXXX') self.assertEquals(dupc.insert(), 0) # Insert of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 6)) + cursor.set_key(ds.key(6)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 201)) + cursor.set_key(ds.key(201)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.insert(), 0) def test_overwrite_remove(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds.populate() # Remove of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) self.assertEquals(cursor.remove(), 0) # Remove of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 6)) + cursor.set_key(ds.key(6)) self.assertEquals(cursor.remove(), 0) # Remove of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) self.assertEquals(cursor.remove(), wiredtiger.WT_NOTFOUND) # Remove of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 201)) + cursor.set_key(ds.key(201)) self.assertEquals(cursor.remove(), 0) def test_overwrite_update(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds.populate() # Update of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), 0) # Update of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 6)) + cursor.set_key(ds.key(6)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), 0) # Update of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND) # Update of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 201)) + cursor.set_key(ds.key(201)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_readonly03.py b/src/third_party/wiredtiger/test/suite/test_readonly03.py index a5607fcfe80..f30c591ca59 100644 --- a/src/third_party/wiredtiger/test/suite/test_readonly03.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly03.py @@ -31,9 +31,9 @@ # all return ENOTSUP. # -from helper import simple_populate -from suite_subprocess import suite_subprocess import os, sys, wiredtiger, wttest +from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:test_readonly03' @@ -61,7 +61,8 @@ class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): create_params = 'key_format=i,value_format=i' entries = 10 # Create a database and a table. - simple_populate(self, self.uri, create_params, entries) + SimpleDataSet(self, self.uri, entries, key_format='i', + value_format='i').populate() # # Now close and reopen. Note that the connection function diff --git a/src/third_party/wiredtiger/test/suite/test_rebalance.py b/src/third_party/wiredtiger/test/suite/test_rebalance.py index acb73d82a13..2d160bafec0 100644 --- a/src/third_party/wiredtiger/test/suite/test_rebalance.py +++ b/src/third_party/wiredtiger/test/suite/test_rebalance.py @@ -28,7 +28,7 @@ import os, time import wiredtiger, wttest -from helper import complex_populate, simple_populate +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # test_rebalance.py @@ -38,7 +38,7 @@ class test_rebalance(wttest.WiredTigerTestCase): # Use small pages so we generate some internal layout # Setup LSM so multiple chunks are present - config = 'key_format=S,allocation_size=512,internal_page_max=512' + \ + config = 'allocation_size=512,internal_page_max=512' + \ ',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)' scenarios = make_scenarios([ @@ -48,9 +48,10 @@ class test_rebalance(wttest.WiredTigerTestCase): ]) # Populate an object, then rebalance it. - def rebalance(self, populate, with_cursor): + def rebalance(self, dataset, with_cursor): uri = self.uri + self.name - populate(self, uri, self.config, 10000) + ds = dataset(self, uri, 10000, config=self.config) + ds.populate() # Force to disk, we don't rebalance in-memory objects. self.reopen_conn() @@ -68,13 +69,13 @@ class test_rebalance(wttest.WiredTigerTestCase): # Test rebalance of an object. def test_rebalance(self): # Simple file or table object. - self.rebalance(simple_populate, False) - self.rebalance(simple_populate, True) + self.rebalance(SimpleDataSet, False) + self.rebalance(SimpleDataSet, True) # A complex, multi-file table object. if self.uri == "table:": - self.rebalance(complex_populate, False) - self.rebalance(complex_populate, True) + self.rebalance(ComplexDataSet, False) + self.rebalance(ComplexDataSet, True) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig01.py b/src/third_party/wiredtiger/test/suite/test_reconfig01.py index fb3fb7edac6..e76becac76a 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig01.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig01.py @@ -28,7 +28,7 @@ import time import wiredtiger, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_reconfig01.py # Smoke-test the connection reconfiguration operations. @@ -70,7 +70,7 @@ class test_reconfig01(wttest.WiredTigerTestCase): # Take all the defaults. uri = "lsm:test_reconfig" nrecs = 10 - simple_populate(self, uri, 'key_format=S', nrecs) + SimpleDataSet(self, uri, nrecs).populate() # Sleep to make sure all threads are started. time.sleep(2) # Now that an LSM tree exists, reconfigure LSM manager threads. @@ -78,7 +78,7 @@ class test_reconfig01(wttest.WiredTigerTestCase): self.conn.reconfigure("lsm_manager=(worker_thread_max=10)") # Generate some work nrecs = 20 - simple_populate(self, uri, 'key_format=S', nrecs) + SimpleDataSet(self, uri, nrecs).populate() # Now reconfigure fewer threads. self.conn.reconfigure("lsm_manager=(worker_thread_max=3)") diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig02.py b/src/third_party/wiredtiger/test/suite/test_reconfig02.py index 9d9ac220aa7..36a78a1805f 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig02.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig02.py @@ -28,7 +28,6 @@ import fnmatch, os, time import wiredtiger, wttest -from helper import simple_populate # test_reconfig02.py # Smoke-test the connection reconfiguration operations. diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig03.py b/src/third_party/wiredtiger/test/suite/test_reconfig03.py index e9d39ea5a76..0019bf4814e 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig03.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig03.py @@ -28,7 +28,7 @@ import fnmatch, os, time import wiredtiger, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_reconfig03.py # Test the connection reconfiguration operations used in the MongoDB @@ -41,16 +41,16 @@ class test_reconfig03(wttest.WiredTigerTestCase): # can run after we've made modifications. def test_reconfig03_mdb(self): entries = 10000 - simple_populate(self, self.uri, 'key_format=S', entries) + SimpleDataSet(self, self.uri, entries).populate() time.sleep(1) self.conn.reconfigure("eviction_target=81") - simple_populate(self, self.uri, 'key_format=S', entries * 2) + SimpleDataSet(self, self.uri, entries * 2).populate() time.sleep(1) self.conn.reconfigure("cache_size=81M") - simple_populate(self, self.uri, 'key_format=S', entries * 3) + SimpleDataSet(self, self.uri, entries * 3).populate() time.sleep(1) self.conn.reconfigure("eviction_dirty_target=82") - simple_populate(self, self.uri, 'key_format=S', entries * 4) + SimpleDataSet(self, self.uri, entries * 4).populate() time.sleep(1) self.conn.reconfigure("shared_cache=(chunk=11MB, name=bar, reserve=12MB, size=1G)") diff --git a/src/third_party/wiredtiger/test/suite/test_rename.py b/src/third_party/wiredtiger/test/suite/test_rename.py index 1979bbb802a..4e3af8e13e0 100644 --- a/src/third_party/wiredtiger/test/suite/test_rename.py +++ b/src/third_party/wiredtiger/test/suite/test_rename.py @@ -28,9 +28,8 @@ import os, time import wiredtiger, wttest -from helper import confirm_does_not_exist,\ - complex_populate, complex_populate_check,\ - simple_populate, simple_populate_check +from helper import confirm_does_not_exist +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # test_rename.py @@ -46,10 +45,13 @@ class test_rename(wttest.WiredTigerTestCase): # Populate and object, and rename it a couple of times, confirming the # old name doesn't exist and the new name has the right contents. - def rename(self, populate, check, with_cursor): + def rename(self, dataset, with_cursor): uri1 = self.uri + self.name1 uri2 = self.uri + self.name2 - populate(self, uri1, 'key_format=S', 10) + # Only populate uri1, we keep ds2 for checking. + ds1 = dataset(self, uri1, 10) + ds2 = dataset(self, uri2, 10) + ds1.populate() # Open cursors should cause failure. if with_cursor: @@ -60,24 +62,24 @@ class test_rename(wttest.WiredTigerTestCase): self.session.rename(uri1, uri2, None) confirm_does_not_exist(self, uri1) - check(self, uri2, 10) + ds2.check() self.session.rename(uri2, uri1, None) confirm_does_not_exist(self, uri2) - check(self, uri1, 10) + ds1.check() self.session.drop(uri1) # Test rename of an object. def test_rename(self): # Simple, one-file file or table object. - self.rename(simple_populate, simple_populate_check, False) - self.rename(simple_populate, simple_populate_check, True) + self.rename(SimpleDataSet, False) + self.rename(SimpleDataSet, True) # A complex, multi-file table object. if self.uri == "table:": - self.rename(complex_populate, complex_populate_check, False) - self.rename(complex_populate, complex_populate_check, True) + self.rename(ComplexDataSet, False) + self.rename(ComplexDataSet, True) def test_rename_dne(self): uri1 = self.uri + self.name1 diff --git a/src/third_party/wiredtiger/test/suite/test_shared_cache01.py b/src/third_party/wiredtiger/test/suite/test_shared_cache01.py index 896a1e569ef..70560a625ee 100644 --- a/src/third_party/wiredtiger/test/suite/test_shared_cache01.py +++ b/src/third_party/wiredtiger/test/suite/test_shared_cache01.py @@ -31,7 +31,6 @@ import os import shutil import wiredtiger, wttest from wttest import unittest -from helper import key_populate, simple_populate # test_shared_cache01.py # Checkpoint tests diff --git a/src/third_party/wiredtiger/test/suite/test_shared_cache02.py b/src/third_party/wiredtiger/test/suite/test_shared_cache02.py index 4afccbe59f7..7cde6c86695 100644 --- a/src/third_party/wiredtiger/test/suite/test_shared_cache02.py +++ b/src/third_party/wiredtiger/test/suite/test_shared_cache02.py @@ -31,7 +31,6 @@ import os import shutil import wiredtiger, wttest from wttest import unittest -from helper import key_populate, simple_populate # test_shared_cache02.py # Shared cache tests diff --git a/src/third_party/wiredtiger/test/suite/test_split.py b/src/third_party/wiredtiger/test/suite/test_split.py index 28bf6bc59b0..411778f21ae 100644 --- a/src/third_party/wiredtiger/test/suite/test_split.py +++ b/src/third_party/wiredtiger/test/suite/test_split.py @@ -32,9 +32,6 @@ import wiredtiger, wttest from wiredtiger import stat -from helper import confirm_empty,\ - key_populate, value_populate, simple_populate,\ - complex_populate, complex_value_populate # Test splits class test_split(wttest.WiredTigerTestCase): diff --git a/src/third_party/wiredtiger/test/suite/test_stat01.py b/src/third_party/wiredtiger/test/suite/test_stat01.py index 1ad51ee9882..2b04a3cbcd5 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat01.py +++ b/src/third_party/wiredtiger/test/suite/test_stat01.py @@ -28,7 +28,7 @@ import helper, wiredtiger, wttest from wiredtiger import stat -from helper import key_populate, simple_populate +from wtdataset import SimpleDataSet, simple_key from wtscenario import make_scenarios # test_stat01.py @@ -89,8 +89,8 @@ class test_stat01(wttest.WiredTigerTestCase): # Test simple connection statistics. def test_basic_conn_stats(self): # Build an object and force some writes. - config = self.config + ',key_format=' + self.keyfmt - simple_populate(self, self.uri, config, 1000) + SimpleDataSet(self, self.uri, 1000, + config=self.config, key_format=self.keyfmt).populate() self.session.checkpoint(None) # See that we can get a specific stat value by its key and verify its @@ -113,7 +113,7 @@ class test_stat01(wttest.WiredTigerTestCase): value = "" for i in range(1, self.nentries): value = value + 1000 * "a" - cursor[key_populate(cursor, i)] = value + cursor[simple_key(cursor, i)] = value cursor.close() # Force the object to disk, otherwise we can't check the overflow count. @@ -140,9 +140,10 @@ class test_stat01(wttest.WiredTigerTestCase): # Test simple per-checkpoint statistics. def test_checkpoint_stats(self): + ds = SimpleDataSet(self, self.uri, self.nentries, + config=self.config, key_format=self.keyfmt) for name in ('first', 'second', 'third'): - config = self.config + ',key_format=' + self.keyfmt - helper.simple_populate(self, self.uri, config, self.nentries) + ds.populate() self.session.checkpoint('name=' + name) cursor = self.session.open_cursor( 'statistics:' + self.uri, None, 'checkpoint=' + name) diff --git a/src/third_party/wiredtiger/test/suite/test_stat02.py b/src/third_party/wiredtiger/test/suite/test_stat02.py index 047d2c74499..cecda7f1ddc 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat02.py +++ b/src/third_party/wiredtiger/test/suite/test_stat02.py @@ -28,22 +28,21 @@ import itertools, wiredtiger, wttest from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, SimpleLSMDataSet, ComplexDataSet, \ + ComplexLSMDataSet from wtscenario import make_scenarios from wiredtiger import stat -from helper import complex_populate, complex_populate_lsm, simple_populate # test_stat02.py # Statistics cursor configurations. class test_stat_cursor_config(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_config' uri = [ - ('file', dict(uri='file:' + pfx, pop=simple_populate, cfg='')), - ('table', dict(uri='table:' + pfx, pop=simple_populate, cfg='')), - ('table-lsm', - dict(uri='table:' + pfx, pop=simple_populate, cfg=',type=lsm')), - ('complex', dict(uri='table:' + pfx, pop=complex_populate, cfg='')), - ('complex-lsm', - dict(uri='table:' + pfx, pop=complex_populate_lsm, cfg='')) + ('file', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('table', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('table-lsm', dict(uri='table:' + pfx, dataset=SimpleLSMDataSet)), + ('complex', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('complex-lsm', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet)) ] data_config = [ ('none', dict(data_config='none', ok=[])), @@ -66,7 +65,7 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): # For each database/cursor configuration, confirm the right combinations # succeed or fail. def test_stat_cursor_config(self): - self.pop(self, self.uri, 'key_format=S' + self.cfg, 100) + self.dataset(self, self.uri, 100).populate() config = 'statistics=(' if self.cursor_config != 'empty': config = config + self.cursor_config @@ -85,7 +84,7 @@ class test_stat_cursor_conn_clear(wttest.WiredTigerTestCase): def test_stat_cursor_conn_clear(self): uri = 'table:' + self.pfx - complex_populate(self, uri, 'key_format=S', 100) + ComplexDataSet(self, uri, 100).populate() # cursor_insert should clear # cache_bytes_dirty should not clear @@ -103,17 +102,17 @@ class test_stat_cursor_dsrc_clear(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_dsrc_clear' uri = [ - ('dsrc_clear_1', dict(uri='file:' + pfx, pop=simple_populate)), - ('dsrc_clear_2', dict(uri='table:' + pfx, pop=simple_populate)), - ('dsrc_clear_3', dict(uri='table:' + pfx, pop=complex_populate)), - ('dsrc_clear_4', dict(uri='table:' + pfx, pop=complex_populate_lsm)) + ('dsrc_clear_1', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('dsrc_clear_2', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('dsrc_clear_3', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('dsrc_clear_4', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(uri) conn_config = 'statistics=(all)' def test_stat_cursor_dsrc_clear(self): - self.pop(self, self.uri, 'key_format=S', 100) + self.dataset(self, self.uri, 100).populate() # cursor_insert should clear # @@ -132,17 +131,17 @@ class test_stat_cursor_fast(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_fast' uri = [ - ('fast_1', dict(uri='file:' + pfx, pop=simple_populate)), - ('fast_2', dict(uri='table:' + pfx, pop=simple_populate)), - ('fast_3', dict(uri='table:' + pfx, pop=complex_populate)), - ('fast_4', dict(uri='table:' + pfx, pop=complex_populate_lsm)) + ('fast_1', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('fast_2', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('fast_3', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('fast_4', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(uri) conn_config = 'statistics=(all)' def test_stat_cursor_fast(self): - self.pop(self, self.uri, 'key_format=S', 100) + self.dataset(self, self.uri, 100).populate() # A "fast" cursor shouldn't see the underlying btree statistics. # Check "fast" first, otherwise we get a copy of the statistics @@ -174,17 +173,17 @@ class test_stat_cursor_dsrc_error(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_dsrc_error' uri = [ - ('dsrc_error_1', dict(uri='file:' + pfx, pop=simple_populate)), - ('dsrc_error_2', dict(uri='table:' + pfx, pop=simple_populate)), - ('dsrc_error_3', dict(uri='table:' + pfx, pop=complex_populate)), - ('dsrc_error_4', dict(uri='table:' + pfx, pop=complex_populate_lsm)) + ('dsrc_error_1', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('dsrc_error_2', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('dsrc_error_3', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('dsrc_error_4', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(uri) conn_config = 'statistics=(all)' def test_stat_cursor_dsrc_error(self): - self.pop(self, self.uri, 'key_format=S', 100) + self.dataset(self, self.uri, 100).populate() args = ['all', 'fast'] for i in list(itertools.permutations(args, 2)): config = 'statistics=(' + i[0] + ',' + i[1] + ')' @@ -200,7 +199,7 @@ class test_stat_cursor_dsrc_cache_walk(wttest.WiredTigerTestCase): conn_config = 'statistics=(none)' def test_stat_cursor_dsrc_cache_walk(self): - simple_populate(self, self.uri, 'key_format=S', 100) + SimpleDataSet(self, self.uri, 100).populate() # Ensure that it's an error to get cache_walk stats if none is set msg = '/doesn\'t match the database statistics/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_stat03.py b/src/third_party/wiredtiger/test/suite/test_stat03.py index 1570ac4c9d8..7e5cf46ef13 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat03.py +++ b/src/third_party/wiredtiger/test/suite/test_stat03.py @@ -30,10 +30,7 @@ import itertools, wiredtiger, wttest from suite_subprocess import suite_subprocess from wiredtiger import stat -from helper import complex_populate, complex_populate_lsm, simple_populate -from helper import key_populate, complex_value_populate, value_populate -from helper import complex_populate_colgroup_count, complex_populate_index_count -from helper import complex_populate_colgroup_name, complex_populate_index_name +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_stat03.py @@ -41,14 +38,11 @@ from wtscenario import make_scenarios class test_stat_cursor_reset(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_reset' uri = [ - ('file-simple', - dict(uri='file:' + pfx, pop=simple_populate)), - ('table-simple', - dict(uri='table:' + pfx, pop=simple_populate)), - ('table-complex', - dict(uri='table:' + pfx, pop=complex_populate)), - ('table-complex-lsm', - dict(uri='table:' + pfx, pop=complex_populate_lsm)), + ('file-simple', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('table-simple', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('table-complex', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table:' + pfx, + dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(uri) @@ -59,25 +53,23 @@ class test_stat_cursor_reset(wttest.WiredTigerTestCase): 'statistics:' + uri, None, 'statistics=(all)') def test_stat_cursor_reset(self): + n = 100 + ds = self.dataset(self, self.uri, n) + ds.populate() + # The number of btree_entries reported is influenced by the # number of column groups and indices. Each insert will have # a multiplied effect. - if self.pop == simple_populate: + if self.dataset == SimpleDataSet: multiplier = 1 # no declared colgroup is like one big colgroup else: - multiplier = complex_populate_colgroup_count() + \ - complex_populate_index_count() - n = 100 - self.pop(self, self.uri, 'key_format=S', n) + multiplier = ds.colgroup_count() + ds.index_count() statc = self.stat_cursor(self.uri) self.assertEqual(statc[stat.dsrc.btree_entries][2], n * multiplier) c = self.session.open_cursor(self.uri) - c.set_key(key_populate(c, 200)) - if self.pop == simple_populate: - c.set_value(value_populate(c, 200)) - else: - c.set_value(tuple(complex_value_populate(c, 200))) + c.set_key(ds.key(200)) + c.set_value(ds.value(200)) c.insert() # Test that cursor reset re-loads the values. @@ -88,14 +80,12 @@ class test_stat_cursor_reset(wttest.WiredTigerTestCase): # For applications with indices and/or column groups, verify # that there is a way to count the base number of entries. - if self.pop != simple_populate: + if self.dataset != SimpleDataSet: statc.close() - statc = self.stat_cursor( - complex_populate_index_name(self, self.uri, 0)) + statc = self.stat_cursor(ds.index_name(0)) self.assertEqual(statc[stat.dsrc.btree_entries][2], n) statc.close() - statc = self.stat_cursor( - complex_populate_colgroup_name(self, self.uri, 0)) + statc = self.stat_cursor(ds.colgroup_name(0)) self.assertEqual(statc[stat.dsrc.btree_entries][2], n) statc.close() diff --git a/src/third_party/wiredtiger/test/suite/test_stat05.py b/src/third_party/wiredtiger/test/suite/test_stat05.py index 62562f78ed6..ef4d65e85e4 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat05.py +++ b/src/third_party/wiredtiger/test/suite/test_stat05.py @@ -30,8 +30,7 @@ import itertools, wiredtiger, wttest from suite_subprocess import suite_subprocess from wtscenario import make_scenarios from wiredtiger import stat -from helper import complex_populate, complex_populate_lsm, simple_populate -from helper import complex_value_populate, key_populate, value_populate +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet # test_stat05.py # Statistics cursor using size only @@ -40,17 +39,17 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): conn_config = 'statistics=(fast)' uri = [ - ('file', dict(uri='file:' + pfx, pop=simple_populate, cfg='')), - ('table', dict(uri='table:' + pfx, pop=simple_populate, cfg='')), - ('inmem', dict(uri='table:' + pfx, pop=simple_populate, cfg='', + ('file', dict(uri='file:' + pfx, dataset=SimpleDataSet, cfg='')), + ('table', dict(uri='table:' + pfx, dataset=SimpleDataSet, cfg='')), + ('inmem', dict(uri='table:' + pfx, dataset=SimpleDataSet, cfg='', conn_config = 'in_memory,statistics=(fast)')), - ('table-lsm', dict(uri='table:' + pfx, pop=simple_populate, - cfg=',type=lsm,lsm=(chunk_size=1MB,merge_min=2)', + ('table-lsm', dict(uri='table:' + pfx, dataset=SimpleDataSet, + cfg='lsm=(chunk_size=1MB,merge_min=2)', conn_config = 'statistics=(fast),eviction_dirty_target=99,eviction_dirty_trigger=99')), - ('complex', dict(uri='table:' + pfx, pop=complex_populate, cfg='')), + ('complex', dict(uri='table:' + pfx, dataset=ComplexDataSet, cfg='')), ('complex-lsm', - dict(uri='table:' + pfx, pop=complex_populate_lsm, - cfg=',lsm=(chunk_size=1MB,merge_min=2)', + dict(uri='table:' + pfx, dataset=ComplexLSMDataSet, + cfg='lsm=(chunk_size=1MB,merge_min=2)', conn_config = 'statistics=(fast),eviction_dirty_target=99,eviction_dirty_trigger=99')), ] @@ -68,17 +67,14 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): # the cursor open succeeds. Insert enough data that LSM tables to need to # switch and merge. def test_stat_cursor_size(self): - self.pop(self, self.uri, 'key_format=S' + self.cfg, 100) + ds = self.dataset(self, self.uri, 100, config=self.cfg) + ds.populate() self.openAndWalkStatCursor() cursor = self.session.open_cursor(self.uri, None) for i in range(100, 40000 + 1): if i % 100 == 0: self.openAndWalkStatCursor() - if self.pop == simple_populate: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) - else: - cursor[key_populate(cursor, i)] = \ - tuple(complex_value_populate(cursor, i)) + cursor[ds.key(i)] = ds.value(i) cursor.close() self.openAndWalkStatCursor() diff --git a/src/third_party/wiredtiger/test/suite/test_truncate01.py b/src/third_party/wiredtiger/test/suite/test_truncate01.py index 37cd1f12c79..2319eeddbef 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate01.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate01.py @@ -31,9 +31,8 @@ # import wiredtiger, wttest -from helper import confirm_empty,\ - key_populate, value_populate, simple_populate,\ - complex_populate, complex_value_populate +from helper import confirm_empty +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # Test truncation arguments. @@ -49,7 +48,7 @@ class test_truncate_arguments(wttest.WiredTigerTestCase): # either cursor specified, expect errors. def test_truncate_bad_args(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=S', 100) + SimpleDataSet(self, uri, 100).populate() msg = '/either a URI or start/stop cursors/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, None, None, None), msg) @@ -64,11 +63,12 @@ class test_truncate_arguments(wttest.WiredTigerTestCase): uri = self.type + self.name msg = '/requires key be set/' - simple_populate(self, uri, 'key_format=S', 100) + ds = SimpleDataSet(self, uri, 100) + ds.populate() c1 = self.session.open_cursor(uri, None) c2 = self.session.open_cursor(uri, None) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, @@ -89,13 +89,13 @@ class test_truncate_uri(wttest.WiredTigerTestCase): uri = self.type + self.name # A simple, one-file file or table object. - simple_populate(self, uri, 'key_format=S', 100) + SimpleDataSet(self, uri, 100).populate() self.session.truncate(uri, None, None, None) confirm_empty(self, uri) self.session.drop(uri, None) if self.type == "table:": - complex_populate(self, uri, 'key_format=S', 100) + ComplexDataSet(self, uri, 100).populate() self.session.truncate(uri, None, None, None) confirm_empty(self, uri) self.session.drop(uri, None) @@ -118,16 +118,17 @@ class test_truncate_cursor_order(wttest.WiredTigerTestCase): # Test an illegal order, then confirm that equal cursors works. def test_truncate_cursor_order(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt) + ds.populate() c1 = self.session.open_cursor(uri, None) c2 = self.session.open_cursor(uri, None) - c1.set_key(key_populate(c1, 20)) - c2.set_key(key_populate(c2, 10)) + c1.set_key(ds.key(20)) + c2.set_key(ds.key(10)) msg = '/the start cursor position is after the stop cursor position/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg) - c2.set_key(key_populate(c2, 20)) + c2.set_key(ds.key(20)) self.session.truncate(None, c1, c2, None) # Test truncation of cursors past the end of the object. @@ -150,22 +151,24 @@ class test_truncate_cursor_end(wttest.WiredTigerTestCase): uri = self.type + self.name # A simple, one-file file or table object. - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt) + ds.populate() c1 = self.session.open_cursor(uri, None) - c1.set_key(key_populate(c1, 1000)) + c1.set_key(ds.key(1000)) c2 = self.session.open_cursor(uri, None) - c2.set_key(key_populate(c2, 2000)) + c2.set_key(ds.key(2000)) self.session.truncate(None, c1, c2, None) self.assertEquals(c1.close(), 0) self.assertEquals(c2.close(), 0) self.session.drop(uri) if self.type == "table:": - complex_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = ComplexDataSet(self, uri, 100, key_format=self.keyfmt) + ds.populate() c1 = self.session.open_cursor(uri, None) - c1.set_key(key_populate(c1, 1000)) + c1.set_key(ds.key(1000)) c2 = self.session.open_cursor(uri, None) - c2.set_key(key_populate(c2, 2000)) + c2.set_key(ds.key(2000)) self.session.truncate(None, c1, c2, None) self.assertEquals(c1.close(), 0) self.assertEquals(c2.close(), 0) @@ -179,13 +182,12 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # The underlying table routines don't easily support 8t value types, limit # those tests to file objects. types = [ - ('file', dict(type='file:',\ - config='allocation_size=512,leaf_page_max=512,key_format=')), - ('file8t', dict(type='file:',\ - config='allocation_size=512,\ - leaf_page_max=512,value_format=8t,key_format=')), - ('table', dict(type='table:',\ - config='allocation_size=512,leaf_page_max=512,key_format=')), + ('file', dict(type='file:', valuefmt='S', + config='allocation_size=512,leaf_page_max=512')), + ('file8t', dict(type='file:', valuefmt='8t', + config='allocation_size=512,leaf_page_max=512')), + ('table', dict(type='table:', valuefmt='S', + config='allocation_size=512,leaf_page_max=512')), ] keyfmt = [ ('integer', dict(keyfmt='i')), @@ -204,18 +206,18 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): scenarios = make_scenarios(types, keyfmt, size, reopen) # Set a cursor key. - def cursorKey(self, uri, key): + def cursorKey(self, ds, uri, key): if key == -1: return None cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, key)) + cursor.set_key(ds.key(key)) return cursor # Truncate a range using cursors, and check the results. - def truncateRangeAndCheck(self, uri, begin, end, expected): + def truncateRangeAndCheck(self, ds, uri, begin, end, expected): self.pr('truncateRangeAndCheck: ' + str(begin) + ',' + str(end)) - cur1 = self.cursorKey(uri, begin) - cur2 = self.cursorKey(uri, end) + cur1 = self.cursorKey(ds, uri, begin) + cur2 = self.cursorKey(ds, uri, end) self.session.truncate(None, cur1, cur2, None) if not cur1: begin = 1 @@ -234,7 +236,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # Check the expected values against the object. cursor = self.session.open_cursor(uri, None) for i in range(begin, end + 1): - expected[key_populate(cursor, i)] = [0] + expected[ds.key(i)] = [0] for k, v in expected.iteritems(): cursor.set_key(k) if v == [0] and \ @@ -306,6 +308,10 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): (self.skip, self.skip) # middle to same middle ] + # Using this data set to compare only, it doesn't create or populate. + ds = SimpleDataSet(self, uri, 0, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.config) + # Build the layout we're going to test total = self.nentries for begin_skipped,begin_insert,end_skipped,end_insert in layout: @@ -332,15 +338,16 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # Create the object. self.session.create( - uri, 'value_format=S,' + self.config + self.keyfmt) + uri, self.config + ',key_format=' + self.keyfmt + + ',value_format=' + self.valuefmt) # Insert the records that aren't skipped or inserted. start = begin_skipped + begin_insert stop = self.nentries - (end_skipped + end_insert) cursor = self.session.open_cursor(uri, None) for i in range(start + 1, stop + 1): - k = key_populate(cursor, i) - v = value_populate(cursor, i) + k = ds.key(i) + v = ds.value(i) cursor[k] = v expected[k] = [v] cursor.close() @@ -355,33 +362,33 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): start = 0 for i in range(0, begin_skipped): start += 1 - k = key_populate(cursor, start) + k = ds.key(start) expected[k] = [0] # Optionally insert leading records. for i in range(0, begin_insert): start += 1 - k = key_populate(cursor, start) - v = value_populate(cursor, start) + k = ds.key(start) + v = ds.value(start) cursor[k] = v expected[k] = [v] # Optionally insert trailing skipped records. for i in range(0, end_skipped): stop += 1 - k = key_populate(cursor, stop) + k = ds.key(stop) expected[k] = [0] # Optionally insert trailing records. for i in range(0, end_insert): stop += 1 - k = key_populate(cursor, stop) - v = value_populate(cursor, stop) + k = ds.key(stop) + v = ds.value(stop) cursor[k] = v expected[k] = [v] cursor.close() - self.truncateRangeAndCheck(uri, begin, end, expected) + self.truncateRangeAndCheck(ds, uri, begin, end, expected) self.session.drop(uri, None) # Test truncation of complex tables using cursors. We can't do the kind of @@ -423,16 +430,16 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): ''' # Create the object. - complex_populate( - self, uri, self.config + self.keyfmt, self.nentries) + ds = ComplexDataSet(self, uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() # Build a dictionary of what the object should look like for # later comparison cursor = self.session.open_cursor(uri, None) expected = {} for i in range(1, self.nentries + 1): - expected[key_populate(cursor, i)] = \ - complex_value_populate(cursor, i) + expected[ds.key(i)] = ds.comparable_value(i) cursor.close() # Optionally close and re-open the object to get a disk image @@ -440,7 +447,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): if self.reopen: self.reopen_conn() - self.truncateRangeAndCheck(uri, begin, end, expected) + self.truncateRangeAndCheck(ds, uri, begin, end, expected) self.session.drop(uri, None) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_truncate02.py b/src/third_party/wiredtiger/test/suite/test_truncate02.py index 7184cd67242..73fed362354 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate02.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate02.py @@ -31,7 +31,7 @@ # import wiredtiger, wttest -from helper import key_populate, value_populate, simple_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_truncate_fast_delete @@ -46,8 +46,7 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): # of individual pages in the file. types = [ ('file', dict(type='file:', config=\ - 'allocation_size=512,leaf_page_max=512,' +\ - 'value_format=S,key_format=')), + 'allocation_size=512,leaf_page_max=512')), ] # This is all about testing the btree layer, not the schema layer, test @@ -123,15 +122,17 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): ''' # Create the object. - simple_populate(self, uri, self.config + self.keyfmt, self.nentries) + ds = SimpleDataSet(self, uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() # Optionally add a few overflow records so we block fast delete on # those pages. if self.overflow: cursor = self.session.open_cursor(uri, None, 'overwrite=false') for i in range(1, self.nentries, 3123): - cursor.set_key(key_populate(cursor, i)) - cursor.set_value(value_populate(cursor, i)) + cursor.set_key(ds.key(i)) + cursor.set_value(ds.value(i)) cursor.update() cursor.close() @@ -143,21 +144,21 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, 'overwrite=false') if self.readbefore: for i in range(1, self.nentries, 737): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) cursor.search() if self.writebefore: for i in range(1, self.nentries, 988): - cursor.set_key(key_populate(cursor, i)) - cursor.set_value(value_populate(cursor, i + 100)) + cursor.set_key(ds.key(i)) + cursor.set_value(ds.value(i + 100)) cursor.update() cursor.close() # Begin a transaction, and truncate a big range of rows. self.session.begin_transaction(None) start = self.session.open_cursor(uri, None) - start.set_key(key_populate(start, 10)) + start.set_key(ds.key(10)) end = self.session.open_cursor(uri, None) - end.set_key(key_populate(end, self.nentries - 10)) + end.set_key(ds.key(self.nentries - 10)) self.session.truncate(None, start, end, None) start.close() end.close() @@ -167,12 +168,12 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, 'overwrite=false') if self.readafter: for i in range(1, self.nentries, 1123): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) cursor.search() if self.writeafter: for i in range(1, self.nentries, 621): - cursor.set_key(key_populate(cursor, i)) - cursor.set_value(value_populate(cursor, i + 100)) + cursor.set_key(ds.key(i)) + cursor.set_value(ds.value(i + 100)) cursor.update() cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_truncate03.py b/src/third_party/wiredtiger/test/suite/test_truncate03.py index 49044ac5690..2b4628950b3 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate03.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate03.py @@ -30,7 +30,7 @@ # session level operations on tables import wiredtiger, wttest -from helper import key_populate, simple_populate, value_populate +from wtdataset import SimpleDataSet # A standalone test case that exercises address-deleted cells. class test_truncate_address_deleted(wttest.WiredTigerTestCase): @@ -39,15 +39,15 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # Use a small page size and lots of keys because we want to create lots # of individual pages in the file. nentries = 10000 - config = 'allocation_size=512,' +\ - 'leaf_page_max=512,value_format=S,key_format=S' + config = 'allocation_size=512,leaf_page_max=512' # address_deleted routine: # Create an object that has a bunch of address-deleted cells on disk. # Recover the object, and turn the address-deleted cells into free pages. def address_deleted(self): # Create the object, force it to disk, and verify the object. - simple_populate(self, self.uri, self.config, self.nentries) + ds = SimpleDataSet(self, self.uri, self.nentries, config=self.config) + ds.populate() self.reopen_conn() self.session.verify(self.uri) @@ -59,9 +59,9 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # Truncate a big range of rows; the leaf pages aren't in memory, so # leaf page references will be deleted without being read. start = self.session.open_cursor(self.uri, None) - start.set_key(key_populate(start, 10)) + start.set_key(ds.key(10)) end = self.session.open_cursor(self.uri, None) - end.set_key(key_populate(end, self.nentries - 10)) + end.set_key(ds.key(self.nentries - 10)) self.session.truncate(None, start, end, None) self.assertEqual(start.close(), 0) self.assertEqual(end.close(), 0) @@ -78,7 +78,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # we get a good look at all the internal pages and the address-deleted # cells. cursor = self.session.open_cursor(self.uri, None) - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) cursor.set_value("changed value") self.assertEqual(cursor.update(), 0) cursor.reset() @@ -88,12 +88,13 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # Checkpoint, freeing the pages. self.session.checkpoint() + return ds # Test object creation, recovery, and conversion of address-deleted cells # into free pages. def test_truncate_address_deleted_free(self): # Create the object on disk. - self.address_deleted() + ds = self.address_deleted() # That's all just verify that worked. self.session.verify(self.uri) @@ -103,7 +104,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # empty pages by a reader after the underlying leaf pages are removed. def test_truncate_address_deleted_empty_page(self): # Create the object on disk. - self.address_deleted() + ds = self.address_deleted() # Open a cursor and update pages in the middle of the range, forcing # creation of empty pages (once the underlying leaf page is freed, we @@ -111,11 +112,11 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # the value as well as write the page and get it back. cursor = self.session.open_cursor(self.uri, None) for i in range(3000, 7000, 137): - k = key_populate(cursor, i) + k = ds.key(i) v = 'changed value: ' + str(i) cursor[k] = v for i in range(3000, 7000, 137): - k = key_populate(cursor, i) + k = ds.key(i) v = 'changed value: ' + str(i) cursor.set_key(k) self.assertEqual(cursor.search(), 0) @@ -128,7 +129,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None) for i in range(3000, 7000, 137): - k = key_populate(cursor, i) + k = ds.key(i) v = 'changed value: ' + str(i) cursor.set_key(k) self.assertEqual(cursor.search(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_txn06.py b/src/third_party/wiredtiger/test/suite/test_txn06.py index e4636e40e2e..2bff97f6aac 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn06.py +++ b/src/third_party/wiredtiger/test/suite/test_txn06.py @@ -30,7 +30,7 @@ # Transactions: test long-running snapshots from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wiredtiger, wttest class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): @@ -47,7 +47,7 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): def test_long_running(self): # Populate a table - simple_populate(self, self.source_uri, 'key_format=S', self.nrows) + SimpleDataSet(self, self.source_uri, self.nrows).populate() # Now scan the table and copy the rows into a new table c_src = self.session.create(self.uri, "key_format=S") diff --git a/src/third_party/wiredtiger/test/suite/test_txn11.py b/src/third_party/wiredtiger/test/suite/test_txn11.py index dfb02799434..147bf3a76c0 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn11.py +++ b/src/third_party/wiredtiger/test/suite/test_txn11.py @@ -31,7 +31,7 @@ import fnmatch, os, time from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wttest class test_txn11(wttest.WiredTigerTestCase, suite_subprocess): @@ -64,7 +64,7 @@ class test_txn11(wttest.WiredTigerTestCase, suite_subprocess): def test_ops(self): # Populate a table - simple_populate(self, self.source_uri, 'key_format=S', self.nrows) + SimpleDataSet(self, self.source_uri, self.nrows).populate() # Run forced checkpoints self.run_checkpoints() diff --git a/src/third_party/wiredtiger/test/suite/test_upgrade.py b/src/third_party/wiredtiger/test/suite/test_upgrade.py index 1c2e3f6cda8..4eb6a9e6817 100644 --- a/src/third_party/wiredtiger/test/suite/test_upgrade.py +++ b/src/third_party/wiredtiger/test/suite/test_upgrade.py @@ -28,7 +28,7 @@ import os, time import wiredtiger, wttest -from helper import complex_populate, simple_populate +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # test_upgrade.py @@ -42,9 +42,9 @@ class test_upgrade(wttest.WiredTigerTestCase): ]) # Populate an object, then upgrade it. - def upgrade(self, populate, with_cursor): + def upgrade(self, dataset, with_cursor): uri = self.uri + self.name - populate(self, uri, 'key_format=S', 10) + dataset(self, uri, 10).populate() # Open cursors should cause failure. if with_cursor: @@ -59,13 +59,13 @@ class test_upgrade(wttest.WiredTigerTestCase): # Test upgrade of an object. def test_upgrade(self): # Simple file or table object. - self.upgrade(simple_populate, False) - self.upgrade(simple_populate, True) + self.upgrade(SimpleDataSet, False) + self.upgrade(SimpleDataSet, True) # A complex, multi-file table object. if self.uri == "table:": - self.upgrade(complex_populate, False) - self.upgrade(complex_populate, True) + self.upgrade(ComplexDataSet, False) + self.upgrade(ComplexDataSet, True) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util02.py b/src/third_party/wiredtiger/test/suite/test_util02.py index 13f52d6aeea..7aa24605ed1 100644 --- a/src/third_party/wiredtiger/test/suite/test_util02.py +++ b/src/third_party/wiredtiger/test/suite/test_util02.py @@ -29,8 +29,8 @@ import string, os import wiredtiger, wttest from suite_subprocess import suite_subprocess +from wtdataset import ComplexDataSet from wtscenario import make_scenarios -from helper import complex_populate # test_util02.py # Utilities: wt load @@ -169,7 +169,7 @@ class test_load_commandline(wttest.WiredTigerTestCase, suite_subprocess): def load_commandline(self, args, fail): errfile= "errfile" - complex_populate(self, self.uri, "key_format=S,value_format=S", 20) + ComplexDataSet(self, self.uri, 20).populate() self.runWt(["dump", self.uri], outfilename="dump.out") loadargs = ["load", "-f", "dump.out"] + args self.runWt(loadargs, errfilename=errfile, failure=fail) diff --git a/src/third_party/wiredtiger/test/suite/test_util13.py b/src/third_party/wiredtiger/test/suite/test_util13.py index 9804dc700ba..7890d4fdb1b 100644 --- a/src/third_party/wiredtiger/test/suite/test_util13.py +++ b/src/third_party/wiredtiger/test/suite/test_util13.py @@ -28,11 +28,9 @@ import os, re, string from suite_subprocess import suite_subprocess -import itertools, wiredtiger, wttest +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet +import wiredtiger, wttest -from helper import complex_populate_cgconfig, complex_populate_cgconfig_lsm -from helper import simple_populate -from helper import complex_populate_check, simple_populate_check from wtscenario import make_scenarios # test_util13.py @@ -52,25 +50,23 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): # Select table configuration settings that are not the default. # types = [ - ('file-simple', dict(uri='file:' + pfx, pop=simple_populate, - populate_check=simple_populate_check, - table_config='prefix_compression_min=3', cfg='')), - ('lsm-simple', dict(uri='lsm:' + pfx, pop=simple_populate, - populate_check=simple_populate_check, - table_config='lsm=(bloom_bit_count=29)', - cfg='bloom_bit_count=29')), - ('table-simple', dict(uri='table:' + pfx, pop=simple_populate, - populate_check=simple_populate_check, - table_config='split_pct=50', cfg='')), + ('file-simple', dict(uri='file:' + pfx, dataset=SimpleDataSet, + table_config='prefix_compression_min=3', cfg='', + cg_config='')), + ('lsm-simple', dict(uri='lsm:' + pfx, dataset=SimpleDataSet, + table_config='lsm=(bloom_bit_count=29)', cfg='bloom_bit_count=29', + cg_config='')), + ('table-simple', dict(uri='table:' + pfx, dataset=SimpleDataSet, + table_config='split_pct=50', cfg='', + cg_config='')), ('table-complex', - dict(uri='table:' + pfx, pop=complex_populate_cgconfig, - populate_check=complex_populate_check, - table_config='allocation_size=512B', cfg='')), + dict(uri='table:' + pfx, dataset=ComplexDataSet, + table_config='allocation_size=512B', cfg='', + cg_config='allocation_size=512B')), ('table-complex-lsm', - dict(uri='table:' + pfx, pop=complex_populate_cgconfig_lsm, - populate_check=complex_populate_check, - table_config='lsm=(merge_max=5)', - cfg='merge_max=5')), + dict(uri='table:' + pfx, dataset=ComplexLSMDataSet, + table_config='lsm=(merge_max=5)', cfg='merge_max=5', + cg_config='lsm=(merge_max=5)')) ] scenarios = make_scenarios(types) @@ -83,7 +79,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): #print "compare_config Expected config " #print expected_cfg cfg_orig = actual_cfg - if self.pop != simple_populate: + if self.dataset != SimpleDataSet: # # If we have a complex config, strip out the colgroups and # columns from the config. Doing so allows us to keep the @@ -127,7 +123,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): break return True - def load_recheck(self, expect_subset, dump_out): + def load_recheck(self, ds, expect_subset, dump_out): newdump = "newdump.out" os.mkdir(self.dir) self.runWt(['-h', self.dir, 'load', '-f', dump_out]) @@ -135,7 +131,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): conn = self.wiredtiger_open(self.dir) session = conn.open_session() cursor = session.open_cursor(self.uri, None, None) - self.populate_check + ds.check() conn.close() dumpargs = ["-h"] dumpargs.append(self.dir) @@ -150,8 +146,9 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): # The number of btree_entries reported is influenced by the # number of column groups and indices. Each insert will have # a multiplied effect. - self.pop(self, self.uri, - 'key_format=S,value_format=S,' + self.table_config, self.nentries) + ds = self.dataset(self, self.uri, self.nentries, + config=self.table_config, cgconfig=self.cg_config) + ds.populate() ver = wiredtiger.wiredtiger_version() verstring = str(ver[1]) + '.' + str(ver[2]) + '.' + str(ver[3]) @@ -166,7 +163,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): expectout.write('Header\n') expectout.write(self.uri + '\n') # Check the config on the colgroup itself for complex tables. - if self.pop != simple_populate: + if self.dataset != SimpleDataSet: expectout.write('key_format=S\n') expectout.write('colgroup:' + self.pfx + ':cgroup1\n') if self.cfg == '': @@ -182,7 +179,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): self.runWt(dumpargs, outfilename=outfile) self.assertTrue(self.compare_files(expectfile, outfile)) - self.assertTrue(self.load_recheck(expectfile, outfile)) + self.assertTrue(self.load_recheck(ds, expectfile, outfile)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wtdataset.py b/src/third_party/wiredtiger/test/suite/wtdataset.py new file mode 100644 index 00000000000..cb77735510c --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/wtdataset.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +class BaseDataSet(object): + """ + BaseDataSet is an abstract base class for other *DataSet classes. + An object of this type should not be created directly. These classes + represent test data sets that can be used to populate tables and + to check the contents of existing tables. + """ + def __init__(self, testcase, uri, rows, **kwargs): + self.testcase = testcase + self.uri = uri + self.rows = rows + self.key_format = kwargs.get('key_format', 'S') + self.value_format = kwargs.get('value_format', 'S') + self.config = kwargs.get('config', '') + + def create(self): + self.testcase.session.create(self.uri, 'key_format=' + self.key_format + + ',value_format=' + self.value_format + + ',' + self.config) + + def fill(self): + c = self.testcase.session.open_cursor(self.uri, None) + for i in range(1, self.rows + 1): + c[self.key(i)] = self.value(i) + c.close() + + def postfill(self): + pass + + @classmethod + def is_lsm(cls): + return False + + def populate(self): + self.testcase.pr('populate: ' + self.uri + ' with ' + + str(self.rows) + ' rows') + self.create() + self.fill() + self.postfill() + + # Create a key for a Simple or Complex data set. + @staticmethod + def key_by_format(i, key_format): + if key_format == 'i' or key_format == 'r' or key_format == 'u': + return i + elif key_format == 'S': + return str('%015d' % i) + else: + raise AssertionError( + 'key: object has unexpected format: ' + key_format) + + # Create a value for a Simple data set. + @staticmethod + def value_by_format(i, value_format): + if value_format == 'i' or value_format == 'r' or value_format == 'u': + return i + elif value_format == 'S': + return str(i) + ': abcdefghijklmnopqrstuvwxyz' + elif value_format == '8t': + value = ( + 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xaa, 0xab, + 0xac, 0xad, 0xae, 0xaf, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, + 0xb7, 0xb8, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf) + return value[i % len(value)] + else: + raise AssertionError( + 'value: object has unexpected format: ' + + value_format) + + # Create a key for this data set. Simple and Complex data sets have + # the same key space. + def key(self, i): + return BaseDataSet.key_by_format(i, self.key_format) + + def check(self): + self.testcase.pr('check: ' + self.uri) + cursor = self.testcase.session.open_cursor(self.uri, None) + self.check_cursor(cursor) + cursor.close() + +class SimpleDataSet(BaseDataSet): + """ + SimpleDataSet creates a table with a single key and value that is + populated with predefined data, up to the requested number of rows. + key_format and value_format may be set in the constructor to + override the simple string defaults. + """ + def __init__(self, testcase, uri, rows, **kwargs): + super(SimpleDataSet, self).__init__(testcase, uri, rows, **kwargs) + + def value(self, i): + return BaseDataSet.value_by_format(i, self.value_format) + + def check_cursor(self, cursor): + i = 0 + for key, val in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + if cursor.value_format == '8t' and val == 0: # deleted + continue + self.testcase.assertEqual(val, self.value(i)) + self.testcase.assertEqual(i, self.rows) + +class SimpleLSMDataSet(SimpleDataSet): + """ + SimpleLSMDataSet is identical to SimpleDataSet, but using LSM files + via the type=lsm configuration. + """ + def __init__(self, testcase, uri, rows, **kwargs): + kwargs['config'] = kwargs.get('config', '') + ',type=lsm' + super(SimpleLSMDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + @classmethod + def is_lsm(cls): + return True + +class SimpleIndexDataSet(SimpleDataSet): + """ + SimpleIndexDataSet is identical to SimpleDataSet, adding one index + that maps the value to the key. + """ + def __init__(self, testcase, uri, rows, **kwargs): + self.indexname = 'index:' + uri.split(":")[1] + ':index1' + self.origconfig = kwargs.get('config', '') + kwargs['config'] = self.origconfig + ',columns=(key0,value0)' + super(SimpleIndexDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + def create(self): + super(SimpleIndexDataSet, self).create() + self.testcase.session.create(self.indexname, 'columns=(value0,key0),' + + self.origconfig) + + def check(self): + BaseDataSet.check(self) + + # Check values in the index. + idxcursor = self.testcase.session.open_cursor(self.indexname) + for i in range(1, self.rows + 1): + k = self.key(i) + v = self.value(i) + ik = (v, k) # The index key is columns=(v,k). + self.testcase.assertEqual(v, idxcursor[ik]) + idxcursor.close() + +class SimpleIndexLSMDataSet(SimpleIndexDataSet): + """ + SimpleIndexLSMDataSet is identical to SimpleIndexDataSet, but + using LSM files. + """ + def __init__(self, testcase, uri, rows, **kwargs): + kwargs['config'] = kwargs.get('config', '') + ',type=lsm' + super(SimpleIndexLSMDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + @classmethod + def is_lsm(cls): + return True + +class ComplexDataSet(BaseDataSet): + """ + ComplexDataSet populates a table with a mixed set of indices + and column groups. Some indices are created before the + table is populated, some after. + """ + def __init__(self, testcase, uri, rows, **kwargs): + self.indexlist = [ + ['indx1', 'column2'], + ['indx2', 'column3'], + ['indx3', 'column4'], + ['indx4', 'column2,column4'], + ['indx5', 'column3,column5'], + ['indx6', 'column3,column5,column4']] + self.cglist = [ + ['cgroup1', 'column2'], + ['cgroup2', 'column3'], + ['cgroup3', 'column4'], + ['cgroup4', 'column2,column3'], + ['cgroup5', 'column3,column4'], + ['cgroup6', 'column2,column4,column5']] + self.cgconfig = kwargs.pop('cgconfig', '') + config = kwargs.get('config', '') + config += ',columns=(record,column2,column3,column4,column5),' + \ + 'colgroups=(cgroup1,cgroup2,cgroup3,cgroup4,cgroup5,cgroup6)' + kwargs['config'] = config + kwargs['value_format'] = 'SiSS' + super(ComplexDataSet, self).__init__(testcase, uri, rows, **kwargs) + + def create(self): + config = 'key_format=' + self.key_format + \ + ',value_format=' + self.value_format + ',' + self.config + session = self.testcase.session + ##self.testcase.tty('URI=' + self.uri + 'CONFIG=' + config) + session.create(self.uri, config) + tablepart = self.uri.split(":")[1] + ':' + for cg in self.cglist: + session.create('colgroup:' + tablepart + cg[0], + ',columns=(' + cg[1] + '),' + self.cgconfig) + for index in self.indexlist[0:4]: + session.create('index:' + tablepart + index[0], + ',columns=(' + index[1] + '),' + self.config) + + def postfill(self): + # add some indices after filling the table + tablepart = self.uri.split(":")[1] + ':' + session = self.testcase.session + for index in self.indexlist[4:]: + session.create('index:' + tablepart + index[0], + ',columns=(' + index[1] + ')') + + def colgroup_count(self): + return len(self.cglist) + + def colgroup_name(self, i): + return 'colgroup:' + self.uri.split(":")[1] + ':' + self.cglist[i][0] + + def index_count(self): + return len(self.indexlist) + + def index_name(self, i): + return 'index:' + self.uri.split(":")[1] + ':' + self.indexlist[i][0] + + # A value suitable for checking the value returned by a cursor, as + # cursor.get_value() returns a list. + def comparable_value(self, i): + return [str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%26], + i, + str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%23], + str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%18]] + + # A value suitable for assigning to a cursor, as + # cursor.set_value() expects a tuple when there it is used with + # a single argument and the value is composite. + def value(self, i): + return tuple(self.comparable_value(i)) + + def check_cursor(self, cursor): + i = 0 + for key, s1, i2, s3, s4 in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + v = self.value(i) + self.testcase.assertEqual(s1, v[0]) + self.testcase.assertEqual(i2, v[1]) + self.testcase.assertEqual(s3, v[2]) + self.testcase.assertEqual(s4, v[3]) + self.testcase.assertEqual(i, self.rows) + +class ComplexLSMDataSet(ComplexDataSet): + """ + ComplexLSMDataSet is identical to ComplexDataSet, but using LSM files. + """ + def __init__(self, testcase, uri, rows, **kwargs): + kwargs['cgconfig'] = kwargs.get('cgconfig', '') + ',type=lsm' + super(ComplexLSMDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + @classmethod + def is_lsm(cls): + return True + +# create a key based on a cursor as a shortcut to creating a SimpleDataSet +def simple_key(cursor, i): + return BaseDataSet.key_by_format(i, cursor.key_format) + +# create a value based on a cursor as a shortcut to creating a SimpleDataSet +def simple_value(cursor, i): + return BaseDataSet.value_by_format(i, cursor.value_format) + +# create a key based on a cursor as a shortcut to creating a ComplexDataSet +def complex_key(cursor, i): + return BaseDataSet.key_by_format(i, cursor.key_format) |